http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java index 78b2e8b..ebe613e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.parse.TableSample; import org.apache.hadoop.hive.ql.plan.Explain.Level; -import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; import org.apache.hadoop.hive.serde.serdeConstants; @@ -397,29 +396,4 @@ public class TableScanDesc extends AbstractOperatorDesc { return opProps; } - public class TableScanOperatorExplainVectorization extends OperatorExplainVectorization { - - private final TableScanDesc tableScanDesc; - private final VectorTableScanDesc vectorTableScanDesc; - - public TableScanOperatorExplainVectorization(TableScanDesc tableScanDesc, VectorDesc vectorDesc) { - // Native vectorization supported. - super(vectorDesc, true); - this.tableScanDesc = tableScanDesc; - vectorTableScanDesc = (VectorTableScanDesc) vectorDesc; - } - - @Explain(vectorization = Vectorization.EXPRESSION, displayName = "projectedOutputColumns", explainLevels = { Level.DEFAULT, Level.EXTENDED }) - public String getProjectedOutputColumns() { - return Arrays.toString(vectorTableScanDesc.getProjectedOutputColumns()); - } - } - - @Explain(vectorization = Vectorization.OPERATOR, displayName = "TableScan Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED }) - public TableScanOperatorExplainVectorization getTableScanVectorization() { - if (vectorDesc == null) { - return null; - } - return new TableScanOperatorExplainVectorization(this, vectorDesc); - } }
http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java index a037ea3..7a70e6b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hive.ql.exec.tez.DagUtils; import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.hive.ql.plan.Explain.Level; -import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; + /** * TezWork. This class encapsulates all the work objects that can be executed @@ -49,8 +49,7 @@ import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; * */ @SuppressWarnings("serial") -@Explain(displayName = "Tez", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }, - vectorization = Vectorization.SUMMARY_PATH) +@Explain(displayName = "Tez", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public class TezWork extends AbstractOperatorDesc { public enum VertexType { @@ -108,8 +107,7 @@ public class TezWork extends AbstractOperatorDesc { /** * getWorkMap returns a map of "vertex name" to BaseWork */ - @Explain(displayName = "Vertices", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }, - vectorization = Vectorization.SUMMARY_PATH) + @Explain(displayName = "Vertices", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public Map<String, BaseWork> getWorkMap() { Map<String, BaseWork> result = new LinkedHashMap<String, BaseWork>(); for (BaseWork w: getAllWork()) { @@ -308,8 +306,7 @@ public class TezWork extends AbstractOperatorDesc { } } - @Explain(displayName = "Edges", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }, - vectorization = Vectorization.SUMMARY_PATH) + @Explain(displayName = "Edges", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public Map<String, List<Dependency>> getDependencyMap() { Map<String, List<Dependency>> result = new LinkedHashMap<String, List<Dependency>>(); for (Map.Entry<BaseWork, List<BaseWork>> entry: invertedWorkGraph.entrySet()) { http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorAppMasterEventDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorAppMasterEventDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorAppMasterEventDesc.java deleted file mode 100644 index 2e11321..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorAppMasterEventDesc.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -/** - * VectorAppMasterEventDesc. - * - * Extra parameters beyond AppMasterEventDesc just for the VectorAppMasterEventDescOperator. - * - * We don't extend AppMasterEventDesc because the base OperatorDesc doesn't support - * clone and adding it is a lot work for little gain. - */ -public class VectorAppMasterEventDesc extends AbstractVectorDesc { - - private static long serialVersionUID = 1L; - - public VectorAppMasterEventDesc() { - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorDesc.java index 078408c..3a2efdb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorDesc.java @@ -22,9 +22,4 @@ import java.io.Serializable; public interface VectorDesc extends Serializable, Cloneable { public Object clone() throws CloneNotSupportedException; - - public void setVectorOp(Class<?> vectorOpClass); - - public Class<?> getVectorOpClass(); - } http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorFileSinkDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorFileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorFileSinkDesc.java deleted file mode 100644 index 325ac91..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorFileSinkDesc.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -/** - * VectorFileSinkDesc. - * - * Extra parameters beyond FileSinkDesc just for the VectorFileSinkOperator. - * - * We don't extend FileSinkDesc because the base OperatorDesc doesn't support - * clone and adding it is a lot work for little gain. - */ -public class VectorFileSinkDesc extends AbstractVectorDesc { - - private static long serialVersionUID = 1L; - - public VectorFileSinkDesc() { - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorFilterDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorFilterDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorFilterDesc.java deleted file mode 100644 index 6feed84..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorFilterDesc.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; - -/** - * VectorFilterDesc. - * - * Extra parameters beyond FilterDesc just for the VectorFilterOperator. - * - * We don't extend FilterDesc because the base OperatorDesc doesn't support - * clone and adding it is a lot work for little gain. - */ -public class VectorFilterDesc extends AbstractVectorDesc { - - private static long serialVersionUID = 1L; - - private VectorExpression predicateExpression; - - public VectorFilterDesc() { - } - - public void setPredicateExpression(VectorExpression predicateExpression) { - this.predicateExpression = predicateExpression; - } - - public VectorExpression getPredicateExpression() { - return predicateExpression; - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java index f8554e2..08f8ebf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hive.ql.plan; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; - /** * VectorGroupByDesc. * @@ -62,10 +59,6 @@ public class VectorGroupByDesc extends AbstractVectorDesc { private boolean isVectorOutput; - private VectorExpression[] keyExpressions; - private VectorAggregateExpression[] aggregators; - private int[] projectedOutputColumns; - public VectorGroupByDesc() { this.processingMode = ProcessingMode.NONE; this.isVectorOutput = false; @@ -86,30 +79,6 @@ public class VectorGroupByDesc extends AbstractVectorDesc { this.isVectorOutput = isVectorOutput; } - public void setKeyExpressions(VectorExpression[] keyExpressions) { - this.keyExpressions = keyExpressions; - } - - public VectorExpression[] getKeyExpressions() { - return keyExpressions; - } - - public void setAggregators(VectorAggregateExpression[] aggregators) { - this.aggregators = aggregators; - } - - public VectorAggregateExpression[] getAggregators() { - return aggregators; - } - - public void setProjectedOutputColumns(int[] projectedOutputColumns) { - this.projectedOutputColumns = projectedOutputColumns; - } - - public int[] getProjectedOutputColumns() { - return projectedOutputColumns; - } - /** * Which ProcessingMode for VectorGroupByOperator? * http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorLimitDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorLimitDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorLimitDesc.java deleted file mode 100644 index c9bc45a..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorLimitDesc.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -/** - * VectorLimitDesc. - * - * Extra parameters beyond LimitDesc just for the VectorLimitOperator. - * - * We don't extend LimitDesc because the base OperatorDesc doesn't support - * clone and adding it is a lot work for little gain. - */ -public class VectorLimitDesc extends AbstractVectorDesc { - - private static long serialVersionUID = 1L; - - public VectorLimitDesc() { - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java index 1252140..8ea230f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java @@ -18,13 +18,9 @@ package org.apache.hadoop.hive.ql.plan; -import java.util.List; - import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import com.google.common.base.Preconditions; - /** * VectorGroupByDesc. * @@ -83,38 +79,23 @@ public class VectorMapJoinDesc extends AbstractVectorDesc { } } - public static enum OperatorVariation { - NONE, - INNER_BIG_ONLY, - INNER, - LEFT_SEMI, - OUTER - } - private HashTableImplementationType hashTableImplementationType; private HashTableKind hashTableKind; private HashTableKeyType hashTableKeyType; - private OperatorVariation operatorVariation; private boolean minMaxEnabled; - private VectorMapJoinInfo vectorMapJoinInfo; - public VectorMapJoinDesc() { hashTableImplementationType = HashTableImplementationType.NONE; hashTableKind = HashTableKind.NONE; hashTableKeyType = HashTableKeyType.NONE; - operatorVariation = OperatorVariation.NONE; minMaxEnabled = false; - vectorMapJoinInfo = null; } public VectorMapJoinDesc(VectorMapJoinDesc clone) { this.hashTableImplementationType = clone.hashTableImplementationType; this.hashTableKind = clone.hashTableKind; this.hashTableKeyType = clone.hashTableKeyType; - this.operatorVariation = clone.operatorVariation; this.minMaxEnabled = clone.minMaxEnabled; - this.vectorMapJoinInfo = clone.vectorMapJoinInfo; } public HashTableImplementationType hashTableImplementationType() { @@ -141,14 +122,6 @@ public class VectorMapJoinDesc extends AbstractVectorDesc { this.hashTableKeyType = hashTableKeyType; } - public OperatorVariation operatorVariation() { - return operatorVariation; - } - - public void setOperatorVariation(OperatorVariation operatorVariation) { - this.operatorVariation = operatorVariation; - } - public boolean minMaxEnabled() { return minMaxEnabled; } @@ -156,87 +129,4 @@ public class VectorMapJoinDesc extends AbstractVectorDesc { public void setMinMaxEnabled(boolean minMaxEnabled) { this.minMaxEnabled = minMaxEnabled; } - - public void setVectorMapJoinInfo(VectorMapJoinInfo vectorMapJoinInfo) { - Preconditions.checkState(vectorMapJoinInfo != null); - this.vectorMapJoinInfo = vectorMapJoinInfo; - } - - public VectorMapJoinInfo getVectorMapJoinInfo() { - return vectorMapJoinInfo; - } - - private boolean isVectorizationMapJoinNativeEnabled; - private String engine; - private boolean oneMapJoinCondition; - private boolean hasNullSafes; - private boolean isFastHashTableEnabled; - private boolean isHybridHashJoin; - private boolean supportsKeyTypes; - private List<String> notSupportedKeyTypes; - private boolean isEmptyKey; - private boolean smallTableExprVectorizes; - - public void setIsVectorizationMapJoinNativeEnabled(boolean isVectorizationMapJoinNativeEnabled) { - this.isVectorizationMapJoinNativeEnabled = isVectorizationMapJoinNativeEnabled; - } - public boolean getIsVectorizationMapJoinNativeEnabled() { - return isVectorizationMapJoinNativeEnabled; - } - public void setEngine(String engine) { - this.engine = engine; - } - public String getEngine() { - return engine; - } - public void setOneMapJoinCondition(boolean oneMapJoinCondition) { - this.oneMapJoinCondition = oneMapJoinCondition; - } - public boolean getOneMapJoinCondition() { - return oneMapJoinCondition; - } - public void setHasNullSafes(boolean hasNullSafes) { - this.hasNullSafes = hasNullSafes; - } - public boolean getHasNullSafes() { - return hasNullSafes; - } - public void setSupportsKeyTypes(boolean supportsKeyTypes) { - this.supportsKeyTypes = supportsKeyTypes; - } - public boolean getSupportsKeyTypes() { - return supportsKeyTypes; - } - public void setNotSupportedKeyTypes(List<String> notSupportedKeyTypes) { - this.notSupportedKeyTypes = notSupportedKeyTypes; - } - public List<String> getNotSupportedKeyTypes() { - return notSupportedKeyTypes; - } - public void setIsEmptyKey(boolean isEmptyKey) { - this.isEmptyKey = isEmptyKey; - } - public boolean getIsEmptyKey() { - return isEmptyKey; - } - public void setSmallTableExprVectorizes(boolean smallTableExprVectorizes) { - this.smallTableExprVectorizes = smallTableExprVectorizes; - } - public boolean getSmallTableExprVectorizes() { - return smallTableExprVectorizes; - } - - public void setIsFastHashTableEnabled(boolean isFastHashTableEnabled) { - this.isFastHashTableEnabled = isFastHashTableEnabled; - } - public boolean getIsFastHashTableEnabled() { - return isFastHashTableEnabled; - } - public void setIsHybridHashJoin(boolean isHybridHashJoin) { - this.isHybridHashJoin = isHybridHashJoin; - } - public boolean getIsHybridHashJoin() { - return isHybridHashJoin; - } - } http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java deleted file mode 100644 index 2cf2e72..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java +++ /dev/null @@ -1,169 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import org.apache.hadoop.hive.ql.exec.vector.VectorColumnOutputMapping; -import org.apache.hadoop.hive.ql.exec.vector.VectorColumnSourceMapping; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; - -/** - * VectorMapJoinInfo. - * - * A convenience data structure that has information needed to vectorize map join. - * - * It is created by the Vectorizer when it is determining whether it can specialize so the - * information doesn't have to be recreated again and again by the VectorMapJoinOperator's - * constructors and later during execution. - */ -public class VectorMapJoinInfo { - - private static long serialVersionUID = 1L; - - private int[] bigTableKeyColumnMap; - private String[] bigTableKeyColumnNames; - private TypeInfo[] bigTableKeyTypeInfos; - private VectorExpression[] bigTableKeyExpressions; - - private int[] bigTableValueColumnMap; - private String[] bigTableValueColumnNames; - private TypeInfo[] bigTableValueTypeInfos; - private VectorExpression[] bigTableValueExpressions; - - private VectorColumnOutputMapping bigTableRetainedMapping; - private VectorColumnOutputMapping bigTableOuterKeyMapping; - private VectorColumnSourceMapping smallTableMapping; - - private VectorColumnSourceMapping projectionMapping; - - public VectorMapJoinInfo() { - bigTableKeyColumnMap = null; - bigTableKeyColumnNames = null; - bigTableKeyTypeInfos = null; - bigTableKeyExpressions = null; - - bigTableValueColumnMap = null; - bigTableValueColumnNames = null; - bigTableValueTypeInfos = null; - bigTableValueExpressions = null; - - bigTableRetainedMapping = null; - bigTableOuterKeyMapping = null; - smallTableMapping = null; - - projectionMapping = null; - } - - public int[] getBigTableKeyColumnMap() { - return bigTableKeyColumnMap; - } - - public void setBigTableKeyColumnMap(int[] bigTableKeyColumnMap) { - this.bigTableKeyColumnMap = bigTableKeyColumnMap; - } - - public String[] getBigTableKeyColumnNames() { - return bigTableKeyColumnNames; - } - - public void setBigTableKeyColumnNames(String[] bigTableKeyColumnNames) { - this.bigTableKeyColumnNames = bigTableKeyColumnNames; - } - - public TypeInfo[] getBigTableKeyTypeInfos() { - return bigTableKeyTypeInfos; - } - - public void setBigTableKeyTypeInfos(TypeInfo[] bigTableKeyTypeInfos) { - this.bigTableKeyTypeInfos = bigTableKeyTypeInfos; - } - - public VectorExpression[] getBigTableKeyExpressions() { - return bigTableKeyExpressions; - } - - public void setBigTableKeyExpressions(VectorExpression[] bigTableKeyExpressions) { - this.bigTableKeyExpressions = bigTableKeyExpressions; - } - - - public int[] getBigTableValueColumnMap() { - return bigTableValueColumnMap; - } - - public void setBigTableValueColumnMap(int[] bigTableValueColumnMap) { - this.bigTableValueColumnMap = bigTableValueColumnMap; - } - - public String[] getBigTableValueColumnNames() { - return bigTableValueColumnNames; - } - - public void setBigTableValueColumnNames(String[] bigTableValueColumnNames) { - this.bigTableValueColumnNames = bigTableValueColumnNames; - } - - public TypeInfo[] getBigTableValueTypeInfos() { - return bigTableValueTypeInfos; - } - - public void setBigTableValueTypeInfos(TypeInfo[] bigTableValueTypeInfos) { - this.bigTableValueTypeInfos = bigTableValueTypeInfos; - } - - public VectorExpression[] getBigTableValueExpressions() { - return bigTableValueExpressions; - } - - public void setBigTableValueExpressions(VectorExpression[] bigTableValueExpressions) { - this.bigTableValueExpressions = bigTableValueExpressions; - } - - public void setBigTableRetainedMapping(VectorColumnOutputMapping bigTableRetainedMapping) { - this.bigTableRetainedMapping = bigTableRetainedMapping; - } - - public VectorColumnOutputMapping getBigTableRetainedMapping() { - return bigTableRetainedMapping; - } - - public void setBigTableOuterKeyMapping(VectorColumnOutputMapping bigTableOuterKeyMapping) { - this.bigTableOuterKeyMapping = bigTableOuterKeyMapping; - } - - public VectorColumnOutputMapping getBigTableOuterKeyMapping() { - return bigTableOuterKeyMapping; - } - - public void setSmallTableMapping(VectorColumnSourceMapping smallTableMapping) { - this.smallTableMapping = smallTableMapping; - } - - public VectorColumnSourceMapping getSmallTableMapping() { - return smallTableMapping; - } - - public void setProjectionMapping(VectorColumnSourceMapping projectionMapping) { - this.projectionMapping = projectionMapping; - } - - public VectorColumnSourceMapping getProjectionMapping() { - return projectionMapping; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java index 288a440..c56bff6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorReduceSinkDesc.java @@ -61,72 +61,4 @@ public class VectorReduceSinkDesc extends AbstractVectorDesc { public VectorReduceSinkInfo getVectorReduceSinkInfo() { return vectorReduceSinkInfo; } - - private boolean isVectorizationReduceSinkNativeEnabled; - private String engine; - private boolean acidChange; - private boolean hasBuckets; - private boolean hasTopN; - private boolean useUniformHash; - private boolean hasDistinctColumns; - private boolean isKeyBinarySortable; - private boolean isValueLazyBinary; - - /* - * The following conditions are for native Vector ReduceSink. - */ - public void setIsVectorizationReduceSinkNativeEnabled(boolean isVectorizationReduceSinkNativeEnabled) { - this.isVectorizationReduceSinkNativeEnabled = isVectorizationReduceSinkNativeEnabled; - } - public boolean getIsVectorizationReduceSinkNativeEnabled() { - return isVectorizationReduceSinkNativeEnabled; - } - public void setEngine(String engine) { - this.engine = engine; - } - public String getEngine() { - return engine; - } - public void setAcidChange(boolean acidChange) { - this.acidChange = acidChange; - } - public boolean getAcidChange() { - return acidChange; - } - public void setHasBuckets(boolean hasBuckets) { - this.hasBuckets = hasBuckets; - } - public boolean getHasBuckets() { - return hasBuckets; - } - public void setHasTopN(boolean hasTopN) { - this.hasTopN = hasTopN; - } - public boolean getHasTopN() { - return hasTopN; - } - public void setUseUniformHash(boolean useUniformHash) { - this.useUniformHash = useUniformHash; - } - public boolean getUseUniformHash() { - return useUniformHash; - } - public void setHasDistinctColumns(boolean hasDistinctColumns) { - this.hasDistinctColumns = hasDistinctColumns; - } - public boolean getHasDistinctColumns() { - return hasDistinctColumns; - } - public void setIsKeyBinarySortable(boolean isKeyBinarySortable) { - this.isKeyBinarySortable = isKeyBinarySortable; - } - public boolean getIsKeyBinarySortable() { - return isKeyBinarySortable; - } - public void setIsValueLazyBinary(boolean isValueLazyBinary) { - this.isValueLazyBinary = isValueLazyBinary; - } - public boolean getIsValueLazyBinary() { - return isValueLazyBinary; - } } http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSMBJoinDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSMBJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSMBJoinDesc.java deleted file mode 100644 index ab578cd..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSMBJoinDesc.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -/** - * VectorSMBMapJoinDesc. - * - * Extra parameters beyond SMBMapJoinDesc just for the VectorSMBMapJoinOperator. - * - * We don't extend SMBMapJoinDesc because the base OperatorDesc doesn't support - * clone and adding it is a lot work for little gain. - */ -public class VectorSMBJoinDesc extends AbstractVectorDesc { - - private static long serialVersionUID = 1L; - - public VectorSMBJoinDesc() { - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSelectDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSelectDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSelectDesc.java deleted file mode 100644 index c2c9450..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSelectDesc.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; - -/** - * VectorSelectDesc. - * - * Extra parameters beyond SelectDesc just for the VectorSelectOperator. - * - * We don't extend SelectDesc because the base OperatorDesc doesn't support - * clone and adding it is a lot work for little gain. - */ -public class VectorSelectDesc extends AbstractVectorDesc { - - private static long serialVersionUID = 1L; - - private VectorExpression[] selectExpressions; - private int[] projectedOutputColumns; - - public VectorSelectDesc() { - } - - public void setSelectExpressions(VectorExpression[] selectExpressions) { - this.selectExpressions = selectExpressions; - } - - public VectorExpression[] getSelectExpressions() { - return selectExpressions; - } - - public void setProjectedOutputColumns(int[] projectedOutputColumns) { - this.projectedOutputColumns = projectedOutputColumns; - } - - public int[] getProjectedOutputColumns() { - return projectedOutputColumns; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSparkHashTableSinkDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSparkHashTableSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSparkHashTableSinkDesc.java deleted file mode 100644 index 7fb59db..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorSparkHashTableSinkDesc.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -/** - * VectorHashTableSinkDesc. - * - * Extra parameters beyond HashTableSinkDesc just for the VectorHashTableSinkOperator. - * - * We don't extend HashTableSinkDesc because the base OperatorDesc doesn't support - * clone and adding it is a lot work for little gain. - */ -public class VectorSparkHashTableSinkDesc extends AbstractVectorDesc { - - private static long serialVersionUID = 1L; - - public VectorSparkHashTableSinkDesc() { - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorTableScanDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorTableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorTableScanDesc.java deleted file mode 100644 index 6e5ebe4..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorTableScanDesc.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -/** - * VectorTableScanDesc. - * - * Extra parameters beyond TableScanDesc just for the VectorTableScanOperator. - * - * We don't extend TableScanDesc because the base OperatorDesc doesn't support - * clone and adding it is a lot work for little gain. - */ -public class VectorTableScanDesc extends AbstractVectorDesc { - - private static long serialVersionUID = 1L; - - private int[] projectedOutputColumns; - - public VectorTableScanDesc() { - } - - public void setProjectedOutputColumns(int[] projectedOutputColumns) { - this.projectedOutputColumns = projectedOutputColumns; - } - - public int[] getProjectedOutputColumns() { - return projectedOutputColumns; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorizationCondition.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorizationCondition.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorizationCondition.java deleted file mode 100644 index 32b62e8..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorizationCondition.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public class VectorizationCondition { - - private final boolean flag; - private final String conditionName; - - public VectorizationCondition(boolean flag, String conditionName) { - this.flag = flag; - this.conditionName = conditionName; - } - - public boolean getFlag() { - return flag; - } - - public String getConditionName() { - return conditionName; - } - - public static List<String> getConditionsMet(VectorizationCondition[] conditions) { - List<String> metList = new ArrayList<String>(); - for (VectorizationCondition condition : conditions) { - if (condition.getFlag()) { - metList.add(condition.getConditionName() + " IS true"); - } - } - return metList; - } - - public static List<String> getConditionsNotMet(VectorizationCondition[] conditions) { - List<String> notMetList = new ArrayList<String>(); - for (VectorizationCondition condition : conditions) { - if (!condition.getFlag()) { - notMetList.add(condition.getConditionName() + " IS false"); - } - } - return notMetList; - } - - public static List<String> addBooleans(List<String> conditions, boolean flag) { - ArrayList<String> result = new ArrayList<String>(conditions.size()); - for (String condition : conditions) { - result.add(condition + " IS " + flag); - } - return result; - } - - // Helper method. - public static List<String> getConditionsSupported(boolean isSupported) { - return Arrays.asList("Supported IS " + isSupported); - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java index 22b845d..d3bb84d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java @@ -25,19 +25,13 @@ import junit.framework.Assert; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.FilterOperator; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterExprAndExpr; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColEqualDoubleScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColGreaterLongColumn; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.VectorFilterDesc; import org.junit.Test; /** @@ -95,15 +89,10 @@ public class TestVectorFilterOperator { ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Long.class, "col1", "table", false); List<String> columns = new ArrayList<String>(); columns.add("col1"); + VectorizationContext vc = new VectorizationContext("name", columns); FilterDesc fdesc = new FilterDesc(); fdesc.setPredicate(col1Expr); - - Operator<? extends OperatorDesc> filterOp = - OperatorFactory.get(new CompilationOpContext(), fdesc); - - VectorizationContext vc = new VectorizationContext("name", columns); - - return (VectorFilterOperator) Vectorizer.vectorizeFilterOperator(filterOp, vc); + return new VectorFilterOperator(new CompilationOpContext(), vc, fdesc); } @Test http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java index 086c0b7..f5b5d9d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java @@ -39,20 +39,16 @@ import java.util.Set; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.vector.util.FakeCaptureOutputOperator; import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromConcat; import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromLongIterables; import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromObjectIterables; import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromRepeats; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc; import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc.ProcessingMode; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; @@ -132,11 +128,9 @@ public class TestVectorGroupByOperator { outputColumnNames.add("_col0"); GroupByDesc desc = new GroupByDesc(); - desc.setVectorDesc(new VectorGroupByDesc()); - desc.setOutputColumnNames(outputColumnNames); desc.setAggregators(aggs); - ((VectorGroupByDesc) desc.getVectorDesc()).setProcessingMode(ProcessingMode.GLOBAL); + desc.getVectorDesc().setProcessingMode(ProcessingMode.GLOBAL); return desc; } @@ -152,8 +146,6 @@ public class TestVectorGroupByOperator { outputColumnNames.add("_col0"); GroupByDesc desc = new GroupByDesc(); - desc.setVectorDesc(new VectorGroupByDesc()); - desc.setOutputColumnNames(outputColumnNames); desc.setAggregators(aggs); @@ -170,7 +162,7 @@ public class TestVectorGroupByOperator { TypeInfo keyTypeInfo) { GroupByDesc desc = buildGroupByDescType(ctx, aggregate, GenericUDAFEvaluator.Mode.PARTIAL1, column, dataTypeInfo); - ((VectorGroupByDesc) desc.getVectorDesc()).setProcessingMode(ProcessingMode.HASH); + desc.getVectorDesc().setProcessingMode(ProcessingMode.HASH); ExprNodeDesc keyExp = buildColumnDesc(ctx, key, keyTypeInfo); ArrayList<ExprNodeDesc> keys = new ArrayList<ExprNodeDesc>(); @@ -204,11 +196,7 @@ public class TestVectorGroupByOperator { desc.setMemoryThreshold(treshold); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); @@ -1747,19 +1735,13 @@ public class TestVectorGroupByOperator { } GroupByDesc desc = new GroupByDesc(); - desc.setVectorDesc(new VectorGroupByDesc()); - desc.setOutputColumnNames(outputColumnNames); desc.setAggregators(aggs); desc.setKeys(keysDesc); - ((VectorGroupByDesc) desc.getVectorDesc()).setProcessingMode(ProcessingMode.HASH); + desc.getVectorDesc().setProcessingMode(ProcessingMode.HASH); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); @@ -1864,11 +1846,9 @@ public class TestVectorGroupByOperator { outputColumnNames.add("_col1"); GroupByDesc desc = new GroupByDesc(); - desc.setVectorDesc(new VectorGroupByDesc()); - desc.setOutputColumnNames(outputColumnNames); desc.setAggregators(aggs); - ((VectorGroupByDesc) desc.getVectorDesc()).setProcessingMode(ProcessingMode.HASH); + desc.getVectorDesc().setProcessingMode(ProcessingMode.HASH); ExprNodeDesc keyExp = buildColumnDesc(ctx, "Key", TypeInfoFactory.getPrimitiveTypeInfo(data.getTypes()[0])); @@ -1877,11 +1857,7 @@ public class TestVectorGroupByOperator { desc.setKeys(keysDesc); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); @@ -2276,14 +2252,10 @@ public class TestVectorGroupByOperator { VectorizationContext ctx = new VectorizationContext("name", mapColumnNames); GroupByDesc desc = buildGroupByDescCountStar (ctx); - ((VectorGroupByDesc) desc.getVectorDesc()).setProcessingMode(ProcessingMode.HASH); + desc.getVectorDesc().setProcessingMode(ProcessingMode.HASH); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); @@ -2311,14 +2283,10 @@ public class TestVectorGroupByOperator { VectorizationContext ctx = new VectorizationContext("name", mapColumnNames); GroupByDesc desc = buildGroupByDescType(ctx, "count", GenericUDAFEvaluator.Mode.FINAL, "A", TypeInfoFactory.longTypeInfo); - VectorGroupByDesc vectorDesc = (VectorGroupByDesc) desc.getVectorDesc(); + VectorGroupByDesc vectorDesc = desc.getVectorDesc(); vectorDesc.setProcessingMode(ProcessingMode.GLOBAL); // Use GLOBAL when no key for Reduce. CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); @@ -2350,11 +2318,7 @@ public class TestVectorGroupByOperator { TypeInfoFactory.stringTypeInfo); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); @@ -2386,11 +2350,7 @@ public class TestVectorGroupByOperator { buildGroupByDescType(ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1, "A", TypeInfoFactory.getDecimalTypeInfo(30, 4)); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); @@ -2423,11 +2383,7 @@ public class TestVectorGroupByOperator { TypeInfoFactory.doubleTypeInfo); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); @@ -2458,11 +2414,7 @@ public class TestVectorGroupByOperator { GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1, "A", TypeInfoFactory.longTypeInfo); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(null, null); @@ -2497,11 +2449,7 @@ public class TestVectorGroupByOperator { TypeInfoFactory.longTypeInfo, "Key", TypeInfoFactory.longTypeInfo); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); @@ -2567,11 +2515,7 @@ public class TestVectorGroupByOperator { dataTypeInfo, "Key", TypeInfoFactory.stringTypeInfo); CompilationOpContext cCtx = new CompilationOpContext(); - - Operator<? extends OperatorDesc> groupByOp = OperatorFactory.get(cCtx, desc); - - VectorGroupByOperator vgo = - (VectorGroupByOperator) Vectorizer.vectorizeGroupByOperator(groupByOp, ctx); + VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc); FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(cCtx, vgo); vgo.initialize(hconf, null); http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java index 614b1d1..779177a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -34,7 +33,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; -import org.apache.hadoop.hive.ql.plan.VectorSelectDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPlus; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -53,7 +51,6 @@ public class TestVectorSelectOperator { public ValidatorVectorSelectOperator(CompilationOpContext ctx, VectorizationContext ctxt, OperatorDesc conf) throws HiveException { super(ctx, ctxt, conf); - initializeOp(null); } @@ -118,19 +115,6 @@ public class TestVectorSelectOperator { outputColNames.add("_col1"); selDesc.setOutputColumnNames(outputColNames); - // CONSIDER unwinding ValidatorVectorSelectOperator as a subclass of VectorSelectOperator. - VectorSelectDesc vectorSelectDesc = new VectorSelectDesc(); - selDesc.setVectorDesc(vectorSelectDesc); - List<ExprNodeDesc> selectColList = selDesc.getColList(); - VectorExpression[] vectorSelectExprs = new VectorExpression[selectColList.size()]; - for (int i = 0; i < selectColList.size(); i++) { - ExprNodeDesc expr = selectColList.get(i); - VectorExpression ve = vc.getVectorExpression(expr); - vectorSelectExprs[i] = ve; - } - vectorSelectDesc.setSelectExpressions(vectorSelectExprs); - vectorSelectDesc.setProjectedOutputColumns(new int[] {3, 2}); - ValidatorVectorSelectOperator vso = new ValidatorVectorSelectOperator( new CompilationOpContext(), vc, selDesc); http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java index ccd1059..3295372 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java @@ -96,8 +96,6 @@ public class TestVectorizer { outputColumnNames.add("_col0"); GroupByDesc desc = new GroupByDesc(); - desc.setVectorDesc(new VectorGroupByDesc()); - desc.setOutputColumnNames(outputColumnNames); ArrayList<AggregationDesc> aggDescList = new ArrayList<AggregationDesc>(); aggDescList.add(aggDesc); @@ -108,14 +106,13 @@ public class TestVectorizer { grpByKeys.add(colExprB); desc.setKeys(grpByKeys); - Operator<? extends OperatorDesc> gbyOp = OperatorFactory.get(new CompilationOpContext(), desc); - + GroupByOperator gbyOp = new GroupByOperator(new CompilationOpContext()); + gbyOp.setConf(desc); desc.setMode(GroupByDesc.Mode.HASH); Vectorizer v = new Vectorizer(); - v.testSetCurrentBaseWork(new MapWork()); Assert.assertTrue(v.validateMapWorkOperator(gbyOp, null, false)); - VectorGroupByOperator vectorOp = (VectorGroupByOperator) v.vectorizeOperator(gbyOp, vContext, false, null); + VectorGroupByOperator vectorOp = (VectorGroupByOperator) v.vectorizeOperator(gbyOp, vContext, false); Assert.assertEquals(VectorUDAFSumLong.class, vectorOp.getAggregators()[0].getClass()); VectorUDAFSumLong udaf = (VectorUDAFSumLong) vectorOp.getAggregators()[0]; Assert.assertEquals(FuncAbsLongToLong.class, udaf.getInputExpression().getClass()); @@ -150,9 +147,8 @@ public class TestVectorizer { andExprDesc.setChildren(children3); Vectorizer v = new Vectorizer(); - v.testSetCurrentBaseWork(new MapWork()); - Assert.assertFalse(v.validateExprNodeDesc(andExprDesc, "test", VectorExpressionDescriptor.Mode.FILTER)); - Assert.assertFalse(v.validateExprNodeDesc(andExprDesc, "test", VectorExpressionDescriptor.Mode.PROJECTION)); + Assert.assertFalse(v.validateExprNodeDesc(andExprDesc, VectorExpressionDescriptor.Mode.FILTER)); + Assert.assertFalse(v.validateExprNodeDesc(andExprDesc, VectorExpressionDescriptor.Mode.PROJECTION)); } /** @@ -200,7 +196,6 @@ public class TestVectorizer { map.setConf(mjdesc); Vectorizer vectorizer = new Vectorizer(); - vectorizer.testSetCurrentBaseWork(new MapWork()); Assert.assertTrue(vectorizer.validateMapWorkOperator(map, null, false)); } @@ -217,7 +212,6 @@ public class TestVectorizer { map.setConf(mjdesc); Vectorizer vectorizer = new Vectorizer(); - vectorizer.testSetCurrentBaseWork(new MapWork()); Assert.assertTrue(vectorizer.validateMapWorkOperator(map, null, false)); } } http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part.q b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part.q index 48903d2..7e66cbc 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part.q @@ -1,4 +1,4 @@ -set hive.explain.user=false; +set hive.explain.user=true; set hive.mapred.mode=nonstrict; set hive.cli.print.header=true; SET hive.exec.schema.evolution=true; @@ -38,7 +38,7 @@ alter table part_add_int_permute_select add columns(c int); insert into table part_add_int_permute_select partition(part=1) VALUES (2, 2222, 'new', 3333); -explain vectorization detail +explain select insert_num,part,a,b from part_add_int_permute_select; -- SELECT permutation columns to make sure NULL defaulting works right @@ -61,7 +61,7 @@ alter table part_add_int_string_permute_select add columns(c int, d string); insert into table part_add_int_string_permute_select partition(part=1) VALUES (2, 2222, 'new', 3333, '4444'); -explain vectorization detail +explain select insert_num,part,a,b from part_add_int_string_permute_select; -- SELECT permutation columns to make sure NULL defaulting works right @@ -93,7 +93,7 @@ alter table part_change_string_group_double replace columns (insert_num int, c1 insert into table part_change_string_group_double partition(part=1) SELECT insert_num, double1, double1, double1, 'new' FROM schema_evolution_data WHERE insert_num = 111; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,b from part_change_string_group_double; select insert_num,part,c1,c2,c3,b from part_change_string_group_double; @@ -116,7 +116,7 @@ alter table part_change_date_group_string_group_date_timestamp replace columns(i insert into table part_change_date_group_string_group_date_timestamp partition(part=1) VALUES (111, 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp; @@ -164,7 +164,7 @@ insert into table part_change_numeric_group_string_group_multi_ints_string_group 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group; @@ -207,7 +207,7 @@ insert into table part_change_numeric_group_string_group_floating_string_group p 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group; @@ -249,7 +249,7 @@ insert into table part_change_string_group_string_group_string partition(part=1) 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string; @@ -299,7 +299,7 @@ insert into table part_change_lower_to_higher_numeric_group_tinyint_to_bigint pa 1234.5678, 9876.543, 789.321, 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint; @@ -330,7 +330,7 @@ alter table part_change_lower_to_higher_numeric_group_decimal_to_float replace c insert into table part_change_lower_to_higher_numeric_group_decimal_to_float partition(part=1) VALUES (111, 1234.5678, 9876.543, 1234.5678, 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float; select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float; http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_complex.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_complex.q b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_complex.q index 45afd9d..ac747e6 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_complex.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_complex.q @@ -1,4 +1,4 @@ -set hive.explain.user=false; +set hive.explain.user=true; set hive.mapred.mode=nonstrict; set hive.cli.print.header=true; SET hive.exec.schema.evolution=true; @@ -51,7 +51,7 @@ load data local inpath '../../data/files/schema_evolution/complex_struct1_c.txt' insert into table part_change_various_various_struct1 partition(part=1) select * from complex_struct1_c_txt; - explain vectorization detail +explain select insert_num,part,s1,b from part_change_various_various_struct1; select insert_num,part,s1,b from part_change_various_various_struct1; @@ -111,7 +111,7 @@ load data local inpath '../../data/files/schema_evolution/complex_struct2_d.txt' insert into table part_add_various_various_struct2 partition(part=1) select * from complex_struct2_d_txt; -explain vectorization detail +explain select insert_num,part,b,s2 from part_add_various_various_struct2; select insert_num,part,b,s2 from part_add_various_various_struct2; @@ -155,7 +155,7 @@ load data local inpath '../../data/files/schema_evolution/complex_struct4_c.txt' insert into table part_add_to_various_various_struct4 partition(part=1) select * from complex_struct4_c_txt; -explain vectorization detail +explain select insert_num,part,b,s3 from part_add_to_various_various_struct4; select insert_num,part,b,s3 from part_add_to_various_various_struct4; http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_primitive.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_primitive.q b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_primitive.q index b266a67..d3898a8 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_primitive.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_all_primitive.q @@ -1,4 +1,4 @@ -set hive.explain.user=false; +set hive.explain.user=true; set hive.mapred.mode=nonstrict; set hive.cli.print.header=true; SET hive.exec.schema.evolution=true; @@ -72,7 +72,7 @@ insert into table part_change_various_various_boolean_to_bigint partition(part=1 bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, 'new' FROM schema_evolution_data; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint; @@ -114,7 +114,7 @@ insert into table part_change_various_various_decimal_to_double partition(part=1 double1, double1, double1, double1, double1, double1, double1, double1, double1, double1, double1, 'new' FROM schema_evolution_data_2 WHERE insert_num=111; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double; @@ -138,7 +138,7 @@ alter table part_change_various_various_timestamp replace columns (insert_num in insert into table part_change_various_various_timestamp partition(part=1) SELECT insert_num, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, 'new' FROM schema_evolution_data_2 WHERE insert_num=111; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp; @@ -159,7 +159,7 @@ alter table part_change_various_various_date replace columns (insert_num int, c1 insert into table part_change_various_various_date partition(part=1) SELECT insert_num, date1, date1, date1, date1, 'new' FROM schema_evolution_data_2 WHERE insert_num=111; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date; select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date; @@ -198,7 +198,7 @@ load data local inpath '../../data/files/schema_evolution/same_type1_c.txt' over insert into table part_change_same_type_different_params partition(part=2) select * from same_type1_c_txt; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params; select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params; http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/queries/clientpositive/schema_evol_orc_vec_table.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_table.q b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_table.q index 866942e..ffaa07b 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_table.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_table.q @@ -1,4 +1,4 @@ -set hive.explain.user=false; +set hive.explain.user=true; set hive.cli.print.header=true; SET hive.exec.schema.evolution=true; SET hive.vectorized.use.vectorized.input.format=true; @@ -36,7 +36,7 @@ alter table table_add_int_permute_select add columns(c int); insert into table table_add_int_permute_select VALUES (111, 80000, 'new', 80000); -explain vectorization detail +explain select insert_num,a,b from table_add_int_permute_select; -- SELECT permutation columns to make sure NULL defaulting works right @@ -59,7 +59,7 @@ alter table table_add_int_string_permute_select add columns(c int, d string); insert into table table_add_int_string_permute_select VALUES (111, 80000, 'new', 80000, 'filler'); -explain vectorization detail +explain select insert_num,a,b from table_add_int_string_permute_select; -- SELECT permutation columns to make sure NULL defaulting works right @@ -91,7 +91,7 @@ alter table table_change_string_group_double replace columns (insert_num int, c1 insert into table table_change_string_group_double VALUES (111, 789.321, 789.321, 789.321, 'new'); -explain vectorization detail +explain select insert_num,c1,c2,c3,b from table_change_string_group_double; select insert_num,c1,c2,c3,b from table_change_string_group_double; @@ -158,7 +158,7 @@ insert into table table_change_numeric_group_string_group_multi_ints_string_grou 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group; select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group; @@ -201,7 +201,7 @@ insert into table table_change_numeric_group_string_group_floating_string_group 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group; select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group; http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/queries/clientpositive/schema_evol_text_vec_part.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part.q b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part.q index 77c863a..6582035 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part.q +++ b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part.q @@ -1,4 +1,4 @@ -set hive.explain.user=false; +set hive.explain.user=true; set hive.mapred.mode=nonstrict; set hive.cli.print.header=true; SET hive.exec.schema.evolution=true; @@ -39,7 +39,7 @@ alter table part_add_int_permute_select add columns(c int); insert into table part_add_int_permute_select partition(part=1) VALUES (2, 2222, 'new', 3333); -explain vectorization detail +explain select insert_num,part,a,b from part_add_int_permute_select; -- SELECT permutation columns to make sure NULL defaulting works right @@ -62,7 +62,7 @@ alter table part_add_int_string_permute_select add columns(c int, d string); insert into table part_add_int_string_permute_select partition(part=1) VALUES (2, 2222, 'new', 3333, '4444'); -explain vectorization detail +explain select insert_num,part,a,b from part_add_int_string_permute_select; -- SELECT permutation columns to make sure NULL defaulting works right @@ -94,7 +94,7 @@ alter table part_change_string_group_double replace columns (insert_num int, c1 insert into table part_change_string_group_double partition(part=1) SELECT insert_num, double1, double1, double1, 'new' FROM schema_evolution_data WHERE insert_num = 111; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,b from part_change_string_group_double; select insert_num,part,c1,c2,c3,b from part_change_string_group_double; @@ -117,7 +117,7 @@ alter table part_change_date_group_string_group_date_timestamp replace columns(i insert into table part_change_date_group_string_group_date_timestamp partition(part=1) VALUES (111, 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp; @@ -165,7 +165,7 @@ insert into table part_change_numeric_group_string_group_multi_ints_string_group 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group; @@ -208,7 +208,7 @@ insert into table part_change_numeric_group_string_group_floating_string_group p 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group; @@ -250,7 +250,7 @@ insert into table part_change_string_group_string_group_string partition(part=1) 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string; @@ -300,7 +300,7 @@ insert into table part_change_lower_to_higher_numeric_group_tinyint_to_bigint pa 1234.5678, 9876.543, 789.321, 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint; @@ -331,7 +331,7 @@ alter table part_change_lower_to_higher_numeric_group_decimal_to_float replace c insert into table part_change_lower_to_higher_numeric_group_decimal_to_float partition(part=1) VALUES (111, 1234.5678, 9876.543, 1234.5678, 'new'); -explain vectorization detail +explain select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float; select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float; http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q index 7eb72e0..e38a01e 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q +++ b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q @@ -1,4 +1,4 @@ -set hive.explain.user=false; +set hive.explain.user=true; set hive.mapred.mode=nonstrict; set hive.cli.print.header=true; SET hive.exec.schema.evolution=true; @@ -53,7 +53,7 @@ load data local inpath '../../data/files/schema_evolution/complex_struct1_c.txt' insert into table part_change_various_various_struct1 partition(part=1) select * from complex_struct1_c_txt; -explain vectorization detail +explain select insert_num,part,s1,b from part_change_various_various_struct1; select insert_num,part,s1,b from part_change_various_various_struct1; @@ -113,7 +113,7 @@ load data local inpath '../../data/files/schema_evolution/complex_struct2_d.txt' insert into table part_add_various_various_struct2 partition(part=1) select * from complex_struct2_d_txt; -explain vectorization detail +explain select insert_num,part,b,s2 from part_add_various_various_struct2; select insert_num,part,b,s2 from part_add_various_various_struct2; @@ -157,7 +157,7 @@ load data local inpath '../../data/files/schema_evolution/complex_struct4_c.txt' insert into table part_add_to_various_various_struct4 partition(part=1) select * from complex_struct4_c_txt; -explain vectorization detail +explain select insert_num,part,b,s3 from part_add_to_various_various_struct4; select insert_num,part,b,s3 from part_add_to_various_various_struct4; http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_primitive.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_primitive.q b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_primitive.q index d5c01cd..c9d90c3 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_primitive.q +++ b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_primitive.q @@ -1,4 +1,4 @@ -set hive.explain.user=false; +set hive.explain.user=true; set hive.mapred.mode=nonstrict; set hive.cli.print.header=true; SET hive.exec.schema.evolution=true; @@ -74,7 +74,7 @@ insert into table part_change_various_various_boolean_to_bigint partition(part=1 bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, 'new' FROM schema_evolution_data; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint; @@ -116,7 +116,7 @@ insert into table part_change_various_various_decimal_to_double partition(part=1 double1, double1, double1, double1, double1, double1, double1, double1, double1, double1, double1, 'new' FROM schema_evolution_data_2 WHERE insert_num=111; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double; @@ -140,7 +140,7 @@ alter table part_change_various_various_timestamp replace columns (insert_num in insert into table part_change_various_various_timestamp partition(part=1) SELECT insert_num, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, 'new' FROM schema_evolution_data_2 WHERE insert_num=111; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp; select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp; @@ -161,7 +161,7 @@ alter table part_change_various_various_date replace columns (insert_num int, c1 insert into table part_change_various_various_date partition(part=1) SELECT insert_num, date1, date1, date1, date1, 'new' FROM schema_evolution_data_2 WHERE insert_num=111; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date; select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date; @@ -200,7 +200,7 @@ load data local inpath '../../data/files/schema_evolution/same_type1_c.txt' over insert into table part_change_same_type_different_params partition(part=2) select * from same_type1_c_txt; -explain vectorization detail +explain select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params; select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params; http://git-wip-us.apache.org/repos/asf/hive/blob/ad6ce078/ql/src/test/queries/clientpositive/schema_evol_text_vec_table.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vec_table.q b/ql/src/test/queries/clientpositive/schema_evol_text_vec_table.q index bbf03af..7785f87 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_text_vec_table.q +++ b/ql/src/test/queries/clientpositive/schema_evol_text_vec_table.q @@ -1,4 +1,4 @@ -set hive.explain.user=false; +set hive.explain.user=true; set hive.cli.print.header=true; SET hive.exec.schema.evolution=true; SET hive.vectorized.use.vectorized.input.format=false; @@ -38,7 +38,7 @@ alter table table_add_int_permute_select add columns(c int); insert into table table_add_int_permute_select VALUES (111, 80000, 'new', 80000); -explain vectorization detail +explain select insert_num,a,b from table_add_int_permute_select; -- SELECT permutation columns to make sure NULL defaulting works right @@ -61,7 +61,7 @@ alter table table_add_int_string_permute_select add columns(c int, d string); insert into table table_add_int_string_permute_select VALUES (111, 80000, 'new', 80000, 'filler'); -explain vectorization detail +explain select insert_num,a,b from table_add_int_string_permute_select; -- SELECT permutation columns to make sure NULL defaulting works right @@ -93,7 +93,7 @@ alter table table_change_string_group_double replace columns (insert_num int, c1 insert into table table_change_string_group_double VALUES (111, 789.321, 789.321, 789.321, 'new'); -explain vectorization detail +explain select insert_num,c1,c2,c3,b from table_change_string_group_double; select insert_num,c1,c2,c3,b from table_change_string_group_double; @@ -160,7 +160,7 @@ insert into table table_change_numeric_group_string_group_multi_ints_string_grou 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group; select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group; @@ -203,7 +203,7 @@ insert into table table_change_numeric_group_string_group_floating_string_group 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); -explain vectorization detail +explain select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group; select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group;