HIVE-17534 : Add a config to turn off parquet vectorization (Vihang Karajgaonkar, reviewed by Matt McCline)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cda7e01c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cda7e01c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cda7e01c Branch: refs/heads/branch-2 Commit: cda7e01c5af29ced5931df10d8f8204fab523d5e Parents: 5f11ba9 Author: Vihang Karajgaonkar <[email protected]> Authored: Tue Oct 10 15:40:55 2017 -0700 Committer: Vihang Karajgaonkar <[email protected]> Committed: Mon Oct 16 08:47:39 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 3 + .../test/resources/testconfiguration.properties | 1 + .../apache/hadoop/hive/ql/exec/Utilities.java | 17 + .../hive/ql/optimizer/physical/LlapDecider.java | 7 +- .../hive/ql/optimizer/physical/Vectorizer.java | 53 +- .../vectorization_input_format_excludes.q | 169 +++ .../vectorization_input_format_excludes.q.out | 1376 ++++++++++++++++++ .../vectorization_input_format_excludes.q.out | 1336 +++++++++++++++++ 8 files changed, 2950 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/cda7e01c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 3d3f3b2..af4058a 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2776,6 +2776,9 @@ public class HiveConf extends Configuration { HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT("hive.vectorized.use.vectorized.input.format", true, "This flag should be set to true to enable vectorizing with vectorized input file format capable SerDe.\n" + "The default value is true."), + HIVE_VECTORIZATION_VECTORIZED_INPUT_FILE_FORMAT_EXCLUDES("hive.vectorized.input.format.excludes","", + "This configuration should be set to fully described input format class names for which \n" + + " vectorized input format should not be used for vectorized execution."), HIVE_VECTORIZATION_USE_VECTOR_DESERIALIZE("hive.vectorized.use.vector.serde.deserialize", true, "This flag should be set to true to enable vectorizing rows using vector deserialize.\n" + "The default value is true."), http://git-wip-us.apache.org/repos/asf/hive/blob/cda7e01c/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 8c8f397..639ffa8 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -1338,6 +1338,7 @@ spark.query.files=add_part_multiple.q, \ vectorization_9.q, \ vectorization_decimal_date.q, \ vectorization_div0.q, \ + vectorization_input_format_excludes.q, \ vectorization_nested_udf.q, \ vectorization_not.q, \ vectorization_part.q, \ http://git-wip-us.apache.org/repos/asf/hive/blob/cda7e01c/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 9569ff4..e90385d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -81,6 +81,7 @@ import org.apache.hadoop.hive.ql.io.merge.MergeFileMapper; import org.apache.hadoop.hive.ql.io.merge.MergeFileWork; import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanMapper; import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork; +import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateMapper; import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateWork; import org.apache.hadoop.hive.ql.log.PerfLogger; @@ -3695,6 +3696,22 @@ public final class Utilities { return VectorizedInputFormatInterface.class.isAssignableFrom(inputFormatClass); } + public static Collection<Class<?>> getClassNamesFromConfig(HiveConf hiveConf, ConfVars confVar) { + String[] classNames = org.apache.hadoop.util.StringUtils.getStrings(HiveConf.getVar(hiveConf, + confVar)); + if (classNames == null) return new ArrayList<>(0); + Collection<Class<?>> classList = new ArrayList<Class<?>>(classNames.length); + for (String className : classNames) { + if (className == null || className.isEmpty()) continue; + try { + classList.add(Class.forName(className)); + } catch (Exception ex) { + LOG.warn("Cannot create class " + className + " for " + confVar.varname + " checks"); + } + } + return classList; + } + public static void addSchemaEvolutionToTableScanOperator(Table table, TableScanOperator tableScanOp) { String colNames = MetaStoreUtils.getColumnNamesFromFieldSchema(table.getSd().getCols()); http://git-wip-us.apache.org/repos/asf/hive/blob/cda7e01c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index a694cf8..edf63e5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -75,6 +75,7 @@ import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.TezWork; +import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -433,9 +434,11 @@ public class LlapDecider implements PhysicalPlanResolver { private boolean checkInputsVectorized(MapWork mapWork) { boolean mayWrap = HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_NONVECTOR_WRAPPER_ENABLED); + Collection<Class<?>> excludedInputFormats = Utilities.getClassNamesFromConfig(conf, ConfVars.HIVE_VECTORIZATION_VECTORIZED_INPUT_FILE_FORMAT_EXCLUDES); for (PartitionDesc pd : mapWork.getPathToPartitionInfo().values()) { - if (Utilities.isInputFileFormatVectorized(pd) || (mayWrap - && HiveInputFormat.canWrapForLlap(pd.getInputFileFormatClass(), true))) { + if ((Utilities.isInputFileFormatVectorized(pd) && !excludedInputFormats + .contains(pd.getInputFileFormatClass())) || (mayWrap && HiveInputFormat + .canWrapForLlap(pd.getInputFileFormatClass(), true))) { continue; } LOG.info("Input format: " + pd.getInputFileFormatClassName() http://git-wip-us.apache.org/repos/asf/hive/blob/cda7e01c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 8f0241b..11e623a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -24,6 +24,7 @@ import java.io.Serializable; import java.lang.annotation.Annotation; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -269,6 +270,7 @@ public class Vectorizer implements PhysicalPlanResolver { private BaseWork currentBaseWork; private Operator<? extends OperatorDesc> currentOperator; + private Collection<Class<?>> vectorizedInputFormatExcludes; public void testSetCurrentBaseWork(BaseWork testBaseWork) { currentBaseWork = testBaseWork; @@ -734,13 +736,14 @@ public class Vectorizer implements PhysicalPlanResolver { if (useVectorizedInputFileFormat) { - if (isInputFileFormatVectorized) { + if (isInputFileFormatVectorized && !isInputFormatExcluded(inputFileFormatClassName, + vectorizedInputFormatExcludes)) { + pd.setVectorPartitionDesc(VectorPartitionDesc + .createVectorizedInputFileFormat(inputFileFormatClassName, + Utilities.isInputFileFormatSelfDescribing(pd))); - pd.setVectorPartitionDesc( - VectorPartitionDesc.createVectorizedInputFileFormat( - inputFileFormatClassName, Utilities.isInputFileFormatSelfDescribing(pd))); - - enabledConditionsMetSet.add(HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT.varname); + enabledConditionsMetSet + .add(HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT.varname); return true; } // Fall through and look for other options... @@ -819,7 +822,6 @@ public class Vectorizer implements PhysicalPlanResolver { // inspect-able Object[] row to a VectorizedRowBatch in the VectorMapOperator. if (useRowDeserialize) { - pd.setVectorPartitionDesc( VectorPartitionDesc.createRowDeserialize( inputFileFormatClassName, @@ -828,12 +830,18 @@ public class Vectorizer implements PhysicalPlanResolver { enabledConditionsMetSet.add(HiveConf.ConfVars.HIVE_VECTORIZATION_USE_ROW_DESERIALIZE.varname); return true; - } if (isInputFileFormatVectorized) { - Preconditions.checkState(!useVectorizedInputFileFormat); - enabledConditionsNotMetList.add(HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT.varname); + if(useVectorizedInputFileFormat) { + enabledConditionsNotMetList.add( + HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT.varname + " IS true AND " + + HiveConf.ConfVars.HIVE_VECTORIZATION_VECTORIZED_INPUT_FILE_FORMAT_EXCLUDES.varname + + " NOT CONTAINS " + inputFileFormatClassName); + } else { + enabledConditionsNotMetList + .add(HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT.varname); + } } else { // Only offer these when the input file format is not the fast vectorized formats. if (isVectorDeserializeEligable) { @@ -848,6 +856,23 @@ public class Vectorizer implements PhysicalPlanResolver { return false; } + private boolean isInputFormatExcluded(String inputFileFormatClassName, Collection<Class<?>> excludes) { + Class<?> ifClass = null; + try { + ifClass = Class.forName(inputFileFormatClassName); + } catch (ClassNotFoundException e) { + LOG.warn("Cannot verify class for " + inputFileFormatClassName, e); + return true; + } + if(excludes == null || excludes.isEmpty()) { + return false; + } + for (Class<?> badClass : excludes) { + if (badClass.isAssignableFrom(ifClass)) return true; + } + return false; + } + private ImmutablePair<Boolean, Boolean> validateInputFormatAndSchemaEvolution(MapWork mapWork, String alias, TableScanOperator tableScanOperator, VectorTaskColumnInfo vectorTaskColumnInfo) throws SemanticException { @@ -1658,6 +1683,9 @@ public class Vectorizer implements PhysicalPlanResolver { useVectorizedInputFileFormat = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT); + if(useVectorizedInputFileFormat) { + initVectorizedInputFormatExcludeClasses(); + } useVectorDeserialize = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_USE_VECTOR_DESERIALIZE); @@ -1691,6 +1719,11 @@ public class Vectorizer implements PhysicalPlanResolver { return physicalContext; } + private void initVectorizedInputFormatExcludeClasses() { + vectorizedInputFormatExcludes = Utilities.getClassNamesFromConfig(hiveConf, + HiveConf.ConfVars.HIVE_VECTORIZATION_VECTORIZED_INPUT_FILE_FORMAT_EXCLUDES); + } + private void setOperatorNotSupported(Operator<? extends OperatorDesc> op) { OperatorDesc desc = op.getConf(); Annotation note = AnnotationUtils.getAnnotation(desc.getClass(), Explain.class); http://git-wip-us.apache.org/repos/asf/hive/blob/cda7e01c/ql/src/test/queries/clientpositive/vectorization_input_format_excludes.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/vectorization_input_format_excludes.q b/ql/src/test/queries/clientpositive/vectorization_input_format_excludes.q new file mode 100644 index 0000000..78d8249 --- /dev/null +++ b/ql/src/test/queries/clientpositive/vectorization_input_format_excludes.q @@ -0,0 +1,169 @@ +set hive.fetch.task.conversion=none; +set hive.vectorized.use.row.serde.deserialize=false; +set hive.vectorized.use.vector.serde.deserialize=true; +set hive.vectorized.execution.enabled=true; +set hive.vectorized.execution.reduce.enabled=true; + +-- SORT_QUERY_RESULTS + +create table if not exists alltypes_parquet ( + cint int, + ctinyint tinyint, + csmallint smallint, + cfloat float, + cdouble double, + cstring1 string) stored as parquet; + +insert overwrite table alltypes_parquet + select cint, + ctinyint, + csmallint, + cfloat, + cdouble, + cstring1 + from alltypesorc; + +-- test native fileinputformat vectorization + +explain vectorization select * + from alltypes_parquet + where cint = 528534767 + limit 10; + +select * + from alltypes_parquet + where cint = 528534767 + limit 10; + +explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint; + +select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint; + +-- exclude MapredParquetInputFormat from vectorization, this should cause mapwork vectorization to be disabled +set hive.vectorized.input.format.excludes=org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; + +explain vectorization select * + from alltypes_parquet + where cint = 528534767 + limit 10; + +select * + from alltypes_parquet + where cint = 528534767 + limit 10; + +explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint; + +select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint; + + +-- unset hive.vectorized.input.format.excludes and confirm if vectorizer vectorizes the mapwork +set hive.vectorized.input.format.excludes=; + +explain vectorization select * + from alltypes_parquet + where cint = 528534767 + limit 10; + +select * + from alltypes_parquet + where cint = 528534767 + limit 10; + +explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint; + +select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint; + + +-- check if hive.vectorized.input.format.excludes work with non-parquet inputformats +set hive.vectorized.input.format.excludes=org.apache.hadoop.hive.ql.io.orc.OrcInputFormat,org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; +set hive.vectorized.use.row.serde.deserialize=false; +set hive.vectorized.use.vector.serde.deserialize=false; + + +create table if not exists alltypes_orc ( + cint int, + ctinyint tinyint, + csmallint smallint, + cfloat float, + cdouble double, + cstring1 string) stored as orc; + +insert overwrite table alltypes_orc + select cint, + ctinyint, + csmallint, + cfloat, + cdouble, + cstring1 + from alltypesorc; + +explain vectorization select * + from alltypes_orc + where cint = 528534767 + limit 10; + +select * + from alltypes_orc + where cint = 528534767 + limit 10; + +explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_orc + group by ctinyint; + +select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_orc + group by ctinyint; http://git-wip-us.apache.org/repos/asf/hive/blob/cda7e01c/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out b/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out new file mode 100644 index 0000000..bf6c9fc --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out @@ -0,0 +1,1376 @@ +PREHOOK: query: create table if not exists alltypes_parquet ( + cint int, + ctinyint tinyint, + csmallint smallint, + cfloat float, + cdouble double, + cstring1 string) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypes_parquet +POSTHOOK: query: create table if not exists alltypes_parquet ( + cint int, + ctinyint tinyint, + csmallint smallint, + cfloat float, + cdouble double, + cstring1 string) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypes_parquet +PREHOOK: query: insert overwrite table alltypes_parquet + select cint, + ctinyint, + csmallint, + cfloat, + cdouble, + cstring1 + from alltypesorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@alltypes_parquet +POSTHOOK: query: insert overwrite table alltypes_parquet + select cint, + ctinyint, + csmallint, + cfloat, + cdouble, + cstring1 + from alltypesorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@alltypes_parquet +POSTHOOK: Lineage: alltypes_parquet.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: alltypes_parquet.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: alltypes_parquet.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: alltypes_parquet.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: alltypes_parquet.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: alltypes_parquet.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: explain vectorization select * + from alltypes_parquet + where cint = 528534767 + limit 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select * + from alltypes_parquet + where cint = 528534767 + limit 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypes_parquet + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint = 528534767) (type: boolean) + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 528534767 (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 60 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 60 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: select * + from alltypes_parquet + where cint = 528534767 + limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +POSTHOOK: query: select * + from alltypes_parquet + where cint = 528534767 + limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +528534767 -11 -15431 -11.0 -15431.0 cvLH6Eat2yFsyy7p +528534767 -28 -15813 -28.0 -15813.0 cvLH6Eat2yFsyy7p +528534767 -34 15007 -34.0 15007.0 cvLH6Eat2yFsyy7p +528534767 -50 -13326 -50.0 -13326.0 cvLH6Eat2yFsyy7p +528534767 27 -7824 27.0 -7824.0 cvLH6Eat2yFsyy7p +528534767 29 7021 29.0 7021.0 cvLH6Eat2yFsyy7p +528534767 31 -9566 31.0 -9566.0 cvLH6Eat2yFsyy7p +528534767 31 4963 31.0 4963.0 cvLH6Eat2yFsyy7p +528534767 61 -15549 61.0 -15549.0 cvLH6Eat2yFsyy7p +528534767 NULL -4213 NULL -4213.0 cvLH6Eat2yFsyy7p +PREHOOK: query: explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypes_parquet + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double) + outputColumnNames: ctinyint, cint, csmallint, cstring1, cfloat, cdouble + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble) + keys: ctinyint (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + groupByVectorOutput: false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true + notVectorizedReason: Aggregation Function UDF avg parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,input:float> of Column[VALUE._col3] not supported + vectorized: false + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4) + keys: KEY._col0 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +-1 626923679 -15441 36 -1.0486250072717667 8786.246963933321 +-10 626923679 -15384 28 -10.0 8850.451610567823 +-11 626923679 -15659 32 -11.0 10453.738567408038 +-12 626923679 -16373 22 -12.0 10173.15707541171 +-13 626923679 -15446 30 -13.0 8907.942987576693 +-14 626923679 -13884 22 -14.0 10125.818731386042 +-15 626923679 -16036 24 -15.0 9450.506254395024 +-16 626923679 -15154 21 -16.0 8884.207393686478 +-17 626923679 -15922 19 -17.0 9944.104273894172 +-18 626923679 -14863 24 -18.0 9638.430684071413 +-19 626923679 -15935 25 -19.0 9967.22240685782 +-2 626923679 -16277 20 -2.0 10800.090249507177 +-20 626923679 -16126 24 -20.0 9868.92268080106 +-21 626923679 -16017 27 -21.0 9480.349236669877 +-22 626923679 -14701 22 -22.0 8809.230165774987 +-23 626923679 -16355 36 -23.345263230173213 9401.831290253447 +-24 626923679 -16311 26 -24.0 9386.736402961187 +-25 626923679 -15862 24 -25.0 9778.256724727018 +-26 626923679 -15686 15 -26.0 10874.523900405318 +-27 626923679 -14984 20 -27.0 8465.29660255097 +-28 626923679 -15813 20 -28.0 9616.869413270924 +-29 626923679 -14747 26 -29.0 9052.945656011721 +-3 626923679 -13632 16 -3.0 8836.215573422822 +-30 626923679 -14863 23 -30.0 9193.941914019653 +-31 626923679 -15915 22 -31.0 9187.596784112568 +-32 626923679 -15866 25 -32.0 9535.546396775915 +-33 626923679 -12779 21 -33.0 8854.331159704514 +-34 626923679 -15450 29 -34.0 8708.243526705026 +-35 626923679 -16059 23 -35.0 10136.580492864763 +-36 626923679 -16208 23 -36.0 8773.547684436919 +-37 626923679 -14780 17 -37.0 10368.905538788269 +-38 626923679 -14914 28 -38.0 8767.375358291503 +-39 626923679 -15612 19 -39.0 9765.551806305297 +-4 626923679 -16207 21 -4.0 9682.726604102581 +-40 626923679 -14678 23 -40.0 9883.334986561835 +-41 626923679 -12606 21 -41.0 9034.40949481481 +-42 626923679 -16025 14 -42.0 9692.646755759979 +-43 626923679 -15607 27 -43.0 8715.255026265124 +-44 626923679 -15667 21 -44.0 10334.01810499552 +-45 626923679 -15027 21 -45.0 8567.489593562543 +-46 626923679 -12427 21 -46.0 9182.943188188632 +-47 626923679 -16096 19 -47.0 9011.009178780589 +-48 626923679 -15462 26 -48.0 9913.883371354861 +-49 626923679 -14831 23 -49.0 9894.429191738676 +-5 626923679 -15780 24 -5.0 10599.227726422314 +-50 626923679 -14320 27 -50.0 8548.827748002343 +-51 1073680599 -15734 1028 -51.0 9531.569305177045 +-52 626923679 -16369 30 -52.0 8625.06871423408 +-53 626923679 -15445 19 -53.0 9387.739325499799 +-54 626923679 -14815 23 -54.0 9614.154026896626 +-55 626923679 -13381 26 -55.0 9157.562103946742 +-56 626923679 -11999 33 -56.0 9490.842152672341 +-57 626923679 -14893 32 -57.0 8572.083461570477 +-58 626923679 -15169 20 -58.0 9549.096672008198 +-59 626923679 -15789 28 -59.0 9829.790704244733 +-6 626923679 -15980 30 -6.0 10262.829252317424 +-60 626923679 -15792 24 -60.0 9892.656196775464 +-61 626923679 -15142 22 -61.0 9357.236187870849 +-62 626923679 -15992 24 -62.0 9004.593091474135 +-63 626923679 -12516 16 -63.0 9263.605837223322 +-64 626923679 -15920 21 -64.0 9254.456539277186 +-7 626923679 -14584 23 -7.0 9946.605446407746 +-8 626923679 -14678 18 -8.0 9976.831992670684 +-9 626923679 -15329 31 -9.0 8999.391457373968 +0 626923679 -14254 24 0.0 10057.5018088718 +1 626923679 -14610 30 1.0 10016.486277900643 +10 626923679 -15887 26 10.0 9104.820520135108 +11 1072654057 -14696 1035 11.0 9531.018991371746 +12 626923679 -14642 18 12.0 9696.038286378725 +13 626923679 -14771 26 13.0 8128.265919972384 +14 626923679 -13367 28 14.0 9074.674998750581 +15 626923679 -16339 28 15.0 9770.473400901916 +16 626923679 -14001 26 16.0 10130.883606275334 +17 626923679 -16109 22 16.73235294865627 1353416.3383574807 +18 626923679 -15779 21 18.0 10820.004053788869 +19 626923679 -16049 21 19.0 9423.560227007669 +2 626923679 -16227 25 2.0 10083.276127543355 +20 626923679 -15149 21 20.0 11161.893298093504 +21 626923679 -15931 23 21.0 9683.044864861204 +22 626923679 -16280 26 22.0 9693.155720861765 +23 626923679 -15514 24 23.0 8542.419116415425 +24 626923679 -15086 24 24.0 9661.203790645088 +25 626923679 -11349 23 25.0 8888.959012093468 +26 626923679 -14516 29 26.0 9123.125508880432 +27 626923679 -14965 24 27.0 9802.871860196345 +28 626923679 -14455 20 28.0 9283.289383115296 +29 626923679 -15892 16 29.0 9874.046501817154 +3 626923679 -16339 30 3.0 10483.526375885149 +30 626923679 -14111 27 30.0 10066.520234676527 +31 626923679 -15960 24 31.0 10427.970184550613 +32 626923679 -14044 24 32.0 8376.464579403413 +33 626923679 -14642 29 40.61776386607777 1304429.5939037625 +34 626923679 -15059 28 34.0 8756.731536033676 +35 626923679 -16153 27 35.0 10351.008404963042 +36 626923679 -15912 20 36.0 9475.257975138164 +37 626923679 -12081 24 37.0 9017.860034890362 +38 626923679 -15248 29 38.0 9900.256257785535 +39 626923679 -14887 28 39.0 10513.343644635232 +4 626923679 -15999 29 4.0 9516.189702058042 +40 626923679 -15861 22 40.0 9283.318678549174 +41 626923679 -13480 21 41.0 9016.291129937847 +42 626923679 -15834 28 42.0 10318.01399719996 +43 626923679 -15703 28 43.0 8757.796089055722 +44 626923679 -11185 16 44.0 9425.076634933797 +45 626923679 -15228 18 45.0 9459.968668643689 +46 626923679 -15187 22 46.0 9685.908173160062 +47 626923679 -16324 22 47.0 9822.220821743611 +48 626923679 -16372 29 48.0 10079.286173063345 +49 626923679 -15923 27 49.0 9850.111848934683 +5 626923679 -16169 31 5.0 11114.001902469323 +50 626923679 -16236 21 50.0 9398.176197406601 +51 626923679 -15790 17 51.0 9220.075799194028 +52 626923679 -15450 20 52.0 9261.723648435052 +53 626923679 -16217 30 53.0 9895.247408969733 +54 626923679 -15245 16 54.0 9789.50878424882 +55 626923679 -15887 21 55.0 9826.38569192808 +56 626923679 -12631 21 56.0 8860.917133763547 +57 626923679 -15620 25 57.0 9413.99393840875 +58 626923679 -13627 20 58.0 9083.529665947459 +59 626923679 -16076 17 59.0 10117.44967077967 +6 626923679 -15948 30 6.0 9644.247255286113 +60 626923679 -13606 23 60.0 8346.267436552042 +61 626923679 -15894 29 61.0 8785.714950987198 +62 626923679 -14307 17 62.0 9491.752726667326 +7 626923679 -15839 25 7.0 10077.151640330823 +8 1070764888 -15778 1034 8.0 9562.355155774725 +9 626923679 -13629 25 9.0 10157.217948808622 +NULL 1073418988 -16379 3115 NULL 305051.4870777435 +PREHOOK: query: explain vectorization select * + from alltypes_parquet + where cint = 528534767 + limit 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select * + from alltypes_parquet + where cint = 528534767 + limit 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypes_parquet + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint = 528534767) (type: boolean) + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 528534767 (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 60 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 60 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map Vectorization: + enabled: false + enabledConditionsNotMet: hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: select * + from alltypes_parquet + where cint = 528534767 + limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +POSTHOOK: query: select * + from alltypes_parquet + where cint = 528534767 + limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +528534767 -11 -15431 -11.0 -15431.0 cvLH6Eat2yFsyy7p +528534767 -28 -15813 -28.0 -15813.0 cvLH6Eat2yFsyy7p +528534767 -34 15007 -34.0 15007.0 cvLH6Eat2yFsyy7p +528534767 -50 -13326 -50.0 -13326.0 cvLH6Eat2yFsyy7p +528534767 27 -7824 27.0 -7824.0 cvLH6Eat2yFsyy7p +528534767 29 7021 29.0 7021.0 cvLH6Eat2yFsyy7p +528534767 31 -9566 31.0 -9566.0 cvLH6Eat2yFsyy7p +528534767 31 4963 31.0 4963.0 cvLH6Eat2yFsyy7p +528534767 61 -15549 61.0 -15549.0 cvLH6Eat2yFsyy7p +528534767 NULL -4213 NULL -4213.0 cvLH6Eat2yFsyy7p +PREHOOK: query: explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypes_parquet + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double) + outputColumnNames: ctinyint, cint, csmallint, cstring1, cfloat, cdouble + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble) + keys: ctinyint (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>) + Map Vectorization: + enabled: false + enabledConditionsNotMet: hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + Reducer 2 + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true + notVectorizedReason: Aggregation Function UDF avg parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,input:float> of Column[VALUE._col3] not supported + vectorized: false + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4) + keys: KEY._col0 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +-1 626923679 -15441 36 -1.0486250072717667 8786.246963933321 +-10 626923679 -15384 28 -10.0 8850.451610567823 +-11 626923679 -15659 32 -11.0 10453.738567408038 +-12 626923679 -16373 22 -12.0 10173.15707541171 +-13 626923679 -15446 30 -13.0 8907.942987576693 +-14 626923679 -13884 22 -14.0 10125.818731386042 +-15 626923679 -16036 24 -15.0 9450.506254395024 +-16 626923679 -15154 21 -16.0 8884.207393686478 +-17 626923679 -15922 19 -17.0 9944.104273894172 +-18 626923679 -14863 24 -18.0 9638.430684071413 +-19 626923679 -15935 25 -19.0 9967.22240685782 +-2 626923679 -16277 20 -2.0 10800.090249507177 +-20 626923679 -16126 24 -20.0 9868.92268080106 +-21 626923679 -16017 27 -21.0 9480.349236669877 +-22 626923679 -14701 22 -22.0 8809.230165774987 +-23 626923679 -16355 36 -23.345263230173213 9401.831290253447 +-24 626923679 -16311 26 -24.0 9386.736402961187 +-25 626923679 -15862 24 -25.0 9778.256724727018 +-26 626923679 -15686 15 -26.0 10874.523900405318 +-27 626923679 -14984 20 -27.0 8465.29660255097 +-28 626923679 -15813 20 -28.0 9616.869413270924 +-29 626923679 -14747 26 -29.0 9052.945656011721 +-3 626923679 -13632 16 -3.0 8836.215573422822 +-30 626923679 -14863 23 -30.0 9193.941914019653 +-31 626923679 -15915 22 -31.0 9187.596784112568 +-32 626923679 -15866 25 -32.0 9535.546396775915 +-33 626923679 -12779 21 -33.0 8854.331159704514 +-34 626923679 -15450 29 -34.0 8708.243526705026 +-35 626923679 -16059 23 -35.0 10136.580492864763 +-36 626923679 -16208 23 -36.0 8773.547684436919 +-37 626923679 -14780 17 -37.0 10368.905538788269 +-38 626923679 -14914 28 -38.0 8767.375358291503 +-39 626923679 -15612 19 -39.0 9765.551806305297 +-4 626923679 -16207 21 -4.0 9682.726604102581 +-40 626923679 -14678 23 -40.0 9883.334986561835 +-41 626923679 -12606 21 -41.0 9034.40949481481 +-42 626923679 -16025 14 -42.0 9692.646755759979 +-43 626923679 -15607 27 -43.0 8715.255026265124 +-44 626923679 -15667 21 -44.0 10334.01810499552 +-45 626923679 -15027 21 -45.0 8567.489593562543 +-46 626923679 -12427 21 -46.0 9182.943188188632 +-47 626923679 -16096 19 -47.0 9011.009178780589 +-48 626923679 -15462 26 -48.0 9913.883371354861 +-49 626923679 -14831 23 -49.0 9894.429191738676 +-5 626923679 -15780 24 -5.0 10599.227726422314 +-50 626923679 -14320 27 -50.0 8548.827748002343 +-51 1073680599 -15734 1028 -51.0 9531.569305177045 +-52 626923679 -16369 30 -52.0 8625.06871423408 +-53 626923679 -15445 19 -53.0 9387.739325499799 +-54 626923679 -14815 23 -54.0 9614.154026896626 +-55 626923679 -13381 26 -55.0 9157.562103946742 +-56 626923679 -11999 33 -56.0 9490.842152672341 +-57 626923679 -14893 32 -57.0 8572.083461570477 +-58 626923679 -15169 20 -58.0 9549.096672008198 +-59 626923679 -15789 28 -59.0 9829.790704244733 +-6 626923679 -15980 30 -6.0 10262.829252317424 +-60 626923679 -15792 24 -60.0 9892.656196775464 +-61 626923679 -15142 22 -61.0 9357.236187870849 +-62 626923679 -15992 24 -62.0 9004.593091474135 +-63 626923679 -12516 16 -63.0 9263.605837223322 +-64 626923679 -15920 21 -64.0 9254.456539277186 +-7 626923679 -14584 23 -7.0 9946.605446407746 +-8 626923679 -14678 18 -8.0 9976.831992670684 +-9 626923679 -15329 31 -9.0 8999.391457373968 +0 626923679 -14254 24 0.0 10057.5018088718 +1 626923679 -14610 30 1.0 10016.486277900643 +10 626923679 -15887 26 10.0 9104.820520135108 +11 1072654057 -14696 1035 11.0 9531.018991371746 +12 626923679 -14642 18 12.0 9696.038286378725 +13 626923679 -14771 26 13.0 8128.265919972384 +14 626923679 -13367 28 14.0 9074.674998750581 +15 626923679 -16339 28 15.0 9770.473400901916 +16 626923679 -14001 26 16.0 10130.883606275334 +17 626923679 -16109 22 16.73235294865627 1353416.3383574807 +18 626923679 -15779 21 18.0 10820.004053788869 +19 626923679 -16049 21 19.0 9423.560227007669 +2 626923679 -16227 25 2.0 10083.276127543355 +20 626923679 -15149 21 20.0 11161.893298093504 +21 626923679 -15931 23 21.0 9683.044864861204 +22 626923679 -16280 26 22.0 9693.155720861765 +23 626923679 -15514 24 23.0 8542.419116415425 +24 626923679 -15086 24 24.0 9661.203790645088 +25 626923679 -11349 23 25.0 8888.959012093468 +26 626923679 -14516 29 26.0 9123.125508880432 +27 626923679 -14965 24 27.0 9802.871860196345 +28 626923679 -14455 20 28.0 9283.289383115296 +29 626923679 -15892 16 29.0 9874.046501817154 +3 626923679 -16339 30 3.0 10483.526375885149 +30 626923679 -14111 27 30.0 10066.520234676527 +31 626923679 -15960 24 31.0 10427.970184550613 +32 626923679 -14044 24 32.0 8376.464579403413 +33 626923679 -14642 29 40.61776386607777 1304429.5939037625 +34 626923679 -15059 28 34.0 8756.731536033676 +35 626923679 -16153 27 35.0 10351.008404963042 +36 626923679 -15912 20 36.0 9475.257975138164 +37 626923679 -12081 24 37.0 9017.860034890362 +38 626923679 -15248 29 38.0 9900.256257785535 +39 626923679 -14887 28 39.0 10513.343644635232 +4 626923679 -15999 29 4.0 9516.189702058042 +40 626923679 -15861 22 40.0 9283.318678549174 +41 626923679 -13480 21 41.0 9016.291129937847 +42 626923679 -15834 28 42.0 10318.01399719996 +43 626923679 -15703 28 43.0 8757.796089055722 +44 626923679 -11185 16 44.0 9425.076634933797 +45 626923679 -15228 18 45.0 9459.968668643689 +46 626923679 -15187 22 46.0 9685.908173160062 +47 626923679 -16324 22 47.0 9822.220821743611 +48 626923679 -16372 29 48.0 10079.286173063345 +49 626923679 -15923 27 49.0 9850.111848934683 +5 626923679 -16169 31 5.0 11114.001902469323 +50 626923679 -16236 21 50.0 9398.176197406601 +51 626923679 -15790 17 51.0 9220.075799194028 +52 626923679 -15450 20 52.0 9261.723648435052 +53 626923679 -16217 30 53.0 9895.247408969733 +54 626923679 -15245 16 54.0 9789.50878424882 +55 626923679 -15887 21 55.0 9826.38569192808 +56 626923679 -12631 21 56.0 8860.917133763547 +57 626923679 -15620 25 57.0 9413.99393840875 +58 626923679 -13627 20 58.0 9083.529665947459 +59 626923679 -16076 17 59.0 10117.44967077967 +6 626923679 -15948 30 6.0 9644.247255286113 +60 626923679 -13606 23 60.0 8346.267436552042 +61 626923679 -15894 29 61.0 8785.714950987198 +62 626923679 -14307 17 62.0 9491.752726667326 +7 626923679 -15839 25 7.0 10077.151640330823 +8 1070764888 -15778 1034 8.0 9562.355155774725 +9 626923679 -13629 25 9.0 10157.217948808622 +NULL 1073418988 -16379 3115 NULL 305051.4870777435 +PREHOOK: query: explain vectorization select * + from alltypes_parquet + where cint = 528534767 + limit 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select * + from alltypes_parquet + where cint = 528534767 + limit 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypes_parquet + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint = 528534767) (type: boolean) + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 528534767 (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 60 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 60 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: select * + from alltypes_parquet + where cint = 528534767 + limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +POSTHOOK: query: select * + from alltypes_parquet + where cint = 528534767 + limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +528534767 -11 -15431 -11.0 -15431.0 cvLH6Eat2yFsyy7p +528534767 -28 -15813 -28.0 -15813.0 cvLH6Eat2yFsyy7p +528534767 -34 15007 -34.0 15007.0 cvLH6Eat2yFsyy7p +528534767 -50 -13326 -50.0 -13326.0 cvLH6Eat2yFsyy7p +528534767 27 -7824 27.0 -7824.0 cvLH6Eat2yFsyy7p +528534767 29 7021 29.0 7021.0 cvLH6Eat2yFsyy7p +528534767 31 -9566 31.0 -9566.0 cvLH6Eat2yFsyy7p +528534767 31 4963 31.0 4963.0 cvLH6Eat2yFsyy7p +528534767 61 -15549 61.0 -15549.0 cvLH6Eat2yFsyy7p +528534767 NULL -4213 NULL -4213.0 cvLH6Eat2yFsyy7p +PREHOOK: query: explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypes_parquet + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double) + outputColumnNames: ctinyint, cint, csmallint, cstring1, cfloat, cdouble + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble) + keys: ctinyint (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 12288 Data size: 73728 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + groupByVectorOutput: false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true + notVectorizedReason: Aggregation Function UDF avg parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,input:float> of Column[VALUE._col3] not supported + vectorized: false + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4) + keys: KEY._col0 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6144 Data size: 36864 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_parquet + group by ctinyint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypes_parquet +#### A masked pattern was here #### +-1 626923679 -15441 36 -1.0486250072717667 8786.246963933321 +-10 626923679 -15384 28 -10.0 8850.451610567823 +-11 626923679 -15659 32 -11.0 10453.738567408038 +-12 626923679 -16373 22 -12.0 10173.15707541171 +-13 626923679 -15446 30 -13.0 8907.942987576693 +-14 626923679 -13884 22 -14.0 10125.818731386042 +-15 626923679 -16036 24 -15.0 9450.506254395024 +-16 626923679 -15154 21 -16.0 8884.207393686478 +-17 626923679 -15922 19 -17.0 9944.104273894172 +-18 626923679 -14863 24 -18.0 9638.430684071413 +-19 626923679 -15935 25 -19.0 9967.22240685782 +-2 626923679 -16277 20 -2.0 10800.090249507177 +-20 626923679 -16126 24 -20.0 9868.92268080106 +-21 626923679 -16017 27 -21.0 9480.349236669877 +-22 626923679 -14701 22 -22.0 8809.230165774987 +-23 626923679 -16355 36 -23.345263230173213 9401.831290253447 +-24 626923679 -16311 26 -24.0 9386.736402961187 +-25 626923679 -15862 24 -25.0 9778.256724727018 +-26 626923679 -15686 15 -26.0 10874.523900405318 +-27 626923679 -14984 20 -27.0 8465.29660255097 +-28 626923679 -15813 20 -28.0 9616.869413270924 +-29 626923679 -14747 26 -29.0 9052.945656011721 +-3 626923679 -13632 16 -3.0 8836.215573422822 +-30 626923679 -14863 23 -30.0 9193.941914019653 +-31 626923679 -15915 22 -31.0 9187.596784112568 +-32 626923679 -15866 25 -32.0 9535.546396775915 +-33 626923679 -12779 21 -33.0 8854.331159704514 +-34 626923679 -15450 29 -34.0 8708.243526705026 +-35 626923679 -16059 23 -35.0 10136.580492864763 +-36 626923679 -16208 23 -36.0 8773.547684436919 +-37 626923679 -14780 17 -37.0 10368.905538788269 +-38 626923679 -14914 28 -38.0 8767.375358291503 +-39 626923679 -15612 19 -39.0 9765.551806305297 +-4 626923679 -16207 21 -4.0 9682.726604102581 +-40 626923679 -14678 23 -40.0 9883.334986561835 +-41 626923679 -12606 21 -41.0 9034.40949481481 +-42 626923679 -16025 14 -42.0 9692.646755759979 +-43 626923679 -15607 27 -43.0 8715.255026265124 +-44 626923679 -15667 21 -44.0 10334.01810499552 +-45 626923679 -15027 21 -45.0 8567.489593562543 +-46 626923679 -12427 21 -46.0 9182.943188188632 +-47 626923679 -16096 19 -47.0 9011.009178780589 +-48 626923679 -15462 26 -48.0 9913.883371354861 +-49 626923679 -14831 23 -49.0 9894.429191738676 +-5 626923679 -15780 24 -5.0 10599.227726422314 +-50 626923679 -14320 27 -50.0 8548.827748002343 +-51 1073680599 -15734 1028 -51.0 9531.569305177045 +-52 626923679 -16369 30 -52.0 8625.06871423408 +-53 626923679 -15445 19 -53.0 9387.739325499799 +-54 626923679 -14815 23 -54.0 9614.154026896626 +-55 626923679 -13381 26 -55.0 9157.562103946742 +-56 626923679 -11999 33 -56.0 9490.842152672341 +-57 626923679 -14893 32 -57.0 8572.083461570477 +-58 626923679 -15169 20 -58.0 9549.096672008198 +-59 626923679 -15789 28 -59.0 9829.790704244733 +-6 626923679 -15980 30 -6.0 10262.829252317424 +-60 626923679 -15792 24 -60.0 9892.656196775464 +-61 626923679 -15142 22 -61.0 9357.236187870849 +-62 626923679 -15992 24 -62.0 9004.593091474135 +-63 626923679 -12516 16 -63.0 9263.605837223322 +-64 626923679 -15920 21 -64.0 9254.456539277186 +-7 626923679 -14584 23 -7.0 9946.605446407746 +-8 626923679 -14678 18 -8.0 9976.831992670684 +-9 626923679 -15329 31 -9.0 8999.391457373968 +0 626923679 -14254 24 0.0 10057.5018088718 +1 626923679 -14610 30 1.0 10016.486277900643 +10 626923679 -15887 26 10.0 9104.820520135108 +11 1072654057 -14696 1035 11.0 9531.018991371746 +12 626923679 -14642 18 12.0 9696.038286378725 +13 626923679 -14771 26 13.0 8128.265919972384 +14 626923679 -13367 28 14.0 9074.674998750581 +15 626923679 -16339 28 15.0 9770.473400901916 +16 626923679 -14001 26 16.0 10130.883606275334 +17 626923679 -16109 22 16.73235294865627 1353416.3383574807 +18 626923679 -15779 21 18.0 10820.004053788869 +19 626923679 -16049 21 19.0 9423.560227007669 +2 626923679 -16227 25 2.0 10083.276127543355 +20 626923679 -15149 21 20.0 11161.893298093504 +21 626923679 -15931 23 21.0 9683.044864861204 +22 626923679 -16280 26 22.0 9693.155720861765 +23 626923679 -15514 24 23.0 8542.419116415425 +24 626923679 -15086 24 24.0 9661.203790645088 +25 626923679 -11349 23 25.0 8888.959012093468 +26 626923679 -14516 29 26.0 9123.125508880432 +27 626923679 -14965 24 27.0 9802.871860196345 +28 626923679 -14455 20 28.0 9283.289383115296 +29 626923679 -15892 16 29.0 9874.046501817154 +3 626923679 -16339 30 3.0 10483.526375885149 +30 626923679 -14111 27 30.0 10066.520234676527 +31 626923679 -15960 24 31.0 10427.970184550613 +32 626923679 -14044 24 32.0 8376.464579403413 +33 626923679 -14642 29 40.61776386607777 1304429.5939037625 +34 626923679 -15059 28 34.0 8756.731536033676 +35 626923679 -16153 27 35.0 10351.008404963042 +36 626923679 -15912 20 36.0 9475.257975138164 +37 626923679 -12081 24 37.0 9017.860034890362 +38 626923679 -15248 29 38.0 9900.256257785535 +39 626923679 -14887 28 39.0 10513.343644635232 +4 626923679 -15999 29 4.0 9516.189702058042 +40 626923679 -15861 22 40.0 9283.318678549174 +41 626923679 -13480 21 41.0 9016.291129937847 +42 626923679 -15834 28 42.0 10318.01399719996 +43 626923679 -15703 28 43.0 8757.796089055722 +44 626923679 -11185 16 44.0 9425.076634933797 +45 626923679 -15228 18 45.0 9459.968668643689 +46 626923679 -15187 22 46.0 9685.908173160062 +47 626923679 -16324 22 47.0 9822.220821743611 +48 626923679 -16372 29 48.0 10079.286173063345 +49 626923679 -15923 27 49.0 9850.111848934683 +5 626923679 -16169 31 5.0 11114.001902469323 +50 626923679 -16236 21 50.0 9398.176197406601 +51 626923679 -15790 17 51.0 9220.075799194028 +52 626923679 -15450 20 52.0 9261.723648435052 +53 626923679 -16217 30 53.0 9895.247408969733 +54 626923679 -15245 16 54.0 9789.50878424882 +55 626923679 -15887 21 55.0 9826.38569192808 +56 626923679 -12631 21 56.0 8860.917133763547 +57 626923679 -15620 25 57.0 9413.99393840875 +58 626923679 -13627 20 58.0 9083.529665947459 +59 626923679 -16076 17 59.0 10117.44967077967 +6 626923679 -15948 30 6.0 9644.247255286113 +60 626923679 -13606 23 60.0 8346.267436552042 +61 626923679 -15894 29 61.0 8785.714950987198 +62 626923679 -14307 17 62.0 9491.752726667326 +7 626923679 -15839 25 7.0 10077.151640330823 +8 1070764888 -15778 1034 8.0 9562.355155774725 +9 626923679 -13629 25 9.0 10157.217948808622 +NULL 1073418988 -16379 3115 NULL 305051.4870777435 +PREHOOK: query: create table if not exists alltypes_orc ( + cint int, + ctinyint tinyint, + csmallint smallint, + cfloat float, + cdouble double, + cstring1 string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypes_orc +POSTHOOK: query: create table if not exists alltypes_orc ( + cint int, + ctinyint tinyint, + csmallint smallint, + cfloat float, + cdouble double, + cstring1 string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypes_orc +PREHOOK: query: insert overwrite table alltypes_orc + select cint, + ctinyint, + csmallint, + cfloat, + cdouble, + cstring1 + from alltypesorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@alltypes_orc +POSTHOOK: query: insert overwrite table alltypes_orc + select cint, + ctinyint, + csmallint, + cfloat, + cdouble, + cstring1 + from alltypesorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@alltypes_orc +POSTHOOK: Lineage: alltypes_orc.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: alltypes_orc.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: alltypes_orc.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: alltypes_orc.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: alltypes_orc.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: alltypes_orc.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: explain vectorization select * + from alltypes_orc + where cint = 528534767 + limit 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select * + from alltypes_orc + where cint = 528534767 + limit 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypes_orc + Statistics: Num rows: 12288 Data size: 1110042 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cint = 528534767) (type: boolean) + Statistics: Num rows: 6144 Data size: 555021 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 528534767 (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6144 Data size: 555021 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 900 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 900 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map Vectorization: + enabled: false + enabledConditionsNotMet: hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.orc.OrcInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: select * + from alltypes_orc + where cint = 528534767 + limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypes_orc +#### A masked pattern was here #### +POSTHOOK: query: select * + from alltypes_orc + where cint = 528534767 + limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypes_orc +#### A masked pattern was here #### +528534767 -11 -15431 -11.0 -15431.0 cvLH6Eat2yFsyy7p +528534767 -28 -15813 -28.0 -15813.0 cvLH6Eat2yFsyy7p +528534767 -34 15007 -34.0 15007.0 cvLH6Eat2yFsyy7p +528534767 -50 -13326 -50.0 -13326.0 cvLH6Eat2yFsyy7p +528534767 27 -7824 27.0 -7824.0 cvLH6Eat2yFsyy7p +528534767 29 7021 29.0 7021.0 cvLH6Eat2yFsyy7p +528534767 31 -9566 31.0 -9566.0 cvLH6Eat2yFsyy7p +528534767 31 4963 31.0 4963.0 cvLH6Eat2yFsyy7p +528534767 61 -15549 61.0 -15549.0 cvLH6Eat2yFsyy7p +528534767 NULL -4213 NULL -4213.0 cvLH6Eat2yFsyy7p +PREHOOK: query: explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_orc + group by ctinyint +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_orc + group by ctinyint +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypes_orc + Statistics: Num rows: 12288 Data size: 1110042 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double) + outputColumnNames: ctinyint, cint, csmallint, cstring1, cfloat, cdouble + Statistics: Num rows: 12288 Data size: 1110042 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(cint), min(csmallint), count(cstring1), avg(cfloat), stddev_pop(cdouble) + keys: ctinyint (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12288 Data size: 1110042 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 12288 Data size: 1110042 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: struct<count:bigint,sum:double,input:float>), _col5 (type: struct<count:bigint,sum:double,variance:double>) + Map Vectorization: + enabled: false + enabledConditionsNotMet: hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.orc.OrcInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + Reducer 2 + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true + notVectorizedReason: Aggregation Function UDF avg parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,input:float> of Column[VALUE._col3] not supported + vectorized: false + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0), min(VALUE._col1), count(VALUE._col2), avg(VALUE._col3), stddev_pop(VALUE._col4) + keys: KEY._col0 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 6144 Data size: 555021 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6144 Data size: 555021 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_orc + group by ctinyint +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypes_orc +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint, + max(cint), + min(csmallint), + count(cstring1), + avg(cfloat), + stddev_pop(cdouble) + from alltypes_orc + group by ctinyint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypes_orc +#### A masked pattern was here #### +-1 626923679 -15441 36 -1.0486250072717667 8786.246963933321 +-10 626923679 -15384 28 -10.0 8850.451610567823 +-11 626923679 -15659 32 -11.0 10453.738567408038 +-12 626923679 -16373 22 -12.0 10173.15707541171 +-13 626923679 -15446 30 -13.0 8907.942987576693 +-14 626923679 -13884 22 -14.0 10125.818731386042 +-15 626923679 -16036 24 -15.0 9450.506254395024 +-16 626923679 -15154 21 -16.0 8884.207393686478 +-17 626923679 -15922 19 -17.0 9944.104273894172 +-18 626923679 -14863 24 -18.0 9638.430684071413 +-19 626923679 -15935 25 -19.0 9967.22240685782 +-2 626923679 -16277 20 -2.0 10800.090249507177 +-20 626923679 -16126 24 -20.0 9868.92268080106 +-21 626923679 -16017 27 -21.0 9480.349236669877 +-22 626923679 -14701 22 -22.0 8809.230165774987 +-23 626923679 -16355 36 -23.345263230173213 9401.831290253447 +-24 626923679 -16311 26 -24.0 9386.736402961187 +-25 626923679 -15862 24 -25.0 9778.256724727018 +-26 626923679 -15686 15 -26.0 10874.523900405318 +-27 626923679 -14984 20 -27.0 8465.29660255097 +-28 626923679 -15813 20 -28.0 9616.869413270924 +-29 626923679 -14747 26 -29.0 9052.945656011721 +-3 626923679 -13632 16 -3.0 8836.215573422822 +-30 626923679 -14863 23 -30.0 9193.941914019653 +-31 626923679 -15915 22 -31.0 9187.596784112568 +-32 626923679 -15866 25 -32.0 9535.546396775915 +-33 626923679 -12779 21 -33.0 8854.331159704514 +-34 626923679 -15450 29 -34.0 8708.243526705026 +-35 626923679 -16059 23 -35.0 10136.580492864763 +-36 626923679 -16208 23 -36.0 8773.547684436919 +-37 626923679 -14780 17 -37.0 10368.905538788269 +-38 626923679 -14914 28 -38.0 8767.375358291503 +-39 626923679 -15612 19 -39.0 9765.551806305297 +-4 626923679 -16207 21 -4.0 9682.726604102581 +-40 626923679 -14678 23 -40.0 9883.334986561835 +-41 626923679 -12606 21 -41.0 9034.40949481481 +-42 626923679 -16025 14 -42.0 9692.646755759979 +-43 626923679 -15607 27 -43.0 8715.255026265124 +-44 626923679 -15667 21 -44.0 10334.01810499552 +-45 626923679 -15027 21 -45.0 8567.489593562543 +-46 626923679 -12427 21 -46.0 9182.943188188632 +-47 626923679 -16096 19 -47.0 9011.009178780589 +-48 626923679 -15462 26 -48.0 9913.883371354861 +-49 626923679 -14831 23 -49.0 9894.429191738676 +-5 626923679 -15780 24 -5.0 10599.227726422314 +-50 626923679 -14320 27 -50.0 8548.827748002343 +-51 1073680599 -15734 1028 -51.0 9531.569305177045 +-52 626923679 -16369 30 -52.0 8625.06871423408 +-53 626923679 -15445 19 -53.0 9387.739325499799 +-54 626923679 -14815 23 -54.0 9614.154026896626 +-55 626923679 -13381 26 -55.0 9157.562103946742 +-56 626923679 -11999 33 -56.0 9490.842152672341 +-57 626923679 -14893 32 -57.0 8572.083461570477 +-58 626923679 -15169 20 -58.0 9549.096672008198 +-59 626923679 -15789 28 -59.0 9829.790704244733 +-6 626923679 -15980 30 -6.0 10262.829252317424 +-60 626923679 -15792 24 -60.0 9892.656196775464 +-61 626923679 -15142 22 -61.0 9357.236187870849 +-62 626923679 -15992 24 -62.0 9004.593091474135 +-63 626923679 -12516 16 -63.0 9263.605837223322 +-64 626923679 -15920 21 -64.0 9254.456539277186 +-7 626923679 -14584 23 -7.0 9946.605446407746 +-8 626923679 -14678 18 -8.0 9976.831992670684 +-9 626923679 -15329 31 -9.0 8999.391457373968 +0 626923679 -14254 24 0.0 10057.5018088718 +1 626923679 -14610 30 1.0 10016.486277900643 +10 626923679 -15887 26 10.0 9104.820520135108 +11 1072654057 -14696 1035 11.0 9531.018991371746 +12 626923679 -14642 18 12.0 9696.038286378725 +13 626923679 -14771 26 13.0 8128.265919972384 +14 626923679 -13367 28 14.0 9074.674998750581 +15 626923679 -16339 28 15.0 9770.473400901916 +16 626923679 -14001 26 16.0 10130.883606275334 +17 626923679 -16109 22 16.73235294865627 1353416.3383574807 +18 626923679 -15779 21 18.0 10820.004053788869 +19 626923679 -16049 21 19.0 9423.560227007669 +2 626923679 -16227 25 2.0 10083.276127543355 +20 626923679 -15149 21 20.0 11161.893298093504 +21 626923679 -15931 23 21.0 9683.044864861204 +22 626923679 -16280 26 22.0 9693.155720861765 +23 626923679 -15514 24 23.0 8542.419116415425 +24 626923679 -15086 24 24.0 9661.203790645088 +25 626923679 -11349 23 25.0 8888.959012093468 +26 626923679 -14516 29 26.0 9123.125508880432 +27 626923679 -14965 24 27.0 9802.871860196345 +28 626923679 -14455 20 28.0 9283.289383115296 +29 626923679 -15892 16 29.0 9874.046501817154 +3 626923679 -16339 30 3.0 10483.526375885149 +30 626923679 -14111 27 30.0 10066.520234676527 +31 626923679 -15960 24 31.0 10427.970184550613 +32 626923679 -14044 24 32.0 8376.464579403413 +33 626923679 -14642 29 40.61776386607777 1304429.5939037625 +34 626923679 -15059 28 34.0 8756.731536033676 +35 626923679 -16153 27 35.0 10351.008404963042 +36 626923679 -15912 20 36.0 9475.257975138164 +37 626923679 -12081 24 37.0 9017.860034890362 +38 626923679 -15248 29 38.0 9900.256257785535 +39 626923679 -14887 28 39.0 10513.343644635232 +4 626923679 -15999 29 4.0 9516.189702058042 +40 626923679 -15861 22 40.0 9283.318678549174 +41 626923679 -13480 21 41.0 9016.291129937847 +42 626923679 -15834 28 42.0 10318.01399719996 +43 626923679 -15703 28 43.0 8757.796089055722 +44 626923679 -11185 16 44.0 9425.076634933797 +45 626923679 -15228 18 45.0 9459.968668643689 +46 626923679 -15187 22 46.0 9685.908173160062 +47 626923679 -16324 22 47.0 9822.220821743611 +48 626923679 -16372 29 48.0 10079.286173063345 +49 626923679 -15923 27 49.0 9850.111848934683 +5 626923679 -16169 31 5.0 11114.001902469323 +50 626923679 -16236 21 50.0 9398.176197406601 +51 626923679 -15790 17 51.0 9220.075799194028 +52 626923679 -15450 20 52.0 9261.723648435052 +53 626923679 -16217 30 53.0 9895.247408969733 +54 626923679 -15245 16 54.0 9789.50878424882 +55 626923679 -15887 21 55.0 9826.38569192808 +56 626923679 -12631 21 56.0 8860.917133763547 +57 626923679 -15620 25 57.0 9413.99393840875 +58 626923679 -13627 20 58.0 9083.529665947459 +59 626923679 -16076 17 59.0 10117.44967077967 +6 626923679 -15948 30 6.0 9644.247255286113 +60 626923679 -13606 23 60.0 8346.267436552042 +61 626923679 -15894 29 61.0 8785.714950987198 +62 626923679 -14307 17 62.0 9491.752726667326 +7 626923679 -15839 25 7.0 10077.151640330823 +8 1070764888 -15778 1034 8.0 9562.355155774725 +9 626923679 -13629 25 9.0 10157.217948808622 +NULL 1073418988 -16379 3115 NULL 305051.4870777435
