http://git-wip-us.apache.org/repos/asf/hive/blob/92fbe256/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out index 2c7c57a..4d3e41a 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out @@ -113,46 +113,70 @@ STAGE PLANS: Group By Operator aggregations: avg(cint), sum(cdouble), stddev_pop(cint), stddev_samp(csmallint), var_samp(cint), avg(cfloat), stddev_samp(cint), min(ctinyint), count(csmallint) Group By Vectorization: - aggregators: VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double>, VectorUDAFSumDouble(col 5) -> double, VectorUDAFStdPopLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdSampLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgDouble(col 4) -> struct<count:bigint,sum:double>, VectorUDAFStdSampLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMinLong(col 0) -> tinyint, VectorUDAFCount(col 1) -> bigint + aggregators: VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double,input:bigint>, VectorUDAFSumDouble(col 5) -> double, VectorUDAFStdPopLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdSampLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgDouble(col 4) -> struct<count:bigint,sum:double,input:double>, VectorUDAFStdSampLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMinLong(col 0) -> tinyint, VectorUDAFCount(col 1) -> bigint className: VectorGroupByOperator - vectorOutput: false + groupByMode: HASH + vectorOutput: true native: false + vectorProcessingMode: HASH projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8] - vectorOutputConditionsNotMet: Vector output of VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdPopLong(col 2) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdSampLong(col 1) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFAvgDouble(col 4) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdSampLong(col 2) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>), _col1 (type: double), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,input:float>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: tinyint), _col8 (type: bigint) Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - groupByVectorOutput: false + groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false usesVectorUDFAdaptor: false vectorized: true Reducer 2 + Execution mode: vectorized Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: Aggregation Function UDF avg parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,input:int> of Column[VALUE._col0] not supported - vectorized: false + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reduce Operator Tree: Group By Operator aggregations: avg(VALUE._col0), sum(VALUE._col1), stddev_pop(VALUE._col2), stddev_samp(VALUE._col3), var_samp(VALUE._col4), avg(VALUE._col5), stddev_samp(VALUE._col6), min(VALUE._col7), count(VALUE._col8) + Group By Vectorization: + aggregators: VectorUDAFAvgFinal(col 0) -> double, VectorUDAFSumDouble(col 1) -> double, VectorUDAFStdPopFinal(col 2) -> double, VectorUDAFStdSampFinal(col 3) -> double, VectorUDAFVarSampFinal(col 4) -> double, VectorUDAFAvgFinal(col 5) -> double, VectorUDAFStdSampFinal(col 6) -> double, VectorUDAFMinLong(col 7) -> tinyint, VectorUDAFCountMerge(col 8) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + vectorOutput: true + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8] mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double), (_col0 + -3728.0) (type: double), (- (_col0 + -3728.0)) (type: double), (- (- (_col0 + -3728.0))) (type: double), ((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) (type: double), _col1 (type: double), (- _col0) (type: double), _col2 (type: double), (((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) * (- (- (_col0 + -3728.0)))) (type: double), _col3 (type: double), (- _col2) (type: double), (_col2 - (- (- (_col0 + -3728.0)))) (type: double), ((_col2 - (- (- (_col0 + -3728.0)))) * _col2) (type: double), _col4 (type: double), _col5 (type: double), (10.175 - _col4) (type: double), (- (10.175 - _col4)) (type: double), ((- _col2) / -563.0) (type: double), _col6 (type: double), (- ((- _col2) / -563.0)) (type: double), (_col0 / _col1) (type: double), _col7 (type: tinyint), _col8 (type: bigint), (UDFToDouble(_col7) / ((- _col2) / -563.0)) (type: double), (- (_col0 / _col1)) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 9, 11, 10, 14, 1, 12, 2, 15, 3, 13, 17, 16, 4, 5, 18, 20, 21, 6, 19, 22, 7, 8, 24, 25] + selectExpressions: DoubleColAddDoubleScalar(col 0, val -3728.0) -> 9:double, DoubleColUnaryMinus(col 10)(children: DoubleColAddDoubleScalar(col 0, val -3728.0) -> 10:double) -> 11:double, DoubleColUnaryMinus(col 12)(children: DoubleColUnaryMinus(col 10)(children: DoubleColAddDoubleScalar(col 0, val -3728.0) -> 10:double) -> 12:double) -> 10:double, DoubleColMultiplyDoubleColumn(col 12, col 13)(children: DoubleColUnaryMinus(col 13)(children: DoubleColUnaryMinus(col 12)(children: DoubleColAddDoubleScalar(col 0, val -3728.0) -> 12:double) -> 13:double) -> 12:double, DoubleColAddDoubleScalar(col 0, val -3728.0) -> 13:double) -> 14:double, DoubleColUnaryMinus(col 0) -> 12:double, DoubleColMultiplyDoubleColumn(col 16, col 13)(children: DoubleColMultiplyDoubleColumn(col 13, col 15)(children: DoubleColUnaryMinus(col 15)(children: DoubleColUnaryMinus(col 13)(children: DoubleColAddDoubleScalar(col 0, val -3728.0) -> 13:double) -> 15:double) -> 13:double, DoubleColAddDoub leScalar(col 0, val -3728.0) -> 15:double) -> 16:double, DoubleColUnaryMinus(col 15)(children: DoubleColUnaryMinus(col 13)(children: DoubleColAddDoubleScalar(col 0, val -3728.0) -> 13:double) -> 15:double) -> 13:double) -> 15:double, DoubleColUnaryMinus(col 2) -> 13:double, DoubleColSubtractDoubleColumn(col 2, col 16)(children: DoubleColUnaryMinus(col 17)(children: DoubleColUnaryMinus(col 16)(children: DoubleColAddDoubleScalar(col 0, val -3728.0) -> 16:double) -> 17:double) -> 16:double) -> 17:double, DoubleColMultiplyDoubleColumn(col 18, col 2)(children: DoubleColSubtractDoubleColumn(col 2, col 16)(children: DoubleColUnaryMinus(col 18)(children: DoubleColUnaryMinus(col 16)(children: DoubleColAddDoubleScalar(col 0, val -3728.0) -> 16:double) -> 18:double) -> 16:double) -> 18:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val 10.175, col 4) -> 18:double, DoubleColUnaryMinus(col 19)(children: DoubleScalarSubtractDoubleColumn(val 10.175, col 4) -> 19:double) -> 20:double, Doubl eColDivideDoubleScalar(col 19, val -563.0)(children: DoubleColUnaryMinus(col 2) -> 19:double) -> 21:double, DoubleColUnaryMinus(col 22)(children: DoubleColDivideDoubleScalar(col 19, val -563.0)(children: DoubleColUnaryMinus(col 2) -> 19:double) -> 22:double) -> 19:double, DoubleColDivideDoubleColumn(col 0, col 1) -> 22:double, DoubleColDivideDoubleColumn(col 23, col 25)(children: CastLongToDouble(col 7) -> 23:double, DoubleColDivideDoubleScalar(col 24, val -563.0)(children: DoubleColUnaryMinus(col 2) -> 24:double) -> 25:double) -> 24:double, DoubleColUnaryMinus(col 23)(children: DoubleColDivideDoubleColumn(col 0, col 1) -> 23:double) -> 25:double Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -351,46 +375,70 @@ STAGE PLANS: Group By Operator aggregations: max(cint), var_pop(cbigint), stddev_pop(csmallint), max(cdouble), avg(ctinyint), min(cint), min(cdouble), stddev_samp(csmallint), var_samp(cint) Group By Vectorization: - aggregators: VectorUDAFMaxLong(col 2) -> int, VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxDouble(col 5) -> double, VectorUDAFAvgLong(col 0) -> struct<count:bigint,sum:double>, VectorUDAFMinLong(col 2) -> int, VectorUDAFMinDouble(col 5) -> double, VectorUDAFStdSampLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double> + aggregators: VectorUDAFMaxLong(col 2) -> int, VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxDouble(col 5) -> double, VectorUDAFAvgLong(col 0) -> struct<count:bigint,sum:double,input:bigint>, VectorUDAFMinLong(col 2) -> int, VectorUDAFMinDouble(col 5) -> double, VectorUDAFStdSampLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double> className: VectorGroupByOperator - vectorOutput: false + groupByMode: HASH + vectorOutput: true native: false + vectorProcessingMode: HASH projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8] - vectorOutputConditionsNotMet: Vector output of VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdPopLong(col 1) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFAvgLong(col 0) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdSampLong(col 1) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,input:tinyint>), _col5 (type: int), _col6 (type: double), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>) Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - groupByVectorOutput: false + groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false usesVectorUDFAdaptor: false vectorized: true Reducer 2 + Execution mode: vectorized Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: Aggregation Function UDF var_pop parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col1] not supported - vectorized: false + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), var_pop(VALUE._col1), stddev_pop(VALUE._col2), max(VALUE._col3), avg(VALUE._col4), min(VALUE._col5), min(VALUE._col6), stddev_samp(VALUE._col7), var_samp(VALUE._col8) + Group By Vectorization: + aggregators: VectorUDAFMaxLong(col 0) -> int, VectorUDAFVarPopFinal(col 1) -> double, VectorUDAFStdPopFinal(col 2) -> double, VectorUDAFMaxDouble(col 3) -> double, VectorUDAFAvgFinal(col 4) -> double, VectorUDAFMinLong(col 5) -> int, VectorUDAFMinDouble(col 6) -> double, VectorUDAFStdSampFinal(col 7) -> double, VectorUDAFVarSampFinal(col 8) -> double + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + vectorOutput: true + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8] mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), (UDFToDouble(_col0) / -3728.0) (type: double), (_col0 * -3728) (type: int), _col1 (type: double), (- (_col0 * -3728)) (type: int), _col2 (type: double), (-563 % (_col0 * -3728)) (type: int), (_col1 / _col2) (type: double), (- _col2) (type: double), _col3 (type: double), _col4 (type: double), (_col2 - 10.175) (type: double), _col5 (type: int), (UDFToDouble((_col0 * -3728)) % (_col2 - 10.175)) (type: double), (- _col3) (type: double), _col6 (type: double), (_col3 % -26.28) (type: double), _col7 (type: double), (- (UDFToDouble(_col0) / -3728.0)) (type: double), ((- (_col0 * -3728)) % (-563 % (_col0 * -3728))) (type: int), ((UDFToDouble(_col0) / -3728.0) - _col4) (type: double), (- (_col0 * -3728)) (type: int), _col8 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 10, 11, 1, 13, 2, 14, 9, 15, 3, 4, 16, 5, 19, 17, 6, 18, 7, 20, 12, 21, 23, 8] + selectExpressions: DoubleColDivideDoubleScalar(col 9, val -3728.0)(children: CastLongToDouble(col 0) -> 9:double) -> 10:double, LongColMultiplyLongScalar(col 0, val -3728) -> 11:long, LongColUnaryMinus(col 12)(children: LongColMultiplyLongScalar(col 0, val -3728) -> 12:long) -> 13:long, LongScalarModuloLongColumn(val -563, col 12)(children: LongColMultiplyLongScalar(col 0, val -3728) -> 12:long) -> 14:long, DoubleColDivideDoubleColumn(col 1, col 2) -> 9:double, DoubleColUnaryMinus(col 2) -> 15:double, DoubleColSubtractDoubleScalar(col 2, val 10.175) -> 16:double, DoubleColModuloDoubleColumn(col 17, col 18)(children: CastLongToDouble(col 12)(children: LongColMultiplyLongScalar(col 0, val -3728) -> 12:long) -> 17:double, DoubleColSubtractDoubleScalar(col 2, val 10.175) -> 18:double) -> 19:double, DoubleColUnaryMinus(col 3) -> 17:double, DoubleColModuloDoubleScalar(col 3, val -26.28) -> 18:double, DoubleColUnaryMinus(col 21)(children: DoubleColDivideDoubleScalar(c ol 20, val -3728.0)(children: CastLongToDouble(col 0) -> 20:double) -> 21:double) -> 20:double, LongColModuloLongColumn(col 22, col 23)(children: LongColUnaryMinus(col 12)(children: LongColMultiplyLongScalar(col 0, val -3728) -> 12:long) -> 22:long, LongScalarModuloLongColumn(val -563, col 12)(children: LongColMultiplyLongScalar(col 0, val -3728) -> 12:long) -> 23:long) -> 12:long, DoubleColSubtractDoubleColumn(col 24, col 4)(children: DoubleColDivideDoubleScalar(col 21, val -3728.0)(children: CastLongToDouble(col 0) -> 21:double) -> 24:double) -> 21:double, LongColUnaryMinus(col 22)(children: LongColMultiplyLongScalar(col 0, val -3728) -> 22:long) -> 23:long Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -473,7 +521,7 @@ WHERE (((cbigint <= 197) POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### --20301111 5445.576984978541 -1626869520 7.9684972882908944E16 1626869520 NULL -563 NULL NULL NULL -8.935323383084578 NULL -1069736047 NULL NULL NULL NULL NULL -5445.576984978541 511 5454.512308361625 1626869520 7.2647256545687792E16 +-20301111 5445.576984978541 -1626869520 7.9684972882908944E16 1626869520 NULL -563 NULL NULL NULL -8.935323383084578 NULL -1069736047 NULL NULL NULL NULL NULL -5445.576984978541 -58 5454.512308361625 1626869520 7.2647256545687792E16 PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT VAR_POP(cbigint), (-(VAR_POP(cbigint))), @@ -581,46 +629,70 @@ STAGE PLANS: Group By Operator aggregations: var_pop(cbigint), count(), max(ctinyint), stddev_pop(csmallint), max(cint), stddev_samp(cdouble), count(ctinyint), avg(ctinyint) Group By Vectorization: - aggregators: VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFCountStar(*) -> bigint, VectorUDAFMaxLong(col 0) -> tinyint, VectorUDAFStdPopLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxLong(col 2) -> int, VectorUDAFStdSampDouble(col 5) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFCount(col 0) -> bigint, VectorUDAFAvgLong(col 0) -> struct<count:bigint,sum:double> + aggregators: VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFCountStar(*) -> bigint, VectorUDAFMaxLong(col 0) -> tinyint, VectorUDAFStdPopLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxLong(col 2) -> int, VectorUDAFStdSampDouble(col 5) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFCount(col 0) -> bigint, VectorUDAFAvgLong(col 0) -> struct<count:bigint,sum:double,input:bigint> className: VectorGroupByOperator - vectorOutput: false + groupByMode: HASH + vectorOutput: true native: false + vectorProcessingMode: HASH projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7] - vectorOutputConditionsNotMet: Vector output of VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdPopLong(col 1) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdSampDouble(col 5) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFAvgLong(col 0) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: struct<count:bigint,sum:double,variance:double>), _col1 (type: bigint), _col2 (type: tinyint), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: int), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: bigint), _col7 (type: struct<count:bigint,sum:double,input:tinyint>) Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - groupByVectorOutput: false + groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false usesVectorUDFAdaptor: false vectorized: true Reducer 2 + Execution mode: vectorized Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: Aggregation Function UDF var_pop parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col0] not supported - vectorized: false + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reduce Operator Tree: Group By Operator aggregations: var_pop(VALUE._col0), count(VALUE._col1), max(VALUE._col2), stddev_pop(VALUE._col3), max(VALUE._col4), stddev_samp(VALUE._col5), count(VALUE._col6), avg(VALUE._col7) + Group By Vectorization: + aggregators: VectorUDAFVarPopFinal(col 0) -> double, VectorUDAFCountMerge(col 1) -> bigint, VectorUDAFMaxLong(col 2) -> tinyint, VectorUDAFStdPopFinal(col 3) -> double, VectorUDAFMaxLong(col 4) -> int, VectorUDAFStdSampFinal(col 5) -> double, VectorUDAFCountMerge(col 6) -> bigint, VectorUDAFAvgFinal(col 7) -> double + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + vectorOutput: true + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7] mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double), (- _col0) (type: double), (_col0 - (- _col0)) (type: double), _col1 (type: bigint), (CAST( _col1 AS decimal(19,0)) % 79.553) (type: decimal(5,3)), _col2 (type: tinyint), (UDFToDouble(_col1) - (- _col0)) (type: double), (- (- _col0)) (type: double), (-1.0 % (- _col0)) (type: double), _col1 (type: bigint), (- _col1) (type: bigint), _col3 (type: double), (- (- (- _col0))) (type: double), (762 * (- _col1)) (type: bigint), _col4 (type: int), (UDFToLong(_col2) + (762 * (- _col1))) (type: bigint), ((- _col0) + UDFToDouble(_col4)) (type: double), _col5 (type: double), ((- _col1) % _col1) (type: bigint), _col6 (type: bigint), _col7 (type: double), (-3728 % (UDFToLong(_col2) + (762 * (- _col1)))) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 8, 10, 1, 12, 2, 14, 13, 15, 1, 16, 3, 9, 19, 4, 18, 22, 5, 23, 6, 7, 24] + selectExpressions: DoubleColUnaryMinus(col 0) -> 8:double, DoubleColSubtractDoubleColumn(col 0, col 9)(children: DoubleColUnaryMinus(col 0) -> 9:double) -> 10:double, DecimalColModuloDecimalScalar(col 11, val 79.553)(children: CastLongToDecimal(col 1) -> 11:decimal(19,0)) -> 12:decimal(5,3), DoubleColSubtractDoubleColumn(col 9, col 13)(children: CastLongToDouble(col 1) -> 9:double, DoubleColUnaryMinus(col 0) -> 13:double) -> 14:double, DoubleColUnaryMinus(col 9)(children: DoubleColUnaryMinus(col 0) -> 9:double) -> 13:double, DoubleScalarModuloDoubleColumn(val -1.0, col 9)(children: DoubleColUnaryMinus(col 0) -> 9:double) -> 15:double, LongColUnaryMinus(col 1) -> 16:long, DoubleColUnaryMinus(col 17)(children: DoubleColUnaryMinus(col 9)(children: DoubleColUnaryMinus(col 0) -> 9:double) -> 17:double) -> 9:double, LongScalarMultiplyLongColumn(val 762, col 18)(children: LongColUnaryMinus(col 1) -> 18:long) -> 19:long, LongColAddLongColumn(col 2, col 20)(children: co l 2, LongScalarMultiplyLongColumn(val 762, col 18)(children: LongColUnaryMinus(col 1) -> 18:long) -> 20:long) -> 18:long, DoubleColAddDoubleColumn(col 17, col 21)(children: DoubleColUnaryMinus(col 0) -> 17:double, CastLongToDouble(col 4) -> 21:double) -> 22:double, LongColModuloLongColumn(col 20, col 1)(children: LongColUnaryMinus(col 1) -> 20:long) -> 23:long, LongScalarModuloLongColumn(val -3728, col 20)(children: LongColAddLongColumn(col 2, col 24)(children: col 2, LongScalarMultiplyLongColumn(val 762, col 20)(children: LongColUnaryMinus(col 1) -> 20:long) -> 24:long) -> 20:long) -> 24:long Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -790,46 +862,70 @@ STAGE PLANS: Group By Operator aggregations: avg(ctinyint), max(cbigint), stddev_samp(cint), var_pop(cint), var_pop(cbigint), max(cfloat) Group By Vectorization: - aggregators: VectorUDAFAvgLong(col 0) -> struct<count:bigint,sum:double>, VectorUDAFMaxLong(col 3) -> bigint, VectorUDAFStdSampLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarPopLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxDouble(col 4) -> float + aggregators: VectorUDAFAvgLong(col 0) -> struct<count:bigint,sum:double,input:bigint>, VectorUDAFMaxLong(col 3) -> bigint, VectorUDAFStdSampLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarPopLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFMaxDouble(col 4) -> float className: VectorGroupByOperator - vectorOutput: false + groupByMode: HASH + vectorOutput: true native: false + vectorProcessingMode: HASH projectedOutputColumns: [0, 1, 2, 3, 4, 5] - vectorOutputConditionsNotMet: Vector output of VectorUDAFAvgLong(col 0) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdSampLong(col 2) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarPopLong(col 2) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: struct<count:bigint,sum:double,input:tinyint>), _col1 (type: bigint), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: float) Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - groupByVectorOutput: false + groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false usesVectorUDFAdaptor: false vectorized: true Reducer 2 + Execution mode: vectorized Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: Aggregation Function UDF avg parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,input:tinyint> of Column[VALUE._col0] not supported - vectorized: false + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reduce Operator Tree: Group By Operator aggregations: avg(VALUE._col0), max(VALUE._col1), stddev_samp(VALUE._col2), var_pop(VALUE._col3), var_pop(VALUE._col4), max(VALUE._col5) + Group By Vectorization: + aggregators: VectorUDAFAvgFinal(col 0) -> double, VectorUDAFMaxLong(col 1) -> bigint, VectorUDAFStdSampFinal(col 2) -> double, VectorUDAFVarPopFinal(col 3) -> double, VectorUDAFVarPopFinal(col 4) -> double, VectorUDAFMaxDouble(col 5) -> float + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + vectorOutput: true + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumns: [0, 1, 2, 3, 4, 5] mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: double), (_col0 + 6981.0) (type: double), ((_col0 + 6981.0) + _col0) (type: double), _col1 (type: bigint), (((_col0 + 6981.0) + _col0) / _col0) (type: double), (- (_col0 + 6981.0)) (type: double), _col2 (type: double), (_col0 % (- (_col0 + 6981.0))) (type: double), _col3 (type: double), _col4 (type: double), (- _col1) (type: bigint), (UDFToDouble((- _col1)) / _col2) (type: double), _col5 (type: float), (_col4 * -26.28) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 6, 8, 1, 7, 10, 2, 9, 3, 4, 12, 14, 5, 11] + selectExpressions: DoubleColAddDoubleScalar(col 0, val 6981.0) -> 6:double, DoubleColAddDoubleColumn(col 7, col 0)(children: DoubleColAddDoubleScalar(col 0, val 6981.0) -> 7:double) -> 8:double, DoubleColDivideDoubleColumn(col 9, col 0)(children: DoubleColAddDoubleColumn(col 7, col 0)(children: DoubleColAddDoubleScalar(col 0, val 6981.0) -> 7:double) -> 9:double) -> 7:double, DoubleColUnaryMinus(col 9)(children: DoubleColAddDoubleScalar(col 0, val 6981.0) -> 9:double) -> 10:double, DoubleColModuloDoubleColumn(col 0, col 11)(children: DoubleColUnaryMinus(col 9)(children: DoubleColAddDoubleScalar(col 0, val 6981.0) -> 9:double) -> 11:double) -> 9:double, LongColUnaryMinus(col 1) -> 12:long, DoubleColDivideDoubleColumn(col 11, col 2)(children: CastLongToDouble(col 13)(children: LongColUnaryMinus(col 1) -> 13:long) -> 11:double) -> 14:double, DoubleColMultiplyDoubleScalar(col 4, val -26.28) -> 11:double Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -2098,11 +2194,12 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFStdSampLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFSumLong(col 3) -> bigint, VectorUDAFVarPopLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator - vectorOutput: false + groupByMode: HASH + vectorOutput: true keyExpressions: col 1 native: false + vectorProcessingMode: HASH projectedOutputColumns: [0, 1, 2, 3] - vectorOutputConditionsNotMet: Vector output of VectorUDAFStdSampLong(col 1) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarPopLong(col 0) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false keys: csmallint (type: smallint) mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4 @@ -2111,26 +2208,42 @@ STAGE PLANS: key expressions: _col0 (type: smallint) sort order: + Map-reduce partition columns: _col0 (type: smallint) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 2503 Data size: 76841 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: bigint), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: bigint) Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - groupByVectorOutput: false + groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false usesVectorUDFAdaptor: false vectorized: true Reducer 2 + Execution mode: vectorized Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: Aggregation Function UDF stddev_samp parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col0] not supported - vectorized: false + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reduce Operator Tree: Group By Operator aggregations: stddev_samp(VALUE._col0), sum(VALUE._col1), var_pop(VALUE._col2), count(VALUE._col3) + Group By Vectorization: + aggregators: VectorUDAFStdSampFinal(col 1) -> double, VectorUDAFSumLong(col 2) -> bigint, VectorUDAFVarPopFinal(col 3) -> double, VectorUDAFCountMerge(col 4) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + vectorOutput: true + keyExpressions: col 0 + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumns: [0, 1, 2, 3] keys: KEY._col0 (type: smallint) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 @@ -2138,10 +2251,19 @@ STAGE PLANS: Select Operator expressions: _col0 (type: smallint), (UDFToInteger(_col0) % -75) (type: int), _col1 (type: double), (-1.389 / CAST( _col0 AS decimal(5,0))) (type: decimal(10,9)), _col2 (type: bigint), (UDFToDouble((UDFToInteger(_col0) % -75)) / UDFToDouble(_col2)) (type: double), (- (UDFToInteger(_col0) % -75)) (type: int), _col3 (type: double), (- (- (UDFToInteger(_col0) % -75))) (type: int), _col4 (type: bigint), (_col4 - -89010) (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 5, 1, 7, 2, 11, 12, 3, 8, 4, 13] + selectExpressions: LongColModuloLongScalar(col 0, val -75)(children: col 0) -> 5:long, DecimalScalarDivideDecimalColumn(val -1.389, col 6)(children: CastLongToDecimal(col 0) -> 6:decimal(5,0)) -> 7:decimal(10,9), DoubleColDivideDoubleColumn(col 9, col 10)(children: CastLongToDouble(col 8)(children: LongColModuloLongScalar(col 0, val -75)(children: col 0) -> 8:long) -> 9:double, CastLongToDouble(col 2) -> 10:double) -> 11:double, LongColUnaryMinus(col 8)(children: LongColModuloLongScalar(col 0, val -75)(children: col 0) -> 8:long) -> 12:long, LongColUnaryMinus(col 13)(children: LongColUnaryMinus(col 8)(children: LongColModuloLongScalar(col 0, val -75)(children: col 0) -> 8:long) -> 13:long) -> 8:long, LongColSubtractLongScalar(col 4, val -89010) -> 13:long Statistics: Num rows: 1251 Data size: 38405 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: double), _col3 (type: decimal(10,9)), _col4 (type: bigint), _col5 (type: double), _col6 (type: int), _col7 (type: double), _col8 (type: int), _col9 (type: bigint), _col10 (type: bigint) sort order: +++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1251 Data size: 38405 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 Reducer 3 @@ -2350,11 +2472,12 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFVarSampDouble(col 5) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFCount(col 4) -> bigint, VectorUDAFSumDouble(col 4) -> double, VectorUDAFVarPopDouble(col 5) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopDouble(col 5) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFSumDouble(col 5) -> double className: VectorGroupByOperator - vectorOutput: false + groupByMode: HASH + vectorOutput: true keyExpressions: col 5 native: false + vectorProcessingMode: HASH projectedOutputColumns: [0, 1, 2, 3, 4, 5] - vectorOutputConditionsNotMet: Vector output of VectorUDAFVarSampDouble(col 5) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarPopDouble(col 5) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdPopDouble(col 5) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false keys: cdouble (type: double) mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 @@ -2363,26 +2486,42 @@ STAGE PLANS: key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 2654 Data size: 81476 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: bigint), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: double) Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - groupByVectorOutput: false + groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false usesVectorUDFAdaptor: false vectorized: true Reducer 2 + Execution mode: vectorized Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: Aggregation Function UDF var_samp parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col0] not supported - vectorized: false + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reduce Operator Tree: Group By Operator aggregations: var_samp(VALUE._col0), count(VALUE._col1), sum(VALUE._col2), var_pop(VALUE._col3), stddev_pop(VALUE._col4), sum(VALUE._col5) + Group By Vectorization: + aggregators: VectorUDAFVarSampFinal(col 1) -> double, VectorUDAFCountMerge(col 2) -> bigint, VectorUDAFSumDouble(col 3) -> double, VectorUDAFVarPopFinal(col 4) -> double, VectorUDAFStdPopFinal(col 5) -> double, VectorUDAFSumDouble(col 6) -> double + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + vectorOutput: true + keyExpressions: col 0 + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumns: [0, 1, 2, 3, 4, 5] keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 @@ -2390,10 +2529,19 @@ STAGE PLANS: Select Operator expressions: _col0 (type: double), _col1 (type: double), (2563.58 * _col1) (type: double), (- _col1) (type: double), _col2 (type: bigint), ((2563.58 * _col1) + -5638.15) (type: double), ((- _col1) * ((2563.58 * _col1) + -5638.15)) (type: double), _col3 (type: double), _col4 (type: double), (_col0 - (- _col1)) (type: double), _col5 (type: double), (_col0 + _col1) (type: double), (_col0 * 762.0) (type: double), _col6 (type: double), (-863.257 % (_col0 * 762.0)) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 1, 7, 8, 2, 10, 11, 3, 4, 12, 5, 9, 13, 6, 15] + selectExpressions: DoubleScalarMultiplyDoubleColumn(val 2563.58, col 1) -> 7:double, DoubleColUnaryMinus(col 1) -> 8:double, DoubleColAddDoubleScalar(col 9, val -5638.15)(children: DoubleScalarMultiplyDoubleColumn(val 2563.58, col 1) -> 9:double) -> 10:double, DoubleColMultiplyDoubleColumn(col 9, col 12)(children: DoubleColUnaryMinus(col 1) -> 9:double, DoubleColAddDoubleScalar(col 11, val -5638.15)(children: DoubleScalarMultiplyDoubleColumn(val 2563.58, col 1) -> 11:double) -> 12:double) -> 11:double, DoubleColSubtractDoubleColumn(col 0, col 9)(children: DoubleColUnaryMinus(col 1) -> 9:double) -> 12:double, DoubleColAddDoubleColumn(col 0, col 1) -> 9:double, DoubleColMultiplyDoubleScalar(col 0, val 762.0) -> 13:double, DoubleScalarModuloDoubleColumn(val -863.257, col 14)(children: DoubleColMultiplyDoubleScalar(col 0, val 762.0) -> 14:double) -> 15:double Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double) sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double) Reducer 3 @@ -2644,13 +2792,14 @@ STAGE PLANS: Group By Operator aggregations: stddev_pop(cint), avg(csmallint), count(), min(ctinyint), var_samp(csmallint), var_pop(cfloat), avg(cint), var_samp(cfloat), avg(cfloat), min(cdouble), var_pop(csmallint), stddev_pop(ctinyint), sum(cint) Group By Vectorization: - aggregators: VectorUDAFStdPopLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgLong(col 1) -> struct<count:bigint,sum:double>, VectorUDAFCountStar(*) -> bigint, VectorUDAFMinLong(col 0) -> tinyint, VectorUDAFVarSampLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double>, VectorUDAFVarSampDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgDouble(col 4) -> struct<count:bigint,sum:double>, VectorUDAFMinDouble(col 5) -> double, VectorUDAFVarPopLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFSumLong(col 2) -> bigint + aggregators: VectorUDAFStdPopLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgLong(col 1) -> struct<count:bigint,sum:double,input:bigint>, VectorUDAFCountStar(*) -> bigint, VectorUDAFMinLong(col 0) -> tinyint, VectorUDAFVarSampLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFVarPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double,input:bigint>, VectorUDAFVarSampDouble(col 4) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgDouble(col 4) -> struct<count:bigint,sum:double,input:double>, VectorUDAFMinDouble(col 5) -> double, VectorUDAFVarPopLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFSumLong(col 2) -> bigint className: VectorGroupByOperator - vectorOutput: false + groupByMode: HASH + vectorOutput: true keyExpressions: col 8, col 6 native: false + vectorProcessingMode: HASH projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] - vectorOutputConditionsNotMet: Vector output of VectorUDAFStdPopLong(col 2) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFAvgLong(col 1) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarSampLong(col 1) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarPopDouble(col 4) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarSampDouble(col 4) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFAvgDouble(col 4) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarPopLong(col 1) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdPopLong(col 0) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false keys: ctimestamp1 (type: timestamp), cstring1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 @@ -2659,26 +2808,42 @@ STAGE PLANS: key expressions: _col0 (type: timestamp), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: timestamp), _col1 (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,input:smallint>), _col4 (type: bigint), _col5 (type: tinyint), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,input:int>), _col9 (type: struct<count:bigint,sum:double,variance:double>), _col10 (type: struct<count:bigint,sum:double,input:float>), _col11 (type: double), _col12 (type: struct<count:bigint,sum:double,variance:double>), _col13 (type: struct<count:bigint,sum:double,variance:double>), _col14 (type: bigint) Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - groupByVectorOutput: false + groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false usesVectorUDFAdaptor: false vectorized: true Reducer 2 + Execution mode: vectorized Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: Aggregation Function UDF stddev_pop parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col0] not supported - vectorized: false + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reduce Operator Tree: Group By Operator aggregations: stddev_pop(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), min(VALUE._col3), var_samp(VALUE._col4), var_pop(VALUE._col5), avg(VALUE._col6), var_samp(VALUE._col7), avg(VALUE._col8), min(VALUE._col9), var_pop(VALUE._col10), stddev_pop(VALUE._col11), sum(VALUE._col12) + Group By Vectorization: + aggregators: VectorUDAFStdPopFinal(col 2) -> double, VectorUDAFAvgFinal(col 3) -> double, VectorUDAFCountMerge(col 4) -> bigint, VectorUDAFMinLong(col 5) -> tinyint, VectorUDAFVarSampFinal(col 6) -> double, VectorUDAFVarPopFinal(col 7) -> double, VectorUDAFAvgFinal(col 8) -> double, VectorUDAFVarSampFinal(col 9) -> double, VectorUDAFAvgFinal(col 10) -> double, VectorUDAFMinDouble(col 11) -> double, VectorUDAFVarPopFinal(col 12) -> double, VectorUDAFStdPopFinal(col 13) -> double, VectorUDAFSumLong(col 14) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + vectorOutput: true + keyExpressions: col 0, col 1 + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] keys: KEY._col0 (type: timestamp), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 @@ -2686,10 +2851,19 @@ STAGE PLANS: Select Operator expressions: _col0 (type: timestamp), _col1 (type: string), _col2 (type: double), (_col2 * 10.175) (type: double), (- _col2) (type: double), _col3 (type: double), (- _col2) (type: double), (-26.28 - _col2) (type: double), _col4 (type: bigint), (- _col4) (type: bigint), ((-26.28 - _col2) * (- _col2)) (type: double), _col5 (type: tinyint), (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4))) (type: double), (- (_col2 * 10.175)) (type: double), _col6 (type: double), (_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) (type: double), (- (- _col2)) (type: double), (UDFToDouble((- _col4)) / _col2) (type: double), _col7 (type: double), (10.175 / _col3) (type: double), _col8 (type: double), _col9 (type: double), ((_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) - (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) (type: double), (- (- (_col2 * 10.175))) (type: double), _col10 (type: double), (((_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) - (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) * 10.175) (type: double), (10.175 % (10.175 / _col3)) (type: double), (- _col5) (type: tinyint), _col11 (type: double), _col12 (type: double), (- ((-26.28 - _col2) * (- _col2))) (type: double), ((- _col2) % _col10) (type: double), (-26.28 / CAST( (- _col5) AS decimal(3,0))) (type: decimal(8,6)), _col13 (type: double), _col14 (type: bigint), ((_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) / _col7) (type: double), (- (- _col4)) (type: bigint), _col4 (type: bigint), ((_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) % -26.28) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 1, 2, 15, 16, 3, 17, 18, 4, 19, 22, 5, 21, 23, 6, 20, 26, 27, 7, 25, 8, 9, 29, 28, 10, 30, 32, 24, 11, 12, 31, 34, 37, 13, 14, 38, 40, 4, 39] + selectExpressions: DoubleColMultiplyDoubleScalar(col 2, val 10.175) -> 15:double, DoubleColUnaryMinus(col 2) -> 16:double, DoubleColUnaryMinus(col 2) -> 17:double, DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 18:double, LongColUnaryMinus(col 4) -> 19:long, DoubleColMultiplyDoubleColumn(col 20, col 21)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 20:double, DoubleColUnaryMinus(col 2) -> 21:double) -> 22:double, DoubleColMultiplyDoubleColumn(col 23, col 20)(children: DoubleColMultiplyDoubleColumn(col 20, col 21)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 20:double, DoubleColUnaryMinus(col 2) -> 21:double) -> 23:double, CastLongToDouble(col 24)(children: LongColUnaryMinus(col 4) -> 24:long) -> 20:double) -> 21:double, DoubleColUnaryMinus(col 20)(children: DoubleColMultiplyDoubleScalar(col 2, val 10.175) -> 20:double) -> 23:double, DoubleColAddDoubleColumn(col 6, col 25)(children: DoubleColMultiplyDoubleColumn(col 26, col 20)(children: DoubleColMultiplyDoubleColumn(col 20, col 25)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 20:double, DoubleColUnaryMinus(col 2) -> 25:double) -> 26:double, CastLongToDouble(col 24)(children: LongColUnaryMinus(col 4) -> 24:long) -> 20:double) -> 25:double) -> 20:double, DoubleColUnaryMinus(col 25)(children: DoubleColUnaryMinus(col 2) -> 25:double) -> 26:double, DoubleColDivideDoubleColumn(col 25, col 2)(children: CastLongToDouble(col 24)(children: LongColUnaryMinus(col 4) -> 24:long) -> 25:double) -> 27:double, DoubleScalarDivideDoubleColumn(val 10.175, col 3) -> 25:double, DoubleColSubtractDoubleColumn(col 28, col 30)(children: DoubleColAddDoubleColumn(col 6, col 29)(children: DoubleColMultiplyDoubleColumn(col 30, col 28)(children: DoubleColMultiplyDoubleColumn(col 28, col 29)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 28:double, DoubleColUnaryMinus(col 2) -> 29:double) -> 30:double, CastLongToDouble(col 24)(children: LongColUnaryMinus(col 4) -> 24:long) -> 28:double) -> 29:double) -> 28:double, DoubleColMultiplyDoubleColumn(col 31, col 29)(children: DoubleColMultiplyDoubleColumn(col 29, col 30)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 29:double, DoubleColUnaryMinus(col 2) -> 30:double) -> 31:double, CastLongToDouble(col 24)(children: LongColUnaryMinus(col 4) -> 24:long) -> 29:double) -> 30:double) -> 29:double, DoubleColUnaryMinus(col 30)(children: DoubleColUnaryMinus(col 28)(children: DoubleColMultiplyDoubleScalar(col 2, val 10.175) -> 28:double) -> 30:double) -> 28:double, DoubleColMultiplyDoubleScalar(col 31, val 10.175)(children: DoubleColSubtractDoubleColumn(col 30, col 32)(children: DoubleColAddDoubleColumn(col 6, col 31)(children: DoubleColMultiplyDoubleColumn(col 32, col 30)(children: DoubleColMultiplyDoubleColumn(col 30, col 31)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 30:double, DoubleColUnaryMinus(col 2) -> 31:double) -> 32:double, Cast LongToDouble(col 24)(children: LongColUnaryMinus(col 4) -> 24:long) -> 30:double) -> 31:double) -> 30:double, DoubleColMultiplyDoubleColumn(col 33, col 31)(children: DoubleColMultiplyDoubleColumn(col 31, col 32)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 31:double, DoubleColUnaryMinus(col 2) -> 32:double) -> 33:double, CastLongToDouble(col 24)(children: LongColUnaryMinus(col 4) -> 24:long) -> 31:double) -> 32:double) -> 31:double) -> 30:double, DoubleScalarModuloDoubleColumn(val 10.175, col 31)(children: DoubleScalarDivideDoubleColumn(val 10.175, col 3) -> 31:double) -> 32:double, LongColUnaryMinus(col 5) -> 24:long, DoubleColUnaryMinus(col 34)(children: DoubleColMultiplyDoubleColumn(col 31, col 33)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 31:double, DoubleColUnaryMinus(col 2) -> 33:double) -> 34:double) -> 31:double, DoubleColModuloDoubleColumn(col 33, col 10)(children: DoubleColUnaryMinus(col 2) -> 33:double) -> 34:double, DecimalScala rDivideDecimalColumn(val -26.28, col 36)(children: CastLongToDecimal(col 35)(children: LongColUnaryMinus(col 5) -> 35:long) -> 36:decimal(3,0)) -> 37:decimal(8,6), DoubleColDivideDoubleColumn(col 33, col 7)(children: DoubleColAddDoubleColumn(col 6, col 38)(children: DoubleColMultiplyDoubleColumn(col 39, col 33)(children: DoubleColMultiplyDoubleColumn(col 33, col 38)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 33:double, DoubleColUnaryMinus(col 2) -> 38:double) -> 39:double, CastLongToDouble(col 35)(children: LongColUnaryMinus(col 4) -> 35:long) -> 33:double) -> 38:double) -> 33:double) -> 38:double, LongColUnaryMinus(col 35)(children: LongColUnaryMinus(col 4) -> 35:long) -> 40:long, DoubleColModuloDoubleScalar(col 33, val -26.28)(children: DoubleColAddDoubleColumn(col 6, col 39)(children: DoubleColMultiplyDoubleColumn(col 41, col 33)(children: DoubleColMultiplyDoubleColumn(col 33, col 39)(children: DoubleScalarSubtractDoubleColumn(val -26.28, col 2) -> 33:double , DoubleColUnaryMinus(col 2) -> 39:double) -> 41:double, CastLongToDouble(col 35)(children: LongColUnaryMinus(col 4) -> 35:long) -> 33:double) -> 39:double) -> 33:double) -> 39:double Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: timestamp), _col1 (type: string), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: bigint), _col9 (type: bigint), _col10 (type: double), _col11 (type: tinyint), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: double), _col19 (type: double), _col20 (type: double), _col21 (type: double), _col22 (type: double), _col23 (type: double), _col24 (type: double), _col25 (type: double), _col26 (type: double), _col27 (type: tinyint), _col28 (type: double), _col29 (type: double), _col30 (type: double), _col31 (type: double), _col32 (type: decimal(8,6)), _col33 (type: double), _col34 (type: bigint), _col35 (type: double), _col36 (type: bigint), _col37 (type: bigint), _col38 (type: double) sort order: +++++++++++++++++++++++++++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 Reducer 3 @@ -3021,13 +3195,14 @@ STAGE PLANS: Group By Operator aggregations: max(cfloat), sum(cbigint), var_samp(cint), avg(cdouble), min(cbigint), var_pop(cbigint), sum(cint), stddev_samp(ctinyint), stddev_pop(csmallint), avg(cint) Group By Vectorization: - aggregators: VectorUDAFMaxDouble(col 4) -> float, VectorUDAFSumLong(col 3) -> bigint, VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgDouble(col 5) -> struct<count:bigint,sum:double>, VectorUDAFMinLong(col 3) -> bigint, VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFSumLong(col 2) -> bigint, VectorUDAFStdSampLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double> + aggregators: VectorUDAFMaxDouble(col 4) -> float, VectorUDAFSumLong(col 3) -> bigint, VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgDouble(col 5) -> struct<count:bigint,sum:double,input:double>, VectorUDAFMinLong(col 3) -> bigint, VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFSumLong(col 2) -> bigint, VectorUDAFStdSampLong(col 0) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFStdPopLong(col 1) -> struct<count:bigint,sum:double,variance:double>, VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double,input:bigint> className: VectorGroupByOperator - vectorOutput: false + groupByMode: HASH + vectorOutput: true keyExpressions: col 10 native: false + vectorProcessingMode: HASH projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - vectorOutputConditionsNotMet: Vector output of VectorUDAFVarSampLong(col 2) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFAvgDouble(col 5) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFVarPopLong(col 3) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdSampLong(col 0) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFStdPopLong(col 1) -> struct<count:bigint,sum:double,variance:double> output type STRUCT requires PRIMITIVE IS false, Vector output of VectorUDAFAvgLong(col 2) -> struct<count:bigint,sum:double> output type STRUCT requires PRIMITIVE IS false keys: cboolean1 (type: boolean) mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 @@ -3036,26 +3211,42 @@ STAGE PLANS: key expressions: _col0 (type: boolean) sort order: + Map-reduce partition columns: _col0 (type: boolean) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 10239 Data size: 314333 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: float), _col2 (type: bigint), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,input:double>), _col5 (type: bigint), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: bigint), _col8 (type: struct<count:bigint,sum:double,variance:double>), _col9 (type: struct<count:bigint,sum:double,variance:double>), _col10 (type: struct<count:bigint,sum:double,input:int>) Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - groupByVectorOutput: false + groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false usesVectorUDFAdaptor: false vectorized: true Reducer 2 + Execution mode: vectorized Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: Aggregation Function UDF var_samp parameter expression for GROUPBY operator: Data type struct<count:bigint,sum:double,variance:double> of Column[VALUE._col2] not supported - vectorized: false + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), sum(VALUE._col1), var_samp(VALUE._col2), avg(VALUE._col3), min(VALUE._col4), var_pop(VALUE._col5), sum(VALUE._col6), stddev_samp(VALUE._col7), stddev_pop(VALUE._col8), avg(VALUE._col9) + Group By Vectorization: + aggregators: VectorUDAFMaxDouble(col 1) -> float, VectorUDAFSumLong(col 2) -> bigint, VectorUDAFVarSampFinal(col 3) -> double, VectorUDAFAvgFinal(col 4) -> double, VectorUDAFMinLong(col 5) -> bigint, VectorUDAFVarPopFinal(col 6) -> double, VectorUDAFSumLong(col 7) -> bigint, VectorUDAFStdSampFinal(col 8) -> double, VectorUDAFStdPopFinal(col 9) -> double, VectorUDAFAvgFinal(col 10) -> double + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + vectorOutput: true + keyExpressions: col 0 + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] keys: KEY._col0 (type: boolean) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 @@ -3063,10 +3254,19 @@ STAGE PLANS: Select Operator expressions: _col0 (type: boolean), _col1 (type: float), (- _col1) (type: float), (-26.28 / UDFToDouble(_col1)) (type: double), _col2 (type: bigint), (CAST( _col2 AS decimal(19,0)) - 10.175) (type: decimal(23,3)), _col3 (type: double), (_col3 % UDFToDouble(_col1)) (type: double), (10.175 + (- _col1)) (type: float), _col4 (type: double), (UDFToDouble((CAST( _col2 AS decimal(19,0)) - 10.175)) + _col3) (type: double), _col5 (type: bigint), _col6 (type: double), (- (10.175 + (- _col1))) (type: float), (79.553 / _col6) (type: double), (_col3 % (79.553 / _col6)) (type: double), _col7 (type: bigint), _col8 (type: double), (-1.389 * CAST( _col5 AS decimal(19,0))) (type: decimal(24,3)), (CAST( _col7 AS decimal(19,0)) - (-1.389 * CAST( _col5 AS decimal(19,0)))) (type: decimal(25,3)), _col9 (type: double), (- (CAST( _col7 AS decimal(19,0)) - (-1.389 * CAST( _col5 AS decimal(19,0))))) (type: decimal(25,3)), _col10 (type: double), (- _col10) (type: double), (_col10 * UDFToDoubl e(_col7)) (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 1, 11, 12, 2, 14, 3, 15, 17, 4, 19, 5, 6, 16, 20, 22, 7, 8, 23, 26, 9, 28, 10, 21, 30] + selectExpressions: DoubleColUnaryMinus(col 1) -> 11:double, DoubleScalarDivideDoubleColumn(val -26.28, col 1)(children: col 1) -> 12:double, DecimalColSubtractDecimalScalar(col 13, val 10.175)(children: CastLongToDecimal(col 2) -> 13:decimal(19,0)) -> 14:decimal(23,3), DoubleColModuloDoubleColumn(col 3, col 1)(children: col 1) -> 15:double, DoubleScalarAddDoubleColumn(val 10.175000190734863, col 16)(children: DoubleColUnaryMinus(col 1) -> 16:double) -> 17:double, DoubleColAddDoubleColumn(col 16, col 3)(children: CastDecimalToDouble(col 18)(children: DecimalColSubtractDecimalScalar(col 13, val 10.175)(children: CastLongToDecimal(col 2) -> 13:decimal(19,0)) -> 18:decimal(23,3)) -> 16:double) -> 19:double, DoubleColUnaryMinus(col 20)(children: DoubleScalarAddDoubleColumn(val 10.175000190734863, col 16)(children: DoubleColUnaryMinus(col 1) -> 16:double) -> 20:double) -> 16:double, DoubleScalarDivideDoubleColumn(val 79.553, col 6) -> 20:double, DoubleColModuloDouble Column(col 3, col 21)(children: DoubleScalarDivideDoubleColumn(val 79.553, col 6) -> 21:double) -> 22:double, DecimalScalarMultiplyDecimalColumn(val -1.389, col 13)(children: CastLongToDecimal(col 5) -> 13:decimal(19,0)) -> 23:decimal(24,3), DecimalColSubtractDecimalColumn(col 13, col 25)(children: CastLongToDecimal(col 7) -> 13:decimal(19,0), DecimalScalarMultiplyDecimalColumn(val -1.389, col 24)(children: CastLongToDecimal(col 5) -> 24:decimal(19,0)) -> 25:decimal(24,3)) -> 26:decimal(25,3), FuncNegateDecimalToDecimal(col 27)(children: DecimalColSubtractDecimalColumn(col 13, col 25)(children: CastLongToDecimal(col 7) -> 13:decimal(19,0), DecimalScalarMultiplyDecimalColumn(val -1.389, col 24)(children: CastLongToDecimal(col 5) -> 24:decimal(19,0)) -> 25:decimal(24,3)) -> 27:decimal(25,3)) -> 28:decimal(25,3), DoubleColUnaryMinus(col 10) -> 21:double, DoubleColMultiplyDoubleColumn(col 10, col 29)(children: CastLongToDouble(col 7) -> 29:double) -> 30:double Statistics: Num rows: 5119 Data size: 157151 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: boolean) sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 5119 Data size: 157151 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: float), _col2 (type: float), _col3 (type: double), _col4 (type: bigint), _col5 (type: decimal(23,3)), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: bigint), _col12 (type: double), _col13 (type: float), _col14 (type: double), _col15 (type: double), _col17 (type: bigint), _col18 (type: double), _col19 (type: decimal(24,3)), _col20 (type: decimal(25,3)), _col21 (type: double), _col22 (type: decimal(25,3)), _col23 (type: double), _col24 (type: double), _col25 (type: double) Reducer 3 @@ -3238,8 +3438,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator + groupByMode: HASH vectorOutput: true native: false + vectorProcessingMode: HASH projectedOutputColumns: [0] mode: hash outputColumnNames: _col0 @@ -3276,8 +3478,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCountMerge(col 0) -> bigint className: VectorGroupByOperator + groupByMode: MERGEPARTIAL vectorOutput: true native: false + vectorProcessingMode: GLOBAL projectedOutputColumns: [0] mode: mergepartial outputColumnNames: _col0 @@ -3350,8 +3554,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCount(col 0) -> bigint className: VectorGroupByOperator + groupByMode: HASH vectorOutput: true native: false + vectorProcessingMode: HASH projectedOutputColumns: [0] mode: hash outputColumnNames: _col0 @@ -3388,8 +3594,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCountMerge(col 0) -> bigint className: VectorGroupByOperator + groupByMode: MERGEPARTIAL vectorOutput: true native: false + vectorProcessingMode: GLOBAL projectedOutputColumns: [0] mode: mergepartial outputColumnNames: _col0 @@ -3534,8 +3742,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCountStar(*) -> bigint className: VectorGroupByOperator + groupByMode: HASH vectorOutput: true native: false + vectorProcessingMode: HASH projectedOutputColumns: [0] mode: hash outputColumnNames: _col0 @@ -3572,8 +3782,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCountMerge(col 0) -> bigint className: VectorGroupByOperator + groupByMode: MERGEPARTIAL vectorOutput: true native: false + vectorProcessingMode: GLOBAL projectedOutputColumns: [0] mode: mergepartial outputColumnNames: _col0 @@ -3646,8 +3858,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCount(col 0) -> bigint className: VectorGroupByOperator + groupByMode: HASH vectorOutput: true native: false + vectorProcessingMode: HASH projectedOutputColumns: [0] mode: hash outputColumnNames: _col0 @@ -3684,8 +3898,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCountMerge(col 0) -> bigint className: VectorGroupByOperator + groupByMode: MERGEPARTIAL vectorOutput: true native: false + vectorProcessingMode: GLOBAL projectedOutputColumns: [0] mode: mergepartial outputColumnNames: _col0 @@ -3758,8 +3974,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCount(col 2) -> bigint className: VectorGroupByOperator + groupByMode: HASH vectorOutput: true native: false + vectorProcessingMode: HASH projectedOutputColumns: [0] mode: hash outputColumnNames: _col0 @@ -3796,8 +4014,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCountMerge(col 0) -> bigint className: VectorGroupByOperator + groupByMode: MERGEPARTIAL vectorOutput: true native: false + vectorProcessingMode: GLOBAL projectedOutputColumns: [0] mode: mergepartial outputColumnNames: _col0 @@ -3870,8 +4090,10 @@ STAGE PLANS: Group By Vectorization: aggregators: VectorUDAFCount(col 4) -> bigint className: VectorGroupByOperator + groupByMode: HASH
<TRUNCATED>
