Repository: hive Updated Branches: refs/heads/master fc48d7218 -> 9d0f9c07e
http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out index 7dde2ec..2581311 100644 --- a/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out @@ -221,7 +221,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: SELECT operator: Unexpected primitive type category VOID + notVectorizedReason: SELECT operator: Unexpected hive type name void vectorized: false Reducer 2 Execution mode: vectorized, llap http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out index d8ab3c5..b29fd4b 100644 --- a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out @@ -151,7 +151,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: SELECT operator: Unexpected primitive type category VOID + notVectorizedReason: SELECT operator: Unexpected hive type name void vectorized: false Reducer 2 Execution mode: vectorized, llap http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out b/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out index fc13ce1..81c6370 100644 --- a/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_null_projection.q.out @@ -32,10 +32,10 @@ POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@b POSTHOOK: Lineage: b.s SCRIPT [] -PREHOOK: query: explain vectorization expression +PREHOOK: query: explain vectorization detail select NULL from a PREHOOK: type: QUERY -POSTHOOK: query: explain vectorization expression +POSTHOOK: query: explain vectorization detail select NULL from a POSTHOOK: type: QUERY PLAN VECTORIZATION: @@ -56,25 +56,45 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:s:string, 1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>] Select Operator expressions: null (type: void) outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2] + selectExpressions: ConstantVectorExpression(val null) -> 2:void Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap + Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: Select expression for SELECT operator: Vectorizing data type void not supported when mode = PROJECTION - vectorized: false + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: s:string + partitionColumnCount: 0 + scratchColumnTypeNames: [void] Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_nvl.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_nvl.q.out b/ql/src/test/results/clientpositive/llap/vector_nvl.q.out index 2bbbbe4..5a312fc 100644 --- a/ql/src/test/results/clientpositive/llap/vector_nvl.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_nvl.q.out @@ -314,28 +314,44 @@ STAGE PLANS: TableScan alias: alltypesorc Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true Select Operator expressions: null (type: void) outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [13] + selectExpressions: ConstantVectorExpression(val null) -> 13:void Statistics: Num rows: 12288 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 + Limit Vectorization: + className: VectorLimitOperator + native: true Statistics: Num rows: 10 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 10 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap + Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: Select expression for SELECT operator: Vectorizing data type void not supported when mode = PROJECTION - vectorized: false + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out b/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out index ebf6853..f1a4ea3 100644 --- a/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_orc_merge_incompat_schema.q.out @@ -155,14 +155,18 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.orc_create_complex - Execution mode: llap + Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + vectorizationSupportRemovedReasons: [DECIMAL_64 disabled because LLAP is enabled] + featureSupportInUse: [] inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: SELECT operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[strct].a - vectorized: false + allNative: false + usesVectorUDFAdaptor: true + vectorized: true Stage: Stage-2 Dependency Collection http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out b/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out index 7ff4bb1..0b645ab 100644 --- a/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out @@ -733,28 +733,45 @@ STAGE PLANS: alias: nested_tbl_1 Pruned Column Paths: s1.f2, s2.f8 Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 8:string, val foo)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 1:int) -> 8:string), FilterLongColGreaterLongScalar(col 11:int, val 1)(children: VectorUDFAdaptor(size(s2.f8.f10))(children: VectorUDFStructField(col 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 1:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 10:array<int>) -> 11:int), SelectColumnIsTrue(col 11:boolean)(children: VectorUDFAdaptor(s2.f8.f11['key1'])(children: VectorUDFStructField(col 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 2:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:in t) -> 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 12:map<string,boolean>) -> 11:boolean)) predicate: ((s1.f2 = 'foo') and (size(s2.f8.f10) > 1) and s2.f8.f11['key1']) (type: boolean) Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s2.f8 (type: struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [9] + selectExpressions: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 9:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>> Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap + Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: FILTER operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[s2].f8.f10 - vectorized: false + allNative: false + usesVectorUDFAdaptor: true + vectorized: true Stage: Stage-0 Fetch Operator @@ -770,7 +787,7 @@ POSTHOOK: query: SELECT s2.f8 FROM nested_tbl_1 WHERE s1.f2 = 'foo' AND size(s2. POSTHOOK: type: QUERY POSTHOOK: Input: default@nested_tbl_1 #### A masked pattern was here #### -{"f9":true,"f10":[10,11],"f11":{"key2":false,"key1":true}} +{"f9":true,"f10":[10,11],"f11":{"key1":true,"key2":false}} PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT col1, col2 FROM nested_tbl_1 LATERAL VIEW explode(s2.f8.f10) tbl1 AS col1 @@ -957,25 +974,38 @@ STAGE PLANS: alias: nested_tbl_1 Pruned Column Paths: s2.f8.f10, s1.f3.f4 Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true Select Operator expressions: (s2.f8.f10[1] pmod s1.f3.f4) (type: int) outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [13] + selectExpressions: VectorUDFAdaptor((s2.f8.f10[1] pmod s1.f3.f4))(children: ListIndexColScalar(col 9:array<int>, col 1:int)(children: VectorUDFStructField(col 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 1:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 9:array<int>) -> 10:int, VectorUDFStructField(col 11:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 11:struct<f4:int,f5:double>) -> 12:int) -> 13:int Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: llap + Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: SELECT operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[s1].f3.f4 - vectorized: false + allNative: false + usesVectorUDFAdaptor: true + vectorized: true Stage: Stage-0 Fetch Operator @@ -1700,14 +1730,32 @@ STAGE PLANS: alias: t2 Pruned Column Paths: s1.f6, s2.f8.f9 Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsTrue(col 10:boolean)(children: VectorUDFAdaptor((s2.f8.f9 and s1.f6 is not null and s2.f8.f9 is not null))(children: VectorUDFStructField(col 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 0:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 9:boolean, IsNotNull(col 10:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 3:int) -> 10:int) -> 11:boolean, IsNotNull(col 10:boolean)(children: VectorUDFStructField(col 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 0:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 10:boolean) -> 12:boolean) -> 10:boolean) predicate: (s1.f6 is not null and s2.f8.f9 and s2.f8.f9 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: s1.f6 (type: int), (s2.f8.f9 = true) (type: boolean) outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [9, 12] + selectExpressions: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 3:int) -> 9:int, LongColEqualLongScalar(col 11:boolean, val 1)(children: VectorUDFStructField(col 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 0:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 11:boolean) -> 12:boolean Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 9:int, col 12:boolean + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [] keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 @@ -1716,15 +1764,22 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) + Reduce Sink Vectorization: + className: VectorReduceSinkMultiKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap + Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: FILTER operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[s2].f8.f9 - vectorized: false + allNative: false + usesVectorUDFAdaptor: true + vectorized: true Reducer 2 Execution mode: llap Reduce Operator Tree: http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out b/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out index 93065d9..3bc9806 100644 --- a/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_udf_adaptor_1.q.out @@ -148,7 +148,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: SELECT operator: Unexpected primitive type category VOID + notVectorizedReason: SELECT operator: Unexpected hive type name void vectorized: false Stage: Stage-2 @@ -656,7 +656,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: SELECT operator: Unexpected primitive type category VOID + notVectorizedReason: SELECT operator: Unexpected hive type name void vectorized: false Stage: Stage-2 http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vector_case_when_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_case_when_1.q.out b/ql/src/test/results/clientpositive/vector_case_when_1.q.out index f17c462..b2ac291 100644 --- a/ql/src/test/results/clientpositive/vector_case_when_1.q.out +++ b/ql/src/test/results/clientpositive/vector_case_when_1.q.out @@ -213,7 +213,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: SELECT operator: Unexpected primitive type category VOID + notVectorizedReason: SELECT operator: Unexpected hive type name void vectorized: false Reduce Vectorization: enabled: false http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vector_case_when_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_case_when_2.q.out b/ql/src/test/results/clientpositive/vector_case_when_2.q.out index 89c16b0..159c983 100644 --- a/ql/src/test/results/clientpositive/vector_case_when_2.q.out +++ b/ql/src/test/results/clientpositive/vector_case_when_2.q.out @@ -143,7 +143,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: SELECT operator: Unexpected primitive type category VOID + notVectorizedReason: SELECT operator: Unexpected hive type name void vectorized: false Reduce Vectorization: enabled: false http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vector_null_projection.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_null_projection.q.out b/ql/src/test/results/clientpositive/vector_null_projection.q.out index 8e491ff..f0f444f 100644 --- a/ql/src/test/results/clientpositive/vector_null_projection.q.out +++ b/ql/src/test/results/clientpositive/vector_null_projection.q.out @@ -32,10 +32,10 @@ POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@b POSTHOOK: Lineage: b.s SCRIPT [] -PREHOOK: query: explain vectorization expression +PREHOOK: query: explain vectorization detail select NULL from a PREHOOK: type: QUERY -POSTHOOK: query: explain vectorization expression +POSTHOOK: query: explain vectorization detail select NULL from a POSTHOOK: type: QUERY PLAN VECTORIZATION: @@ -53,23 +53,44 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:s:string, 1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>] Select Operator expressions: null (type: void) outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2] + selectExpressions: ConstantVectorExpression(val null) -> 2:void Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: Select expression for SELECT operator: Vectorizing data type void not supported when mode = PROJECTION - vectorized: false + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [] + dataColumns: s:string + partitionColumnCount: 0 + scratchColumnTypeNames: [void] Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vector_nvl.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_nvl.q.out b/ql/src/test/results/clientpositive/vector_nvl.q.out index 11eacc2..467dc7f 100644 --- a/ql/src/test/results/clientpositive/vector_nvl.q.out +++ b/ql/src/test/results/clientpositive/vector_nvl.q.out @@ -299,26 +299,43 @@ STAGE PLANS: TableScan alias: alltypesorc Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true Select Operator expressions: null (type: void) outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [13] + selectExpressions: ConstantVectorExpression(val null) -> 13:void Statistics: Num rows: 12288 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 + Limit Vectorization: + className: VectorLimitOperator + native: true Statistics: Num rows: 10 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false Statistics: Num rows: 10 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: Select expression for SELECT operator: Vectorizing data type void not supported when mode = PROJECTION - vectorized: false + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out b/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out index c9a8fe3..6768c57 100644 --- a/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out +++ b/ql/src/test/results/clientpositive/vectorized_bucketmapjoin1.q.out @@ -115,13 +115,7 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - TableScan Vectorization: - native: true Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0:int) predicate: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator @@ -131,35 +125,21 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col5, _col6 - SMB Map Join Vectorization: - className: VectorSMBMapJoinOperator - native: false Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 1, 2, 3] File Output Operator compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true + notVectorizedReason: MAPJOIN operator: FilterMaps not supported for Vector Pass-Thru SMB MapJoin + vectorized: false Stage: Stage-0 Fetch Operator @@ -202,13 +182,7 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - TableScan Vectorization: - native: true Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0:int) predicate: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator @@ -218,35 +192,21 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col5, _col6 - SMB Map Join Vectorization: - className: VectorSMBMapJoinOperator - native: false Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 1, 2, 3] File Output Operator compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true + notVectorizedReason: MAPJOIN operator: FilterMaps not supported for Vector Pass-Thru SMB MapJoin + vectorized: false Stage: Stage-0 Fetch Operator @@ -289,13 +249,7 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE - TableScan Vectorization: - native: true Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0:int) predicate: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator @@ -305,35 +259,21 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1, _col5, _col6 - SMB Map Join Vectorization: - className: VectorSMBMapJoinOperator - native: false Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 1, 2, 3] File Output Operator compressed: false - File Sink Vectorization: - className: VectorFileSinkOperator - native: false table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [] - featureSupportInUse: [] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: false - usesVectorUDFAdaptor: false - vectorized: true + notVectorizedReason: MAPJOIN operator: FilterMaps not supported for Vector Pass-Thru SMB MapJoin + vectorized: false Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java index a498428..01bd2f6 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java @@ -47,7 +47,8 @@ public abstract class ColumnVector { STRUCT, LIST, MAP, - UNION + UNION, + VOID } public final Type type; http://git-wip-us.apache.org/repos/asf/hive/blob/9d0f9c07/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VoidColumnVector.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VoidColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VoidColumnVector.java new file mode 100644 index 0000000..2108a68 --- /dev/null +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VoidColumnVector.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +/** + * This class represents a void (or no) type column vector. + * + * No value vector(s) needed. Always a NULL. + */ +public class VoidColumnVector extends ColumnVector { + + /** + * Use this constructor by default. All column vectors + * should normally be the default size. + */ + public VoidColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Don't use this except for testing purposes. + * + * @param len + */ + public VoidColumnVector(int len) { + super(Type.VOID, len); + } + + @Override + public void flatten(boolean selectedInUse, int[] sel, int size) { + throw new UnsupportedOperationException(); + } + + @Override + public void setElement(int outputElementNum, int inputElementNum, + ColumnVector inputColVector) { + throw new UnsupportedOperationException(); + } + + @Override + public void copySelected(boolean selectedInUse, int[] sel, int size, + ColumnVector outputColVector) { + throw new UnsupportedOperationException(); + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + throw new UnsupportedOperationException(); + } +}
