http://git-wip-us.apache.org/repos/asf/hive/blob/fc48d721/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out b/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out new file mode 100644 index 0000000..7ff4bb1 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_orc_nested_column_pruning.q.out @@ -0,0 +1,2861 @@ +PREHOOK: query: DROP TABLE IF EXISTS dummy +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS dummy +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE dummy (i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dummy +POSTHOOK: query: CREATE TABLE dummy (i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dummy +PREHOOK: query: INSERT INTO TABLE dummy VALUES (42) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dummy +POSTHOOK: query: INSERT INTO TABLE dummy VALUES (42) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dummy +POSTHOOK: Lineage: dummy.i SCRIPT [] +PREHOOK: query: DROP TABLE IF EXISTS nested_tbl_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS nested_tbl_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE nested_tbl_1 ( + a int, + s1 struct<f1: boolean, f2: string, f3: struct<f4: int, f5: double>, f6: int>, + s2 struct<f7: string, f8: struct<f9 : boolean, f10: array<int>, f11: map<string, boolean>>>, + s3 struct<f12: array<struct<f13:string, f14:int>>>, + s4 map<string, struct<f15:int>>, + s5 struct<f16: array<struct<f17:string, f18:struct<f19:int>>>>, + s6 map<string, struct<f20:array<struct<f21:struct<f22:int>>>>> +) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_tbl_1 +POSTHOOK: query: CREATE TABLE nested_tbl_1 ( + a int, + s1 struct<f1: boolean, f2: string, f3: struct<f4: int, f5: double>, f6: int>, + s2 struct<f7: string, f8: struct<f9 : boolean, f10: array<int>, f11: map<string, boolean>>>, + s3 struct<f12: array<struct<f13:string, f14:int>>>, + s4 map<string, struct<f15:int>>, + s5 struct<f16: array<struct<f17:string, f18:struct<f19:int>>>>, + s6 map<string, struct<f20:array<struct<f21:struct<f22:int>>>>> +) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_tbl_1 +PREHOOK: query: INSERT INTO TABLE nested_tbl_1 SELECT + 1, named_struct('f1', false, 'f2', 'foo', 'f3', named_struct('f4', 4, 'f5', cast(5.0 as double)), 'f6', 4), + named_struct('f7', 'f7', 'f8', named_struct('f9', true, 'f10', array(10, 11), 'f11', map('key1', true, 'key2', false))), + named_struct('f12', array(named_struct('f13', 'foo', 'f14', 14), named_struct('f13', 'bar', 'f14', 28))), + map('key1', named_struct('f15', 1), 'key2', named_struct('f15', 2)), + named_struct('f16', array(named_struct('f17', 'foo', 'f18', named_struct('f19', 14)), named_struct('f17', 'bar', 'f18', named_struct('f19', 28)))), + map('key1', named_struct('f20', array(named_struct('f21', named_struct('f22', 1)))), + 'key2', named_struct('f20', array(named_struct('f21', named_struct('f22', 2))))) +FROM dummy +PREHOOK: type: QUERY +PREHOOK: Input: default@dummy +PREHOOK: Output: default@nested_tbl_1 +POSTHOOK: query: INSERT INTO TABLE nested_tbl_1 SELECT + 1, named_struct('f1', false, 'f2', 'foo', 'f3', named_struct('f4', 4, 'f5', cast(5.0 as double)), 'f6', 4), + named_struct('f7', 'f7', 'f8', named_struct('f9', true, 'f10', array(10, 11), 'f11', map('key1', true, 'key2', false))), + named_struct('f12', array(named_struct('f13', 'foo', 'f14', 14), named_struct('f13', 'bar', 'f14', 28))), + map('key1', named_struct('f15', 1), 'key2', named_struct('f15', 2)), + named_struct('f16', array(named_struct('f17', 'foo', 'f18', named_struct('f19', 14)), named_struct('f17', 'bar', 'f18', named_struct('f19', 28)))), + map('key1', named_struct('f20', array(named_struct('f21', named_struct('f22', 1)))), + 'key2', named_struct('f20', array(named_struct('f21', named_struct('f22', 2))))) +FROM dummy +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dummy +POSTHOOK: Output: default@nested_tbl_1 +POSTHOOK: Lineage: nested_tbl_1.a SIMPLE [] +POSTHOOK: Lineage: nested_tbl_1.s1 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_1.s2 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_1.s3 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_1.s4 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_1.s5 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_1.s6 EXPRESSION [] +PREHOOK: query: DROP TABLE IF EXISTS nested_tbl_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS nested_tbl_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE nested_tbl_2 LIKE nested_tbl_1 +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_tbl_2 +POSTHOOK: query: CREATE TABLE nested_tbl_2 LIKE nested_tbl_1 +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_tbl_2 +PREHOOK: query: INSERT INTO TABLE nested_tbl_2 SELECT + 2, named_struct('f1', true, 'f2', 'bar', 'f3', named_struct('f4', 4, 'f5', cast(6.5 as double)), 'f6', 4), + named_struct('f7', 'f72', 'f8', named_struct('f9', false, 'f10', array(20, 22), 'f11', map('key3', true, 'key4', false))), + named_struct('f12', array(named_struct('f13', 'bar', 'f14', 28), named_struct('f13', 'foo', 'f14', 56))), + map('key3', named_struct('f15', 3), 'key4', named_struct('f15', 4)), + named_struct('f16', array(named_struct('f17', 'bar', 'f18', named_struct('f19', 28)), named_struct('f17', 'foo', 'f18', named_struct('f19', 56)))), + map('key3', named_struct('f20', array(named_struct('f21', named_struct('f22', 3)))), + 'key4', named_struct('f20', array(named_struct('f21', named_struct('f22', 4))))) +FROM dummy +PREHOOK: type: QUERY +PREHOOK: Input: default@dummy +PREHOOK: Output: default@nested_tbl_2 +POSTHOOK: query: INSERT INTO TABLE nested_tbl_2 SELECT + 2, named_struct('f1', true, 'f2', 'bar', 'f3', named_struct('f4', 4, 'f5', cast(6.5 as double)), 'f6', 4), + named_struct('f7', 'f72', 'f8', named_struct('f9', false, 'f10', array(20, 22), 'f11', map('key3', true, 'key4', false))), + named_struct('f12', array(named_struct('f13', 'bar', 'f14', 28), named_struct('f13', 'foo', 'f14', 56))), + map('key3', named_struct('f15', 3), 'key4', named_struct('f15', 4)), + named_struct('f16', array(named_struct('f17', 'bar', 'f18', named_struct('f19', 28)), named_struct('f17', 'foo', 'f18', named_struct('f19', 56)))), + map('key3', named_struct('f20', array(named_struct('f21', named_struct('f22', 3)))), + 'key4', named_struct('f20', array(named_struct('f21', named_struct('f22', 4))))) +FROM dummy +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dummy +POSTHOOK: Output: default@nested_tbl_2 +POSTHOOK: Lineage: nested_tbl_2.a SIMPLE [] +POSTHOOK: Lineage: nested_tbl_2.s1 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_2.s2 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_2.s3 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_2.s4 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_2.s5 EXPRESSION [] +POSTHOOK: Lineage: nested_tbl_2.s6 EXPRESSION [] +PREHOOK: query: EXPLAIN VECTORIZATION +SELECT a FROM nested_tbl_1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION +SELECT a FROM nested_tbl_1 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT a FROM nested_tbl_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT a FROM nested_tbl_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +1 +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f1 FROM nested_tbl_1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f1 FROM nested_tbl_1 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Select Operator + expressions: s1.f1 (type: boolean) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [8] + selectExpressions: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 0:int) -> 8:boolean + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f1 FROM nested_tbl_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f1 FROM nested_tbl_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +false +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f1, s1.f2 FROM nested_tbl_1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f1, s1.f2 FROM nested_tbl_1 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f1, s1.f2 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Select Operator + expressions: s1.f1 (type: boolean), s1.f2 (type: string) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [8, 9] + selectExpressions: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 0:int) -> 8:boolean, VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 1:int) -> 9:string + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f1, s1.f2 FROM nested_tbl_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f1, s1.f2 FROM nested_tbl_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +false foo +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3, s1.f3.f4 FROM nested_tbl_1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3, s1.f3.f4 FROM nested_tbl_1 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f3 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Select Operator + expressions: s1.f3 (type: struct<f4:int,f5:double>), s1.f3.f4 (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [8, 10] + selectExpressions: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>, VectorUDFStructField(col 9:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 9:struct<f4:int,f5:double>) -> 10:int + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f3, s1.f3.f4 FROM nested_tbl_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f3, s1.f3.f4 FROM nested_tbl_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +{"f4":4,"f5":5.0} 4 +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3.f5 FROM nested_tbl_1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3.f5 FROM nested_tbl_1 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f3.f5 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Select Operator + expressions: s1.f3.f5 (type: double) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [9] + selectExpressions: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 1:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:double + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f3.f5 FROM nested_tbl_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f3.f5 FROM nested_tbl_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +5.0 +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3.f4, s2.f8.f9 FROM nested_tbl_1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3.f4, s2.f8.f9 FROM nested_tbl_1 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f3.f4, s2.f8.f9 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Select Operator + expressions: s1.f3.f4 (type: int), s2.f8.f9 (type: boolean) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [9, 11] + selectExpressions: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:int, VectorUDFStructField(col 10:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 0:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 10:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 11:boolean + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f3.f4, s2.f8.f9 FROM nested_tbl_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f3.f4, s2.f8.f9 FROM nested_tbl_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +4 true +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f2 FROM nested_tbl_1 WHERE s1.f1 = FALSE +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f2 FROM nested_tbl_1 WHERE s1.f1 = FALSE +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f2, s1.f1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColEqualLongScalar(col 8:boolean, val 0)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 0:int) -> 8:boolean) + predicate: (s1.f1 = false) (type: boolean) + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1.f2 (type: string) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [9] + selectExpressions: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 1:int) -> 9:string + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f2 FROM nested_tbl_1 WHERE s1.f1 = FALSE +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f2 FROM nested_tbl_1 WHERE s1.f1 = FALSE +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +foo +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3.f5 FROM nested_tbl_1 WHERE s1.f3.f4 = 4 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3.f5 FROM nested_tbl_1 WHERE s1.f3.f4 = 4 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f3.f5, s1.f3.f4 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterLongColEqualLongScalar(col 9:int, val 4)(children: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:int) + predicate: (s1.f3.f4 = 4) (type: boolean) + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1.f3.f5 (type: double) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [10] + selectExpressions: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 1:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 10:double + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f3.f5 FROM nested_tbl_1 WHERE s1.f3.f4 = 4 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f3.f5 FROM nested_tbl_1 WHERE s1.f3.f4 = 4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +5.0 +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s2.f8 FROM nested_tbl_1 WHERE s1.f2 = 'foo' AND size(s2.f8.f10) > 1 AND s2.f8.f11['key1'] = TRUE +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s2.f8 FROM nested_tbl_1 WHERE s1.f2 = 'foo' AND size(s2.f8.f10) > 1 AND s2.f8.f11['key1'] = TRUE +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f2, s2.f8 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((s1.f2 = 'foo') and (size(s2.f8.f10) > 1) and s2.f8.f11['key1']) (type: boolean) + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s2.f8 (type: struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + notVectorizedReason: FILTER operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[s2].f8.f10 + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s2.f8 FROM nested_tbl_1 WHERE s1.f2 = 'foo' AND size(s2.f8.f10) > 1 AND s2.f8.f11['key1'] = TRUE +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s2.f8 FROM nested_tbl_1 WHERE s1.f2 = 'foo' AND size(s2.f8.f10) > 1 AND s2.f8.f11['key1'] = TRUE +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +{"f9":true,"f10":[10,11],"f11":{"key2":false,"key1":true}} +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT col1, col2 FROM nested_tbl_1 +LATERAL VIEW explode(s2.f8.f10) tbl1 AS col1 +LATERAL VIEW explode(s3.f12) tbl2 AS col2 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT col1, col2 FROM nested_tbl_1 +LATERAL VIEW explode(s2.f8.f10) tbl1 AS col1 +LATERAL VIEW explode(s3.f12) tbl2 AS col2 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s2.f8.f10 + Statistics: Num rows: 1 Data size: 3632 Basic stats: COMPLETE Column stats: NONE + Lateral View Forward + Statistics: Num rows: 1 Data size: 3632 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s3 (type: struct<f12:array<struct<f13:string,f14:int>>>) + outputColumnNames: s3 + Statistics: Num rows: 1 Data size: 3632 Basic stats: COMPLETE Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col3, _col10 + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + Lateral View Forward + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col10 (type: int) + outputColumnNames: _col10 + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col10, _col11 + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col10 (type: int), _col11 (type: struct<f13:string,f14:int>) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col3.f12 (type: array<struct<f13:string,f14:int>>) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col10, _col11 + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col10 (type: int), _col11 (type: struct<f13:string,f14:int>) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: s2.f8.f10 (type: array<int>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 3632 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 1 Data size: 3632 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col3, _col10 + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + Lateral View Forward + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col10 (type: int) + outputColumnNames: _col10 + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col10, _col11 + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col10 (type: int), _col11 (type: struct<f13:string,f14:int>) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col3.f12 (type: array<struct<f13:string,f14:int>>) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 2 Data size: 7264 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col10, _col11 + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col10 (type: int), _col11 (type: struct<f13:string,f14:int>) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 14528 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + notVectorizedReason: Lateral View Forward (LATERALVIEWFORWARD) not supported + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT col1, col2 FROM nested_tbl_1 +LATERAL VIEW explode(s2.f8.f10) tbl1 AS col1 +LATERAL VIEW explode(s3.f12) tbl2 AS col2 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT col1, col2 FROM nested_tbl_1 +LATERAL VIEW explode(s2.f8.f10) tbl1 AS col1 +LATERAL VIEW explode(s3.f12) tbl2 AS col2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +10 {"f13":"foo","f14":14} +10 {"f13":"bar","f14":28} +11 {"f13":"foo","f14":14} +11 {"f13":"bar","f14":28} +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT pmod(s2.f8.f10[1], s1.f3.f4) FROM nested_tbl_1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT pmod(s2.f8.f10[1], s1.f3.f4) FROM nested_tbl_1 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s2.f8.f10, s1.f3.f4 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (s2.f8.f10[1] pmod s1.f3.f4) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + notVectorizedReason: SELECT operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[s1].f3.f4 + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT pmod(s2.f8.f10[1], s1.f3.f4) FROM nested_tbl_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT pmod(s2.f8.f10[1], s1.f3.f4) FROM nested_tbl_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +3 +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3.f5, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3.f5 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3.f5, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3.f5 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f3.f5, s1.f3.f4 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Select Operator + expressions: s1.f3.f5 (type: double), s1.f3.f4 (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [9, 10] + selectExpressions: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 1:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:double, VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 10:int + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col1) + Group By Vectorization: + aggregators: VectorUDAFCount(col 10:int) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 9:double + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + keys: _col0 (type: double) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Reduce Sink Vectorization: + className: VectorReduceSinkMultiKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 1:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + keyExpressions: col 0:double + native: false + vectorProcessingMode: MERGE_PARTIAL + projectedOutputColumnNums: [0] + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f3.f5, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3.f5 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f3.f5, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3.f5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +5.0 1 +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f3 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1.f3 (type: struct<f4:int,f5:double>), s1.f3.f4 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col1) + keys: _col0 (type: struct<f4:int,f5:double>) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct<f4:int,f5:double>) + sort order: + + Map-reduce partition columns: _col0 (type: struct<f4:int,f5:double>) + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + notVectorizedReason: Key expression for GROUPBY operator: Vectorizing complex type STRUCT not supported + vectorized: false + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: Key expression for GROUPBY operator: Vectorizing complex type STRUCT not supported + vectorized: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: struct<f4:int,f5:double>) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f3, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f3, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +{"f4":4,"f5":5.0} 1 +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3 ORDER BY s1.f3 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT s1.f3, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3 ORDER BY s1.f3 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_tbl_1 + Pruned Column Paths: s1.f3 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1.f3 (type: struct<f4:int,f5:double>), s1.f3.f4 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(_col1) + keys: _col0 (type: struct<f4:int,f5:double>) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct<f4:int,f5:double>) + sort order: + + Map-reduce partition columns: _col0 (type: struct<f4:int,f5:double>) + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + notVectorizedReason: Key expression for GROUPBY operator: Vectorizing complex type STRUCT not supported + vectorized: false + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: Key expression for GROUPBY operator: Vectorizing complex type STRUCT not supported + vectorized: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: struct<f4:int,f5:double>) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: struct<f4:int,f5:double>) + sort order: + + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct<f4:int,f5:double>), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT s1.f3, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3 ORDER BY s1.f3 +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT s1.f3, count(s1.f3.f4) FROM nested_tbl_1 GROUP BY s1.f3 ORDER BY s1.f3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +{"f4":4,"f5":5.0} 1 +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT t1.s1.f3.f5, t2.s2.f8 +FROM nested_tbl_1 t1 JOIN nested_tbl_2 t2 +ON t1.s1.f3.f4 = t2.s1.f6 +WHERE t2.s2.f8.f9 == FALSE +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT t1.s1.f3.f5, t2.s2.f8 +FROM nested_tbl_1 t1 JOIN nested_tbl_2 t2 +ON t1.s1.f3.f4 = t2.s1.f6 +WHERE t2.s2.f8.f9 == FALSE +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 9:int)(children: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:int) + predicate: s1.f3.f4 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1 (type: struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1] + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0.f3.f4 (type: int) + sort order: + + Map-reduce partition columns: _col0.f3.f4 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyExpressions: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 3 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterExprAndExpr(children: FilterLongColEqualLongScalar(col 9:boolean, val 0)(children: VectorUDFStructField(col 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>, col 0:int)(children: VectorUDFStructField(col 2:struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>, col 1:int) -> 8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) -> 9:boolean), SelectColumnIsNotNull(col 9:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 3:int) -> 9:int)) + predicate: ((s2.f8.f9 = false) and s1.f6 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1 (type: struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>), s2 (type: struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1, 2] + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0.f6 (type: int) + sort order: + + Map-reduce partition columns: _col0.f6 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyExpressions: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 3:int) -> 9:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0.f3.f4 (type: int) + 1 _col0.f6 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 347 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0.f3.f5 (type: double), _col2.f8 (type: struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 347 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 347 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT t1.s1.f3.f5, t2.s2.f8 +FROM nested_tbl_1 t1 JOIN nested_tbl_2 t2 +ON t1.s1.f3.f4 = t2.s1.f6 +WHERE t2.s2.f8.f9 == FALSE +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +PREHOOK: Input: default@nested_tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT t1.s1.f3.f5, t2.s2.f8 +FROM nested_tbl_1 t1 JOIN nested_tbl_2 t2 +ON t1.s1.f3.f4 = t2.s1.f6 +WHERE t2.s2.f8.f9 == FALSE +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +POSTHOOK: Input: default@nested_tbl_2 +#### A masked pattern was here #### +5.0 {"f9":false,"f10":[20,22],"f11":{"key3":true,"key4":false}} +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT t1.s1.f3.f5, t2.s2.f8 +FROM nested_tbl_1 t1 JOIN nested_tbl_1 t2 +ON t1.s1.f3.f4 = t2.s1.f6 +WHERE t2.s2.f8.f9 == TRUE +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT t1.s1.f3.f5, t2.s2.f8 +FROM nested_tbl_1 t1 JOIN nested_tbl_1 t2 +ON t1.s1.f3.f4 = t2.s1.f6 +WHERE t2.s2.f8.f9 == TRUE +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 9:int)(children: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:int) + predicate: s1.f3.f4 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1 (type: struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1] + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0.f3.f4 (type: int) + sort order: + + Map-reduce partition columns: _col0.f3.f4 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyExpressions: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 3 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (s1.f6 is not null and s2.f8.f9) (type: boolean) + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1 (type: struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>), s2 (type: struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0.f6 (type: int) + sort order: + + Map-reduce partition columns: _col0.f6 (type: int) + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: struct<f7:string,f8:struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>>) + Execution mode: llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + notVectorizedReason: FILTER operator: Could not vectorize expression (mode = FILTER): Column[s2].f8.f9 + vectorized: false + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0.f3.f4 (type: int) + 1 _col0.f6 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1 Data size: 347 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0.f3.f5 (type: double), _col2.f8 (type: struct<f9:boolean,f10:array<int>,f11:map<string,boolean>>) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 347 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 347 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT t1.s1.f3.f5, t2.s2.f8 +FROM nested_tbl_1 t1 JOIN nested_tbl_1 t2 +ON t1.s1.f3.f4 = t2.s1.f6 +WHERE t2.s2.f8.f9 == TRUE +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT t1.s1.f3.f5, t2.s2.f8 +FROM nested_tbl_1 t1 JOIN nested_tbl_1 t2 +ON t1.s1.f3.f4 = t2.s1.f6 +WHERE t2.s2.f8.f9 == TRUE +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +5.0 {"f9":true,"f10":[10,11],"f11":{"key1":true,"key2":false}} +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT t1.s1.f3.f5 +FROM nested_tbl_1 t1 LEFT SEMI JOIN nested_tbl_1 t2 +ON t1.s1.f3.f4 = t2.s1.f6 AND t2.s2.f8.f9 == TRUE +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +SELECT t1.s1.f3.f5 +FROM nested_tbl_1 t1 LEFT SEMI JOIN nested_tbl_1 t2 +ON t1.s1.f3.f4 = t2.s1.f6 AND t2.s2.f8.f9 == TRUE +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 9:int)(children: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:int) + predicate: s1.f3.f4 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1 (type: struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>), s1.f3.f4 (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [1, 9] + selectExpressions: VectorUDFStructField(col 8:struct<f4:int,f5:double>, col 0:int)(children: VectorUDFStructField(col 1:struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>, col 2:int) -> 8:struct<f4:int,f5:double>) -> 9:int + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int), true (type: boolean) + sort order: ++ + Map-reduce partition columns: _col1 (type: int), true (type: boolean) + Reduce Sink Vectorization: + className: VectorReduceSinkMultiKeyOperator + keyExpressions: ConstantVectorExpression(val 1) -> 10:boolean + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 316 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<f1:boolean,f2:string,f3:struct<f4:int,f5:double>,f6:int>) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Map 3 + Map Operator Tree: + TableScan + alias: t2 + Pruned Column Paths: s1.f6, s2.f8.f9 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (s1.f6 is not null and s2.f8.f9 and s2.f8.f9 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: s1.f6 (type: int), (s2.f8.f9 = true) (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: boolean) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: boolean) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) + Statistics: Num rows: 1 Data size: 1468 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + notVectorizedReason: FILTER operator: Unable to vectorize custom UDF. Encountered unsupported expr desc : Column[s2].f8.f9 + vectorized: false + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col1 (type: int), true (type: boolean) + 1 _col0 (type: int), _col1 (type: boolean) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 347 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0.f3.f5 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 347 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 347 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT t1.s1.f3.f5 +FROM nested_tbl_1 t1 LEFT SEMI JOIN nested_tbl_1 t2 +ON t1.s1.f3.f4 = t2.s1.f6 AND t2.s2.f8.f9 == TRUE +PREHOOK: type: QUERY +PREHOOK: Input: default@nested_tbl_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT t1.s1.f3
<TRUNCATED>
