Added: hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out?rev=1627210&view=auto ============================================================================== --- hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out (added) +++ hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out Wed Sep 24 03:16:25 2014 @@ -0,0 +1,1736 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS + +create table src_10 as select * from src limit 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_10 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: -- SORT_QUERY_RESULTS + +create table src_10 as select * from src limit 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_10 +PREHOOK: query: create table src_lv1 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_lv1 +POSTHOOK: query: create table src_lv1 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_lv1 +PREHOOK: query: create table src_lv2 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_lv2 +POSTHOOK: query: create table src_lv2 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_lv2 +PREHOOK: query: create table src_lv3 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_lv3 +POSTHOOK: query: create table src_lv3 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_lv3 +PREHOOK: query: -- 2LV +-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-FS[12] +-- -SEL[3]-UDTF[4]-LVJ[5] +-- -LVF[6]-SEL[7]-LVJ[10]-SEL[13]-FS[14] +-- -SEL[8]-UDTF[9]-LVJ[10] +explain +from src_10 +insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C +insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C +PREHOOK: type: QUERY +POSTHOOK: query: -- 2LV +-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-FS[12] +-- -SEL[3]-UDTF[4]-LVJ[5] +-- -LVF[6]-SEL[7]-LVJ[10]-SEL[13]-FS[14] +-- -SEL[8]-UDTF[9]-LVJ[10] +explain +from src_10 +insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C +insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-4 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-4, Stage-5 + Stage-0 depends on stages: Stage-3 + Stage-6 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-7 depends on stages: Stage-1 + Stage-5 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: src_10 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + Lateral View Forward + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + Select Operator + expressions: array((key + 1),(key + 2)) (type: array<double>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-6 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + + Stage: Stage-7 + Stats-Aggr Operator + + Stage: Stage-5 + Spark +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + Lateral View Forward + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + Select Operator + expressions: array((key + 3),(key + 4)) (type: array<double>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + +PREHOOK: query: from src_10 +insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C +insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C +PREHOOK: type: QUERY +PREHOOK: Input: default@src_10 +PREHOOK: Output: default@src_lv1 +PREHOOK: Output: default@src_lv2 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: from src_10 +insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C +insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_10 +POSTHOOK: Output: default@src_lv1 +POSTHOOK: Output: default@src_lv2 +POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +PREHOOK: query: select * from src_lv1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +165 166.0 +165 167.0 +238 239.0 +238 240.0 +255 256.0 +255 257.0 +27 28.0 +27 29.0 +278 279.0 +278 280.0 +311 312.0 +311 313.0 +409 410.0 +409 411.0 +484 485.0 +484 486.0 +86 87.0 +86 88.0 +98 100.0 +98 99.0 +PREHOOK: query: select * from src_lv2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +165 168.0 +165 169.0 +238 241.0 +238 242.0 +255 258.0 +255 259.0 +27 30.0 +27 31.0 +278 281.0 +278 282.0 +311 314.0 +311 315.0 +409 412.0 +409 413.0 +484 487.0 +484 488.0 +86 89.0 +86 90.0 +98 101.0 +98 102.0 +PREHOOK: query: -- 2(LV+GBY) +-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16] +-- -SEL[3]-UDTF[4]-LVJ[5] +-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22] +-- -SEL[8]-UDTF[9]-LVJ[10] +explain +from src_10 +insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key +PREHOOK: type: QUERY +POSTHOOK: query: -- 2(LV+GBY) +-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16] +-- -SEL[3]-UDTF[4]-LVJ[5] +-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22] +-- -SEL[8]-UDTF[9]-LVJ[10] +explain +from src_10 +insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-4 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-4, Stage-5 + Stage-0 depends on stages: Stage-3 + Stage-6 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-3 + Stage-7 depends on stages: Stage-1 + Stage-5 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-2 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: src_10 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Spark + Edges: + Reducer 4 <- Map 1 (GROUP) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + Lateral View Forward + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col5) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Select Operator + expressions: array((key + 1),(key + 2)) (type: array<double>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col5) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-6 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + + Stage: Stage-7 + Stats-Aggr Operator + + Stage: Stage-5 + Spark + Edges: + Reducer 5 <- Map 2 (GROUP) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + Lateral View Forward + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col5) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Select Operator + expressions: array((key + 3),(key + 4)) (type: array<double>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col5) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + +PREHOOK: query: from src_10 +insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src_10 +PREHOOK: Output: default@src_lv1 +PREHOOK: Output: default@src_lv2 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: from src_10 +insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_10 +POSTHOOK: Output: default@src_lv1 +POSTHOOK: Output: default@src_lv2 +POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +PREHOOK: query: select * from src_lv1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +165 333.0 +238 479.0 +255 513.0 +27 57.0 +278 559.0 +311 625.0 +409 821.0 +484 971.0 +86 175.0 +98 199.0 +PREHOOK: query: select * from src_lv2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +165 337.0 +238 483.0 +255 517.0 +27 61.0 +278 563.0 +311 629.0 +409 825.0 +484 975.0 +86 179.0 +98 203.0 +PREHOOK: query: -- (LV+GBY) + RS:2GBY +-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[6]-GBY[7]-RS[8]-GBY[9]-SEL[10]-FS[11] +-- -SEL[3]-UDTF[4]-LVJ[5] +-- -FIL[12]-SEL[13]-RS[14]-FOR[15]-FIL[16]-GBY[17]-SEL[18]-FS[19] +-- -FIL[20]-GBY[21]-SEL[22]-FS[23] +explain +from src_10 +insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, count(value) where key > 200 group by key +insert overwrite table src_lv3 select key, count(value) where key < 200 group by key +PREHOOK: type: QUERY +POSTHOOK: query: -- (LV+GBY) + RS:2GBY +-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[6]-GBY[7]-RS[8]-GBY[9]-SEL[10]-FS[11] +-- -SEL[3]-UDTF[4]-LVJ[5] +-- -FIL[12]-SEL[13]-RS[14]-FOR[15]-FIL[16]-GBY[17]-SEL[18]-FS[19] +-- -FIL[20]-GBY[21]-SEL[22]-FS[23] +explain +from src_10 +insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, count(value) where key > 200 group by key +insert overwrite table src_lv3 select key, count(value) where key < 200 group by key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-5 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-5, Stage-6 + Stage-0 depends on stages: Stage-4 + Stage-7 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-4 + Stage-8 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-4 + Stage-9 depends on stages: Stage-2 + Stage-6 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-3 + Spark +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: src_10 + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-5 + Spark + Edges: + Reducer 4 <- Map 1 (GROUP) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + Lateral View Forward + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(_col5) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: double) + Select Operator + expressions: array((key + 1),(key + 2)) (type: array<double>) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + UDTF Operator + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(_col5) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: double) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-4 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-7 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + + Stage: Stage-8 + Stats-Aggr Operator + + Stage: Stage-2 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv3 + + Stage: Stage-9 + Stats-Aggr Operator + + Stage: Stage-6 + Spark + Edges: + Reducer 5 <- Map 2 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + Filter Operator + predicate: ((key > 200) or (key < 200)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: value (type: string) + Reducer 5 + Reduce Operator Tree: + Forward + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: (KEY._col0 > 200) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + Filter Operator + predicate: (KEY._col0 < 200) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv3 + +PREHOOK: query: from src_10 +insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, count(value) where key > 200 group by key +insert overwrite table src_lv3 select key, count(value) where key < 200 group by key +PREHOOK: type: QUERY +PREHOOK: Input: default@src_10 +PREHOOK: Output: default@src_lv1 +PREHOOK: Output: default@src_lv2 +PREHOOK: Output: default@src_lv3 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: from src_10 +insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, count(value) where key > 200 group by key +insert overwrite table src_lv3 select key, count(value) where key < 200 group by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_10 +POSTHOOK: Output: default@src_lv1 +POSTHOOK: Output: default@src_lv2 +POSTHOOK: Output: default@src_lv3 +POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from src_lv1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +165 333.0 +238 479.0 +255 513.0 +27 57.0 +278 559.0 +311 625.0 +409 821.0 +484 971.0 +86 175.0 +98 199.0 +PREHOOK: query: select * from src_lv2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +238 1 +255 1 +278 1 +311 1 +409 1 +484 1 +PREHOOK: query: select * from src_lv3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv3 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv3 +#### A masked pattern was here #### +165 1 +27 1 +86 1 +98 1 +PREHOOK: query: -- todo: shared distinct columns (should work with hive.optimize.multigroupby.common.distincts) +-- 2(LV+GBY) + RS:2GBY +-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16] +-- -SEL[3]-UDTF[4]-LVJ[5] +-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22] +-- -SEL[8]-UDTF[9]-LVJ[10] +-- -SEL[23]-GBY[24]-RS[25]-GBY[26]-SEL[27]-FS[28] +explain +from src_10 +insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C +insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C +insert overwrite table src_lv3 select value, sum(distinct key) group by value +PREHOOK: type: QUERY +POSTHOOK: query: -- todo: shared distinct columns (should work with hive.optimize.multigroupby.common.distincts) +-- 2(LV+GBY) + RS:2GBY +-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16] +-- -SEL[3]-UDTF[4]-LVJ[5] +-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22] +-- -SEL[8]-UDTF[9]-LVJ[10] +-- -SEL[23]-GBY[24]-RS[25]-GBY[26]-SEL[27]-FS[28] +explain +from src_10 +insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C +insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C +insert overwrite table src_lv3 select value, sum(distinct key) group by value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-5 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-5, Stage-6, Stage-7 + Stage-0 depends on stages: Stage-4 + Stage-8 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-4 + Stage-9 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-4 + Stage-10 depends on stages: Stage-2 + Stage-6 depends on stages: Stage-3 + Stage-7 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-3 + Spark +#### A masked pattern was here #### + Vertices: + Map 4 + Map Operator Tree: + TableScan + alias: src_10 + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-5 + Spark + Edges: + Reducer 5 <- Map 1 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + Lateral View Forward + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col5 (type: double), _col0 (type: string) + outputColumnNames: _col5, _col0 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT _col0) + keys: _col5 (type: double), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: array((key + 1),(key + 2)) (type: array<double>) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + UDTF Operator + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col5 (type: double), _col0 (type: string) + outputColumnNames: _col5, _col0 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT _col0) + keys: _col5 (type: double), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: sum(DISTINCT KEY._col1:0._col0) + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: double), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-4 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-8 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + + Stage: Stage-9 + Stats-Aggr Operator + + Stage: Stage-2 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv3 + + Stage: Stage-10 + Stats-Aggr Operator + + Stage: Stage-6 + Spark + Edges: + Reducer 6 <- Map 2 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + Lateral View Forward + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col5 (type: double), _col0 (type: string) + outputColumnNames: _col5, _col0 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT _col0) + keys: _col5 (type: double), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: array((key + 3),(key + 4)) (type: array<double>) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + UDTF Operator + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col5 (type: double), _col0 (type: string) + outputColumnNames: _col5, _col0 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT _col0) + keys: _col5 (type: double), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: sum(DISTINCT KEY._col1:0._col0) + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: double), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + + Stage: Stage-7 + Spark + Edges: + Reducer 7 <- Map 3 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT key) + keys: value (type: string), key (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Reducer 7 + Reduce Operator Tree: + Group By Operator + aggregations: sum(DISTINCT KEY._col1:0._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv3 + +PREHOOK: query: from src_10 +insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C +insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C +insert overwrite table src_lv3 select value, sum(distinct key) group by value +PREHOOK: type: QUERY +PREHOOK: Input: default@src_10 +PREHOOK: Output: default@src_lv1 +PREHOOK: Output: default@src_lv2 +PREHOOK: Output: default@src_lv3 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: from src_10 +insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C +insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C +insert overwrite table src_lv3 select value, sum(distinct key) group by value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_10 +POSTHOOK: Output: default@src_lv1 +POSTHOOK: Output: default@src_lv2 +POSTHOOK: Output: default@src_lv3 +POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: select * from src_lv1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +100.0 98.0 +166.0 165.0 +167.0 165.0 +239.0 238.0 +240.0 238.0 +256.0 255.0 +257.0 255.0 +279.0 278.0 +28.0 27.0 +280.0 278.0 +29.0 27.0 +312.0 311.0 +313.0 311.0 +410.0 409.0 +411.0 409.0 +485.0 484.0 +486.0 484.0 +87.0 86.0 +88.0 86.0 +99.0 98.0 +PREHOOK: query: select * from src_lv2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +101.0 98.0 +102.0 98.0 +168.0 165.0 +169.0 165.0 +241.0 238.0 +242.0 238.0 +258.0 255.0 +259.0 255.0 +281.0 278.0 +282.0 278.0 +30.0 27.0 +31.0 27.0 +314.0 311.0 +315.0 311.0 +412.0 409.0 +413.0 409.0 +487.0 484.0 +488.0 484.0 +89.0 86.0 +90.0 86.0 +PREHOOK: query: select * from src_lv3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv3 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv3 +#### A masked pattern was here #### +val_165 165.0 +val_238 238.0 +val_255 255.0 +val_27 27.0 +val_278 278.0 +val_311 311.0 +val_409 409.0 +val_484 484.0 +val_86 86.0 +val_98 98.0 +PREHOOK: query: create table src_lv4 (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_lv4 +POSTHOOK: query: create table src_lv4 (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_lv4 +PREHOOK: query: -- Common distincts optimization works across non-lateral view queries, but not across lateral view multi inserts +explain +from src_10 +insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key +insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value +insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value +PREHOOK: type: QUERY +POSTHOOK: query: -- Common distincts optimization works across non-lateral view queries, but not across lateral view multi inserts +explain +from src_10 +insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key +insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value +insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-6 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-6, Stage-7, Stage-8 + Stage-0 depends on stages: Stage-5 + Stage-9 depends on stages: Stage-0 + Stage-1 depends on stages: Stage-5 + Stage-10 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-5 + Stage-11 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-5 + Stage-12 depends on stages: Stage-3 + Stage-7 depends on stages: Stage-4 + Stage-8 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-4 + Spark +#### A masked pattern was here #### + Vertices: + Map 4 + Map Operator Tree: + TableScan + alias: src_10 + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Spark + Edges: + Reducer 5 <- Map 1 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + Lateral View Forward + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT _col5) + keys: _col0 (type: string), _col5 (type: double) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: array((key + 1),(key + 2)) (type: array<double>) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + UDTF Operator + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT _col5) + keys: _col0 (type: string), _col5 (type: double) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: sum(DISTINCT KEY._col1:0._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-5 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv1 + + Stage: Stage-9 + Stats-Aggr Operator + + Stage: Stage-1 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + + Stage: Stage-10 + Stats-Aggr Operator + + Stage: Stage-2 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv3 + + Stage: Stage-11 + Stats-Aggr Operator + + Stage: Stage-3 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv4 + + Stage: Stage-12 + Stats-Aggr Operator + + Stage: Stage-7 + Spark + Edges: + Reducer 6 <- Map 2 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + Lateral View Forward + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT _col5) + keys: _col0 (type: string), _col5 (type: double) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: array((key + 3),(key + 4)) (type: array<double>) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + UDTF Operator + Statistics: Num rows: 0 Data size: 114 Basic stats: PARTIAL Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col5 (type: double) + outputColumnNames: _col0, _col5 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT _col5) + keys: _col0 (type: string), _col5 (type: double) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 228 Basic stats: PARTIAL Column stats: NONE + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: sum(DISTINCT KEY._col1:0._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv2 + + Stage: Stage-8 + Spark + Edges: + Reducer 7 <- Map 3 (GROUP SORT) +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + Filter Operator + predicate: ((key > 200) or (key < 200)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string), key (type: string) + sort order: ++ + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reducer 7 + Reduce Operator Tree: + Forward + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: (KEY._col1:0._col0 > 200) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT KEY._col1:0._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv3 + Filter Operator + predicate: (KEY._col1:0._col0 < 200) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: sum(DISTINCT KEY._col1:0._col0) + keys: KEY._col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_lv4 + +PREHOOK: query: from src_10 +insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key +insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value +insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value +PREHOOK: type: QUERY +PREHOOK: Input: default@src_10 +PREHOOK: Output: default@src_lv1 +PREHOOK: Output: default@src_lv2 +PREHOOK: Output: default@src_lv3 +PREHOOK: Output: default@src_lv4 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: from src_10 +insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key +insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key +insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value +insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_10 +POSTHOOK: Output: default@src_lv1 +POSTHOOK: Output: default@src_lv2 +POSTHOOK: Output: default@src_lv3 +POSTHOOK: Output: default@src_lv4 +POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.null, ] +POSTHOOK: Lineage: src_lv4.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv4.value EXPRESSION [(src_10)src_10.null, ] +PREHOOK: query: select * from src_lv1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv1 +#### A masked pattern was here #### +165 333.0 +238 479.0 +255 513.0 +27 57.0 +278 559.0 +311 625.0 +409 821.0 +484 971.0 +86 175.0 +98 199.0 +PREHOOK: query: select * from src_lv2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv2 +#### A masked pattern was here #### +165 337.0 +238 483.0 +255 517.0 +27 61.0 +278 563.0 +311 629.0 +409 825.0 +484 975.0 +86 179.0 +98 203.0 +PREHOOK: query: select * from src_lv3 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv3 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv3 +#### A masked pattern was here #### +val_238 238.0 +val_255 255.0 +val_278 278.0 +val_311 311.0 +val_409 409.0 +val_484 484.0 +PREHOOK: query: select * from src_lv4 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_lv4 +#### A masked pattern was here #### +POSTHOOK: query: select * from src_lv4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_lv4 +#### A masked pattern was here #### +val_165 165.0 +val_27 27.0 +val_86 86.0 +val_98 98.0
