http://git-wip-us.apache.org/repos/asf/hive/blob/f562dfb5/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out new file mode 100644 index 0000000..3facce7 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out @@ -0,0 +1,1029 @@ +PREHOOK: query: drop table calendar +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table calendar +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE calendar (year int, month int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@calendar +POSTHOOK: query: CREATE TABLE calendar (year int, month int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@calendar +PREHOOK: query: insert into calendar values (2010, 10), (2011, 11), (2012, 12) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@calendar +POSTHOOK: query: insert into calendar values (2010, 10), (2011, 11), (2012, 12) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@calendar +POSTHOOK: Lineage: calendar.month EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: calendar.year EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted calendar +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@calendar +POSTHOOK: query: desc formatted calendar +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@calendar +# col_name data_type comment + +year int +month int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 21 + totalSize 24 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: analyze table calendar compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +PREHOOK: Output: default@calendar +POSTHOOK: query: analyze table calendar compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +POSTHOOK: Output: default@calendar +PREHOOK: query: desc formatted calendar +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@calendar +POSTHOOK: query: desc formatted calendar +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@calendar +# col_name data_type comment + +year int +month int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 21 + totalSize 24 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select count(1) from calendar +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from calendar +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select max(year) from calendar +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(year) from calendar +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: calendar + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: year (type: int) + outputColumnNames: year + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(year) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(year) from calendar +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: select max(year) from calendar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +2012 +PREHOOK: query: select max(month) from calendar +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: select max(month) from calendar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +12 +PREHOOK: query: analyze table calendar compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: analyze table calendar compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +PREHOOK: query: desc formatted calendar +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@calendar +POSTHOOK: query: desc formatted calendar +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@calendar +# col_name data_type comment + +year int +month int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"month\":\"true\",\"year\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 21 + totalSize 24 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select max(year) from calendar +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(year) from calendar +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(year) from calendar +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: select max(year) from calendar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +2012 +PREHOOK: query: insert into calendar values (2015, 15) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@calendar +POSTHOOK: query: insert into calendar values (2015, 15) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@calendar +POSTHOOK: Lineage: calendar.month EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: calendar.year EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted calendar +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@calendar +POSTHOOK: query: desc formatted calendar +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@calendar +# col_name data_type comment + +year int +month int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 4 + rawDataSize 28 + totalSize 32 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select max(year) from calendar +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(year) from calendar +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: calendar + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: year (type: int) + outputColumnNames: year + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(year) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(year) from calendar +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: select max(year) from calendar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +2015 +PREHOOK: query: explain select max(month) from calendar +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(month) from calendar +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: calendar + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: month (type: int) + outputColumnNames: month + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(month) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(month) from calendar +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: select max(month) from calendar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +15 +PREHOOK: query: analyze table calendar compute statistics for columns year +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: analyze table calendar compute statistics for columns year +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +PREHOOK: query: desc formatted calendar +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@calendar +POSTHOOK: query: desc formatted calendar +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@calendar +# col_name data_type comment + +year int +month int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"year\":\"true\"}} + numFiles 2 + numRows 4 + rawDataSize 28 + totalSize 32 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select max(year) from calendar +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(year) from calendar +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(year) from calendar +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: select max(year) from calendar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +2015 +PREHOOK: query: explain select max(month) from calendar +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(month) from calendar +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: calendar + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: month (type: int) + outputColumnNames: month + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(month) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(month) from calendar +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: select max(month) from calendar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +15 +PREHOOK: query: analyze table calendar compute statistics for columns month +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: analyze table calendar compute statistics for columns month +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +PREHOOK: query: desc formatted calendar +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@calendar +POSTHOOK: query: desc formatted calendar +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@calendar +# col_name data_type comment + +year int +month int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"month\":\"true\",\"year\":\"true\"}} + numFiles 2 + numRows 4 + rawDataSize 28 + totalSize 32 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select max(month) from calendar +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(month) from calendar +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(month) from calendar +PREHOOK: type: QUERY +PREHOOK: Input: default@calendar +#### A masked pattern was here #### +POSTHOOK: query: select max(month) from calendar +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendar +#### A masked pattern was here #### +15 +PREHOOK: query: CREATE TABLE calendarp (`year` int) partitioned by (p int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@calendarp +POSTHOOK: query: CREATE TABLE calendarp (`year` int) partitioned by (p int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@calendarp +PREHOOK: query: insert into table calendarp partition (p=1) values (2010), (2011), (2012) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__3 +PREHOOK: Output: default@calendarp@p=1 +POSTHOOK: query: insert into table calendarp partition (p=1) values (2010), (2011), (2012) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__3 +POSTHOOK: Output: default@calendarp@p=1 +POSTHOOK: Lineage: calendarp PARTITION(p=1).year EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted calendarp partition (p=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@calendarp +POSTHOOK: query: desc formatted calendarp partition (p=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@calendarp +# col_name data_type comment + +year int + +# Partition Information +# col_name data_type comment + +p int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: calendarp +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 12 + totalSize 15 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select max(year) from calendarp where p=1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(year) from calendarp where p=1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: calendarp + Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: year (type: int) + outputColumnNames: year + Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(year) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(year) from calendarp where p=1 +PREHOOK: type: QUERY +PREHOOK: Input: default@calendarp +PREHOOK: Input: default@calendarp@p=1 +#### A masked pattern was here #### +POSTHOOK: query: select max(year) from calendarp where p=1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendarp +POSTHOOK: Input: default@calendarp@p=1 +#### A masked pattern was here #### +2012 +PREHOOK: query: analyze table calendarp partition (p=1) compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@calendarp +PREHOOK: Input: default@calendarp@p=1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table calendarp partition (p=1) compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendarp +POSTHOOK: Input: default@calendarp@p=1 +#### A masked pattern was here #### +PREHOOK: query: desc formatted calendarp partition (p=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@calendarp +POSTHOOK: query: desc formatted calendarp partition (p=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@calendarp +# col_name data_type comment + +year int + +# Partition Information +# col_name data_type comment + +p int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: calendarp +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"year\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 12 + totalSize 15 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select max(year) from calendarp where p=1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(year) from calendarp where p=1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: insert into table calendarp partition (p=1) values (2015) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__4 +PREHOOK: Output: default@calendarp@p=1 +POSTHOOK: query: insert into table calendarp partition (p=1) values (2015) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__4 +POSTHOOK: Output: default@calendarp@p=1 +POSTHOOK: Lineage: calendarp PARTITION(p=1).year EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted calendarp partition (p=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@calendarp +POSTHOOK: query: desc formatted calendarp partition (p=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@calendarp +# col_name data_type comment + +year int + +# Partition Information +# col_name data_type comment + +p int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: calendarp +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 4 + rawDataSize 16 + totalSize 20 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select max(year) from calendarp where p=1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select max(year) from calendarp where p=1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: calendarp + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: year (type: int) + outputColumnNames: year + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(year) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(year) from calendarp where p=1 +PREHOOK: type: QUERY +PREHOOK: Input: default@calendarp +PREHOOK: Input: default@calendarp@p=1 +#### A masked pattern was here #### +POSTHOOK: query: select max(year) from calendarp where p=1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@calendarp +POSTHOOK: Input: default@calendarp@p=1 +#### A masked pattern was here #### +2015 +PREHOOK: query: create table t (key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t (key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table t +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table t +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t +PREHOOK: query: desc formatted t +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t +POSTHOOK: query: desc formatted t +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: analyze table t compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@t +PREHOOK: Output: default@t +POSTHOOK: query: analyze table t compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +POSTHOOK: Output: default@t +PREHOOK: query: desc formatted t +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@t +POSTHOOK: query: desc formatted t +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@t +# col_name data_type comment + +key string +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1
http://git-wip-us.apache.org/repos/asf/hive/blob/f562dfb5/ql/src/test/results/clientpositive/llap/column_access_stats.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/column_access_stats.q.out b/ql/src/test/results/clientpositive/llap/column_access_stats.q.out new file mode 100644 index 0000000..5a66b0d --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/column_access_stats.q.out @@ -0,0 +1,938 @@ +PREHOOK: query: -- SORT_QUERY_RESULTS +-- This test is used for testing the ColumnAccessAnalyzer + +CREATE TABLE T1(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +PREHOOK: query: CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T2 +PREHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T3 +PREHOOK: query: CREATE TABLE T4(key STRING, val STRING) PARTITIONED BY (p STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T4 +PREHOOK: query: -- Simple select queries +SELECT key FROM T1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +Table:default@t1 +Columns:key + +1 +2 +3 +7 +8 +8 +PREHOOK: query: SELECT key, val FROM T1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +Table:default@t1 +Columns:key,val + +1 11 +2 12 +3 13 +7 17 +8 18 +8 28 +PREHOOK: query: SELECT 1 FROM T1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +1 +1 +1 +1 +1 +1 +PREHOOK: query: SELECT key, val from T4 where p=1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t4 +#### A masked pattern was here #### +Table:default@t4 +Columns:key,p,val + +PREHOOK: query: SELECT val FROM T4 where p=1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t4 +#### A masked pattern was here #### +Table:default@t4 +Columns:p,val + +PREHOOK: query: SELECT p, val FROM T4 where p=1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t4 +#### A masked pattern was here #### +Table:default@t4 +Columns:p,val + +PREHOOK: query: -- More complicated select queries +EXPLAIN SELECT key FROM (SELECT key, val FROM T1) subq1 +PREHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: t1 + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT key FROM (SELECT key, val FROM T1) subq1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +Table:default@t1 +Columns:key + +1 +2 +3 +7 +8 +8 +PREHOOK: query: EXPLAIN SELECT k FROM (SELECT key as k, val as v FROM T1) subq1 +PREHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: t1 + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT k FROM (SELECT key as k, val as v FROM T1) subq1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +Table:default@t1 +Columns:key + +1 +2 +3 +7 +8 +8 +PREHOOK: query: SELECT key + 1 as k FROM T1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +Table:default@t1 +Columns:key + +2.0 +3.0 +4.0 +8.0 +9.0 +9.0 +PREHOOK: query: SELECT key + val as k FROM T1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +Table:default@t1 +Columns:key,val + +12.0 +14.0 +16.0 +24.0 +26.0 +36.0 +PREHOOK: query: -- Work with union +EXPLAIN +SELECT * FROM ( +SELECT key as c FROM T1 + UNION ALL +SELECT val as c FROM T1 +) subq1 +PREHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: val (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( +SELECT key as c FROM T1 + UNION ALL +SELECT val as c FROM T1 +) subq1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +Table:default@t1 +Columns:key,val + +1 +11 +12 +13 +17 +18 +2 +28 +3 +7 +8 +8 +PREHOOK: query: EXPLAIN +SELECT * FROM ( +SELECT key as c FROM T1 + UNION ALL +SELECT key as c FROM T1 +) subq1 +PREHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM ( +SELECT key as c FROM T1 + UNION ALL +SELECT key as c FROM T1 +) subq1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +Table:default@t1 +Columns:key + +1 +1 +2 +2 +3 +3 +7 +7 +8 +8 +8 +8 +PREHOOK: query: -- Work with insert overwrite +FROM T1 +INSERT OVERWRITE TABLE T2 SELECT key, count(1) GROUP BY key +INSERT OVERWRITE TABLE T3 SELECT key, sum(val) GROUP BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t2 +PREHOOK: Output: default@t3 +Table:default@t1 +Columns:key,val + +PREHOOK: query: -- Simple joins +SELECT * +FROM T1 JOIN T2 +ON T1.key = T2.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +Table:default@t1 +Columns:key,val + +Table:default@t2 +Columns:key,val + +1 11 1 1 +2 12 2 1 +3 13 3 1 +7 17 7 1 +8 18 8 2 +8 28 8 2 +PREHOOK: query: EXPLAIN +SELECT T1.key +FROM T1 JOIN T2 +ON T1.key = T2.key +PREHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT T1.key +FROM T1 JOIN T2 +ON T1.key = T2.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +Table:default@t1 +Columns:key + +Table:default@t2 +Columns:key + +1 +2 +3 +7 +8 +8 +PREHOOK: query: SELECT * +FROM T1 JOIN T2 +ON T1.key = T2.key AND T1.val = T2.val +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +Table:default@t1 +Columns:key,val + +Table:default@t2 +Columns:key,val + +PREHOOK: query: -- Map join +SELECT /*+ MAPJOIN(a) */ * +FROM T1 a JOIN T2 b +ON a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +Table:default@t1 +Columns:key,val + +Table:default@t2 +Columns:key,val + +1 11 1 1 +2 12 2 1 +3 13 3 1 +7 17 7 1 +8 18 8 2 +8 28 8 2 +PREHOOK: query: -- More joins +EXPLAIN +SELECT * +FROM T1 JOIN T2 +ON T1.key = T2.key AND T1.val = 3 and T2.val = 3 +PREHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(val) = 3.0) and key is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(val) = 3.0) and key is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM T1 JOIN T2 +ON T1.key = T2.key AND T1.val = 3 and T2.val = 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +Table:default@t1 +Columns:key,val + +Table:default@t2 +Columns:key,val + +PREHOOK: query: EXPLAIN +SELECT subq1.val +FROM +( + SELECT val FROM T1 WHERE key = 5 +) subq1 +JOIN +( + SELECT val FROM T2 WHERE key = 6 +) subq2 +ON subq1.val = subq2.val +PREHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) = 5.0) and val is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: val (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) = 6.0) and val is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: val (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT subq1.val +FROM +( + SELECT val FROM T1 WHERE key = 5 +) subq1 +JOIN +( + SELECT val FROM T2 WHERE key = 6 +) subq2 +ON subq1.val = subq2.val +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +Table:default@t1 +Columns:key,val + +Table:default@t2 +Columns:key,val + +PREHOOK: query: -- Join followed by join +EXPLAIN +SELECT * +FROM +( + SELECT subq1.key as key + FROM + ( + SELECT key, val FROM T1 + ) subq1 + JOIN + ( + SELECT key, 'teststring' as val FROM T2 + ) subq2 + ON subq1.key = subq2.key +) T4 +JOIN T3 +ON T3.key = T4.key +PREHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: t3 + Statistics: Num rows: 5 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 17 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM +( + SELECT subq1.key as key + FROM + ( + SELECT key, val FROM T1 + ) subq1 + JOIN + ( + SELECT key, 'teststring' as val FROM T2 + ) subq2 + ON subq1.key = subq2.key +) T4 +JOIN T3 +ON T3.key = T4.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +Table:default@t1 +Columns:key + +Table:default@t2 +Columns:key + +Table:default@t3 +Columns:key,val + +1 1 11.0 +2 2 12.0 +3 3 13.0 +7 7 17.0 +8 8 46.0 +8 8 46.0 +PREHOOK: query: -- for partitioned table +SELECT * FROM srcpart TABLESAMPLE (10 ROWS) +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +#### A masked pattern was here #### +Table:default@srcpart +Columns:ds,hr,key,value + +165 val_165 2008-04-08 11 +238 val_238 2008-04-08 11 +255 val_255 2008-04-08 11 +27 val_27 2008-04-08 11 +278 val_278 2008-04-08 11 +311 val_311 2008-04-08 11 +409 val_409 2008-04-08 11 +484 val_484 2008-04-08 11 +86 val_86 2008-04-08 11 +98 val_98 2008-04-08 11 +PREHOOK: query: SELECT key,ds FROM srcpart TABLESAMPLE (10 ROWS) WHERE hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +#### A masked pattern was here #### +Table:default@srcpart +Columns:ds,hr,key + +165 2008-04-08 +238 2008-04-08 +255 2008-04-08 +27 2008-04-08 +278 2008-04-08 +311 2008-04-08 +409 2008-04-08 +484 2008-04-08 +86 2008-04-08 +98 2008-04-08 +PREHOOK: query: SELECT value FROM srcpart TABLESAMPLE (10 ROWS) WHERE ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +Table:default@srcpart +Columns:ds,value + +val_165 +val_238 +val_255 +val_27 +val_278 +val_311 +val_409 +val_484 +val_86 +val_98
