Repository: hive Updated Branches: refs/heads/master-txnstats f2d5ac221 -> 31ee8704e
HIVE-20219 : verify that analyze and analyze for columns manage txn stats state correctly (Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/31ee8704 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/31ee8704 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/31ee8704 Branch: refs/heads/master-txnstats Commit: 31ee8704e0dc5f8b9ef9530dbff4d3d1332be863 Parents: f2d5ac2 Author: sergey <[email protected]> Authored: Fri Jul 20 17:41:32 2018 -0700 Committer: sergey <[email protected]> Committed: Fri Jul 20 17:41:32 2018 -0700 ---------------------------------------------------------------------- ql/src/test/queries/clientpositive/acid_stats.q | 15 - .../test/queries/clientpositive/acid_stats5.q | 68 ++ .../results/clientpositive/acid_stats.q.out | 140 --- .../results/clientpositive/acid_stats5.q.out | 849 +++++++++++++++++++ 4 files changed, 917 insertions(+), 155 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/31ee8704/ql/src/test/queries/clientpositive/acid_stats.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/acid_stats.q b/ql/src/test/queries/clientpositive/acid_stats.q index 15eb930..8fed89a 100644 --- a/ql/src/test/queries/clientpositive/acid_stats.q +++ b/ql/src/test/queries/clientpositive/acid_stats.q @@ -30,18 +30,3 @@ explain select count(key) from stats_part; drop table stats_part; --- test the case where we insert without updating stats... just in case - -create table stats2(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only"); -insert into table stats2 values (1, "foo"); -explain select count(*) from stats2; -insert into table stats2 values (2, "bar"); -explain select count(*) from stats2; -desc formatted stats2 key; - -set hive.stats.autogather=false; -set hive.stats.column.autogather=false; -insert into table stats2 values (1, "baz"); -explain select count(*) from stats2; - -drop table stats2; http://git-wip-us.apache.org/repos/asf/hive/blob/31ee8704/ql/src/test/queries/clientpositive/acid_stats5.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/acid_stats5.q b/ql/src/test/queries/clientpositive/acid_stats5.q new file mode 100644 index 0000000..be6a581 --- /dev/null +++ b/ql/src/test/queries/clientpositive/acid_stats5.q @@ -0,0 +1,68 @@ +set hive.stats.dbclass=fs; +set hive.stats.fetch.column.stats=true; +set datanucleus.cache.collections=false; + +set hive.merge.mapfiles=false; +set hive.merge.mapredfiles=false; + +set hive.stats.autogather=true; +set hive.stats.column.autogather=true; +set hive.compute.query.using.stats=true; +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; + +set hive.fetch.task.conversion=none; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.query.results.cache.enabled=false; + +-- Test various scenarios where stats become invalid; verify they are invalid, and that analyze works. + +create table stats2(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only"); +insert into table stats2 values (1, "foo"); +insert into table stats2 values (2, "bar"); +explain select count(*) from stats2; +explain select min(key) from stats2; + +set hive.stats.autogather=false; +set hive.stats.column.autogather=false; +insert into table stats2 values (3, "baz"); +set hive.stats.autogather=true; +set hive.stats.column.autogather=true; +desc formatted stats2; +desc formatted stats2 key; +explain select count(*) from stats2; +explain select count(distinct key) from stats2; + +analyze table stats2 compute statistics; +desc formatted stats2; +desc formatted stats2 key; +explain select count(*) from stats2; +explain select min(key) from stats2; + +analyze table stats2 compute statistics for columns; +desc formatted stats2; +desc formatted stats2 key; +explain select count(*) from stats2; +explain select min(key) from stats2; + + +truncate table stats2; +desc formatted stats2; +desc formatted stats2 key; +explain select count(*) from stats2; +explain select count(distinct key) from stats2; + +analyze table stats2 compute statistics; +desc formatted stats2; +desc formatted stats2 key; +explain select count(*) from stats2; +explain select min(key) from stats2; + +analyze table stats2 compute statistics for columns; +desc formatted stats2; +desc formatted stats2 key; +explain select count(*) from stats2; +explain select min(key) from stats2; + +drop table stats2; http://git-wip-us.apache.org/repos/asf/hive/blob/31ee8704/ql/src/test/results/clientpositive/acid_stats.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/acid_stats.q.out b/ql/src/test/results/clientpositive/acid_stats.q.out index 8dcfdfb..02dee99 100644 --- a/ql/src/test/results/clientpositive/acid_stats.q.out +++ b/ql/src/test/results/clientpositive/acid_stats.q.out @@ -84,143 +84,3 @@ POSTHOOK: query: drop table stats_part POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@stats_part POSTHOOK: Output: default@stats_part -PREHOOK: query: create table stats2(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only") -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@stats2 -POSTHOOK: query: create table stats2(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only") -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@stats2 -PREHOOK: query: insert into table stats2 values (1, "foo") -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@stats2 -POSTHOOK: query: insert into table stats2 values (1, "foo") -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@stats2 -POSTHOOK: Lineage: stats2.key SCRIPT [] -POSTHOOK: Lineage: stats2.value SCRIPT [] -PREHOOK: query: explain select count(*) from stats2 -PREHOOK: type: QUERY -POSTHOOK: query: explain select count(*) from stats2 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -PREHOOK: query: insert into table stats2 values (2, "bar") -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@stats2 -POSTHOOK: query: insert into table stats2 values (2, "bar") -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@stats2 -POSTHOOK: Lineage: stats2.key SCRIPT [] -POSTHOOK: Lineage: stats2.value SCRIPT [] -PREHOOK: query: explain select count(*) from stats2 -PREHOOK: type: QUERY -POSTHOOK: query: explain select count(*) from stats2 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -PREHOOK: query: desc formatted stats2 key -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stats2 -POSTHOOK: query: desc formatted stats2 key -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stats2 -col_name key -data_type int -min 1 -max 2 -num_nulls 0 -distinct_count 2 -avg_col_len -max_col_len -num_trues -num_falses -bitVector HL -comment from deserializer -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} -PREHOOK: query: insert into table stats2 values (1, "baz") -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@stats2 -POSTHOOK: query: insert into table stats2 values (1, "baz") -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@stats2 -POSTHOOK: Lineage: stats2.key SCRIPT [] -POSTHOOK: Lineage: stats2.value SCRIPT [] -PREHOOK: query: explain select count(*) from stats2 -PREHOOK: type: QUERY -POSTHOOK: query: explain select count(*) from stats2 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: stats2 - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: drop table stats2 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@stats2 -PREHOOK: Output: default@stats2 -POSTHOOK: query: drop table stats2 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@stats2 -POSTHOOK: Output: default@stats2 http://git-wip-us.apache.org/repos/asf/hive/blob/31ee8704/ql/src/test/results/clientpositive/acid_stats5.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/acid_stats5.q.out b/ql/src/test/results/clientpositive/acid_stats5.q.out new file mode 100644 index 0000000..99b19bb --- /dev/null +++ b/ql/src/test/results/clientpositive/acid_stats5.q.out @@ -0,0 +1,849 @@ +PREHOOK: query: create table stats2(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats2 +POSTHOOK: query: create table stats2(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats2 +PREHOOK: query: insert into table stats2 values (1, "foo") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats2 +POSTHOOK: query: insert into table stats2 values (1, "foo") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats2 +POSTHOOK: Lineage: stats2.key SCRIPT [] +POSTHOOK: Lineage: stats2.value SCRIPT [] +PREHOOK: query: insert into table stats2 values (2, "bar") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats2 +POSTHOOK: query: insert into table stats2 values (2, "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats2 +POSTHOOK: Lineage: stats2.key SCRIPT [] +POSTHOOK: Lineage: stats2.value SCRIPT [] +PREHOOK: query: explain select count(*) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select min(key) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select min(key) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: insert into table stats2 values (3, "baz") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats2 +POSTHOOK: query: insert into table stats2 values (3, "baz") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats2 +POSTHOOK: Lineage: stats2.key SCRIPT [] +POSTHOOK: Lineage: stats2.value SCRIPT [] +PREHOOK: query: desc formatted stats2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +# col_name data_type comment +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + bucketing_version 2 + numFiles 3 + numRows 2 + rawDataSize 10 + totalSize 18 + transactional true + transactional_properties insert_only +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats2 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +col_name key +data_type int +min 1 +max 2 +num_nulls 0 +distinct_count 2 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer +PREHOOK: query: explain select count(*) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: stats2 + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(distinct key) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(distinct key) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: stats2 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: partial2 + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col0) + mode: partial2 + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: analyze table stats2 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats2 +PREHOOK: Output: default@stats2 +POSTHOOK: query: analyze table stats2 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats2 +POSTHOOK: Output: default@stats2 +PREHOOK: query: desc formatted stats2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +# col_name data_type comment +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 3 + numRows 3 + rawDataSize 15 + totalSize 18 + transactional true + transactional_properties insert_only +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats2 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +col_name key +data_type int +min 1 +max 2 +num_nulls 0 +distinct_count 2 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: explain select count(*) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select min(key) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select min(key) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: stats2 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(key) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: analyze table stats2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@stats2 +PREHOOK: Output: default@stats2 +#### A masked pattern was here #### +POSTHOOK: query: analyze table stats2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@stats2 +POSTHOOK: Output: default@stats2 +#### A masked pattern was here #### +PREHOOK: query: desc formatted stats2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +# col_name data_type comment +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + bucketing_version 2 + numFiles 3 + numRows 3 + rawDataSize 15 + totalSize 18 + transactional true + transactional_properties insert_only +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats2 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +col_name key +data_type int +min 1 +max 3 +num_nulls 0 +distinct_count 3 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: explain select count(*) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select min(key) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select min(key) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: truncate table stats2 +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@stats2 +POSTHOOK: query: truncate table stats2 +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@stats2 +PREHOOK: query: desc formatted stats2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +# col_name data_type comment +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 + transactional true + transactional_properties insert_only +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats2 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +col_name key +data_type int +min 1 +max 3 +num_nulls 0 +distinct_count 3 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: explain select count(*) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(distinct key) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(distinct key) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: stats2 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: partial2 + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col0) + mode: partial2 + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: analyze table stats2 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@stats2 +PREHOOK: Output: default@stats2 +POSTHOOK: query: analyze table stats2 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats2 +POSTHOOK: Output: default@stats2 +PREHOOK: query: desc formatted stats2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +# col_name data_type comment +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 + transactional true + transactional_properties insert_only +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats2 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +col_name key +data_type int +min 1 +max 3 +num_nulls 0 +distinct_count 3 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: explain select count(*) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select min(key) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select min(key) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: stats2 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(key) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: analyze table stats2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@stats2 +PREHOOK: Output: default@stats2 +#### A masked pattern was here #### +POSTHOOK: query: analyze table stats2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@stats2 +POSTHOOK: Output: default@stats2 +#### A masked pattern was here #### +PREHOOK: query: desc formatted stats2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +# col_name data_type comment +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 + transactional true + transactional_properties insert_only +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted stats2 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@stats2 +POSTHOOK: query: desc formatted stats2 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@stats2 +col_name key +data_type int +min 1 +max 3 +num_nulls 0 +distinct_count 0 +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: explain select count(*) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: explain select min(key) from stats2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select min(key) from stats2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: drop table stats2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@stats2 +PREHOOK: Output: default@stats2 +POSTHOOK: query: drop table stats2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@stats2 +POSTHOOK: Output: default@stats2
