Repository: hive Updated Branches: refs/heads/master 22e7bffa1 -> 31c1b7199
HIVE-18245 - clean up acid_vectorization_original.q (Eugene Koifman, reviewed by Jason Dere) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/31c1b719 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/31c1b719 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/31c1b719 Branch: refs/heads/master Commit: 31c1b719929298dc85510104ccbf5820a28be3ff Parents: 22e7bff Author: Eugene Koifman <[email protected]> Authored: Mon Dec 11 13:16:05 2017 -0800 Committer: Eugene Koifman <[email protected]> Committed: Mon Dec 11 13:16:05 2017 -0800 ---------------------------------------------------------------------- .../acid_vectorization_original.q | 9 +++----- .../acid_vectorization_original_tez.q | 11 ++++----- .../llap/acid_vectorization_original.q.out | 20 ++++++++-------- .../tez/acid_vectorization_original_tez.q.out | 24 ++++++++++---------- 4 files changed, 29 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/31c1b719/ql/src/test/queries/clientpositive/acid_vectorization_original.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/acid_vectorization_original.q b/ql/src/test/queries/clientpositive/acid_vectorization_original.q index ddf138d..0b91f69 100644 --- a/ql/src/test/queries/clientpositive/acid_vectorization_original.q +++ b/ql/src/test/queries/clientpositive/acid_vectorization_original.q @@ -4,8 +4,6 @@ set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.exec.dynamic.partition.mode=nonstrict; set hive.vectorized.execution.enabled=true; --- enables vectorizaiton of VirtualColumn.ROWID -set hive.vectorized.row.identifier.enabled=true; -- enable ppd set hive.optimize.index.filter=true; @@ -60,13 +58,12 @@ CREATE TABLE over10k_orc_bucketed(t tinyint, -- this produces about 250 distinct values across all 4 equivalence classes select distinct si, si%4 from over10k order by si; --- explain insert into over10k_orc_bucketed select * from over10k cluster by si; --- w/o "cluster by" all data is written to 000000_0 -insert into over10k_orc_bucketed select * from over10k cluster by si; +-- explain insert into over10k_orc_bucketed select * from over10k; +insert into over10k_orc_bucketed select * from over10k; dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/over10k_orc_bucketed; -- create copy_N files -insert into over10k_orc_bucketed select * from over10k cluster by si; +insert into over10k_orc_bucketed select * from over10k; -- this output of this is masked in .out - it is visible in .orig dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/over10k_orc_bucketed; http://git-wip-us.apache.org/repos/asf/hive/blob/31c1b719/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q b/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q index 4d93662..00a3ab2 100644 --- a/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q +++ b/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q @@ -4,8 +4,6 @@ set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.exec.dynamic.partition.mode=nonstrict; set hive.vectorized.execution.enabled=true; --- enables vectorizaiton of VirtualColumn.ROWID -set hive.vectorized.row.identifier.enabled=true; -- enable ppd set hive.optimize.index.filter=true; @@ -60,13 +58,12 @@ CREATE TABLE over10k_orc_bucketed(t tinyint, -- this produces about 250 distinct values across all 4 equivalence classes select distinct si, si%4 from over10k order by si; --- explain insert into over10k_orc_bucketed select * from over10k cluster by si; --- w/o "cluster by" all data is written to 000000_0 -insert into over10k_orc_bucketed select * from over10k cluster by si; +-- explain insert into over10k_orc_bucketed select * from over10k; +insert into over10k_orc_bucketed select * from over10k; dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/over10k_orc_bucketed; -- create copy_N files -insert into over10k_orc_bucketed select * from over10k cluster by si; +insert into over10k_orc_bucketed select * from over10k; -- this output of this is masked in .out - it is visible in .orig dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/over10k_orc_bucketed; @@ -109,7 +106,7 @@ explain select ROW__ID, count(*) from over10k_orc_bucketed group by ROW__ID havi select ROW__ID, count(*) from over10k_orc_bucketed group by ROW__ID having count(*) > 1; -- schedule compactor -alter table over10k_orc_bucketed compact 'major' WITH OVERWRITE TBLPROPERTIES ("compactor.mapreduce.map.memory.mb"="500","compactor.hive.tez.container.size"="500");; +alter table over10k_orc_bucketed compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500'); -- run compactor - this currently fails with http://git-wip-us.apache.org/repos/asf/hive/blob/31c1b719/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out index 1a2aff7..f7022b5 100644 --- a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out @@ -348,11 +348,11 @@ NULL NULL 509 1 510 2 511 3 -PREHOOK: query: insert into over10k_orc_bucketed select * from over10k cluster by si +PREHOOK: query: insert into over10k_orc_bucketed select * from over10k PREHOOK: type: QUERY PREHOOK: Input: default@over10k PREHOOK: Output: default@over10k_orc_bucketed -POSTHOOK: query: insert into over10k_orc_bucketed select * from over10k cluster by si +POSTHOOK: query: insert into over10k_orc_bucketed select * from over10k POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k POSTHOOK: Output: default@over10k_orc_bucketed @@ -369,11 +369,11 @@ POSTHOOK: Lineage: over10k_orc_bucketed.t SIMPLE [(over10k)over10k.FieldSchema(n POSTHOOK: Lineage: over10k_orc_bucketed.ts SIMPLE [(over10k)over10k.FieldSchema(name:ts, type:timestamp, comment:null), ] Found 4 items #### A masked pattern was here #### -PREHOOK: query: insert into over10k_orc_bucketed select * from over10k cluster by si +PREHOOK: query: insert into over10k_orc_bucketed select * from over10k PREHOOK: type: QUERY PREHOOK: Input: default@over10k PREHOOK: Output: default@over10k_orc_bucketed -POSTHOOK: query: insert into over10k_orc_bucketed select * from over10k cluster by si +POSTHOOK: query: insert into over10k_orc_bucketed select * from over10k POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k POSTHOOK: Output: default@over10k_orc_bucketed @@ -541,12 +541,12 @@ POSTHOOK: query: select ROW__ID, t, si, i from over10k_orc_bucketed where b = 42 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_orc_bucketed #### A masked pattern was here #### -{"transactionid":0,"bucketid":536870912,"rowid":104} -3 344 65733 -{"transactionid":0,"bucketid":536870912,"rowid":368} -3 344 65733 -{"transactionid":0,"bucketid":536936448,"rowid":250} 5 501 65585 -{"transactionid":0,"bucketid":536936448,"rowid":512} 5 501 65585 -{"transactionid":0,"bucketid":537067520,"rowid":224} 35 463 65646 -{"transactionid":0,"bucketid":537067520,"rowid":501} 35 463 65646 +{"transactionid":0,"bucketid":536870912,"rowid":36} -3 344 65733 +{"transactionid":0,"bucketid":536870912,"rowid":300} -3 344 65733 +{"transactionid":0,"bucketid":536936448,"rowid":82} 5 501 65585 +{"transactionid":0,"bucketid":536936448,"rowid":344} 5 501 65585 +{"transactionid":0,"bucketid":537067520,"rowid":211} 35 463 65646 +{"transactionid":0,"bucketid":537067520,"rowid":488} 35 463 65646 PREHOOK: query: explain update over10k_orc_bucketed set i = 0 where b = 4294967363 and t < 100 PREHOOK: type: QUERY POSTHOOK: query: explain update over10k_orc_bucketed set i = 0 where b = 4294967363 and t < 100 http://git-wip-us.apache.org/repos/asf/hive/blob/31c1b719/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out index f10af8c..3314d25 100644 --- a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out +++ b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out @@ -348,11 +348,11 @@ NULL NULL 509 1 510 2 511 3 -PREHOOK: query: insert into over10k_orc_bucketed select * from over10k cluster by si +PREHOOK: query: insert into over10k_orc_bucketed select * from over10k PREHOOK: type: QUERY PREHOOK: Input: default@over10k PREHOOK: Output: default@over10k_orc_bucketed -POSTHOOK: query: insert into over10k_orc_bucketed select * from over10k cluster by si +POSTHOOK: query: insert into over10k_orc_bucketed select * from over10k POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k POSTHOOK: Output: default@over10k_orc_bucketed @@ -369,11 +369,11 @@ POSTHOOK: Lineage: over10k_orc_bucketed.t SIMPLE [(over10k)over10k.FieldSchema(n POSTHOOK: Lineage: over10k_orc_bucketed.ts SIMPLE [(over10k)over10k.FieldSchema(name:ts, type:timestamp, comment:null), ] Found 4 items #### A masked pattern was here #### -PREHOOK: query: insert into over10k_orc_bucketed select * from over10k cluster by si +PREHOOK: query: insert into over10k_orc_bucketed select * from over10k PREHOOK: type: QUERY PREHOOK: Input: default@over10k PREHOOK: Output: default@over10k_orc_bucketed -POSTHOOK: query: insert into over10k_orc_bucketed select * from over10k cluster by si +POSTHOOK: query: insert into over10k_orc_bucketed select * from over10k POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k POSTHOOK: Output: default@over10k_orc_bucketed @@ -539,12 +539,12 @@ POSTHOOK: query: select ROW__ID, t, si, i from over10k_orc_bucketed where b = 42 POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_orc_bucketed #### A masked pattern was here #### -{"transactionid":0,"bucketid":536870912,"rowid":104} -3 344 65733 -{"transactionid":0,"bucketid":536870912,"rowid":368} -3 344 65733 -{"transactionid":0,"bucketid":536936448,"rowid":250} 5 501 65585 -{"transactionid":0,"bucketid":536936448,"rowid":512} 5 501 65585 -{"transactionid":0,"bucketid":537067520,"rowid":224} 35 463 65646 -{"transactionid":0,"bucketid":537067520,"rowid":501} 35 463 65646 +{"transactionid":0,"bucketid":536870912,"rowid":36} -3 344 65733 +{"transactionid":0,"bucketid":536870912,"rowid":300} -3 344 65733 +{"transactionid":0,"bucketid":536936448,"rowid":82} 5 501 65585 +{"transactionid":0,"bucketid":536936448,"rowid":344} 5 501 65585 +{"transactionid":0,"bucketid":537067520,"rowid":211} 35 463 65646 +{"transactionid":0,"bucketid":537067520,"rowid":488} 35 463 65646 PREHOOK: query: explain update over10k_orc_bucketed set i = 0 where b = 4294967363 and t < 100 PREHOOK: type: QUERY POSTHOOK: query: explain update over10k_orc_bucketed set i = 0 where b = 4294967363 and t < 100 @@ -710,9 +710,9 @@ POSTHOOK: query: select ROW__ID, count(*) from over10k_orc_bucketed group by ROW POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_orc_bucketed #### A masked pattern was here #### -PREHOOK: query: alter table over10k_orc_bucketed compact 'major' WITH OVERWRITE TBLPROPERTIES ("compactor.mapreduce.map.memory.mb"="500","compactor.hive.tez.container.size"="500") +PREHOOK: query: alter table over10k_orc_bucketed compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500') PREHOOK: type: ALTERTABLE_COMPACT -POSTHOOK: query: alter table over10k_orc_bucketed compact 'major' WITH OVERWRITE TBLPROPERTIES ("compactor.mapreduce.map.memory.mb"="500","compactor.hive.tez.container.size"="500") +POSTHOOK: query: alter table over10k_orc_bucketed compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500') POSTHOOK: type: ALTERTABLE_COMPACT PREHOOK: query: select ROW__ID, t, si, i from over10k_orc_bucketed where b = 4294967363 and t < 100 order by ROW__ID PREHOOK: type: QUERY
