HIVE-19765: Add Parquet specific tests to BlobstoreCliDriver (Sahil Takiar, reviewed by Marta Kuczora)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cbcfab9e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cbcfab9e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cbcfab9e Branch: refs/heads/master Commit: cbcfab9eb1c58e0c389b093af188d6483440beb0 Parents: c2940a0 Author: Sahil Takiar <takiar.sa...@gmail.com> Authored: Fri Jun 1 12:25:01 2018 -0500 Committer: Sahil Takiar <stak...@cloudera.com> Committed: Thu Jul 5 17:47:42 2018 -0500 ---------------------------------------------------------------------- .../queries/clientpositive/parquet_buckets.q | 31 ++ .../clientpositive/parquet_format_nonpart.q | 30 ++ .../clientpositive/parquet_format_part.q | 67 +++ .../parquet_nonstd_partitions_loc.q | 100 ++++ .../queries/clientpositive/ptf_parquetfile.q | 32 ++ .../clientpositive/parquet_buckets.q.out | 183 +++++++ .../clientpositive/parquet_format_nonpart.q.out | 201 +++++++ .../clientpositive/parquet_format_part.q.out | 290 ++++++++++ .../parquet_nonstd_partitions_loc.q.out | 537 +++++++++++++++++++ .../clientpositive/ptf_parquetfile.q.out | 104 ++++ 10 files changed, 1575 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q new file mode 100644 index 0000000..c5a2cd3 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q @@ -0,0 +1,31 @@ +-- Test simple interaction with partitioned bucketed table with paquet format in blobstore + +SET hive.exec.dynamic.partition=true; +SET hive.exec.reducers.max=10; +SET hive.exec.dynamic.partition.mode=nonstrict; + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE parquet_buckets; +CREATE TABLE parquet_buckets (a STRING, value DOUBLE) +PARTITIONED BY (b STRING) +CLUSTERED BY (a) INTO 10 BUCKETS +STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/parquet_buckets'; + +INSERT OVERWRITE TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source; +SELECT * FROM parquet_buckets; + +INSERT INTO TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source; +SELECT * FROM parquet_buckets; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q new file mode 100644 index 0000000..a7827e9 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q @@ -0,0 +1,30 @@ +-- Test INSERT OVERWRITE and INSERT INTO on parquet table in blobstore + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE parquet_table; +CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/parquet_table'; + +INSERT OVERWRITE TABLE parquet_table +SELECT * FROM blobstore_source; + +SELECT * FROM parquet_table; +SELECT a FROM parquet_table GROUP BY a; +SELECT b FROM parquet_table GROUP BY b; +SELECT value FROM parquet_table GROUP BY value; + +INSERT INTO TABLE parquet_table +SELECT * FROM blobstore_source; + +SELECT * FROM parquet_table; +SELECT a FROM parquet_table GROUP BY a; +SELECT b FROM parquet_table GROUP BY b; +SELECT value FROM parquet_table GROUP BY value; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q new file mode 100644 index 0000000..e9924fe --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q @@ -0,0 +1,67 @@ +-- Test INSERT INTO and INSERT OVERWRITE on partitioned parquet table in blobstore + +DROP TABLE src_events; +CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/src_events/'; +LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events; + +DROP TABLE parquet_events; +CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/parquet_events'; + +SET hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; +SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921; +SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121; + +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39; +SELECT COUNT(*) FROM parquet_events; + +INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'; +SELECT COUNT(*) FROM parquet_events; + +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'; +SELECT COUNT(*) FROM parquet_events; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q new file mode 100644 index 0000000..2d20b99 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q @@ -0,0 +1,100 @@ +-- Test table in parquet format with non-standard partition locations in blobstore + +DROP TABLE src_events; +CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/src_events/'; +LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events; + +DROP TABLE parquet_events; +CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_events/'; + +SET hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location +ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1/'; +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; +INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +SET hive.merge.mapfiles=false; + +-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location with hive.merge.mapfiles false +ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2/'; +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles false +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +SET hive.merge.mapfiles=true; + +ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3/'; +INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles true +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change'); +ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change'); +ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change'); +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q b/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q new file mode 100644 index 0000000..fbf5ff3 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q @@ -0,0 +1,32 @@ +-- Test several window functions on a table stored using PARQUET +DROP TABLE part_parquet; +CREATE TABLE part_parquet( + p_partkey int, + p_name string, + p_mfgr string, + p_brand string, + p_type string, + p_size int, + p_container string, + p_retailprice double, + p_comment string +) +STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/ptf_parquetfile/part_parquet'; + +LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet; + +-- Test windowing PTFs with several partitions, using PARQUET storage +SELECT + p_mfgr, p_name, p_size, + RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r, + DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr, + SUM(p_retailprice) OVER ( + PARTITION BY p_mfgr + ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS s1 +FROM NOOP( + ON part_parquet + PARTITION BY p_mfgr + ORDER BY p_name +); http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out new file mode 100644 index 0000000..355958d --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out @@ -0,0 +1,183 @@ +PREHOOK: query: DROP TABLE blobstore_source +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE blobstore_source +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source +PREHOOK: Output: database:default +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: DROP TABLE parquet_buckets +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_buckets +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE) +PARTITIONED BY (b STRING) +CLUSTERED BY (a) INTO 10 BUCKETS +STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_buckets +POSTHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE) +PARTITIONED BY (b STRING) +CLUSTERED BY (a) INTO 10 BUCKETS +STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_buckets +PREHOOK: query: INSERT OVERWRITE TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@parquet_buckets +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@parquet_buckets@b=abc +POSTHOOK: Output: default@parquet_buckets@b=ajss +POSTHOOK: Output: default@parquet_buckets@b=data +POSTHOOK: Output: default@parquet_buckets@b=def +POSTHOOK: Output: default@parquet_buckets@b=djns +POSTHOOK: Output: default@parquet_buckets@b=ne +POSTHOOK: Output: default@parquet_buckets@b=random +POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_buckets +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_buckets +PREHOOK: Input: default@parquet_buckets@b=abc +PREHOOK: Input: default@parquet_buckets@b=ajss +PREHOOK: Input: default@parquet_buckets@b=data +PREHOOK: Input: default@parquet_buckets@b=def +PREHOOK: Input: default@parquet_buckets@b=djns +PREHOOK: Input: default@parquet_buckets@b=ne +PREHOOK: Input: default@parquet_buckets@b=random +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_buckets +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_buckets +POSTHOOK: Input: default@parquet_buckets@b=abc +POSTHOOK: Input: default@parquet_buckets@b=ajss +POSTHOOK: Input: default@parquet_buckets@b=data +POSTHOOK: Input: default@parquet_buckets@b=def +POSTHOOK: Input: default@parquet_buckets@b=djns +POSTHOOK: Input: default@parquet_buckets@b=ne +POSTHOOK: Input: default@parquet_buckets@b=random +#### A masked pattern was here #### +1 10.5 abc +3 90.23232 ajss +6 3.002 data +2 11.5 def +4 89.02002 djns +7 71.9084 ne +5 2.99 random +PREHOOK: query: INSERT INTO TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@parquet_buckets +POSTHOOK: query: INSERT INTO TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@parquet_buckets@b=abc +POSTHOOK: Output: default@parquet_buckets@b=ajss +POSTHOOK: Output: default@parquet_buckets@b=data +POSTHOOK: Output: default@parquet_buckets@b=def +POSTHOOK: Output: default@parquet_buckets@b=djns +POSTHOOK: Output: default@parquet_buckets@b=ne +POSTHOOK: Output: default@parquet_buckets@b=random +POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_buckets +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_buckets +PREHOOK: Input: default@parquet_buckets@b=abc +PREHOOK: Input: default@parquet_buckets@b=ajss +PREHOOK: Input: default@parquet_buckets@b=data +PREHOOK: Input: default@parquet_buckets@b=def +PREHOOK: Input: default@parquet_buckets@b=djns +PREHOOK: Input: default@parquet_buckets@b=ne +PREHOOK: Input: default@parquet_buckets@b=random +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_buckets +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_buckets +POSTHOOK: Input: default@parquet_buckets@b=abc +POSTHOOK: Input: default@parquet_buckets@b=ajss +POSTHOOK: Input: default@parquet_buckets@b=data +POSTHOOK: Input: default@parquet_buckets@b=def +POSTHOOK: Input: default@parquet_buckets@b=djns +POSTHOOK: Input: default@parquet_buckets@b=ne +POSTHOOK: Input: default@parquet_buckets@b=random +#### A masked pattern was here #### +1 10.5 abc +1 10.5 abc +3 90.23232 ajss +3 90.23232 ajss +6 3.002 data +6 3.002 data +2 11.5 def +2 11.5 def +4 89.02002 djns +4 89.02002 djns +7 71.9084 ne +7 71.9084 ne +5 2.99 random +5 2.99 random http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out new file mode 100644 index 0000000..cba6b1a --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out @@ -0,0 +1,201 @@ +PREHOOK: query: DROP TABLE blobstore_source +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE blobstore_source +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source +PREHOOK: Output: database:default +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: DROP TABLE parquet_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_table +POSTHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_table +PREHOOK: query: INSERT OVERWRITE TABLE parquet_table +SELECT * FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@parquet_table +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_table +SELECT * FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@parquet_table +POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_table +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +1 abc 10.5 +2 def 11.5 +3 ajss 90.23232 +4 djns 89.02002 +5 random 2.99 +6 data 3.002 +7 ne 71.9084 +PREHOOK: query: SELECT a FROM parquet_table GROUP BY a +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +7 +PREHOOK: query: SELECT b FROM parquet_table GROUP BY b +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +abc +ajss +data +def +djns +ne +random +PREHOOK: query: SELECT value FROM parquet_table GROUP BY value +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +2.99 +3.002 +10.5 +11.5 +71.9084 +89.02002 +90.23232 +PREHOOK: query: INSERT INTO TABLE parquet_table +SELECT * FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@parquet_table +POSTHOOK: query: INSERT INTO TABLE parquet_table +SELECT * FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@parquet_table +POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_table +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +1 abc 10.5 +2 def 11.5 +3 ajss 90.23232 +4 djns 89.02002 +5 random 2.99 +6 data 3.002 +7 ne 71.9084 +1 abc 10.5 +2 def 11.5 +3 ajss 90.23232 +4 djns 89.02002 +5 random 2.99 +6 data 3.002 +7 ne 71.9084 +PREHOOK: query: SELECT a FROM parquet_table GROUP BY a +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +7 +PREHOOK: query: SELECT b FROM parquet_table GROUP BY b +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +abc +ajss +data +def +djns +ne +random +PREHOOK: query: SELECT value FROM parquet_table GROUP BY value +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +2.99 +3.002 +10.5 +11.5 +71.9084 +89.02002 +90.23232 http://git-wip-us.apache.org/repos/asf/hive/blob/cbcfab9e/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out new file mode 100644 index 0000000..0931e3d --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out @@ -0,0 +1,290 @@ +PREHOOK: query: DROP TABLE src_events +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE src_events +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events +PREHOOK: Output: database:default +PREHOOK: Output: default@src_events +POSTHOOK: query: CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_events +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@src_events +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@src_events +PREHOOK: query: DROP TABLE parquet_events +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_events +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_events +POSTHOOK: query: CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_events +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +200 +PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +50 +PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +100 +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201211' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201211 +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201211' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=201211/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +300 +PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39 +POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +350 +PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +400 +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +350