HIVE-19572: Add option to mask stats and data size in q files (Jesus Camacho Rodriguez, reviewed by Prasanth Jayachandran) (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b329afa0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b329afa0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b329afa0 Branch: refs/heads/master Commit: b329afa0a21af1fc2e520236ec9ea817d5afb4df Parents: cb6dee1 Author: Jesus Camacho Rodriguez <[email protected]> Authored: Wed May 16 16:40:20 2018 -0700 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Wed May 16 16:40:41 2018 -0700 ---------------------------------------------------------------------- ql/src/test/queries/clientpositive/smb_cache.q | 2 + .../clientpositive/tez_dynpart_hashjoin_1.q | 3 + .../tez_vector_dynpart_hashjoin_1.q | 3 + .../results/clientpositive/llap/smb_cache.q.out | 36 ++--- .../llap/tez_dynpart_hashjoin_1.q.out | 160 +++++++++---------- .../llap/tez_vector_dynpart_hashjoin_1.q.out | 160 +++++++++---------- 6 files changed, 186 insertions(+), 178 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b329afa0/ql/src/test/queries/clientpositive/smb_cache.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/smb_cache.q b/ql/src/test/queries/clientpositive/smb_cache.q index 67a9242..d87ba6c 100644 --- a/ql/src/test/queries/clientpositive/smb_cache.q +++ b/ql/src/test/queries/clientpositive/smb_cache.q @@ -1,3 +1,5 @@ +-- MASK_STATS + set hive.explain.user=false; create table bug_201_input_a ( userid int http://git-wip-us.apache.org/repos/asf/hive/blob/b329afa0/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_1.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_1.q b/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_1.q index 16e4d7f..ea3dfce 100644 --- a/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_1.q +++ b/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_1.q @@ -1,4 +1,7 @@ --! qt:dataset:alltypesorc + +-- MASK_STATS + set hive.mapred.mode=nonstrict; set hive.explain.user=false; http://git-wip-us.apache.org/repos/asf/hive/blob/b329afa0/ql/src/test/queries/clientpositive/tez_vector_dynpart_hashjoin_1.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/tez_vector_dynpart_hashjoin_1.q b/ql/src/test/queries/clientpositive/tez_vector_dynpart_hashjoin_1.q index f828471..ecb2249 100644 --- a/ql/src/test/queries/clientpositive/tez_vector_dynpart_hashjoin_1.q +++ b/ql/src/test/queries/clientpositive/tez_vector_dynpart_hashjoin_1.q @@ -1,4 +1,7 @@ --! qt:dataset:alltypesorc + +-- MASK_STATS + set hive.mapred.mode=nonstrict; set hive.explain.user=false; http://git-wip-us.apache.org/repos/asf/hive/blob/b329afa0/ql/src/test/results/clientpositive/llap/smb_cache.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/smb_cache.q.out b/ql/src/test/results/clientpositive/llap/smb_cache.q.out index 2e46575..7c885d1 100644 --- a/ql/src/test/results/clientpositive/llap/smb_cache.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_cache.q.out @@ -216,38 +216,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - Statistics: Num rows: 130 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: userid is not null (type: boolean) - Statistics: Num rows: 130 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: userid (type: int) outputColumnNames: _col0 - Statistics: Num rows: 130 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 130 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 3 Map Operator Tree: TableScan alias: fa - Statistics: Num rows: 527 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: userid is not null (type: boolean) - Statistics: Num rows: 527 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: userid (type: int) outputColumnNames: _col0 - Statistics: Num rows: 527 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 527 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -260,10 +260,10 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 130 Data size: 1040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 130 Data size: 1040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -335,25 +335,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - Statistics: Num rows: 130 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: userid is not null (type: boolean) - Statistics: Num rows: 130 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: userid (type: int) outputColumnNames: _col0 - Statistics: Num rows: 130 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Map Operator Tree: TableScan alias: fa - Statistics: Num rows: 527 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: userid is not null (type: boolean) - Statistics: Num rows: 527 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: userid (type: int) outputColumnNames: _col0 - Statistics: Num rows: 527 Data size: 2108 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Merge Join Operator condition map: Inner Join 0 to 1 @@ -361,10 +361,10 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 130 Data size: 1040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 130 Data size: 1040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat http://git-wip-us.apache.org/repos/asf/hive/blob/b329afa0/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out index 8f0572b..fd6541e 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out @@ -31,19 +31,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: cint BETWEEN 1000000 AND 3000000 (type: boolean) - Statistics: Num rows: 1365 Data size: 343800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1365 Data size: 343800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1365 Data size: 343800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs @@ -51,19 +51,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1019 Data size: 256780 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs @@ -77,11 +77,11 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 2076 Data size: 1148660 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) sort order: + - Statistics: Num rows: 2076 Data size: 1148660 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) Reducer 3 Execution mode: vectorized, llap @@ -89,10 +89,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: double), VALUE._col5 (type: string), VALUE._col6 (type: string), VALUE._col7 (type: timestamp), VALUE._col8 (type: timestamp), VALUE._col9 (type: boolean), VALUE._col10 (type: boolean), VALUE._col11 (type: tinyint), VALUE._col12 (type: smallint), VALUE._col13 (type: int), VALUE._col14 (type: bigint), VALUE._col15 (type: float), VALUE._col16 (type: double), VALUE._col17 (type: string), VALUE._col18 (type: string), VALUE._col19 (type: timestamp), VALUE._col20 (type: timestamp), VALUE._col21 (type: boolean), VALUE._col22 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 2076 Data size: 1148660 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2076 Data size: 1148660 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -163,38 +163,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: cint BETWEEN 1000000 AND 3000000 (type: boolean) - Statistics: Num rows: 1365 Data size: 4080 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cint (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 4080 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 4080 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map 4 Map Operator Tree: TableScan alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cint (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -206,15 +206,15 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 2076 Data size: 16608 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized, llap @@ -223,10 +223,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -291,19 +291,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 12288 Data size: 73396 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: cint BETWEEN 1000000 AND 3000000 (type: boolean) - Statistics: Num rows: 1365 Data size: 8160 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: csmallint (type: smallint), cint (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1365 Data size: 8160 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1365 Data size: 8160 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -311,19 +311,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cint (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1019 Data size: 9144 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -336,18 +336,18 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2076 Data size: 6924 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: _col0 (type: smallint) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 623 Data size: 7064 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: smallint) sort order: + Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 623 Data size: 7064 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reducer 3 Execution mode: vectorized, llap @@ -357,11 +357,11 @@ STAGE PLANS: keys: KEY._col0 (type: smallint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 623 Data size: 7064 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: smallint) sort order: + - Statistics: Num rows: 623 Data size: 7064 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reducer 4 Execution mode: vectorized, llap @@ -369,10 +369,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 623 Data size: 7064 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 623 Data size: 7064 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -442,19 +442,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: cint BETWEEN 1000000 AND 3000000 (type: boolean) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs @@ -462,19 +462,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs @@ -490,12 +490,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: 1 Map 4 - Statistics: Num rows: 1501 Data size: 355334 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Reduce Output Operator key expressions: _col2 (type: int) sort order: + - Statistics: Num rows: 1501 Data size: 355334 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean), _col12 (type: tinyint), _col13 (type: smallint), _col14 (type: int), _col15 (type: bigint), _col16 (type: float), _col17 (type: double), _col18 (type: string), _col19 (type: string), _col20 (type: timestamp), _col21 (type: timestamp), _col22 (type: boolean), _col23 (type: boolean) Reducer 3 Execution mode: vectorized, llap @@ -503,10 +503,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: double), VALUE._col5 (type: string), VALUE._col6 (type: string), VALUE._col7 (type: timestamp), VALUE._col8 (type: timestamp), VALUE._col9 (type: boolean), VALUE._col10 (type: boolean), VALUE._col11 (type: tinyint), VALUE._col12 (type: smallint), VALUE._col13 (type: int), VALUE._col14 (type: bigint), VALUE._col15 (type: float), VALUE._col16 (type: double), VALUE._col17 (type: string), VALUE._col18 (type: string), VALUE._col19 (type: timestamp), VALUE._col20 (type: timestamp), VALUE._col21 (type: boolean), VALUE._col22 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 1501 Data size: 355334 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1501 Data size: 355334 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -577,38 +577,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: cint BETWEEN 1000000 AND 3000000 (type: boolean) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Map 4 Map Operator Tree: TableScan alias: b - Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -622,16 +622,16 @@ STAGE PLANS: 1 KEY.reducesinkkey0 (type: int) input vertices: 1 Map 4 - Statistics: Num rows: 1501 Data size: 355334 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized, llap @@ -640,10 +640,10 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -708,19 +708,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: cint BETWEEN 1000000 AND 3000000 (type: boolean) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Select Operator expressions: csmallint (type: smallint), cint (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -728,19 +728,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1365 Data size: 323031 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -755,19 +755,19 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 5 - Statistics: Num rows: 1501 Data size: 355334 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Group By Operator aggregations: count() keys: _col0 (type: smallint) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1501 Data size: 355334 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: smallint) sort order: + Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 1501 Data size: 355334 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reducer 3 Execution mode: vectorized, llap @@ -777,11 +777,11 @@ STAGE PLANS: keys: KEY._col0 (type: smallint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 750 Data size: 177548 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: smallint) sort order: + - Statistics: Num rows: 750 Data size: 177548 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reducer 4 Execution mode: vectorized, llap @@ -789,10 +789,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 750 Data size: 177548 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 750 Data size: 177548 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
