Repository: hive Updated Branches: refs/heads/master 29332fbf8 -> a9581995d
HIVE-20522: HiveFilterSetOpTransposeRule may throw assertion error due to nullability of fields (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a9581995 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a9581995 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a9581995 Branch: refs/heads/master Commit: a9581995dbd285bb4c8d0668506f4a7cb8f2ab47 Parents: 29332fb Author: Jesus Camacho Rodriguez <[email protected]> Authored: Fri Sep 7 18:20:08 2018 -0700 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Sun Sep 9 08:49:48 2018 -0700 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../rules/HiveFilterSetOpTransposeRule.java | 9 +- .../clientpositive/union_assertion_type.q | 51 +++ .../llap/union_assertion_type.q.out | 335 +++++++++++++++++++ 4 files changed, 391 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a9581995/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 7497af1..3a5aec7 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -744,6 +744,7 @@ minillaplocal.query.files=\ udaf_collect_set_2.q,\ udaf_all_keyword.q,\ udf_coalesce.q,\ + union_assertion_type.q,\ union_fast_stats.q,\ union_rowcounts.q,\ union_remove_26.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/a9581995/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java index 2dd8a77..7ea8154 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java @@ -136,12 +136,11 @@ public class HiveFilterSetOpTransposeRule extends FilterSetOpTransposeRule { // create a new setop whose children are the filters created above SetOp newSetOp = setOp.copy(setOp.getTraitSet(), newSetOpInputs); call.transformTo(newSetOp); - } else if (newSetOpInputs.size() == 1) { - call.transformTo(newSetOpInputs.get(0)); } else { - // we have to keep at least a branch before we support empty values() in - // hive - call.transformTo(lastInput); + // We have to keep at least a branch before we support empty values() in Hive + RelNode result = newSetOpInputs.size() == 1 ? newSetOpInputs.get(0) : lastInput; + call.transformTo( + relBuilder.push(result).convert(filterRel.getRowType(), false).build()); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/a9581995/ql/src/test/queries/clientpositive/union_assertion_type.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/union_assertion_type.q b/ql/src/test/queries/clientpositive/union_assertion_type.q new file mode 100644 index 0000000..0148495 --- /dev/null +++ b/ql/src/test/queries/clientpositive/union_assertion_type.q @@ -0,0 +1,51 @@ +CREATE TABLE union_table_test_n0 (column1 string not null, column2 string not null, column3 string not null); +CREATE TABLE union_table_test_n1 (column1 string, column2 string, column3 string); +INSERT INTO union_table_test_n0 VALUES ('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('10', '11', '12'); +INSERT INTO union_table_test_n1 VALUES ('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('10', '11', '12'); + +EXPLAIN +SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, column3 FROM union_table_test_n0 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n1) x +WHERE x.column3 < '5'; + +SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, column3 FROM union_table_test_n0 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n1) x +WHERE x.column3 < '5'; + +EXPLAIN +SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, '5' as column3 FROM union_table_test_n1 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n0) x +WHERE x.column3 < '5'; + +SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, '5' as column3 FROM union_table_test_n1 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n0) x +WHERE x.column3 < '5'; + +DROP TABLE union_table_test_n0; +DROP TABLE union_table_test_n1; + +CREATE TABLE union_table_test_n3 (k int); +CREATE TABLE union_table_test_n4 (k int); +CREATE TABLE union_table_test_n5 (k int); +INSERT INTO union_table_test_n3 VALUES (1),(3); +INSERT INTO union_table_test_n4 VALUES (1); +INSERT INTO union_table_test_n5 VALUES (1),(3); + +EXPLAIN +SELECT u0.k as key, u0.d1 as data0, u0.d2 as data2 FROM ( + SELECT k,'' as d1,'' as d2 FROM union_table_test_n3 + UNION ALL + SELECT k,'' as d1,'' as d2 FROM union_table_test_n4) u0 +LEFT OUTER JOIN union_table_test_n5 tx1 ON (u0.k = tx1.k AND tx1.k != d1) AND u0.k!=1; + +DROP TABLE union_table_test_n3; +DROP TABLE union_table_test_n4; +DROP TABLE union_table_test_n5; http://git-wip-us.apache.org/repos/asf/hive/blob/a9581995/ql/src/test/results/clientpositive/llap/union_assertion_type.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/union_assertion_type.q.out b/ql/src/test/results/clientpositive/llap/union_assertion_type.q.out new file mode 100644 index 0000000..0420353 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/union_assertion_type.q.out @@ -0,0 +1,335 @@ +PREHOOK: query: CREATE TABLE union_table_test_n0 (column1 string not null, column2 string not null, column3 string not null) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_table_test_n0 +POSTHOOK: query: CREATE TABLE union_table_test_n0 (column1 string not null, column2 string not null, column3 string not null) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_table_test_n0 +PREHOOK: query: CREATE TABLE union_table_test_n1 (column1 string, column2 string, column3 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_table_test_n1 +POSTHOOK: query: CREATE TABLE union_table_test_n1 (column1 string, column2 string, column3 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_table_test_n1 +PREHOOK: query: INSERT INTO union_table_test_n0 VALUES ('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('10', '11', '12') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@union_table_test_n0 +POSTHOOK: query: INSERT INTO union_table_test_n0 VALUES ('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('10', '11', '12') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@union_table_test_n0 +POSTHOOK: Lineage: union_table_test_n0.column1 SCRIPT [] +POSTHOOK: Lineage: union_table_test_n0.column2 SCRIPT [] +POSTHOOK: Lineage: union_table_test_n0.column3 SCRIPT [] +PREHOOK: query: INSERT INTO union_table_test_n1 VALUES ('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('10', '11', '12') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@union_table_test_n1 +POSTHOOK: query: INSERT INTO union_table_test_n1 VALUES ('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('10', '11', '12') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@union_table_test_n1 +POSTHOOK: Lineage: union_table_test_n1.column1 SCRIPT [] +POSTHOOK: Lineage: union_table_test_n1.column2 SCRIPT [] +POSTHOOK: Lineage: union_table_test_n1.column3 SCRIPT [] +PREHOOK: query: EXPLAIN +SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, column3 FROM union_table_test_n0 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n1) x +WHERE x.column3 < '5' +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, column3 FROM union_table_test_n0 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n1) x +WHERE x.column3 < '5' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: union_table_test_n0 + filterExpr: (column3 < '5') (type: boolean) + Filter Operator + predicate: (column3 < '5') (type: boolean) + Select Operator + expressions: column1 (type: string), column2 (type: string), column3 (type: string) + outputColumnNames: _col0, _col1, _col2 + ListSink + +PREHOOK: query: SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, column3 FROM union_table_test_n0 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n1) x +WHERE x.column3 < '5' +PREHOOK: type: QUERY +PREHOOK: Input: default@union_table_test_n0 +PREHOOK: Input: default@union_table_test_n1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, column3 FROM union_table_test_n0 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n1) x +WHERE x.column3 < '5' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_table_test_n0 +POSTHOOK: Input: default@union_table_test_n1 +#### A masked pattern was here #### +1 2 3 +10 11 12 +PREHOOK: query: EXPLAIN +SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, '5' as column3 FROM union_table_test_n1 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n0) x +WHERE x.column3 < '5' +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, '5' as column3 FROM union_table_test_n1 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n0) x +WHERE x.column3 < '5' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: union_table_test_n0 + Filter Operator + predicate: false (type: boolean) + Select Operator + expressions: column1 (type: string), column2 (type: string), '5' (type: string) + outputColumnNames: _col0, _col1, _col2 + ListSink + +PREHOOK: query: SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, '5' as column3 FROM union_table_test_n1 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n0) x +WHERE x.column3 < '5' +PREHOOK: type: QUERY +PREHOOK: Input: default@union_table_test_n0 +PREHOOK: Input: default@union_table_test_n1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT column1, x.column2, x.column3 FROM ( +SELECT column1, column2, '5' as column3 FROM union_table_test_n1 +UNION ALL +SELECT column1, column2, '5' as column3 FROM union_table_test_n0) x +WHERE x.column3 < '5' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@union_table_test_n0 +POSTHOOK: Input: default@union_table_test_n1 +#### A masked pattern was here #### +PREHOOK: query: DROP TABLE union_table_test_n0 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@union_table_test_n0 +PREHOOK: Output: default@union_table_test_n0 +POSTHOOK: query: DROP TABLE union_table_test_n0 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@union_table_test_n0 +POSTHOOK: Output: default@union_table_test_n0 +PREHOOK: query: DROP TABLE union_table_test_n1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@union_table_test_n1 +PREHOOK: Output: default@union_table_test_n1 +POSTHOOK: query: DROP TABLE union_table_test_n1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@union_table_test_n1 +POSTHOOK: Output: default@union_table_test_n1 +PREHOOK: query: CREATE TABLE union_table_test_n3 (k int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_table_test_n3 +POSTHOOK: query: CREATE TABLE union_table_test_n3 (k int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_table_test_n3 +PREHOOK: query: CREATE TABLE union_table_test_n4 (k int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_table_test_n4 +POSTHOOK: query: CREATE TABLE union_table_test_n4 (k int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_table_test_n4 +PREHOOK: query: CREATE TABLE union_table_test_n5 (k int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@union_table_test_n5 +POSTHOOK: query: CREATE TABLE union_table_test_n5 (k int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@union_table_test_n5 +PREHOOK: query: INSERT INTO union_table_test_n3 VALUES (1),(3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@union_table_test_n3 +POSTHOOK: query: INSERT INTO union_table_test_n3 VALUES (1),(3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@union_table_test_n3 +POSTHOOK: Lineage: union_table_test_n3.k SCRIPT [] +PREHOOK: query: INSERT INTO union_table_test_n4 VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@union_table_test_n4 +POSTHOOK: query: INSERT INTO union_table_test_n4 VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@union_table_test_n4 +POSTHOOK: Lineage: union_table_test_n4.k SCRIPT [] +PREHOOK: query: INSERT INTO union_table_test_n5 VALUES (1),(3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@union_table_test_n5 +POSTHOOK: query: INSERT INTO union_table_test_n5 VALUES (1),(3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@union_table_test_n5 +POSTHOOK: Lineage: union_table_test_n5.k SCRIPT [] +Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_1]] in Stage 'Reducer 3' is a cross product +PREHOOK: query: EXPLAIN +SELECT u0.k as key, u0.d1 as data0, u0.d2 as data2 FROM ( + SELECT k,'' as d1,'' as d2 FROM union_table_test_n3 + UNION ALL + SELECT k,'' as d1,'' as d2 FROM union_table_test_n4) u0 +LEFT OUTER JOIN union_table_test_n5 tx1 ON (u0.k = tx1.k AND tx1.k != d1) AND u0.k!=1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT u0.k as key, u0.d1 as data0, u0.d2 as data2 FROM ( + SELECT k,'' as d1,'' as d2 FROM union_table_test_n3 + UNION ALL + SELECT k,'' as d1,'' as d2 FROM union_table_test_n4) u0 +LEFT OUTER JOIN union_table_test_n5 tx1 ON (u0.k = tx1.k AND tx1.k != d1) AND u0.k!=1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 4 <- Union 2 (CONTAINS) + Reducer 3 <- Map 5 (CUSTOM_SIMPLE_EDGE), Union 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: union_table_test_n3 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: k (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: union_table_test_n4 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: k (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: tx1 + Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), '' (type: string), '' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: DROP TABLE union_table_test_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@union_table_test_n3 +PREHOOK: Output: default@union_table_test_n3 +POSTHOOK: query: DROP TABLE union_table_test_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@union_table_test_n3 +POSTHOOK: Output: default@union_table_test_n3 +PREHOOK: query: DROP TABLE union_table_test_n4 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@union_table_test_n4 +PREHOOK: Output: default@union_table_test_n4 +POSTHOOK: query: DROP TABLE union_table_test_n4 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@union_table_test_n4 +POSTHOOK: Output: default@union_table_test_n4 +PREHOOK: query: DROP TABLE union_table_test_n5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@union_table_test_n5 +PREHOOK: Output: default@union_table_test_n5 +POSTHOOK: query: DROP TABLE union_table_test_n5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@union_table_test_n5 +POSTHOOK: Output: default@union_table_test_n5
