This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 310cdd8233e HIVE-26722: HiveFilterSetOpTransposeRule incorrectly
prunes UNION ALL operands. (#3748). (Alessandro Solimando, reviewed by Ayush
Saxena, Simhadri Govindappa)
310cdd8233e is described below
commit 310cdd8233eabb36a8c247160faefa3ea83c2e94
Author: Alessandro Solimando <[email protected]>
AuthorDate: Thu Nov 17 15:20:59 2022 +0100
HIVE-26722: HiveFilterSetOpTransposeRule incorrectly prunes UNION ALL
operands. (#3748). (Alessandro Solimando, reviewed by Ayush Saxena, Simhadri
Govindappa)
---
.../rules/HiveFilterSetOpTransposeRule.java | 18 ++--
.../clientpositive/union_all_filter_transpose.q | 32 +++++++
.../llap/create_view_when_mv_exists2.q.out | 18 ++--
.../llap/union_all_filter_transpose.q.out | 99 ++++++++++++++++++++++
4 files changed, 145 insertions(+), 22 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java
index 930979d2661..192fb682e13 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSetOpTransposeRule.java
@@ -42,8 +42,6 @@ import org.apache.calcite.util.Util;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
-import com.google.common.collect.ImmutableList;
-
public class HiveFilterSetOpTransposeRule extends FilterSetOpTransposeRule {
public static final HiveFilterSetOpTransposeRule INSTANCE =
@@ -66,8 +64,8 @@ public class HiveFilterSetOpTransposeRule extends
FilterSetOpTransposeRule {
* Op1 Op2
*
*
- * It additionally can remove branch(es) of filter if its able to determine
- * that they are going to generate empty result set.
+ * It additionally can remove branch(es) of filter if it's able to determine
+ * that they are going to generate an empty result set.
*/
private HiveFilterSetOpTransposeRule(RelBuilderFactory relBuilderFactory) {
super(relBuilderFactory);
@@ -111,18 +109,14 @@ public class HiveFilterSetOpTransposeRule extends
FilterSetOpTransposeRule {
final RelMetadataQuery mq = call.getMetadataQuery();
final RelOptPredicateList predicates = mq.getPulledUpPredicates(input);
if (predicates != null) {
- ImmutableList.Builder<RexNode> listBuilder = ImmutableList.builder();
- listBuilder.addAll(predicates.pulledUpPredicates);
- listBuilder.add(newCondition);
- RexExecutor executor =
+ final RexExecutor executor =
Util.first(filterRel.getCluster().getPlanner().getExecutor(),
RexUtil.EXECUTOR);
- final RexSimplify simplify = new RexSimplify(rexBuilder,
RelOptPredicateList.EMPTY, executor);
- final RexNode cond = RexUtil.composeConjunction(rexBuilder,
listBuilder.build());
- final RexNode x = simplify.simplifyUnknownAs(cond,
RexUnknownAs.FALSE);
+ final RexSimplify simplify = new RexSimplify(rexBuilder, predicates,
executor);
+ final RexNode x = simplify.simplifyUnknownAs(newCondition,
RexUnknownAs.FALSE);
if (x.isAlwaysFalse()) {
// this is the last branch, and it is always false
// We assume alwaysFalse filter will get pushed down to TS so this
- // branch so it won't read any data.
+ // branch won't read any data.
if (index == setOp.getInputs().size() - 1) {
lastInput = relBuilder.push(input).filter(newCondition).build();
}
diff --git a/ql/src/test/queries/clientpositive/union_all_filter_transpose.q
b/ql/src/test/queries/clientpositive/union_all_filter_transpose.q
new file mode 100644
index 00000000000..0be3128e373
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/union_all_filter_transpose.q
@@ -0,0 +1,32 @@
+# needed to avoid the simplification of CAST(NULL) into NULL
+set hive.cbo.rule.exclusion.regex=ReduceExpressionsRule\(Project\);
+
+CREATE EXTERNAL TABLE t (a string, b string);
+
+INSERT INTO t VALUES ('1000', 'b1');
+INSERT INTO t VALUES ('2000', 'b2');
+
+SELECT * FROM (
+ SELECT
+ a,
+ b
+ FROM t
+ UNION ALL
+ SELECT
+ a,
+ CAST(NULL AS string)
+ FROM t) AS t2
+WHERE a = 1000;
+
+EXPLAIN CBO
+SELECT * FROM (
+ SELECT
+ a,
+ b
+ FROM t
+ UNION ALL
+ SELECT
+ a,
+ CAST(NULL AS string)
+ FROM t) AS t2
+WHERE a = 1000;
diff --git
a/ql/src/test/results/clientpositive/llap/create_view_when_mv_exists2.q.out
b/ql/src/test/results/clientpositive/llap/create_view_when_mv_exists2.q.out
index 857c59b72ef..aaa59d9a07d 100644
--- a/ql/src/test/results/clientpositive/llap/create_view_when_mv_exists2.q.out
+++ b/ql/src/test/results/clientpositive/llap/create_view_when_mv_exists2.q.out
@@ -46,11 +46,10 @@ POSTHOOK: Input: default@t1
POSTHOOK: Output: database:default
POSTHOOK: Output: default@v1
CBO PLAN:
-HiveProject(col0=[CAST(10):INTEGER])
- HiveAggregate(group=[{0}])
- HiveProject($f0=[true])
- HiveFilter(condition=[=($0, 10)])
- HiveTableScan(table=[[default, t1]], table:alias=[t1])
+HiveAggregate(group=[{0}])
+ HiveProject($f0=[CAST(10):INTEGER])
+ HiveFilter(condition=[=($0, 10)])
+ HiveTableScan(table=[[default, t1]], table:alias=[t1])
PREHOOK: query: create view v1 as
select sub.* from (select * from t1 where col0 > 2 union select * from t1
where col0 = 0) sub
@@ -80,11 +79,10 @@ POSTHOOK: Input: default@t1
POSTHOOK: Input: default@v1
#### A masked pattern was here ####
CBO PLAN:
-HiveProject(col0=[CAST(10):INTEGER])
- HiveAggregate(group=[{0}])
- HiveProject($f0=[true])
- HiveFilter(condition=[=($0, 10)])
- HiveTableScan(table=[[default, t1]], table:alias=[t1])
+HiveAggregate(group=[{0}])
+ HiveProject($f0=[CAST(10):INTEGER])
+ HiveFilter(condition=[=($0, 10)])
+ HiveTableScan(table=[[default, t1]], table:alias=[t1])
PREHOOK: query: select * from v1
PREHOOK: type: QUERY
diff --git
a/ql/src/test/results/clientpositive/llap/union_all_filter_transpose.q.out
b/ql/src/test/results/clientpositive/llap/union_all_filter_transpose.q.out
new file mode 100644
index 00000000000..51bc71953f9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/union_all_filter_transpose.q.out
@@ -0,0 +1,99 @@
+PREHOOK: query: CREATE EXTERNAL TABLE t (a string, b string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: CREATE EXTERNAL TABLE t (a string, b string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: INSERT INTO t VALUES ('1000', 'b1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t
+POSTHOOK: query: INSERT INTO t VALUES ('1000', 'b1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.a SCRIPT []
+POSTHOOK: Lineage: t.b SCRIPT []
+PREHOOK: query: INSERT INTO t VALUES ('2000', 'b2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t
+POSTHOOK: query: INSERT INTO t VALUES ('2000', 'b2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.a SCRIPT []
+POSTHOOK: Lineage: t.b SCRIPT []
+PREHOOK: query: SELECT * FROM (
+ SELECT
+ a,
+ b
+ FROM t
+ UNION ALL
+ SELECT
+ a,
+ CAST(NULL AS string)
+ FROM t) AS t2
+WHERE a = 1000
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM (
+ SELECT
+ a,
+ b
+ FROM t
+ UNION ALL
+ SELECT
+ a,
+ CAST(NULL AS string)
+ FROM t) AS t2
+WHERE a = 1000
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+1000 b1
+1000 NULL
+PREHOOK: query: EXPLAIN CBO
+SELECT * FROM (
+ SELECT
+ a,
+ b
+ FROM t
+ UNION ALL
+ SELECT
+ a,
+ CAST(NULL AS string)
+ FROM t) AS t2
+WHERE a = 1000
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO
+SELECT * FROM (
+ SELECT
+ a,
+ b
+ FROM t
+ UNION ALL
+ SELECT
+ a,
+ CAST(NULL AS string)
+ FROM t) AS t2
+WHERE a = 1000
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+Excluded rules: ReduceExpressionsRule\(Project\)
+
+CBO PLAN:
+HiveUnion(all=[true])
+ HiveProject(a=[$0], b=[$1])
+ HiveFilter(condition=[=(CAST($0):DOUBLE, 1000)])
+ HiveTableScan(table=[[default, t]], table:alias=[t])
+ HiveProject(a=[$0], _o__c1=[null:VARCHAR(2147483647) CHARACTER SET
"UTF-16LE"])
+ HiveFilter(condition=[=(CAST($0):DOUBLE, 1000)])
+ HiveTableScan(table=[[default, t]], table:alias=[t])
+
