This is an automated email from the ASF dual-hosted git repository.
chunwei pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new 300f5200fe [CALCITE-5195] ArrayIndexOutOfBoundsException when
inferring more equal conditions from join condition for semi join
300f5200fe is described below
commit 300f5200fe50870da418366613df1cd09921b1b6
Author: chunwei.lcw <[email protected]>
AuthorDate: Wed Jun 22 11:41:13 2022 +0800
[CALCITE-5195] ArrayIndexOutOfBoundsException when inferring more equal
conditions from join condition for semi join
---
.../apache/calcite/rel/rules/FilterJoinRule.java | 53 ++++++-------
.../org/apache/calcite/test/RelOptRulesTest.java | 82 +++++++++++++++++++++
.../org/apache/calcite/test/RelOptRulesTest.xml | 86 ++++++++++++++++++++++
3 files changed, 196 insertions(+), 25 deletions(-)
diff --git
a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
index 0df08263a2..e699a9e1ee 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
@@ -138,7 +138,9 @@ public abstract class FilterJoinRule<C extends
FilterJoinRule.Config>
}
}
- joinFilters = inferJoinEqualConditions(joinFilters, join);
+ if (joinType != JoinRelType.FULL) {
+ joinFilters = inferJoinEqualConditions(joinFilters, join);
+ }
// Try to push down filters in ON clause. A ON clause filter can only be
// pushed down if it does not affect the non-matching set, i.e. it is
@@ -249,10 +251,10 @@ public abstract class FilterJoinRule<C extends
FilterJoinRule.Config>
*/
protected List<RexNode> inferJoinEqualConditions(List<RexNode> rexNodes,
Join join) {
final List<RexNode> result = new ArrayList<>(rexNodes.size());
- final List<Set<Integer>> equalSets = splitEqualSets(rexNodes, result);
+ final List<Set<RexInputRef>> equalSets = splitEqualSets(rexNodes, result);
boolean needOptimize = false;
- for (Set<Integer> set : equalSets) {
+ for (Set<RexInputRef> set : equalSets) {
if (set.size() > 2) {
needOptimize = true;
break;
@@ -274,8 +276,8 @@ public abstract class FilterJoinRule<C extends
FilterJoinRule.Config>
* @param leftNodes where the conditions not feasible for equal sets are put
* @return the equal sets
*/
- private List<Set<Integer>> splitEqualSets(List<RexNode> rexNodes,
List<RexNode> leftNodes) {
- final List<Set<Integer>> equalSets = new ArrayList<>();
+ private List<Set<RexInputRef>> splitEqualSets(List<RexNode> rexNodes,
List<RexNode> leftNodes) {
+ final List<Set<RexInputRef>> equalSets = new ArrayList<>();
for (RexNode rexNode : rexNodes) {
if (rexNode.isA(SqlKind.EQUALS)) {
final RexNode op1 = ((RexCall) rexNode).getOperands().get(0);
@@ -283,9 +285,9 @@ public abstract class FilterJoinRule<C extends
FilterJoinRule.Config>
if (op1 instanceof RexInputRef && op2 instanceof RexInputRef) {
final RexInputRef in1 = (RexInputRef) op1;
final RexInputRef in2 = (RexInputRef) op2;
- Set<Integer> set = null;
- for (Set<Integer> s : equalSets) {
- if (s.contains(in1.getIndex()) || s.contains(in2.getIndex())) {
+ Set<RexInputRef> set = null;
+ for (Set<RexInputRef> s : equalSets) {
+ if (s.contains(in1) || s.contains(in2)) {
set = s;
break;
}
@@ -295,8 +297,8 @@ public abstract class FilterJoinRule<C extends
FilterJoinRule.Config>
set = new LinkedHashSet<>();
equalSets.add(set);
}
- set.add(in1.getIndex());
- set.add(in2.getIndex());
+ set.add(in1);
+ set.add(in2);
} else {
leftNodes.add(rexNode);
}
@@ -315,18 +317,19 @@ public abstract class FilterJoinRule<C extends
FilterJoinRule.Config>
* @param equalSets the equal sets
* @return the newly constructed conditions from equal sets
*/
- private List<RexNode> constructConditionFromEqualSets(Join join,
List<Set<Integer>> equalSets) {
+ private List<RexNode> constructConditionFromEqualSets(Join join,
+ List<Set<RexInputRef>> equalSets) {
final RexBuilder rexBuilder = join.getCluster().getRexBuilder();
final List<RexNode> result = new ArrayList<>();
- final int leftRowCount = join.getLeft().getRowType().getFieldCount();
- for (Set<Integer> set : equalSets) {
- final List<Integer> leftSet = new ArrayList<>();
- final List<Integer> rightSet = new ArrayList<>();
- for (int i : set) {
- if (i < leftRowCount) {
- leftSet.add(i);
+ final int leftFieldCount = join.getLeft().getRowType().getFieldCount();
+ for (Set<RexInputRef> set : equalSets) {
+ final List<RexInputRef> leftSet = new ArrayList<>();
+ final List<RexInputRef> rightSet = new ArrayList<>();
+ for (RexInputRef ref : set) {
+ if (ref.getIndex() < leftFieldCount) {
+ leftSet.add(ref);
} else {
- rightSet.add(i);
+ rightSet.add(ref);
}
}
// Add left side conditions.
@@ -334,8 +337,8 @@ public abstract class FilterJoinRule<C extends
FilterJoinRule.Config>
for (int i = 1; i < leftSet.size(); ++i) {
result.add(
rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
- rexBuilder.makeInputRef(join, leftSet.get(0)),
- rexBuilder.makeInputRef(join, leftSet.get(i))));
+ leftSet.get(0),
+ leftSet.get(i)));
}
}
// Add right side conditions.
@@ -343,16 +346,16 @@ public abstract class FilterJoinRule<C extends
FilterJoinRule.Config>
for (int i = 1; i < rightSet.size(); ++i) {
result.add(
rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
- rexBuilder.makeInputRef(join, rightSet.get(0)),
- rexBuilder.makeInputRef(join, rightSet.get(i))));
+ rightSet.get(0),
+ rightSet.get(i)));
}
}
// Only need one equal condition for each equal set.
if (leftSet.size() > 0 && rightSet.size() > 0) {
result.add(
rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
- rexBuilder.makeInputRef(join, leftSet.get(0)),
- rexBuilder.makeInputRef(join, rightSet.get(0))));
+ leftSet.get(0),
+ rightSet.get(0)));
}
}
diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
index 127662f4a7..b305f8c930 100644
--- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
@@ -2216,6 +2216,88 @@ class RelOptRulesTest extends RelOptTestBase {
.check();
}
+ /** Similar to {@link #testJoinConditionPushdown1()} but full join
+ * from which more equality conditions can not be inferred. */
+ @Test void testJoinConditionPushdown3() {
+ final String sql = "select *\n"
+ + "from emp e full join dept d\n"
+ + "on e.deptno = d.deptno and e.empno = d.deptno";
+ sql(sql)
+ .withRule(CoreRules.FILTER_INTO_JOIN,
+ CoreRules.JOIN_CONDITION_PUSH,
+ CoreRules.PROJECT_MERGE,
+ CoreRules.FILTER_PROJECT_TRANSPOSE)
+ .checkUnchanged();
+ }
+
+ /** Similar to {@link #testJoinConditionPushdown1()} but semi join
+ * from which more equality conditions can be inferred. */
+ @Test void testJoinConditionPushdown4() {
+ final Function<RelBuilder, RelNode> relFn = b -> {
+ RelNode left = b.scan("EMP")
+ .project(
+ b.field("DEPTNO"),
+ b.field("ENAME"))
+ .build();
+ RelNode right = b.scan("DEPT")
+ .project(
+ b.field("DEPTNO"),
+ b.field("DNAME"))
+ .build();
+
+ b.push(left).push(right);
+
+ RexInputRef ref1 = b.field(2, 0, "DEPTNO");
+ RexInputRef ref2 = b.field(2, 1, "DEPTNO");
+ RexInputRef ref3 = b.field(2, 1, "DNAME");
+
+ RexCall cond1 = (RexCall) b.equals(ref1, ref2);
+ RexCall cond2 = (RexCall) b.equals(ref1, ref3);
+
+ RexNode cond = b.and(cond1, cond2);
+ return b.semiJoin(cond)
+ .project(b.field(0))
+ .build();
+ };
+
+ relFn(relFn)
+ .withRule(
+ CoreRules.JOIN_PUSH_EXPRESSIONS,
+ CoreRules.JOIN_CONDITION_PUSH,
+ CoreRules.SEMI_JOIN_PROJECT_TRANSPOSE,
+ CoreRules.JOIN_REDUCE_EXPRESSIONS,
+ CoreRules.FILTER_REDUCE_EXPRESSIONS)
+ .check();
+ }
+
+ /** Similar to {@link #testJoinConditionPushdown1()} but left join
+ * from which more equality conditions can not be inferred. */
+ @Test void testJoinConditionPushdown5() {
+ final String sql = "select *\n"
+ + "from emp e left join dept d\n"
+ + "on e.deptno = d.deptno and e.empno = d.deptno";
+ sql(sql)
+ .withRule(CoreRules.FILTER_INTO_JOIN,
+ CoreRules.JOIN_CONDITION_PUSH,
+ CoreRules.PROJECT_MERGE,
+ CoreRules.FILTER_PROJECT_TRANSPOSE)
+ .checkUnchanged();
+ }
+
+ /** Similar to {@link #testJoinConditionPushdown1()} but right join
+ * from which more equality conditions can be inferred. */
+ @Test void testJoinConditionPushdown6() {
+ final String sql = "select *\n"
+ + "from emp e right join dept d\n"
+ + "on e.deptno = d.deptno and e.empno = d.deptno";
+ sql(sql)
+ .withRule(CoreRules.FILTER_INTO_JOIN,
+ CoreRules.JOIN_CONDITION_PUSH,
+ CoreRules.PROJECT_MERGE,
+ CoreRules.FILTER_PROJECT_TRANSPOSE)
+ .check();
+ }
+
/** Tests that filters are combined if they are identical. */
@Test void testMergeFilter() {
final String sql = "select name from (\n"
diff --git
a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
index 2e72d43700..87e5489f57 100644
--- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
@@ -4537,6 +4537,92 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2],
MGR=[$3], HIREDATE=[$4], SAL=[$
LogicalFilter(condition=[=($7, $0)])
LogicalTableScan(table=[[CATALOG, SALES, EMP]])
LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testJoinConditionPushdown3">
+ <Resource name="sql">
+ <![CDATA[select *
+from emp e full join dept d
+on e.deptno = d.deptno and e.empno = d.deptno
+]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4],
SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], DEPTNO0=[$9], NAME=[$10])
+ LogicalJoin(condition=[AND(=($7, $9), =($0, $9))], joinType=[full])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testJoinConditionPushdown4">
+ <Resource name="sql">
+ <![CDATA[select *
+from emp e semi join dept d
+on e.deptno = d.deptno and e.empno = d.deptno
+]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(DEPTNO=[$0])
+ LogicalJoin(condition=[AND(=($0, $2), =($0, $3))], joinType=[semi])
+ LogicalProject(DEPTNO=[$7], ENAME=[$1])
+ LogicalTableScan(table=[[scott, EMP]])
+ LogicalProject(DEPTNO=[$0], DNAME=[$1])
+ LogicalTableScan(table=[[scott, DEPT]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalProject(DEPTNO=[$0])
+ LogicalProject(DEPTNO=[$7], ENAME=[$1])
+ LogicalJoin(condition=[=($7, $8)], joinType=[semi])
+ LogicalTableScan(table=[[scott, EMP]])
+ LogicalFilter(condition=[=($0, $1)])
+ LogicalProject(DEPTNO=[$0], DNAME=[$1])
+ LogicalTableScan(table=[[scott, DEPT]])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testJoinConditionPushdown5">
+ <Resource name="sql">
+ <![CDATA[select *
+from emp e left join dept d
+on e.deptno = d.deptno and e.empno = d.deptno
+]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4],
SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], DEPTNO0=[$9], NAME=[$10])
+ LogicalJoin(condition=[AND(=($7, $9), =($0, $9))], joinType=[left])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testJoinConditionPushdown6">
+ <Resource name="sql">
+ <![CDATA[select *
+from emp e right join dept d
+on e.deptno = d.deptno and e.empno = d.deptno
+]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4],
SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], DEPTNO0=[$9], NAME=[$10])
+ LogicalJoin(condition=[AND(=($7, $9), =($0, $9))], joinType=[right])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4],
SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], DEPTNO0=[$9], NAME=[$10])
+ LogicalJoin(condition=[=($7, $9)], joinType=[right])
+ LogicalFilter(condition=[=($7, $0)])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
]]>
</Resource>
</TestCase>
