This is an automated email from the ASF dual-hosted git repository.
hyuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new f26d92f [CALCITE-3004] RexOver is incorrectly pushed down in
ProjectSetOpTransposeRule and ProjectCorrelateTransposeRule (Chunwei Lei)
f26d92f is described below
commit f26d92f956c4f8b59bafbf7cfc7431f3d4d536f9
Author: Chunwei Lei <[email protected]>
AuthorDate: Tue Apr 16 13:24:51 2019 +0800
[CALCITE-3004] RexOver is incorrectly pushed down in
ProjectSetOpTransposeRule and ProjectCorrelateTransposeRule (Chunwei Lei)
---
.../rel/rules/ProjectCorrelateTransposeRule.java | 5 +-
.../rel/rules/ProjectSetOpTransposeRule.java | 45 +++++++++++------
.../org/apache/calcite/test/RelOptRulesTest.java | 21 ++++++++
.../org/apache/calcite/test/RelOptRulesTest.xml | 59 ++++++++++++++++++++++
4 files changed, 112 insertions(+), 18 deletions(-)
diff --git
a/core/src/main/java/org/apache/calcite/rel/rules/ProjectCorrelateTransposeRule.java
b/core/src/main/java/org/apache/calcite/rel/rules/ProjectCorrelateTransposeRule.java
index adeac9e..56d572e 100644
---
a/core/src/main/java/org/apache/calcite/rel/rules/ProjectCorrelateTransposeRule.java
+++
b/core/src/main/java/org/apache/calcite/rel/rules/ProjectCorrelateTransposeRule.java
@@ -30,6 +30,7 @@ import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCorrelVariable;
import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.BitSets;
@@ -43,10 +44,10 @@ import java.util.Map;
/**
* Push Project under Correlate to apply on Correlate's left and right child
*/
-public class ProjectCorrelateTransposeRule extends RelOptRule {
+public class ProjectCorrelateTransposeRule extends RelOptRule {
public static final ProjectCorrelateTransposeRule INSTANCE =
- new ProjectCorrelateTransposeRule(expr -> true,
+ new ProjectCorrelateTransposeRule(expr -> !(expr instanceof RexOver),
RelFactories.LOGICAL_BUILDER);
//~ Instance fields --------------------------------------------------------
diff --git
a/core/src/main/java/org/apache/calcite/rel/rules/ProjectSetOpTransposeRule.java
b/core/src/main/java/org/apache/calcite/rel/rules/ProjectSetOpTransposeRule.java
index 6190556..ce5adf9 100644
---
a/core/src/main/java/org/apache/calcite/rel/rules/ProjectSetOpTransposeRule.java
+++
b/core/src/main/java/org/apache/calcite/rel/rules/ProjectSetOpTransposeRule.java
@@ -24,6 +24,7 @@ import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.core.SetOp;
import org.apache.calcite.rel.logical.LogicalProject;
import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexOver;
import org.apache.calcite.tools.RelBuilderFactory;
import java.util.ArrayList;
@@ -40,7 +41,7 @@ import java.util.List;
*/
public class ProjectSetOpTransposeRule extends RelOptRule {
public static final ProjectSetOpTransposeRule INSTANCE =
- new ProjectSetOpTransposeRule(expr -> false,
+ new ProjectSetOpTransposeRule(expr -> !(expr instanceof RexOver),
RelFactories.LOGICAL_BUILDER);
//~ Instance fields --------------------------------------------------------
@@ -90,23 +91,35 @@ public class ProjectSetOpTransposeRule extends RelOptRule {
List<RelNode> newSetOpInputs = new ArrayList<>();
int[] adjustments = pushProject.getAdjustments();
- // push the projects completely below the setop; this
- // is different from pushing below a join, where we decompose
- // to try to keep expensive expressions above the join,
- // because UNION ALL does not have any filtering effect,
- // and it is the only operator this rule currently acts on
- for (RelNode input : setOp.getInputs()) {
- // be lazy: produce two ProjectRels, and let another rule
- // merge them (could probably just clone origProj instead?)
- Project p = pushProject.createProjectRefsAndExprs(input, true, false);
- newSetOpInputs.add(pushProject.createNewProject(p, adjustments));
+ final RelNode node;
+ if (RexOver.containsOver(origProj.getProjects(), null)) {
+ // should not push over past setop but can push its operand down.
+ for (RelNode input : setOp.getInputs()) {
+ Project p = pushProject.createProjectRefsAndExprs(input, true, false);
+ // make sure that it is not a trivial project to avoid infinite loop.
+ if (p.getRowType().equals(input.getRowType())) {
+ return;
+ }
+ newSetOpInputs.add(p);
+ }
+ SetOp newSetOp =
+ setOp.copy(setOp.getTraitSet(), newSetOpInputs);
+ node = pushProject.createNewProject(newSetOp, adjustments);
+ } else {
+ // push some expressions below the setop; this
+ // is different from pushing below a join, where we decompose
+ // to try to keep expensive expressions above the join,
+ // because UNION ALL does not have any filtering effect,
+ // and it is the only operator this rule currently acts on
+ setOp.getInputs().forEach(input ->
+ newSetOpInputs.add(
+ pushProject.createNewProject(
+ pushProject.createProjectRefsAndExprs(
+ input, true, false), adjustments)));
+ node = setOp.copy(setOp.getTraitSet(), newSetOpInputs);
}
- // create a new setop whose children are the ProjectRels created above
- SetOp newSetOp =
- setOp.copy(setOp.getTraitSet(), newSetOpInputs);
-
- call.transformTo(newSetOp);
+ call.transformTo(node);
}
}
diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
index 664a2e5..2516f72 100644
--- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
@@ -1205,6 +1205,16 @@ public class RelOptRulesTest extends RelOptTestBase {
+ "on e.ename = b.ename and e.deptno = 10");
}
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-3004";>[CALCITE-3004]
+ * Should not push over past union but its operands can since setop
+ * will affect row count</a>. */
+ @Test public void testProjectSetOpTranspose() {
+ checkPlanning(ProjectSetOpTransposeRule.INSTANCE,
+ "select job, sum(sal + 100) over (partition by deptno) from\n"
+ + "(select * from emp e1 union all select * from emp e2)");
+ }
+
@Test public void testProjectCorrelateTransposeDynamic() {
ProjectCorrelateTransposeRule customPCTrans =
new ProjectCorrelateTransposeRule(skipItem,
RelFactories.LOGICAL_BUILDER);
@@ -1337,6 +1347,17 @@ public class RelOptRulesTest extends RelOptTestBase {
+ "unnest(t1.employees) as t2");
}
+ /** As {@link #testProjectSetOpTranspose()};
+ * should not push over past correlate but its operands can since correlate
+ * will affect row count. */
+ @Test public void testProjectCorrelateTransposeWithOver() {
+ checkPlanning(ProjectCorrelateTransposeRule.INSTANCE,
+ "select sum(t1.deptno + 1) over (partition by t1.name),\n"
+ + "count(t2.empno) over ()\n"
+ + "from DEPT_NESTED as t1, "
+ + "unnest(t1.employees) as t2");
+ }
+
/** Tests that the default instance of {@link FilterProjectTransposeRule}
* does not push a Filter that contains a correlating variable.
*
diff --git
a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
index 53f19e2..e90f1b0 100644
--- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
@@ -9778,6 +9778,35 @@ LogicalProject(NAME=[$0], ENAME=[$2])
]]>
</Resource>
</TestCase>
+ <TestCase name="testProjectSetOpTranspose">
+ <Resource name="sql">
+ <![CDATA[select job, sum(sal + 100) over (partition by deptno) from
+(select * from emp e1 union all select * from emp e2)
+]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(JOB=[$2], EXPR$1=[SUM(+($5, 100)) OVER (PARTITION BY $7 RANGE
BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)])
+ LogicalUnion(all=[true])
+ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4],
SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4],
SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalProject(JOB=[$0], EXPR$1=[SUM($2) OVER (PARTITION BY $1 RANGE BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)])
+ LogicalUnion(all=[true])
+ LogicalProject(JOB=[$2], DEPTNO=[$7], +=[+($5, 100)])
+ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3],
HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ LogicalProject(JOB=[$2], DEPTNO=[$7], +=[+($5, 100)])
+ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3],
HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ </TestCase>
<TestCase name="testProjectCorrelateTransposeDynamic">
<Resource name="sql">
<![CDATA[select t1.c_nationkey, t2.a as fake_col2 from
SALES.CUSTOMER as t1, unnest(t1.fake_col) as t2(a)]]>
@@ -9832,6 +9861,36 @@ LogicalProject(NAME=[$0], ENAME=[$2])
]]>
</Resource>
</TestCase>
+ <TestCase name="testProjectCorrelateTransposeWithOver">
+ <Resource name="sql">
+ <![CDATA[select sum(t1.deptno + 1) over (partition by t1.name),
+count(t2.empno) over ()
+from DEPT_NESTED as t1, unnest(t1.employees) as t2
+]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(EXPR$0=[SUM(+($0, 1)) OVER (PARTITION BY $1 RANGE BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)], EXPR$1=[COUNT($4) OVER (RANGE
BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)])
+ LogicalCorrelate(correlation=[$cor0], joinType=[inner],
requiredColumns=[{3}])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED]])
+ Uncollect
+ LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES])
+ LogicalValues(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalProject(EXPR$0=[SUM($2) OVER (PARTITION BY $0 RANGE BETWEEN UNBOUNDED
PRECEDING AND UNBOUNDED FOLLOWING)], EXPR$1=[COUNT($3) OVER (RANGE BETWEEN
UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)])
+ LogicalCorrelate(correlation=[$cor1], joinType=[inner],
requiredColumns=[{1}])
+ LogicalProject(NAME=[$1], EMPLOYEES=[$3], +=[+($0, 1)])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED]])
+ LogicalProject(EMPNO=[$0])
+ Uncollect
+ LogicalProject(EMPLOYEES=[$cor1.EMPLOYEES])
+ LogicalValues(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ </TestCase>
<TestCase name="testProjectCorrelateTransposeRuleLeftCorrelate">
<Resource name="planBefore">
<