Repository: calcite Updated Branches: refs/heads/master f22a9965a -> 83cef58b2
[CALCITE-1802] Add post-aggregation step for Union in materialized view rewriting Fix for expressions on top of view, DISTINCT and additional tests Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/83cef58b Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/83cef58b Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/83cef58b Branch: refs/heads/master Commit: 83cef58b2b51e3e428fa50e532bf6ce6fa8f201c Parents: f22a996 Author: Jesus Camacho Rodriguez <[email protected]> Authored: Wed May 31 19:00:11 2017 +0100 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Wed May 31 19:00:26 2017 +0100 ---------------------------------------------------------------------- .../rel/rules/AbstractMaterializedViewRule.java | 71 +++++++++++--------- .../calcite/test/MaterializationTest.java | 47 +++++++++++++ 2 files changed, 88 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/83cef58b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java index d9e7aa6..9cc8d1d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java @@ -121,16 +121,16 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { /** Whether to generate rewritings containing union if the query results * are contained within the view results. */ - private final boolean generateUnionRewrites; + private final boolean generateUnionRewriting; //~ Constructors ----------------------------------------------------------- /** Creates a AbstractMaterializedViewRule. */ protected AbstractMaterializedViewRule(RelOptRuleOperand operand, RelBuilderFactory relBuilderFactory, String description, - boolean generateUnionRewrites) { + boolean generateUnionRewriting) { super(operand, relBuilderFactory, description); - this.generateUnionRewrites = generateUnionRewrites; + this.generateUnionRewriting = generateUnionRewriting; } /** @@ -382,7 +382,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { computeCompensationPredicates(rexBuilder, simplify, currQEC, queryPreds, queryBasedVEC, viewPreds, queryToViewTableMapping); - if (compensationPreds == null && generateUnionRewrites) { + if (compensationPreds == null && generateUnionRewriting) { // Attempt partial rewriting using union operator. This rewriting // will read some data from the view and the rest of the data from // the query computation. The resulting predicates are expressed @@ -417,8 +417,8 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { // We trigger the unifying method. This method will either create a Project // or an Aggregate operator on top of the view. It will also compute the // output expressions for the query. - final RelNode unionInputView = unify(call.builder(), rexBuilder, mq, matchModality, - view, topProject, node, topViewProject, viewNode, + final RelNode unionInputView = rewriteView(call.builder(), rexBuilder, mq, + matchModality, true, view, topProject, node, topViewProject, viewNode, queryToViewTableMapping, currQEC); if (unionInputView == null) { // Skip it @@ -428,6 +428,10 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { // d. Generate final rewriting (union). final RelNode result = createUnion(call.builder(), rexBuilder, topProject, unionInputQuery, unionInputView); + if (result == null) { + // Skip it + continue; + } call.transformTo(result); } else if (compensationPreds != null) { RexNode compensationColumnsEquiPred = compensationPreds.getLeft(); @@ -485,8 +489,9 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { if (!viewCompensationPred.isAlwaysTrue()) { builder.filter(simplify.simplify(viewCompensationPred)); } - RelNode result = unify(builder, rexBuilder, mq, matchModality, builder.build(), - topProject, node, topViewProject, viewNode, queryToViewTableMapping, currQEC); + final RelNode result = rewriteView(builder, rexBuilder, mq, matchModality, false, + builder.build(), topProject, node, topViewProject, viewNode, + queryToViewTableMapping, currQEC); if (result == null) { // Skip it continue; @@ -543,8 +548,8 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { * on top. If a rewriting can be produced, we return that rewriting. If it cannot * be produced, we will return null. */ - protected abstract RelNode unify(RelBuilder relBuilder, RexBuilder rexBuilder, - RelMetadataQuery mq, MatchModality matchModality, RelNode input, + protected abstract RelNode rewriteView(RelBuilder relBuilder, RexBuilder rexBuilder, + RelMetadataQuery mq, MatchModality matchModality, boolean unionRewriting, RelNode input, Project topProject, RelNode node, Project topViewProject, RelNode viewNode, BiMap<RelTableRef, RelTableRef> queryToViewTableMapping, @@ -558,8 +563,8 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { /** Creates a MaterializedViewJoinRule. */ protected MaterializedViewJoinRule(RelOptRuleOperand operand, RelBuilderFactory relBuilderFactory, String description, - boolean generateUnionRewrites) { - super(operand, relBuilderFactory, description, generateUnionRewrites); + boolean generateUnionRewriting) { + super(operand, relBuilderFactory, description, generateUnionRewriting); } @Override protected boolean isValidPlan(Project topProject, RelNode node, @@ -714,11 +719,12 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { return relBuilder.build(); } - @Override protected RelNode unify( + @Override protected RelNode rewriteView( RelBuilder relBuilder, RexBuilder rexBuilder, RelMetadataQuery mq, MatchModality matchModality, + boolean unionRewriting, RelNode input, Project topProject, RelNode node, @@ -757,13 +763,13 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { /** Rule that matches Project on Join. */ public static class MaterializedViewProjectJoinRule extends MaterializedViewJoinRule { public MaterializedViewProjectJoinRule(RelBuilderFactory relBuilderFactory, - boolean generateUnionRewrites) { + boolean generateUnionRewriting) { super( operand(Project.class, operand(Join.class, any())), relBuilderFactory, "MaterializedViewJoinRule(Project-Join)", - generateUnionRewrites); + generateUnionRewriting); } @Override public void onMatch(RelOptRuleCall call) { @@ -776,13 +782,13 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { /** Rule that matches Project on Filter. */ public static class MaterializedViewProjectFilterRule extends MaterializedViewJoinRule { public MaterializedViewProjectFilterRule(RelBuilderFactory relBuilderFactory, - boolean generateUnionRewrites) { + boolean generateUnionRewriting) { super( operand(Project.class, operand(Filter.class, any())), relBuilderFactory, "MaterializedViewJoinRule(Project-Filter)", - generateUnionRewrites); + generateUnionRewriting); } @Override public void onMatch(RelOptRuleCall call) { @@ -795,12 +801,12 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { /** Rule that matches Join. */ public static class MaterializedViewOnlyJoinRule extends MaterializedViewJoinRule { public MaterializedViewOnlyJoinRule(RelBuilderFactory relBuilderFactory, - boolean generateUnionRewrites) { + boolean generateUnionRewriting) { super( operand(Join.class, any()), relBuilderFactory, "MaterializedViewJoinRule(Join)", - generateUnionRewrites); + generateUnionRewriting); } @Override public void onMatch(RelOptRuleCall call) { @@ -812,12 +818,12 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { /** Rule that matches Filter. */ public static class MaterializedViewOnlyFilterRule extends MaterializedViewJoinRule { public MaterializedViewOnlyFilterRule(RelBuilderFactory relBuilderFactory, - boolean generateUnionRewrites) { + boolean generateUnionRewriting) { super( operand(Filter.class, any()), relBuilderFactory, "MaterializedViewJoinRule(Filter)", - generateUnionRewrites); + generateUnionRewriting); } @Override public void onMatch(RelOptRuleCall call) { @@ -834,8 +840,8 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { /** Creates a MaterializedViewAggregateRule. */ protected MaterializedViewAggregateRule(RelOptRuleOperand operand, RelBuilderFactory relBuilderFactory, String description, - boolean generateUnionRewrites) { - super(operand, relBuilderFactory, description, generateUnionRewrites); + boolean generateUnionRewriting) { + super(operand, relBuilderFactory, description, generateUnionRewriting); } @Override protected boolean isValidPlan(Project topProject, RelNode node, @@ -1032,6 +1038,10 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { final List<AggCall> aggregateCalls = new ArrayList<>(); for (int i = 0; i < aggregate.getAggCallList().size(); i++) { AggregateCall aggCall = aggregate.getAggCallList().get(i); + if (aggCall.isDistinct()) { + // Cannot ROLLUP distinct + return null; + } aggregateCalls.add( relBuilder.aggregateCall( SubstitutionVisitor.getRollup(aggCall.getAggregation()), @@ -1053,11 +1063,12 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { return result; } - @Override protected RelNode unify( + @Override protected RelNode rewriteView( RelBuilder relBuilder, RexBuilder rexBuilder, RelMetadataQuery mq, MatchModality matchModality, + boolean unionRewriting, RelNode input, Project topProject, RelNode node, @@ -1070,7 +1081,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { // Get group by references and aggregate call input references needed ImmutableBitSet.Builder indexes = ImmutableBitSet.builder(); ImmutableBitSet references = null; - if (topProject != null) { + if (topProject != null && !unionRewriting) { // We have a Project on top, gather only what is needed final RelOptUtil.InputFinder inputFinder = new RelOptUtil.InputFinder(new LinkedHashSet<RelDataTypeField>()); @@ -1289,7 +1300,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { // we use the mapping to resolve the position of the expression in the // node. final List<RexNode> topExprs = new ArrayList<>(); - if (topProject != null) { + if (topProject != null && !unionRewriting) { topExprs.addAll(topProject.getChildExps()); } else { // Add all @@ -1340,13 +1351,13 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { /** Rule that matches Project on Aggregate. */ public static class MaterializedViewProjectAggregateRule extends MaterializedViewAggregateRule { public MaterializedViewProjectAggregateRule(RelBuilderFactory relBuilderFactory, - boolean generateUnionRewrites) { + boolean generateUnionRewriting) { super( operand(Project.class, operand(Aggregate.class, any())), relBuilderFactory, "MaterializedViewAggregateRule(Project-Aggregate)", - generateUnionRewrites); + generateUnionRewriting); } @Override public void onMatch(RelOptRuleCall call) { @@ -1359,12 +1370,12 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { /** Rule that matches Aggregate. */ public static class MaterializedViewOnlyAggregateRule extends MaterializedViewAggregateRule { public MaterializedViewOnlyAggregateRule(RelBuilderFactory relBuilderFactory, - boolean generateUnionRewrites) { + boolean generateUnionRewriting) { super( operand(Aggregate.class, any()), relBuilderFactory, "MaterializedViewAggregateRule(Aggregate)", - generateUnionRewrites); + generateUnionRewriting); } @Override public void onMatch(RelOptRuleCall call) { http://git-wip-us.apache.org/repos/asf/calcite/blob/83cef58b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java index 115f0f3..685f290 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java @@ -1528,6 +1528,53 @@ public class MaterializationTest { HR_FKUK_MODEL); } + @Test public void testJoinAggregateMaterializationAggregateFuncs11() { + checkMaterialize( + "select \"depts\".\"deptno\", \"dependents\".\"empid\", count(\"emps\".\"salary\") as s\n" + + "from \"depts\"\n" + + "join \"dependents\" on (\"depts\".\"name\" = \"dependents\".\"name\")\n" + + "join \"locations\" on (\"locations\".\"name\" = \"dependents\".\"name\")\n" + + "join \"emps\" on (\"emps\".\"deptno\" = \"depts\".\"deptno\")\n" + + "where \"depts\".\"deptno\" > 11 and \"depts\".\"deptno\" < 19\n" + + "group by \"depts\".\"deptno\", \"dependents\".\"empid\"", + "select \"dependents\".\"empid\", count(\"emps\".\"salary\") + 1\n" + + "from \"depts\"\n" + + "join \"dependents\" on (\"depts\".\"name\" = \"dependents\".\"name\")\n" + + "join \"locations\" on (\"locations\".\"name\" = \"dependents\".\"name\")\n" + + "join \"emps\" on (\"emps\".\"deptno\" = \"depts\".\"deptno\")\n" + + "where \"depts\".\"deptno\" > 10 and \"depts\".\"deptno\" < 20\n" + + "group by \"dependents\".\"empid\"", + HR_FKUK_MODEL, + CalciteAssert.checkResultContains( + "PLAN=EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[+($t1, $t2)], " + + "empid=[$t0], EXPR$1=[$t3])\n" + + " EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)])", + "EnumerableUnion(all=[true])", + "EnumerableAggregate(group=[{2}], agg#0=[COUNT()])", + "EnumerableAggregate(group=[{1}], agg#0=[$SUM0($2)])", + "EnumerableTableScan(table=[[hr, m0]])", + "expr#13=[OR($t10, $t12)], expr#14=[AND($t6, $t8, $t13)]")); + } + + @Test public void testJoinAggregateMaterializationAggregateFuncs12() { + checkNoMaterialize( + "select \"depts\".\"deptno\", \"dependents\".\"empid\", count(distinct \"emps\".\"salary\") as s\n" + + "from \"depts\"\n" + + "join \"dependents\" on (\"depts\".\"name\" = \"dependents\".\"name\")\n" + + "join \"locations\" on (\"locations\".\"name\" = \"dependents\".\"name\")\n" + + "join \"emps\" on (\"emps\".\"deptno\" = \"depts\".\"deptno\")\n" + + "where \"depts\".\"deptno\" > 11 and \"depts\".\"deptno\" < 19\n" + + "group by \"depts\".\"deptno\", \"dependents\".\"empid\"", + "select \"dependents\".\"empid\", count(distinct \"emps\".\"salary\") + 1\n" + + "from \"depts\"\n" + + "join \"dependents\" on (\"depts\".\"name\" = \"dependents\".\"name\")\n" + + "join \"locations\" on (\"locations\".\"name\" = \"dependents\".\"name\")\n" + + "join \"emps\" on (\"emps\".\"deptno\" = \"depts\".\"deptno\")\n" + + "where \"depts\".\"deptno\" > 10 and \"depts\".\"deptno\" < 20\n" + + "group by \"dependents\".\"empid\"", + HR_FKUK_MODEL); + } + @Test public void testJoinMaterialization4() { checkMaterialize( "select \"empid\" \"deptno\" from \"emps\"\n"
