Repository: calcite Updated Branches: refs/heads/master 04c5ff9e8 -> 576c1a1ee
[CALCITE-1182] Add ProjectRemoveRule to pre-processing program of materialization substitution Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/576c1a1e Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/576c1a1e Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/576c1a1e Branch: refs/heads/master Commit: 576c1a1eeb138726387e55680f4a1bd2aa7ed023 Parents: 04c5ff9 Author: maryannxue <[email protected]> Authored: Tue Apr 5 23:05:44 2016 -0400 Committer: maryannxue <[email protected]> Committed: Tue Apr 5 23:05:44 2016 -0400 ---------------------------------------------------------------------- .../MaterializedViewSubstitutionVisitor.java | 180 ++++++++++++++++--- .../calcite/plan/SubstitutionVisitor.java | 75 ++++---- .../calcite/plan/volcano/VolcanoPlanner.java | 1 + .../calcite/test/MaterializationTest.java | 13 ++ 4 files changed, 215 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/576c1a1e/core/src/main/java/org/apache/calcite/plan/MaterializedViewSubstitutionVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/plan/MaterializedViewSubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/MaterializedViewSubstitutionVisitor.java index 9a95c49..e1779e2 100644 --- a/core/src/main/java/org/apache/calcite/plan/MaterializedViewSubstitutionVisitor.java +++ b/core/src/main/java/org/apache/calcite/plan/MaterializedViewSubstitutionVisitor.java @@ -34,6 +34,8 @@ public class MaterializedViewSubstitutionVisitor extends SubstitutionVisitor { ImmutableList.<UnifyRule>builder() .addAll(DEFAULT_RULES) .add(ProjectToProjectUnifyRule1.INSTANCE) + .add(FilterToFilterUnifyRule1.INSTANCE) + .add(FilterToProjectUnifyRule1.INSTANCE) .build(); public MaterializedViewSubstitutionVisitor(RelNode target_, RelNode query_) { @@ -45,7 +47,20 @@ public class MaterializedViewSubstitutionVisitor extends SubstitutionVisitor { } /** - * Project to Project Unify rule. + * Implementation of {@link UnifyRule} that matches a {@link MutableProject} + * to a {@link MutableProject} where the condition of the target relation is + * weaker. + * + * <p>Example: target has a weaker condition and contains all columns selected + * by query</p> + * <ul> + * <li>query: Project(projects: [$2, $0]) + * Filter(condition: >($1, 20)) + * Scan(table: [hr, emps])</li> + * <li>target: Project(projects: [$0, $1, $2]) + * Filter(condition: >($1, 10)) + * Scan(table: [hr, emps])</li> + * </ul> */ private static class ProjectToProjectUnifyRule1 extends AbstractUnifyRule { public static final ProjectToProjectUnifyRule1 INSTANCE = @@ -109,35 +124,156 @@ public class MaterializedViewSubstitutionVisitor extends SubstitutionVisitor { } return null; } + } + + /** + * Implementation of {@link UnifyRule} that matches a {@link MutableFilter} + * to a {@link MutableFilter} where the condition of the target relation is + * weaker. + * + * <p>Example: target has a weaker condition</p> + * <ul> + * <li>query: Filter(condition: >($1, 20)) + * Scan(table: [hr, emps])</li> + * <li>target: Filter(condition: >($1, 10)) + * Scan(table: [hr, emps])</li> + * </ul> + */ + private static class FilterToFilterUnifyRule1 extends AbstractUnifyRule { + public static final FilterToFilterUnifyRule1 INSTANCE = + new FilterToFilterUnifyRule1(); + + private FilterToFilterUnifyRule1() { + super(operand(MutableFilter.class, query(0)), + operand(MutableFilter.class, target(0)), 1); + } + + public UnifyResult apply(UnifyRuleCall call) { + final MutableFilter query = (MutableFilter) call.query; + final MutableFilter target = (MutableFilter) call.target; + final MutableFilter newFilter = MutableFilter.of(target, query.getCondition()); + return call.result(newFilter); + } + + @Override protected UnifyRuleCall match(SubstitutionVisitor visitor, + MutableRel query, MutableRel target) { + if (queryOperand.matches(visitor, query)) { + if (targetOperand.matches(visitor, target)) { + if (visitor.isWeaker(query, target)) { + return visitor.new UnifyRuleCall(this, query, target, + copy(visitor.slots, slotCount)); + } + } + } + return null; + } + } + + /** + * Implementation of {@link UnifyRule} that matches a {@link MutableFilter} + * to a {@link MutableProject} on top of a {@link MutableFilter} where the + * condition of the target relation is weaker. + * + * <p>Example: target has a weaker condition and is a permutation projection + * of its child relation</p> + * <ul> + * <li>query: Filter(condition: >($1, 20)) + * Scan(table: [hr, emps])</li> + * <li>target: Project(projects: [$1, $0, $2, $3, $4]) + * Filter(condition: >($1, 10)) + * Scan(table: [hr, emps])</li> + * </ul> + */ + private static class FilterToProjectUnifyRule1 extends AbstractUnifyRule { + public static final FilterToProjectUnifyRule1 INSTANCE = + new FilterToProjectUnifyRule1(); + + private FilterToProjectUnifyRule1() { + super( + operand(MutableFilter.class, query(0)), + operand(MutableProject.class, + operand(MutableFilter.class, target(0))), 1); + } + + public UnifyResult apply(UnifyRuleCall call) { + final MutableRel query = call.query; + + final List<RelDataTypeField> oldFieldList = + query.getRowType().getFieldList(); + final List<RelDataTypeField> newFieldList = + call.target.getRowType().getFieldList(); + List<RexNode> newProjects; + try { + newProjects = transformRex( + (List<RexNode>) call.getCluster().getRexBuilder().identityProjects( + query.getRowType()), + oldFieldList, newFieldList); + } catch (MatchFailed e) { + return null; + } + + final MutableProject newProject = + MutableProject.of( + query.getRowType(), call.target, newProjects); - private RexNode transformRex(RexNode node, - final List<RelDataTypeField> oldFields, - final List<RelDataTypeField> newFields) { - List<RexNode> nodes = - transformRex(ImmutableList.of(node), oldFields, newFields); - return nodes.get(0); + final MutableRel newProject2 = MutableRels.strip(newProject); + return call.result(newProject2); } - private List<RexNode> transformRex( - List<RexNode> nodes, - final List<RelDataTypeField> oldFields, - final List<RelDataTypeField> newFields) { - RexShuttle shuttle = new RexShuttle() { - @Override public RexNode visitInputRef(RexInputRef ref) { - RelDataTypeField f = oldFields.get(ref.getIndex()); - for (int index = 0; index < newFields.size(); index++) { - RelDataTypeField newf = newFields.get(index); - if (f.getKey().equals(newf.getKey()) - && f.getValue() == newf.getValue()) { - return new RexInputRef(index, f.getValue()); + @Override protected UnifyRuleCall match(SubstitutionVisitor visitor, + MutableRel query, MutableRel target) { + assert query instanceof MutableFilter && target instanceof MutableProject; + + if (queryOperand.matches(visitor, query)) { + if (targetOperand.matches(visitor, target)) { + if (visitor.isWeaker(query, ((MutableProject) target).getInput())) { + final MutableFilter filter = (MutableFilter) query; + RexNode newCondition; + try { + newCondition = transformRex(filter.getCondition(), + filter.getInput().getRowType().getFieldList(), + target.getRowType().getFieldList()); + } catch (MatchFailed e) { + return null; } + final MutableFilter newFilter = MutableFilter.of(target, + newCondition); + return visitor.new UnifyRuleCall(this, query, newFilter, + copy(visitor.slots, slotCount)); } - throw MatchFailed.INSTANCE; } - }; - return shuttle.apply(nodes); + } + return null; } } + + private static RexNode transformRex(RexNode node, + final List<RelDataTypeField> oldFields, + final List<RelDataTypeField> newFields) { + List<RexNode> nodes = + transformRex(ImmutableList.of(node), oldFields, newFields); + return nodes.get(0); + } + + private static List<RexNode> transformRex( + List<RexNode> nodes, + final List<RelDataTypeField> oldFields, + final List<RelDataTypeField> newFields) { + RexShuttle shuttle = new RexShuttle() { + @Override public RexNode visitInputRef(RexInputRef ref) { + RelDataTypeField f = oldFields.get(ref.getIndex()); + for (int index = 0; index < newFields.size(); index++) { + RelDataTypeField newf = newFields.get(index); + if (f.getKey().equals(newf.getKey()) + && f.getValue() == newf.getValue()) { + return new RexInputRef(index, f.getValue()); + } + } + throw MatchFailed.INSTANCE; + } + }; + return shuttle.apply(nodes); + } } // End MaterializedViewSubstitutionVisitor.java http://git-wip-us.apache.org/repos/asf/calcite/blob/576c1a1e/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java index 6aa935b..6045df4 100644 --- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java +++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java @@ -79,6 +79,7 @@ import org.slf4j.Logger; import java.util.AbstractList; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -147,12 +148,12 @@ public class SubstitutionVisitor { protected static final ImmutableList<UnifyRule> DEFAULT_RULES = ImmutableList.<UnifyRule>of( -// TrivialRule.INSTANCE, + TrivialRule.INSTANCE, ScanToProjectUnifyRule.INSTANCE, ProjectToProjectUnifyRule.INSTANCE, FilterToProjectUnifyRule.INSTANCE, // ProjectToFilterUnifyRule.INSTANCE, - FilterToFilterUnifyRule.INSTANCE, +// FilterToFilterUnifyRule.INSTANCE, AggregateToAggregateUnifyRule.INSTANCE, AggregateOnProjectToAggregateUnifyRule.INSTANCE); @@ -515,7 +516,10 @@ public class SubstitutionVisitor { // Replace previous equivalents with new equivalents, higher up // the tree. for (int i = 0; i < rule.slotCount; i++) { - equivalents.removeAll(slots[i]); + Collection<MutableRel> equi = equivalents.get(slots[i]); + if (!equi.isEmpty()) { + equivalents.remove(slots[i], equi.iterator().next()); + } } assert result.result.rowType.equals(result.call.query.rowType) : Pair.of(result.result, result.call.query); @@ -1137,6 +1141,9 @@ public class SubstitutionVisitor { MutableProject project) { LOGGER.trace("SubstitutionVisitor: invert:\nmodel: {}\ninput: {}\nproject: {}\n", model, input, project); + if (project.getProjects().size() < model.getRowType().getFieldCount()) { + throw MatchFailed.INSTANCE; + } final List<RexNode> exprList = new ArrayList<>(); final RexBuilder rexBuilder = model.cluster.getRexBuilder(); for (RelDataTypeField field : model.getRowType().getFieldList()) { @@ -2209,6 +2216,38 @@ public class SubstitutionVisitor { } } + /** Returns if one rel is weaker than another. */ + protected boolean isWeaker(MutableRel rel0, MutableRel rel) { + if (rel0 == rel || equivalents.get(rel0).contains(rel)) { + return false; + } + + if (!(rel0 instanceof MutableFilter) + || !(rel instanceof MutableFilter)) { + return false; + } + + if (!rel.getRowType().equals(rel0.getRowType())) { + return false; + } + + final MutableRel rel0input = ((MutableFilter) rel0).getInput(); + final MutableRel relinput = ((MutableFilter) rel).getInput(); + if (rel0input != relinput + && !equivalents.get(rel0input).contains(relinput)) { + return false; + } + + RexExecutorImpl rexImpl = + (RexExecutorImpl) (rel.cluster.getPlanner().getExecutor()); + RexImplicationChecker rexImplicationChecker = new RexImplicationChecker( + rel.cluster.getRexBuilder(), + rexImpl, rel.getRowType()); + + return rexImplicationChecker.implies(((MutableFilter) rel0).getCondition(), + ((MutableFilter) rel).getCondition()); + } + /** Operand to a {@link UnifyRule}. */ protected abstract static class Operand { protected final Class<? extends MutableRel> clazz; @@ -2324,35 +2363,7 @@ public class SubstitutionVisitor { @Override public boolean isWeaker(SubstitutionVisitor visitor, MutableRel rel) { final MutableRel rel0 = visitor.slots[ordinal]; assert rel0 != null : "QueryOperand should have been called first"; - - if (rel0 == rel || visitor.equivalents.get(rel0).contains(rel)) { - return false; - } - - if (!(rel0 instanceof MutableFilter) - || !(rel instanceof MutableFilter)) { - return false; - } - - if (!rel.getRowType().equals(rel0.getRowType())) { - return false; - } - - final MutableRel rel0input = ((MutableFilter) rel0).getInput(); - final MutableRel relinput = ((MutableFilter) rel).getInput(); - if (rel0input != relinput - && !visitor.equivalents.get(rel0input).contains(relinput)) { - return false; - } - - RexExecutorImpl rexImpl = - (RexExecutorImpl) (rel.cluster.getPlanner().getExecutor()); - RexImplicationChecker rexImplicationChecker = new RexImplicationChecker( - rel.cluster.getRexBuilder(), - rexImpl, rel.getRowType()); - - return rexImplicationChecker.implies(((MutableFilter) rel0).getCondition(), - ((MutableFilter) rel).getCondition()); + return visitor.isWeaker(rel0, rel); } } http://git-wip-us.apache.org/repos/asf/calcite/blob/576c1a1e/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java index 8cb91b0..baa918b 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java @@ -410,6 +410,7 @@ public class VolcanoPlanner extends AbstractRelOptPlanner { new HepProgramBuilder() .addRuleInstance(FilterProjectTransposeRule.INSTANCE) .addRuleInstance(ProjectMergeRule.INSTANCE) + .addRuleInstance(ProjectRemoveRule.INSTANCE) .build(); final HepPlanner hepPlanner = new HepPlanner(program, getContext()); http://git-wip-us.apache.org/repos/asf/calcite/blob/576c1a1e/core/src/test/java/org/apache/calcite/test/MaterializationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java index 389d3ec..8b7aaf5 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java @@ -436,6 +436,19 @@ public class MaterializationTest { JdbcTest.HR_MODEL); } + /** As {@link #testFilterQueryOnFilterView7()} but columns in materialized + * view are a permutation of columns in the query*/ + @Test public void testFilterQueryOnFilterView14() { + String q = "select * from \"emps\" where (\"salary\" > 1000 " + + "or (\"deptno\" >= 30 and \"salary\" <= 500))"; + String m = "select \"deptno\", \"empid\", \"name\", \"salary\", \"commission\" " + + "from \"emps\" as em where " + + "((\"salary\" < 1111.9 and \"deptno\" > 10)" + + "or (\"empid\" > 400 and \"salary\" > 5000) " + + "or \"salary\" > 500)"; + checkMaterialize(m, q); + } + /** As {@link #testFilterQueryOnFilterView13()} but using alias * and condition of query is stronger*/ @Test public void testAlias() {
