[CALCITE-890] Register all combinations of materialization substitutions (Maryann Xue)
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/9d0fef31 Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/9d0fef31 Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/9d0fef31 Branch: refs/heads/branch-release Commit: 9d0fef31725e73912b37fff79037a3258bca7fa6 Parents: 2c339be Author: Julian Hyde <[email protected]> Authored: Fri Oct 23 13:08:42 2015 -0700 Committer: Julian Hyde <[email protected]> Committed: Fri Oct 23 14:36:42 2015 -0700 ---------------------------------------------------------------------- core/pom.xml | 5 ++ .../calcite/plan/volcano/VolcanoPlanner.java | 61 +++++++++++++++----- .../calcite/test/MaterializationTest.java | 56 ++++++++++++++++++ 3 files changed, 107 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/9d0fef31/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 65adb3e..6a62d68 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -56,6 +56,11 @@ limitations under the License. <artifactId>commons-dbcp</artifactId> </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/calcite/blob/9d0fef31/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java index e57a760..4cc513a 100644 --- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java +++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java @@ -360,7 +360,8 @@ public class VolcanoPlanner extends AbstractRelOptPlanner { registerImpl(rel, root.set); } - private void useMaterialization(RelOptMaterialization materialization) { + private RelNode useMaterialization(RelNode root, + RelOptMaterialization materialization, boolean firstRun) { // Try to rewrite the original root query in terms of the materialized // query. If that is possible, register the remnant query as equivalent // to the root. @@ -368,22 +369,23 @@ public class VolcanoPlanner extends AbstractRelOptPlanner { // This call modifies originalRoot. Doesn't look like originalRoot should be mutable though. // Need to check. - RelNode sub = substitute(originalRoot, materialization); + RelNode sub = substitute(root, materialization); if (sub != null) { - // TODO: try to substitute other materializations in the remnant. - // Useful for big queries, e.g. - // (t1 group by c1) join (t2 group by c2). Hook.SUB.run(sub); - registerImpl(sub, root.set); - return; + registerImpl(sub, this.root.set); + return sub; + } + + if (firstRun) { + RelSubset subset = registerImpl(materialization.queryRel, null); + RelNode tableRel2 = + RelOptUtil.createCastRel( + materialization.tableRel, + materialization.queryRel.getRowType(), + true); + registerImpl(tableRel2, subset.set); } - RelSubset subset = registerImpl(materialization.queryRel, null); - RelNode tableRel2 = - RelOptUtil.createCastRel( - materialization.tableRel, - materialization.queryRel.getRowType(), - true); - registerImpl(tableRel2, subset.set); + return null; } private RelNode substitute( @@ -417,6 +419,33 @@ public class VolcanoPlanner extends AbstractRelOptPlanner { .go(materialization.tableRel); } + // Register all possible combinations of materialization substitution. + // Useful for big queries, e.g. + // (t1 group by c1) join (t2 group by c2). + private void useMaterializations(RelNode root, + List<RelOptMaterialization> materializations, boolean firstRun) { + for (RelOptMaterialization m : materializations) { + RelNode sub = useMaterialization(root, m, firstRun); + if (sub != null) { + useMaterializations(sub, materializations, false); + } else { + // Based on the assumption that a substitution itself won't trigger another + // substitution, if a materialization is not matched here it won't be useful + // in any subsequent matching for the current level of recursion or this level + // down. So we can safely remove the unmatched materialization from the remnant + // list for the current root. + List<RelOptMaterialization> newList = + Lists.newArrayListWithExpectedSize(materializations.size() - 1); + for (RelOptMaterialization elem : materializations) { + if (elem != m) { + newList.add(elem); + } + } + materializations = newList; + } + } + } + private void useApplicableMaterializations() { // Avoid using materializations while populating materializations! final CalciteConnectionConfig config = @@ -454,6 +483,7 @@ public class VolcanoPlanner extends AbstractRelOptPlanner { final Graphs.FrozenGraph<List<String>, DefaultEdge> frozenGraph = Graphs.makeImmutable(usesGraph); final Set<RelOptTable> queryTables = findTables(originalRoot); + final List<RelOptMaterialization> applicableMaterializations = Lists.newArrayList(); for (RelOptMaterialization materialization : materializations) { if (materialization.starTable != null) { // Materialization is a tile in a lattice. We will deal with it shortly. @@ -461,10 +491,11 @@ public class VolcanoPlanner extends AbstractRelOptPlanner { } if (materialization.table != null) { if (usesTable(materialization.table, queryTables, frozenGraph)) { - useMaterialization(materialization); + applicableMaterializations.add(materialization); } } } + useMaterializations(originalRoot, applicableMaterializations, true); // Use a lattice if the query uses at least the central (fact) table of the // lattice. http://git-wip-us.apache.org/repos/asf/calcite/blob/9d0fef31/core/src/test/java/org/apache/calcite/test/MaterializationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java index 61d49dd..07d5a44 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java @@ -29,6 +29,9 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.util.JsonBuilder; +import org.apache.calcite.util.Util; + +import org.apache.commons.lang3.StringUtils; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; @@ -38,6 +41,7 @@ import org.junit.Test; import java.math.BigDecimal; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -815,6 +819,58 @@ public class MaterializationTest { Prepare.THREAD_TRIM.set(false); } } + + @Test public void testSingleMaterializationMultiUsage() { + String q = "select *\n" + + "from (select * from \"emps\" where \"empid\" < 300)\n" + + "join (select * from \"emps\" where \"empid\" < 200) using (\"empid\")"; + try { + Prepare.THREAD_TRIM.set(true); + MaterializationService.setThreadLocal(); + CalciteAssert.that() + .withMaterializations(JdbcTest.HR_MODEL, + "m0", "select * from \"emps\" where \"empid\" < 500") + .query(q) + .enableMaterializations(true) + .explainMatches("", new Function<ResultSet, Void>() { + public Void apply(ResultSet s) { + try { + final String actual = Util.toLinux(CalciteAssert.toString(s)); + final String scan = "EnumerableTableScan(table=[[hr, m0]])"; + assertTrue(actual + " should have had two occurrences of " + scan, + StringUtils.countMatches(actual, scan) == 2); + return null; + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + }) + .sameResultWithMaterializationsDisabled(); + } finally { + Prepare.THREAD_TRIM.set(false); + } + } + + @Test public void testMultiMaterializationMultiUsage() { + String q = "select *\n" + + "from (select * from \"emps\" where \"empid\" < 300)\n" + + "join (select \"deptno\", count(*) as c from \"emps\" group by \"deptno\") using (\"deptno\")"; + try { + Prepare.THREAD_TRIM.set(true); + MaterializationService.setThreadLocal(); + CalciteAssert.that() + .withMaterializations(JdbcTest.HR_MODEL, + "m0", "select \"deptno\", count(*) as c, sum(\"empid\") as s from \"emps\" group by \"deptno\"", + "m1", "select * from \"emps\" where \"empid\" < 500") + .query(q) + .enableMaterializations(true) + .explainContains("EnumerableTableScan(table=[[hr, m0]])") + .explainContains("EnumerableTableScan(table=[[hr, m1]])") + .sameResultWithMaterializationsDisabled(); + } finally { + Prepare.THREAD_TRIM.set(false); + } + } } // End MaterializationTest.java
