This is an automated email from the ASF dual-hosted git repository. jhyde pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 3ab722b5cccf3357f5c6fddf796ccf612d978cc2 Author: Julian Hyde <[email protected]> AuthorDate: Mon Jun 17 09:44:49 2019 -0700 [CALCITE-3131] In LatticeSuggester, record whether columns are used as "dimensions" or "measures" --- .../org/apache/calcite/materialize/Lattice.java | 40 +++++++++++++- .../calcite/materialize/LatticeSuggester.java | 53 ++++++++++-------- .../calcite/materialize/LatticeSuggesterTest.java | 62 +++++++++++++++++++++- 3 files changed, 130 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/materialize/Lattice.java b/core/src/main/java/org/apache/calcite/materialize/Lattice.java index 005aa90..f729bf7 100644 --- a/core/src/main/java/org/apache/calcite/materialize/Lattice.java +++ b/core/src/main/java/org/apache/calcite/materialize/Lattice.java @@ -61,7 +61,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; import com.google.common.collect.Ordering; import java.util.ArrayList; @@ -98,13 +100,15 @@ public class Lattice { public final double rowCountEstimate; public final ImmutableList<Measure> defaultMeasures; public final ImmutableList<Tile> tiles; + public final ImmutableListMultimap<Integer, Boolean> columnUses; public final LatticeStatisticProvider statisticProvider; private Lattice(CalciteSchema rootSchema, LatticeRootNode rootNode, boolean auto, boolean algorithm, long algorithmMaxMillis, LatticeStatisticProvider.Factory statisticProviderFactory, @Nullable Double rowCountEstimate, ImmutableList<Column> columns, - ImmutableSortedSet<Measure> defaultMeasures, ImmutableList<Tile> tiles) { + ImmutableSortedSet<Measure> defaultMeasures, ImmutableList<Tile> tiles, + ImmutableListMultimap<Integer, Boolean> columnUses) { this.rootSchema = rootSchema; this.rootNode = Objects.requireNonNull(rootNode); this.columns = Objects.requireNonNull(columns); @@ -113,6 +117,7 @@ public class Lattice { this.algorithmMaxMillis = algorithmMaxMillis; this.defaultMeasures = defaultMeasures.asList(); // unique and sorted this.tiles = Objects.requireNonNull(tiles); + this.columnUses = columnUses; assert isValid(Litmus.THROW); @@ -478,6 +483,24 @@ public class Lattice { return -1; } + /** Returns whether every use of a column is as an argument to a measure. + * + * <p>For example, in the query + * {@code select sum(x + y), sum(a + b) from t group by x + y} + * the expression "x + y" is used once as an argument to a measure, + * and once as a dimension. + * + * <p>Therefore, in a lattice created from that one query, + * {@code isAlwaysMeasure} for the derived column corresponding to "x + y" + * returns false, and for "a + b" returns true. + * + * @param column Column or derived column + * @return Whether all uses are as arguments to aggregate functions + */ + public boolean isAlwaysMeasure(Column column) { + return !columnUses.get(column.ordinal).contains(false); + } + /** Edge in the temporary graph. */ private static class Edge extends DefaultEdge { public static final DirectedGraph.EdgeFactory<Vertex, Edge> FACTORY = @@ -750,6 +773,8 @@ public class Lattice { new TreeSet<>(); private final ImmutableList.Builder<Tile> tileListBuilder = ImmutableList.builder(); + private final Multimap<Integer, Boolean> columnUses = + LinkedHashMultimap.create(); private final CalciteSchema rootSchema; private boolean algorithm = false; private long algorithmMaxMillis = -1; @@ -903,7 +928,7 @@ public class Lattice { return new Lattice(rootSchema, rootNode, auto, algorithm, algorithmMaxMillis, statisticProvider, rowCountEstimate, columnBuilder.build(), ImmutableSortedSet.copyOf(defaultMeasureSet), - tileListBuilder.build()); + tileListBuilder.build(), ImmutableListMultimap.copyOf(columnUses)); } /** Resolves the arguments of a @@ -1047,6 +1072,17 @@ public class Lattice { }); } + /** Records a use of a column. + * + * @param column Column + * @param measure Whether this use is as an argument to a measure; + * e.g. "sum(x + y)" is a measure use of the expression + * "x + y"; "group by x + y" is not + */ + public void use(Column column, boolean measure) { + columnUses.put(column.ordinal, measure); + } + /** Work space for fixing up a tree of mutable nodes. */ private static class Fixer { final Set<String> aliases = new HashSet<>(); diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java index c9c3658..f532eb3 100644 --- a/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java +++ b/core/src/main/java/org/apache/calcite/materialize/LatticeSuggester.java @@ -63,6 +63,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -215,20 +216,23 @@ public class LatticeSuggester { new Lattice.Measure(measure.aggregate, measure.distinct, measure.name, Lists.transform(measure.arguments, colRef -> { + final Lattice.Column column; if (colRef instanceof BaseColRef) { final BaseColRef baseColRef = (BaseColRef) colRef; final MutableNode node = nodes.get(baseColRef.t); final int table = flatNodes.indexOf(node); - return latticeBuilder.column(table, baseColRef.c); + column = latticeBuilder.column(table, baseColRef.c); } else if (colRef instanceof DerivedColRef) { final DerivedColRef derivedColRef = (DerivedColRef) colRef; final String alias = deriveAlias(measure, derivedColRef); - return latticeBuilder.expression(derivedColRef.e, alias, + column = latticeBuilder.expression(derivedColRef.e, alias, derivedColRef.tableAliases()); } else { throw new AssertionError("expression in measure"); } + latticeBuilder.use(column, true); + return column; }))); } @@ -239,6 +243,7 @@ public class LatticeSuggester { final Lattice.Column expression = latticeBuilder.expression(derivedColRef.e, derivedColRef.alias, derivedColRef.tableAliases()); + latticeBuilder.use(expression, false); } } @@ -323,12 +328,8 @@ public class LatticeSuggester { final CalciteSchema rootSchema = CalciteSchema.createRootSchema(false); final Lattice.Builder builder = new Lattice.Builder(space, rootSchema, mutableNode); - for (Lattice.Measure measure : bestMatch.defaultMeasures) { - builder.addMeasure(measure.copy(mapper(bestMatch, builder))); - } - for (Lattice.Measure measure : lattice.defaultMeasures) { - builder.addMeasure(measure.copy(mapper(lattice, builder))); - } + copyMeasures(builder, bestMatch); + copyMeasures(builder, lattice); final Lattice lattice2 = builder.build(); latticeMap.remove(bestMatch.toString()); obsoleteLatticeMap.put(bestMatch, lattice2); @@ -342,19 +343,29 @@ public class LatticeSuggester { return lattice; } - private java.util.function.Function<Lattice.Column, Lattice.Column> mapper( - final Lattice lattice, final Lattice.Builder builder) { - return (Lattice.Column c) -> { - if (c instanceof Lattice.BaseColumn) { - Lattice.BaseColumn baseColumn = (Lattice.BaseColumn) c; - Pair<Path, Integer> p = lattice.columnToPathOffset(baseColumn); - return builder.pathOffsetToColumn(p.left, p.right); - } else { - final Lattice.DerivedColumn derivedColumn = (Lattice.DerivedColumn) c; - return builder.expression(derivedColumn.e, derivedColumn.alias, - derivedColumn.tables); - } - }; + /** Copies measures and column usages from an existing lattice into a builder, + * using a mapper to translate old-to-new columns, so that the new lattice can + * inherit from the old. */ + private void copyMeasures(Lattice.Builder builder, Lattice lattice) { + final Function<Lattice.Column, Lattice.Column> mapper = + (Lattice.Column c) -> { + if (c instanceof Lattice.BaseColumn) { + Lattice.BaseColumn baseColumn = (Lattice.BaseColumn) c; + Pair<Path, Integer> p = lattice.columnToPathOffset(baseColumn); + return builder.pathOffsetToColumn(p.left, p.right); + } else { + final Lattice.DerivedColumn derivedColumn = (Lattice.DerivedColumn) c; + return builder.expression(derivedColumn.e, derivedColumn.alias, + derivedColumn.tables); + } + }; + for (Lattice.Measure measure : lattice.defaultMeasures) { + builder.addMeasure(measure.copy(mapper)); + } + for (Map.Entry<Integer, Boolean> entry : lattice.columnUses.entries()) { + final Lattice.Column column = lattice.columns.get(entry.getKey()); + builder.use(mapper.apply(column), entry.getValue()); + } } private int matchQuality(Lattice lattice, Lattice target) { diff --git a/core/src/test/java/org/apache/calcite/materialize/LatticeSuggesterTest.java b/core/src/test/java/org/apache/calcite/materialize/LatticeSuggesterTest.java index 339cfad..22e4ab8 100644 --- a/core/src/test/java/org/apache/calcite/materialize/LatticeSuggesterTest.java +++ b/core/src/test/java/org/apache/calcite/materialize/LatticeSuggesterTest.java @@ -499,8 +499,66 @@ public class LatticeSuggesterTest { .collect(Collectors.toList()); assertThat(derivedColumns.size(), is(2)); final List<String> tables = ImmutableList.of("customer"); - assertThat(derivedColumns.get(0).tables, is(tables)); - assertThat(derivedColumns.get(1).tables, is(tables)); + checkDerivedColumn(lattice, tables, derivedColumns, 0, "$f2", true); + checkDerivedColumn(lattice, tables, derivedColumns, 1, "full_name", false); + } + + /** As {@link #testExpression()} but with multiple queries. + * Some expressions are measures in one query and dimensions in another. */ + @Test public void testExpressionEvolution() throws Exception { + final Tester t = new Tester().foodmart().withEvolve(true); + + // q0 uses n10 as a measure, n11 as a measure, n12 as a dimension + final String q0 = "select\n" + + " \"num_children_at_home\" + 12 as \"n12\",\n" + + " sum(\"num_children_at_home\" + 10) as \"n10\",\n" + + " sum(\"num_children_at_home\" + 11) as \"n11\",\n" + + " count(*) as c\n" + + "from \"customer\"\n" + + "group by \"num_children_at_home\" + 12"; + // q1 uses n10 as a dimension, n12 as a measure + final String q1 = "select\n" + + " \"num_children_at_home\" + 10 as \"n10\",\n" + + " \"num_children_at_home\" + 14 as \"n14\",\n" + + " sum(\"num_children_at_home\" + 12) as \"n12\",\n" + + " sum(\"num_children_at_home\" + 13) as \"n13\"\n" + + "from \"customer\"\n" + + "group by \"num_children_at_home\" + 10," + + " \"num_children_at_home\" + 14"; + // n10 = [measure, dimension] -> not always measure + // n11 = [measure, _] -> always measure + // n12 = [dimension, measure] -> not always measure + // n13 = [_, measure] -> always measure + // n14 = [_, dimension] -> not always measure + t.addQuery(q0); + t.addQuery(q1); + assertThat(t.s.latticeMap.size(), is(1)); + final String l0 = + "customer:[COUNT(), SUM(n10), SUM(n11), SUM(n12), SUM(n13)]"; + assertThat(Iterables.getOnlyElement(t.s.latticeMap.keySet()), + is(l0)); + final Lattice lattice = Iterables.getOnlyElement(t.s.latticeMap.values()); + final List<Lattice.DerivedColumn> derivedColumns = lattice.columns.stream() + .filter(c -> c instanceof Lattice.DerivedColumn) + .map(c -> (Lattice.DerivedColumn) c) + .collect(Collectors.toList()); + assertThat(derivedColumns.size(), is(5)); + final List<String> tables = ImmutableList.of("customer"); + + checkDerivedColumn(lattice, tables, derivedColumns, 0, "n10", false); + checkDerivedColumn(lattice, tables, derivedColumns, 1, "n11", true); + checkDerivedColumn(lattice, tables, derivedColumns, 2, "n12", false); + checkDerivedColumn(lattice, tables, derivedColumns, 3, "n13", true); + checkDerivedColumn(lattice, tables, derivedColumns, 4, "n14", false); + } + + private void checkDerivedColumn(Lattice lattice, List<String> tables, + List<Lattice.DerivedColumn> derivedColumns, + int index, String name, boolean alwaysMeasure) { + final Lattice.DerivedColumn dc0 = derivedColumns.get(index); + assertThat(dc0.tables, is(tables)); + assertThat(dc0.alias, is(name)); + assertThat(lattice.isAlwaysMeasure(dc0), is(alwaysMeasure)); } @Test public void testExpressionInJoin() throws Exception {
