This is an automated email from the ASF dual-hosted git repository.
yanlin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new 46fb263 [CALCITE-4735] Enhance Aggregate-Materialization, when
aggcall of query could be expressed by target's grouping (xurenhe)
46fb263 is described below
commit 46fb2632ea4457e5c0f87d42536ae2d5cdf5f07d
Author: xurenhe <[email protected]>
AuthorDate: Thu Sep 16 20:13:47 2021 +0800
[CALCITE-4735] Enhance Aggregate-Materialization, when aggcall of query
could be expressed by target's grouping (xurenhe)
---
.../apache/calcite/plan/SubstitutionVisitor.java | 124 +++++++++++++++------
.../MaterializedViewSubstitutionVisitorTest.java | 79 +++++++++++++
2 files changed, 170 insertions(+), 33 deletions(-)
diff --git
a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
index c0899fa..8044fca 100644
--- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
+++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
@@ -61,6 +61,7 @@ import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.ControlFlowException;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Litmus;
+import org.apache.calcite.util.Optionality;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.mapping.Mapping;
@@ -1906,18 +1907,39 @@ public class SubstitutionVisitor {
int i = targetGroupByIndexList.indexOf(inputIndex);
projects.add(i);
}
+ final List<AggregateCall> targetGroupGenAggCalls = new ArrayList<>();
for (AggregateCall aggregateCall : query.aggCalls) {
int i = target.aggCalls.indexOf(aggregateCall);
if (i < 0) {
- return null;
+ final AggregateCall newAggCall = genAggCallWithTargetGrouping(
+ aggregateCall, targetGroupByIndexList);
+ if (newAggCall == null) {
+ return null;
+ } else {
+ // Here, we create a new `MutableAggregate` to return.
+ // So, we record this new agg-call.
+ targetGroupGenAggCalls.add(newAggCall);
+ }
+ } else {
+ if (!targetGroupGenAggCalls.isEmpty()) {
+ // Here, we didn't build target's agg-call by ref of mv's agg-call,
+ // if some agg-call is generated by target's grouping.
+ // So, we return null to stop it.
+ return null;
+ }
+ projects.add(groupCount + i + projOffset);
}
- projects.add(groupCount + i + projOffset);
}
- List<RexNode> compenProjs = MutableRels.createProjectExprs(target,
projects);
- RexProgram compenRexProgram = RexProgram.create(
- target.rowType, compenProjs, targetCond, query.rowType, rexBuilder);
- result = MutableCalc.of(target, compenRexProgram);
+ if (targetGroupGenAggCalls.isEmpty()) {
+ List<RexNode> compenProjs = MutableRels.createProjectExprs(target,
projects);
+ RexProgram compenRexProgram = RexProgram.create(
+ target.rowType, compenProjs, targetCond, query.rowType,
rexBuilder);
+ result = MutableCalc.of(target, compenRexProgram);
+ } else {
+ result = MutableAggregate.of(target,
+ target.groupSet, target.groupSets, targetGroupGenAggCalls);
+ }
} else if (target.getGroupType() == Aggregate.Group.SIMPLE) {
// Query is coarser level of aggregation. Generate an aggregate.
final Map<Integer, Integer> map = new HashMap<>();
@@ -1935,38 +1957,34 @@ public class SubstitutionVisitor {
}
final List<AggregateCall> aggregateCalls = new ArrayList<>();
for (AggregateCall aggregateCall : query.aggCalls) {
- if (aggregateCall.isDistinct() && aggregateCall.getArgList().size() ==
1) {
- final int aggIndex = aggregateCall.getArgList().get(0);
- final int newIndex = targetGroupByIndexList.indexOf(aggIndex);
- if (newIndex >= 0) {
- aggregateCalls.add(
- AggregateCall.create(aggregateCall.getAggregation(),
- aggregateCall.isDistinct(), aggregateCall.isApproximate(),
- aggregateCall.ignoreNulls(),
- ImmutableList.of(newIndex), -1, aggregateCall.distinctKeys,
- aggregateCall.collation, aggregateCall.type,
- aggregateCall.name));
- continue;
+ AggregateCall newAggCall = null;
+ // 1. try to find rollup agg-call.
+ if (!aggregateCall.isDistinct()) {
+ int i = target.aggCalls.indexOf(aggregateCall);
+ if (i >= 0) {
+ // When an SqlAggFunction does not support roll up, it will return
null,
+ // which means that it cannot do secondary aggregation
+ // and the materialization recognition will fail.
+ final SqlAggFunction aggFunction =
aggregateCall.getAggregation().getRollup();
+ if (aggFunction != null) {
+ newAggCall = AggregateCall.create(aggFunction,
+ aggregateCall.isDistinct(), aggregateCall.isApproximate(),
+ aggregateCall.ignoreNulls(),
+ ImmutableList.of(target.groupSet.cardinality() + i), -1,
+ aggregateCall.distinctKeys, aggregateCall.collation,
+ aggregateCall.type, aggregateCall.name);
+ }
}
- return null;
}
- int i = target.aggCalls.indexOf(aggregateCall);
- if (i < 0) {
- return null;
+ // 2. try to build a new agg-cal by target's grouping.
+ if (newAggCall == null) {
+ newAggCall = genAggCallWithTargetGrouping(aggregateCall,
targetGroupByIndexList);
}
- // When an SqlAggFunction does not support roll up, it will return
null, which means that
- // it cannot do secondary aggregation and the materialization
recognition will fail.
- final SqlAggFunction aggFunction =
aggregateCall.getAggregation().getRollup();
- if (aggFunction == null) {
+ if (newAggCall == null) {
+ // gen agg call fail.
return null;
}
- aggregateCalls.add(
- AggregateCall.create(aggFunction,
- aggregateCall.isDistinct(), aggregateCall.isApproximate(),
- aggregateCall.ignoreNulls(),
- ImmutableList.of(target.groupSet.cardinality() + i), -1,
- aggregateCall.distinctKeys, aggregateCall.collation,
- aggregateCall.type, aggregateCall.name));
+ aggregateCalls.add(newAggCall);
}
if (targetCond != null && !targetCond.isAlwaysTrue()) {
RexProgram compenRexProgram = RexProgram.create(
@@ -1986,6 +2004,46 @@ public class SubstitutionVisitor {
return result;
}
+ /**
+ * Generate agg call by mv's grouping.
+ */
+ private static @Nullable AggregateCall
genAggCallWithTargetGrouping(AggregateCall queryAggCall,
+ List<Integer> targetGroupByIndexes) {
+ final SqlAggFunction aggregation = queryAggCall.getAggregation();
+ final List<Integer> argList = queryAggCall.getArgList();
+ final List<Integer> newArgList = new ArrayList<>();
+ for (Integer arg : argList) {
+ final int newArgIndex = targetGroupByIndexes.indexOf(arg);
+ if (newArgIndex < 0) {
+ return null;
+ }
+ newArgList.add(newArgIndex);
+ }
+ final boolean isAllowBuild;
+ if (newArgList.size() == 0) {
+ // Size of agg-call's args is empty, we stop to build a new agg-call,
+ // eg: count(1) or count(*).
+ isAllowBuild = false;
+ } else if (queryAggCall.isDistinct()) {
+ // Args of agg-call is distinct, we can build a new agg-call.
+ isAllowBuild = true;
+ } else if (aggregation.getDistinctOptionality() == Optionality.IGNORED) {
+ // If attribute of agg-call's distinct could be ignore, we can build a
new agg-call.
+ isAllowBuild = true;
+ } else {
+ isAllowBuild = false;
+ }
+ if (!isAllowBuild) {
+ return null;
+ }
+ return AggregateCall.create(aggregation,
+ queryAggCall.isDistinct(), queryAggCall.isApproximate(),
+ queryAggCall.ignoreNulls(),
+ newArgList, -1, queryAggCall.distinctKeys,
+ queryAggCall.collation, queryAggCall.type,
+ queryAggCall.name);
+ }
+
@Deprecated // to be removed before 2.0
public static @Nullable SqlAggFunction getRollup(SqlAggFunction aggregation)
{
if (aggregation == SqlStdOperatorTable.SUM
diff --git
a/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java
b/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java
index dede0a7..91a85eb 100644
---
a/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java
+++
b/core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java
@@ -1135,6 +1135,85 @@ public class MaterializedViewSubstitutionVisitorTest
extends AbstractMaterialize
sql(mv, query).ok();
}
+ /**
+ * It's match, distinct agg-call could be expressed by mv's grouping.
+ */
+ @Test void testAggDistinctInMvGrouping() {
+ final String mv = ""
+ + "select \"deptno\", \"name\""
+ + "from \"emps\" group by \"deptno\", \"name\"";
+ final String query = ""
+ + "select \"deptno\", \"name\", count(distinct \"name\")"
+ + "from \"emps\" group by \"deptno\", \"name\"";
+ sql(mv, query).ok();
+ }
+
+ /**
+ * It's match, `Optionality.IGNORED` agg-call could be expressed by mv's
grouping.
+ */
+ @Test void testAggOptionalityInMvGrouping() {
+ final String mv = ""
+ + "select \"deptno\", \"salary\""
+ + "from \"emps\" group by \"deptno\", \"salary\"";
+ final String query = ""
+ + "select \"deptno\", \"salary\", max(\"salary\")"
+ + "from \"emps\" group by \"deptno\", \"salary\"";
+ sql(mv, query).ok();
+ }
+
+ /**
+ * It's not match, normal agg-call could be expressed by mv's grouping.
+ * Such as: sum, count
+ */
+ @Test void testAggNormalInMvGrouping() {
+ final String mv = ""
+ + "select \"deptno\", \"salary\""
+ + "from \"emps\" group by \"deptno\", \"salary\"";
+ final String query = ""
+ + "select \"deptno\", sum(\"salary\")"
+ + "from \"emps\" group by \"deptno\"";
+ sql(mv, query).noMat();
+ }
+
+ /**
+ * It's not match, which is count(*) with same grouping.
+ */
+ @Test void testGenerateQueryAggCallByMvGroupingForEmptyArg1() {
+ final String mv = ""
+ + "select \"deptno\""
+ + "from \"emps\" group by \"deptno\"";
+ final String query = ""
+ + "select \"deptno\", count(*)"
+ + "from \"emps\" group by \"deptno\"";
+ sql(mv, query).noMat();
+ }
+
+ /**
+ * It's not match, which is count(*) with rollup grouping.
+ */
+ @Test void testGenerateQueryAggCallByMvGroupingForEmptyArg2() {
+ final String mv = ""
+ + "select \"deptno\", \"commission\", \"salary\""
+ + "from \"emps\" group by \"deptno\", \"commission\", \"salary\"";
+ final String query = ""
+ + "select \"deptno\", \"commission\", count(*)"
+ + "from \"emps\" group by \"deptno\", \"commission\"";
+ sql(mv, query).noMat();
+ }
+
+ /**
+ * It's match, when query's agg-calls could be both rollup and expressed by
mv's grouping.
+ */
+ @Test void testAggCallBothGenByMvGroupingAndRollupOk() {
+ final String mv = ""
+ + "select \"name\", \"deptno\", \"empid\", min(\"commission\")"
+ + "from \"emps\" group by \"name\", \"deptno\", \"empid\"";
+ final String query = ""
+ + "select \"name\", max(\"deptno\"), count(distinct \"empid\"),
min(\"commission\")"
+ + "from \"emps\" group by \"name\"";
+ sql(mv, query).ok();
+ }
+
/** Unit test for logic functions
* {@link org.apache.calcite.plan.SubstitutionVisitor#mayBeSatisfiable} and
* {@link RexUtil#simplify}. */
