Repository: calcite Updated Branches: refs/heads/master e07e8ad9f -> 77a354917
[CALCITE-2035] Allow approximate aggregate functions, and add APPROX_COUNT_DISTINCT There are changes to RelBuilder and AggregateCall APIs, but extensions to SQL grammar will not happen until [CALCITE-1588]. In Druid adapter, HyperLogLog will be used if the call is to APPROX_COUNT_DISTINCT or if approximateDistinctCount=true in connection properties. Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/fe3529d9 Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/fe3529d9 Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/fe3529d9 Branch: refs/heads/master Commit: fe3529d9a6d4d826c6e77fc4c0d7d18bd681bb85 Parents: e07e8ad Author: Julian Hyde <[email protected]> Authored: Mon Nov 6 16:03:13 2017 -0800 Committer: Julian Hyde <[email protected]> Committed: Wed Nov 8 08:41:32 2017 -0800 ---------------------------------------------------------------------- .../org/apache/calcite/plan/RelOptUtil.java | 6 +- .../calcite/plan/SubstitutionVisitor.java | 2 +- .../apache/calcite/rel/core/AggregateCall.java | 60 +++++++++++++++----- .../java/org/apache/calcite/rel/core/Match.java | 4 +- .../org/apache/calcite/rel/core/Window.java | 5 +- .../calcite/rel/externalize/RelJsonReader.java | 2 +- .../rel/rules/AbstractMaterializedViewRule.java | 22 +++---- .../AggregateExpandDistinctAggregatesRule.java | 25 ++++---- .../rel/rules/AggregateFilterTransposeRule.java | 4 +- .../rel/rules/AggregateReduceFunctionsRule.java | 26 +++++---- .../rel/rules/AggregateStarTableRule.java | 6 +- .../rel/rules/AggregateUnionTransposeRule.java | 1 + .../calcite/rel/rules/SubQueryRemoveRule.java | 8 +-- .../calcite/sql/SqlSplittableAggFunction.java | 8 +-- .../calcite/sql/fun/SqlCountAggFunction.java | 13 +---- .../calcite/sql/fun/SqlStdOperatorTable.java | 8 ++- .../apache/calcite/sql2rel/RelFieldTrimmer.java | 3 +- .../calcite/sql2rel/SqlToRelConverter.java | 15 +++-- .../org/apache/calcite/tools/PigRelBuilder.java | 3 +- .../org/apache/calcite/tools/RelBuilder.java | 46 ++++++++++----- .../org/apache/calcite/plan/RelWriterTest.java | 6 +- .../plan/volcano/TraitPropagationTest.java | 2 +- .../calcite/sql/test/SqlOperatorBaseTest.java | 45 +++++++++++++++ .../org/apache/calcite/test/RelBuilderTest.java | 34 +++++------ .../apache/calcite/test/RelMetadataTest.java | 4 +- .../calcite/adapter/druid/DruidQuery.java | 2 +- .../calcite/adapter/druid/DruidRules.java | 6 +- .../calcite/adapter/druid/DruidTable.java | 12 ++-- .../org/apache/calcite/test/DruidAdapterIT.java | 35 +++++++----- site/_docs/algebra.md | 2 +- site/_docs/reference.md | 6 +- 31 files changed, 275 insertions(+), 146 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 625fc73..5021569 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -500,6 +500,7 @@ public abstract class RelOptUtil { final AggregateCall aggCall = AggregateCall.create(SqlStdOperatorTable.MIN, false, + false, ImmutableList.of(0), -1, 0, @@ -578,6 +579,7 @@ public abstract class RelOptUtil { final AggregateCall aggCall = AggregateCall.create(SqlStdOperatorTable.MIN, false, + false, ImmutableList.of(projectedKeyCount), -1, projectedKeyCount, @@ -781,8 +783,8 @@ public abstract class RelOptUtil { for (int i = 0; i < aggCallCnt; i++) { aggCalls.add( AggregateCall.create( - SqlStdOperatorTable.SINGLE_VALUE, false, ImmutableList.of(i), -1, - 0, rel, null, null)); + SqlStdOperatorTable.SINGLE_VALUE, false, false, + ImmutableList.of(i), -1, 0, rel, null, null)); } return LogicalAggregate.create(rel, ImmutableBitSet.of(), null, aggCalls); http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java index ffc036a..7129ccb 100644 --- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java +++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java @@ -1264,7 +1264,7 @@ public class SubstitutionVisitor { } aggregateCalls.add( AggregateCall.create(getRollup(aggregateCall.getAggregation()), - aggregateCall.isDistinct(), + aggregateCall.isDistinct(), aggregateCall.isApproximate(), ImmutableList.of(target.groupSet.cardinality() + i), -1, aggregateCall.type, aggregateCall.name)); } http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java b/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java index 501aba6..f7194f4 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java +++ b/core/src/main/java/org/apache/calcite/rel/core/AggregateCall.java @@ -40,6 +40,7 @@ public class AggregateCall { private final SqlAggFunction aggFunction; private final boolean distinct; + private final boolean approximate; public final RelDataType type; public final String name; @@ -66,7 +67,7 @@ public class AggregateCall { List<Integer> argList, RelDataType type, String name) { - this(aggFunction, distinct, argList, -1, type, name); + this(aggFunction, distinct, false, argList, -1, type, name); } /** @@ -74,6 +75,7 @@ public class AggregateCall { * * @param aggFunction Aggregate function * @param distinct Whether distinct + * @param approximate Whether approximate * @param argList List of ordinals of arguments * @param filterArg Ordinal of filter argument, or -1 * @param type Result type @@ -82,6 +84,7 @@ public class AggregateCall { private AggregateCall( SqlAggFunction aggFunction, boolean distinct, + boolean approximate, List<Integer> argList, int filterArg, RelDataType type, @@ -92,23 +95,32 @@ public class AggregateCall { this.argList = ImmutableList.copyOf(argList); this.filterArg = filterArg; this.distinct = distinct; + this.approximate = approximate; } //~ Methods ---------------------------------------------------------------- - /** Creates an AggregateCall, inferring its type if {@code type} is null. */ @Deprecated // to be removed before 2.0 public static AggregateCall create(SqlAggFunction aggFunction, boolean distinct, List<Integer> argList, int groupCount, RelNode input, RelDataType type, String name) { - return create(aggFunction, distinct, argList, -1, groupCount, input, type, - name); + return create(aggFunction, distinct, false, argList, -1, groupCount, input, + type, name); } - /** Creates an AggregateCall, inferring its type if {@code type} is null. */ + @Deprecated // to be removed before 2.0 public static AggregateCall create(SqlAggFunction aggFunction, boolean distinct, List<Integer> argList, int filterArg, int groupCount, RelNode input, RelDataType type, String name) { + return create(aggFunction, distinct, false, argList, -1, groupCount, input, + type, name); + } + + /** Creates an AggregateCall, inferring its type if {@code type} is null. */ + public static AggregateCall create(SqlAggFunction aggFunction, + boolean distinct, boolean approximate, List<Integer> argList, + int filterArg, int groupCount, + RelNode input, RelDataType type, String name) { if (type == null) { final RelDataTypeFactory typeFactory = input.getCluster().getTypeFactory(); @@ -119,15 +131,23 @@ public class AggregateCall { groupCount, filterArg >= 0); type = aggFunction.inferReturnType(callBinding); } - return create(aggFunction, distinct, argList, filterArg, type, name); + return create(aggFunction, distinct, approximate, argList, filterArg, type, + name); } - /** Creates an AggregateCall. */ + @Deprecated // to be removed before 2.0 public static AggregateCall create(SqlAggFunction aggFunction, boolean distinct, List<Integer> argList, int filterArg, RelDataType type, String name) { - return new AggregateCall(aggFunction, distinct, argList, filterArg, type, - name); + return create(aggFunction, distinct, false, argList, filterArg, type, name); + } + + /** Creates an AggregateCall. */ + public static AggregateCall create(SqlAggFunction aggFunction, + boolean distinct, boolean approximate, List<Integer> argList, + int filterArg, RelDataType type, String name) { + return new AggregateCall(aggFunction, distinct, approximate, argList, + filterArg, type, name); } /** @@ -141,6 +161,16 @@ public class AggregateCall { } /** + * Returns whether this AggregateCall is approximate, as in <code> + * APPROX_COUNT_DISTINCT(empno)</code>. + * + * @return whether approximate + */ + public final boolean isApproximate() { + return approximate; + } + + /** * Returns the aggregate function. * * @return aggregate function @@ -187,8 +217,8 @@ public class AggregateCall { if (Objects.equals(this.name, name)) { return this; } - return new AggregateCall(aggFunction, distinct, argList, filterArg, type, - name); + return new AggregateCall(aggFunction, distinct, approximate, argList, + filterArg, type, name); } public String toString() { @@ -257,8 +287,8 @@ public class AggregateCall { * @return AggregateCall that suits new inputs and GROUP BY columns */ public AggregateCall copy(List<Integer> args, int filterArg) { - return new AggregateCall(aggFunction, distinct, args, filterArg, type, - name); + return new AggregateCall(aggFunction, distinct, approximate, args, + filterArg, type, name); } @Deprecated // to be removed before 2.0 @@ -287,8 +317,8 @@ public class AggregateCall { && filterArg == this.filterArg ? type : null; - return create(aggFunction, distinct, argList, filterArg, newGroupKeyCount, - input, newType, getName()); + return create(aggFunction, distinct, approximate, argList, filterArg, + newGroupKeyCount, input, newType, getName()); } /** Creates a copy of this aggregate call, applying a mapping to its http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/core/Match.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/core/Match.java b/core/src/main/java/org/apache/calcite/rel/core/Match.java index 2278954..2ebafec 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Match.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Match.java @@ -28,8 +28,8 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexPatternFieldRef; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.sql.SqlAggFunction; -import org.apache.calcite.sql.fun.SqlCountAggFunction; import org.apache.calcite.sql.fun.SqlMinMaxAggFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.fun.SqlSumAggFunction; import org.apache.calcite.sql.fun.SqlSumEmptyIsZeroAggFunction; @@ -248,7 +248,7 @@ public abstract class Match extends SingleRel { aggFunction = new SqlMinMaxAggFunction(call.getKind()); break; case COUNT: - aggFunction = new SqlCountAggFunction(); + aggFunction = SqlStdOperatorTable.COUNT; break; default: for (RexNode rex : call.getOperands()) { http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/core/Window.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/core/Window.java b/core/src/main/java/org/apache/calcite/rel/core/Window.java index b9d92ac..1e00bba 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Window.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Window.java @@ -313,8 +313,9 @@ public abstract class Window extends SingleRel { public AggregateCall get(int index) { final RexWinAggCall aggCall = aggCalls.get(index); - return AggregateCall.create((SqlAggFunction) aggCall.getOperator(), - aggCall.distinct, getProjectOrdinals(aggCall.getOperands()), -1, + final SqlAggFunction op = (SqlAggFunction) aggCall.getOperator(); + return AggregateCall.create(op, aggCall.distinct, + false, getProjectOrdinals(aggCall.getOperands()), -1, aggCall.getType(), fieldNames.get(aggCall.ordinal)); } }; http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java index a3d6dd4..695ab0b 100644 --- a/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java +++ b/core/src/main/java/org/apache/calcite/rel/externalize/RelJsonReader.java @@ -279,7 +279,7 @@ public class RelJsonReader { final Integer filterOperand = (Integer) jsonAggCall.get("filter"); final RelDataType type = relJson.toType(cluster.getTypeFactory(), jsonAggCall.get("type")); - return AggregateCall.create(aggregation, distinct, operands, + return AggregateCall.create(aggregation, distinct, false, operands, filterOperand == null ? -1 : filterOperand, type, null); } http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java index d563f8b..e2ea81d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java @@ -1048,12 +1048,10 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { aggregateCalls.add( relBuilder.aggregateCall( SubstitutionVisitor.getRollup(aggCall.getAggregation()), - aggCall.isDistinct(), - null, + aggCall.isDistinct(), aggCall.isApproximate(), null, aggCall.name, - ImmutableList.of( - rexBuilder.makeInputRef( - relBuilder.peek(), aggregate.getGroupCount() + i)))); + rexBuilder.makeInputRef(relBuilder.peek(), + aggregate.getGroupCount() + i))); } RelNode result = relBuilder .aggregate(relBuilder.groupKey(groupSet, null), aggregateCalls) @@ -1251,10 +1249,9 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { aggregateCalls.add( relBuilder.aggregateCall( SubstitutionVisitor.getRollup(queryAggCall.getAggregation()), - queryAggCall.isDistinct(), - null, - queryAggCall.name, - ImmutableList.of(rexBuilder.makeInputRef(input, k)))); + queryAggCall.isDistinct(), queryAggCall.isApproximate(), + null, queryAggCall.name, + rexBuilder.makeInputRef(input, k))); rewritingMapping.set(k, sourceIdx); added = true; break; @@ -1268,10 +1265,9 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule { aggregateCalls.add( relBuilder.aggregateCall( SubstitutionVisitor.getRollup(queryAggCall.getAggregation()), - queryAggCall.isDistinct(), - null, - queryAggCall.name, - ImmutableList.of(rexBuilder.makeInputRef(input, targetIdx)))); + queryAggCall.isDistinct(), queryAggCall.isApproximate(), + null, queryAggCall.name, + rexBuilder.makeInputRef(input, targetIdx))); rewritingMapping.set(targetIdx, sourceIdx); } } http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java index ca95bcc..53068d0 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java @@ -293,7 +293,7 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule { if (!aggCall.isDistinct()) { final AggregateCall newCall = AggregateCall.create(aggCall.getAggregation(), false, - aggCall.getArgList(), -1, + aggCall.isApproximate(), aggCall.getArgList(), -1, ImmutableBitSet.of(bottomGroupSet).cardinality(), relBuilder.peek(), null, aggCall.name); bottomAggregateCalls.add(newCall); @@ -318,9 +318,9 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule { newArgList.add(bottomGroupSet.headSet(arg).size()); } newCall = - AggregateCall.create( - aggCall.getAggregation(), + AggregateCall.create(aggCall.getAggregation(), false, + aggCall.isApproximate(), newArgList, -1, originalGroupSet.cardinality(), @@ -334,12 +334,14 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule { Lists.newArrayList(bottomGroupSet.size() + nonDistinctAggCallProcessedSoFar); if (aggCall.getAggregation().getKind() == SqlKind.COUNT) { newCall = - AggregateCall.create(new SqlSumEmptyIsZeroAggFunction(), false, newArgs, -1, + AggregateCall.create(new SqlSumEmptyIsZeroAggFunction(), false, + aggCall.isApproximate(), newArgs, -1, originalGroupSet.cardinality(), relBuilder.peek(), aggCall.getType(), aggCall.getName()); } else { newCall = - AggregateCall.create(aggCall.getAggregation(), false, newArgs, -1, + AggregateCall.create(aggCall.getAggregation(), false, + aggCall.isApproximate(), newArgs, -1, originalGroupSet.cardinality(), relBuilder.peek(), aggCall.getType(), aggCall.name); } @@ -400,7 +402,7 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule { final Map<ImmutableBitSet, Integer> filters = new LinkedHashMap<>(); final int z = groupCount + distinctAggCalls.size(); distinctAggCalls.add( - AggregateCall.create(SqlStdOperatorTable.GROUPING, false, + AggregateCall.create(SqlStdOperatorTable.GROUPING, false, false, ImmutableIntList.copyOf(fullGroupSet), -1, groupSets.size(), relBuilder.peek(), null, "$g")); for (Ord<ImmutableBitSet> groupSet : Ord.zip(groupSets)) { @@ -446,8 +448,9 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule { .union(aggregate.getGroupSet())); } final AggregateCall newCall = - AggregateCall.create(aggregation, false, newArgList, newFilterArg, - aggregate.getGroupCount(), distinct, null, aggCall.name); + AggregateCall.create(aggregation, false, aggCall.isApproximate(), + newArgList, newFilterArg, aggregate.getGroupCount(), distinct, + null, aggCall.name); newCalls.add(newCall); } @@ -655,7 +658,8 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule { final int newFilterArg = aggCall.filterArg >= 0 ? sourceOf.get(aggCall.filterArg) : -1; final AggregateCall newAggCall = - AggregateCall.create(aggCall.getAggregation(), false, newArgs, + AggregateCall.create(aggCall.getAggregation(), false, + aggCall.isApproximate(), newArgs, newFilterArg, aggCall.getType(), aggCall.getName()); assert refs.get(i) == null; if (n == 0) { @@ -743,7 +747,8 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule { newArgs.add(sourceOf.get(arg)); } final AggregateCall newAggCall = - AggregateCall.create(aggCall.getAggregation(), false, newArgs, -1, + AggregateCall.create(aggCall.getAggregation(), false, + aggCall.isApproximate(), newArgs, -1, aggCall.getType(), aggCall.getName()); newAggCalls.set(i, newAggCall); } http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java index dfe28f5..abe17bc 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java @@ -147,8 +147,8 @@ public class AggregateFilterTransposeRule extends RelOptRule { } topAggCallList.add( AggregateCall.create(rollup, aggregateCall.isDistinct(), - ImmutableList.of(i++), -1, aggregateCall.type, - aggregateCall.name)); + aggregateCall.isApproximate(), ImmutableList.of(i++), -1, + aggregateCall.type, aggregateCall.name)); } final Aggregate topAggregate = aggregate.copy(aggregate.getTraitSet(), newFilter, http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java index a9bdb84..746e1f4 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java @@ -267,6 +267,7 @@ public class AggregateReduceFunctionsRule extends RelOptRule { oldCall.filterArg >= 0); return AggregateCall.create(aggFunction, oldCall.isDistinct(), + oldCall.isApproximate(), ImmutableIntList.of(argOrdinal), oldCall.filterArg, aggFunction.inferReturnType(binding), @@ -287,9 +288,9 @@ public class AggregateReduceFunctionsRule extends RelOptRule { oldAggRel.getInput(), iAvgInput); final AggregateCall sumCall = - AggregateCall.create( - SqlStdOperatorTable.SUM, + AggregateCall.create(SqlStdOperatorTable.SUM, oldCall.isDistinct(), + oldCall.isApproximate(), oldCall.getArgList(), oldCall.filterArg, oldAggRel.getGroupCount(), @@ -297,9 +298,9 @@ public class AggregateReduceFunctionsRule extends RelOptRule { null, null); final AggregateCall countCall = - AggregateCall.create( - SqlStdOperatorTable.COUNT, + AggregateCall.create(SqlStdOperatorTable.COUNT, oldCall.isDistinct(), + oldCall.isApproximate(), oldCall.getArgList(), oldCall.filterArg, oldAggRel.getGroupCount(), @@ -347,12 +348,13 @@ public class AggregateReduceFunctionsRule extends RelOptRule { arg); final AggregateCall sumZeroCall = AggregateCall.create(SqlStdOperatorTable.SUM0, oldCall.isDistinct(), - oldCall.getArgList(), oldCall.filterArg, oldAggRel.getGroupCount(), - oldAggRel.getInput(), null, oldCall.name); + oldCall.isApproximate(), oldCall.getArgList(), oldCall.filterArg, + oldAggRel.getGroupCount(), oldAggRel.getInput(), null, + oldCall.name); final AggregateCall countCall = - AggregateCall.create( - SqlStdOperatorTable.COUNT, + AggregateCall.create(SqlStdOperatorTable.COUNT, oldCall.isDistinct(), + oldCall.isApproximate(), oldCall.getArgList(), oldCall.filterArg, oldAggRel.getGroupCount(), @@ -441,9 +443,9 @@ public class AggregateReduceFunctionsRule extends RelOptRule { ImmutableList.of(sumArgSquaredAggCall.getType())); final AggregateCall sumArgAggCall = - AggregateCall.create( - SqlStdOperatorTable.SUM, + AggregateCall.create(SqlStdOperatorTable.SUM, oldCall.isDistinct(), + oldCall.isApproximate(), ImmutableIntList.of(argOrdinal), oldCall.filterArg, oldAggRel.getGroupCount(), @@ -464,9 +466,9 @@ public class AggregateReduceFunctionsRule extends RelOptRule { SqlStdOperatorTable.MULTIPLY, sumArgCast, sumArgCast); final AggregateCall countArgAggCall = - AggregateCall.create( - SqlStdOperatorTable.COUNT, + AggregateCall.create(SqlStdOperatorTable.COUNT, oldCall.isDistinct(), + oldCall.isApproximate(), oldCall.getArgList(), oldCall.filterArg, oldAggRel.getGroupCount(), http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/rules/AggregateStarTableRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateStarTableRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateStarTableRule.java index 6391a98..51ae97f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateStarTableRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateStarTableRule.java @@ -213,7 +213,8 @@ public class AggregateStarTableRule extends RelOptRule { if (roll == null) { break tryRoll; } - return AggregateCall.create(roll, false, ImmutableList.of(offset + i), -1, + return AggregateCall.create(roll, false, + aggregateCall.isApproximate(), ImmutableList.of(offset + i), -1, groupCount, relBuilder.peek(), null, aggregateCall.name); } @@ -228,7 +229,8 @@ public class AggregateStarTableRule extends RelOptRule { } newArgs.add(z); } - return AggregateCall.create(aggregation, false, newArgs, -1, + return AggregateCall.create(aggregation, false, + aggregateCall.isApproximate(), newArgs, -1, groupCount, relBuilder.peek(), null, aggregateCall.name); } http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/rules/AggregateUnionTransposeRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateUnionTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateUnionTransposeRule.java index 0e07256..abc5fbe 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateUnionTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateUnionTransposeRule.java @@ -170,6 +170,7 @@ public class AggregateUnionTransposeRule extends RelOptRule { } AggregateCall newCall = AggregateCall.create(aggFun, origCall.isDistinct(), + origCall.isApproximate(), ImmutableList.of(groupCount + ord.i), -1, groupCount, input, aggType, origCall.getName()); newCalls.add(newCall); http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java index 1961999..3bed2b3 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java @@ -165,8 +165,8 @@ public abstract class SubQueryRemoveRule extends RelOptRule { ImmutableBitSet.of()); if (unique == null || !unique) { builder.aggregate(builder.groupKey(), - builder.aggregateCall(SqlStdOperatorTable.SINGLE_VALUE, false, null, - null, builder.field(0))); + builder.aggregateCall(SqlStdOperatorTable.SINGLE_VALUE, false, + false, null, null, builder.field(0))); } builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet); return field(builder, inputCount, offset); @@ -292,8 +292,8 @@ public abstract class SubQueryRemoveRule extends RelOptRule { } builder.aggregate(builder.groupKey(), builder.count(false, "c"), - builder.aggregateCall(SqlStdOperatorTable.COUNT, false, null, "ck", - builder.fields())); + builder.aggregateCall(SqlStdOperatorTable.COUNT, false, false, null, + "ck", builder.fields())); builder.as("ct"); if (!variablesSet.isEmpty()) { builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet); http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/sql/SqlSplittableAggFunction.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSplittableAggFunction.java b/core/src/main/java/org/apache/calcite/sql/SqlSplittableAggFunction.java index d776847..6e8e4fe 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSplittableAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSplittableAggFunction.java @@ -117,7 +117,7 @@ public interface SqlSplittableAggFunction { } public AggregateCall other(RelDataTypeFactory typeFactory, AggregateCall e) { - return AggregateCall.create(SqlStdOperatorTable.COUNT, false, + return AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, ImmutableIntList.of(), -1, typeFactory.createSqlType(SqlTypeName.BIGINT), null); } @@ -146,7 +146,7 @@ public interface SqlSplittableAggFunction { throw new AssertionError("unexpected count " + merges); } int ordinal = extra.register(node); - return AggregateCall.create(SqlStdOperatorTable.SUM0, false, + return AggregateCall.create(SqlStdOperatorTable.SUM0, false, false, ImmutableList.of(ordinal), -1, aggregateCall.type, aggregateCall.name); } @@ -229,7 +229,7 @@ public interface SqlSplittableAggFunction { } public AggregateCall other(RelDataTypeFactory typeFactory, AggregateCall e) { - return AggregateCall.create(SqlStdOperatorTable.COUNT, false, + return AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, ImmutableIntList.of(), -1, typeFactory.createSqlType(SqlTypeName.BIGINT), null); } @@ -260,7 +260,7 @@ public interface SqlSplittableAggFunction { throw new AssertionError("unexpected count " + merges); } int ordinal = extra.register(node); - return AggregateCall.create(SqlStdOperatorTable.SUM, false, + return AggregateCall.create(SqlStdOperatorTable.SUM, false, false, ImmutableList.of(ordinal), -1, aggregateCall.type, aggregateCall.name); } http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java index ebfa084..e053294 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlCountAggFunction.java @@ -44,19 +44,12 @@ import java.util.List; public class SqlCountAggFunction extends SqlAggFunction { //~ Constructors ----------------------------------------------------------- - public SqlCountAggFunction() { - super( - "COUNT", - null, - SqlKind.COUNT, - ReturnTypes.BIGINT, - null, + public SqlCountAggFunction(String name) { + super(name, null, SqlKind.COUNT, ReturnTypes.BIGINT, null, SqlValidator.STRICT ? OperandTypes.ANY : OperandTypes.ONE_OR_MORE, - SqlFunctionCategory.NUMERIC, - false, - false); + SqlFunctionCategory.NUMERIC, false, false); } //~ Methods ---------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 9b5273a..f6a762b 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -828,7 +828,13 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { /** * <code>COUNT</code> aggregate function. */ - public static final SqlAggFunction COUNT = new SqlCountAggFunction(); + public static final SqlAggFunction COUNT = new SqlCountAggFunction("COUNT"); + + /** + * <code>APPROX_COUNT_DISTINCT</code> aggregate function. + */ + public static final SqlAggFunction APPROX_COUNT_DISTINCT = + new SqlCountAggFunction("APPROX_COUNT_DISTINCT"); /** * <code>MIN</code> aggregate function. http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java index 0b7af6f..2d7ec59 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -866,7 +866,8 @@ public class RelFieldTrimmer implements ReflectiveVisitor { : relBuilder.field(Mappings.apply(inputMapping, aggCall.filterArg)); RelBuilder.AggCall newAggCall = relBuilder.aggregateCall(aggCall.getAggregation(), - aggCall.isDistinct(), filterArg, aggCall.name, args); + aggCall.isDistinct(), aggCall.isApproximate(), + filterArg, aggCall.name, args); mapping.set(j, groupCount + indicatorCount + newAggCallList.size()); newAggCallList.add(newAggCall); } http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 752743b..bbe7b68 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -1101,9 +1101,9 @@ public class SqlToRelConverter { LogicalAggregate.create(seek, ImmutableBitSet.of(), null, ImmutableList.of( AggregateCall.create(SqlStdOperatorTable.COUNT, false, - ImmutableList.<Integer>of(), -1, longType, null), + false, ImmutableList.<Integer>of(), -1, longType, null), AggregateCall.create(SqlStdOperatorTable.COUNT, false, - args, -1, longType, null))); + false, args, -1, longType, null))); LogicalJoin join = LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true), ImmutableSet.<CorrelationId>of(), JoinRelType.INNER); @@ -4879,19 +4879,26 @@ public class SqlToRelConverter { bb.agg = this; } - final SqlAggFunction aggFunction = + SqlAggFunction aggFunction = (SqlAggFunction) call.getOperator(); - RelDataType type = validator.deriveType(bb.scope, call); + final RelDataType type = validator.deriveType(bb.scope, call); boolean distinct = false; SqlLiteral quantifier = call.getFunctionQuantifier(); if ((null != quantifier) && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) { distinct = true; } + boolean approximate = false; + if (aggFunction == SqlStdOperatorTable.APPROX_COUNT_DISTINCT) { + aggFunction = SqlStdOperatorTable.COUNT; + distinct = true; + approximate = true; + } final AggregateCall aggCall = AggregateCall.create( aggFunction, distinct, + approximate, args, filterArg, type, http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/tools/PigRelBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/tools/PigRelBuilder.java b/core/src/main/java/org/apache/calcite/tools/PigRelBuilder.java index 4451f5c..dcac18b 100644 --- a/core/src/main/java/org/apache/calcite/tools/PigRelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/PigRelBuilder.java @@ -146,7 +146,8 @@ public class PigRelBuilder extends RelBuilder { cluster.getRexBuilder().makeCall(peek(1, 0).getRowType(), SqlStdOperatorTable.ROW, fields()); aggregate(groupKey.e, - aggregateCall(SqlStdOperatorTable.COLLECT, false, null, getAlias(), row)); + aggregateCall(SqlStdOperatorTable.COLLECT, false, false, null, + getAlias(), row)); if (groupKey.i < n - 1) { push(r); List<RexNode> predicates = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/main/java/org/apache/calcite/tools/RelBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java index c131f8e..0b871f9 100644 --- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java +++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java @@ -727,16 +727,30 @@ public class RelBuilder { return groupKey(nodes, indicator, nodeLists); } - /** Creates a call to an aggregate function. */ + @Deprecated // to be removed before 2.0 public AggCall aggregateCall(SqlAggFunction aggFunction, boolean distinct, RexNode filter, String alias, RexNode... operands) { - return aggregateCall(aggFunction, distinct, filter, alias, + return aggregateCall(aggFunction, distinct, false, filter, alias, ImmutableList.copyOf(operands)); } /** Creates a call to an aggregate function. */ public AggCall aggregateCall(SqlAggFunction aggFunction, boolean distinct, + boolean approximate, RexNode filter, String alias, RexNode... operands) { + return aggregateCall(aggFunction, distinct, approximate, filter, alias, + ImmutableList.copyOf(operands)); + } + + @Deprecated // to be removed before 2.0 + public AggCall aggregateCall(SqlAggFunction aggFunction, boolean distinct, RexNode filter, String alias, Iterable<? extends RexNode> operands) { + return aggregateCall(aggFunction, distinct, false, filter, alias, operands); + } + + /** Creates a call to an aggregate function. */ + public AggCall aggregateCall(SqlAggFunction aggFunction, boolean distinct, + boolean approximate, RexNode filter, String alias, + Iterable<? extends RexNode> operands) { if (filter != null) { if (filter.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) { throw RESOURCE.filterMustBeBoolean().ex(); @@ -745,41 +759,43 @@ public class RelBuilder { filter = call(SqlStdOperatorTable.IS_TRUE, filter); } } - return new AggCallImpl(aggFunction, distinct, filter, alias, + return new AggCallImpl(aggFunction, distinct, approximate, filter, alias, ImmutableList.copyOf(operands)); } /** Creates a call to the COUNT aggregate function. */ public AggCall count(boolean distinct, String alias, RexNode... operands) { - return aggregateCall(SqlStdOperatorTable.COUNT, distinct, null, alias, - operands); + return aggregateCall(SqlStdOperatorTable.COUNT, distinct, false, null, + alias, operands); } /** Creates a call to the COUNT(*) aggregate function. */ public AggCall countStar(String alias) { - return aggregateCall(SqlStdOperatorTable.COUNT, false, null, alias); + return aggregateCall(SqlStdOperatorTable.COUNT, false, false, null, alias); } /** Creates a call to the SUM aggregate function. */ public AggCall sum(boolean distinct, String alias, RexNode operand) { - return aggregateCall(SqlStdOperatorTable.SUM, distinct, null, alias, + return aggregateCall(SqlStdOperatorTable.SUM, distinct, false, null, alias, operand); } /** Creates a call to the AVG aggregate function. */ public AggCall avg(boolean distinct, String alias, RexNode operand) { - return aggregateCall( - SqlStdOperatorTable.AVG, distinct, null, alias, operand); + return aggregateCall(SqlStdOperatorTable.AVG, distinct, false, null, alias, + operand); } /** Creates a call to the MIN aggregate function. */ public AggCall min(String alias, RexNode operand) { - return aggregateCall(SqlStdOperatorTable.MIN, false, null, alias, operand); + return aggregateCall(SqlStdOperatorTable.MIN, false, false, null, alias, + operand); } /** Creates a call to the MAX aggregate function. */ public AggCall max(String alias, RexNode operand) { - return aggregateCall(SqlStdOperatorTable.MAX, false, null, alias, operand); + return aggregateCall(SqlStdOperatorTable.MAX, false, false, null, alias, + operand); } // Methods for patterns @@ -1235,7 +1251,8 @@ public class RelBuilder { throw new IllegalArgumentException("FILTER not allowed"); } aggregateCall = - AggregateCall.create(aggCall1.aggFunction, aggCall1.distinct, args, + AggregateCall.create(aggCall1.aggFunction, aggCall1.distinct, + aggCall1.approximate, args, filterArg, groupSet.cardinality(), r, null, aggCall1.alias); } else { aggregateCall = ((AggCallImpl2) aggCall).aggregateCall; @@ -1933,14 +1950,17 @@ public class RelBuilder { private static class AggCallImpl implements AggCall { private final SqlAggFunction aggFunction; private final boolean distinct; + private final boolean approximate; private final RexNode filter; private final String alias; private final ImmutableList<RexNode> operands; - AggCallImpl(SqlAggFunction aggFunction, boolean distinct, RexNode filter, + AggCallImpl(SqlAggFunction aggFunction, boolean distinct, + boolean approximate, RexNode filter, String alias, ImmutableList<RexNode> operands) { this.aggFunction = aggFunction; this.distinct = distinct; + this.approximate = approximate; this.filter = filter; this.alias = alias; this.operands = operands; http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java index 9749b6d..4f4829b 100644 --- a/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java +++ b/core/src/test/java/org/apache/calcite/plan/RelWriterTest.java @@ -141,10 +141,10 @@ public class RelWriterTest { LogicalAggregate.create(filter, ImmutableBitSet.of(0), null, ImmutableList.of( AggregateCall.create(SqlStdOperatorTable.COUNT, - true, ImmutableList.of(1), -1, bigIntType, - "c"), + true, false, ImmutableList.of(1), -1, + bigIntType, "c"), AggregateCall.create(SqlStdOperatorTable.COUNT, - false, ImmutableList.<Integer>of(), -1, + false, false, ImmutableList.<Integer>of(), -1, bigIntType, "d"))); aggregate.explain(writer); return writer.asString(); http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java index 74e675c..8847f93 100644 --- a/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java +++ b/core/src/test/java/org/apache/calcite/plan/volcano/TraitPropagationTest.java @@ -166,7 +166,7 @@ public class TraitPropagationTest { // aggregate on s, count AggregateCall aggCall = AggregateCall.create(SqlStdOperatorTable.COUNT, - false, Collections.singletonList(1), -1, sqlBigInt, "cnt"); + false, false, Collections.singletonList(1), -1, sqlBigInt, "cnt"); RelNode agg = new LogicalAggregate(cluster, cluster.traitSetOf(Convention.NONE), project, false, ImmutableBitSet.of(0), null, Collections.singletonList(aggCall)); http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/test/java/org/apache/calcite/sql/test/SqlOperatorBaseTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/sql/test/SqlOperatorBaseTest.java b/core/src/test/java/org/apache/calcite/sql/test/SqlOperatorBaseTest.java index 7c0be3f..8bd922a 100644 --- a/core/src/test/java/org/apache/calcite/sql/test/SqlOperatorBaseTest.java +++ b/core/src/test/java/org/apache/calcite/sql/test/SqlOperatorBaseTest.java @@ -6203,6 +6203,51 @@ public abstract class SqlOperatorBaseTest { tester.checkAgg("COUNT(DISTINCT 123)", stringValues, 1, (double) 0); } + @Test public void testApproxCountDistinctFunc() { + tester.setFor(SqlStdOperatorTable.COUNT, VM_EXPAND); + tester.checkFails("approx_count_distinct(^*^)", "Unknown identifier '\\*'", + false); + tester.checkType("approx_count_distinct('name')", "BIGINT NOT NULL"); + tester.checkType("approx_count_distinct(1)", "BIGINT NOT NULL"); + tester.checkType("approx_count_distinct(1.2)", "BIGINT NOT NULL"); + tester.checkType("APPROX_COUNT_DISTINCT(DISTINCT 'x')", "BIGINT NOT NULL"); + tester.checkFails("^APPROX_COUNT_DISTINCT()^", + "Invalid number of arguments to function 'APPROX_COUNT_DISTINCT'. " + + "Was expecting 1 arguments", + false); + tester.checkType("approx_count_distinct(1, 2)", "BIGINT NOT NULL"); + tester.checkType("approx_count_distinct(1, 2, 'x', 'y')", + "BIGINT NOT NULL"); + final String[] values = {"0", "CAST(null AS INTEGER)", "1", "0"}; + // currently APPROX_COUNT_DISTINCT(x) returns the same as COUNT(DISTINCT x) + tester.checkAgg( + "APPROX_COUNT_DISTINCT(x)", + values, + 2, + (double) 0); + tester.checkAgg( + "APPROX_COUNT_DISTINCT(CASE x WHEN 0 THEN NULL ELSE -1 END)", + values, + 1, + (double) 0); + // DISTINCT keyword is allowed but has no effect + tester.checkAgg( + "APPROX_COUNT_DISTINCT(DISTINCT x)", + values, + 2, + (double) 0); + + // string values -- note that empty string is not null + final String[] stringValues = { + "'a'", "CAST(NULL AS VARCHAR(1))", "''" + }; + tester.checkAgg("APPROX_COUNT_DISTINCT(x)", stringValues, 2, (double) 0); + tester.checkAgg("APPROX_COUNT_DISTINCT(DISTINCT x)", stringValues, 2, + (double) 0); + tester.checkAgg("APPROX_COUNT_DISTINCT(DISTINCT 123)", stringValues, 1, + (double) 0); + } + @Test public void testSumFunc() { tester.setFor(SqlStdOperatorTable.SUM, VM_EXPAND); tester.checkFails( http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index a17a154..4858841 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -467,7 +467,7 @@ public class RelBuilderTest { builder.literal(20))) .aggregate(builder.groupKey(0, 1, 2), builder.aggregateCall(SqlStdOperatorTable.SUM, - false, null, null, + false, false, null, null, builder.field(0))) .project(builder.field("c"), builder.field("a")) @@ -623,8 +623,8 @@ public class RelBuilderTest { RelNode root = builder.scan("EMP") .aggregate(builder.groupKey(), - builder.aggregateCall(SqlStdOperatorTable.COUNT, true, null, - "C", builder.field("DEPTNO"))) + builder.aggregateCall(SqlStdOperatorTable.COUNT, true, false, + null, "C", builder.field("DEPTNO"))) .build(); assertThat(str(root), is("LogicalAggregate(group=[{}], C=[COUNT(DISTINCT $7)])\n" @@ -645,9 +645,10 @@ public class RelBuilderTest { builder.field(4), builder.field(3)), builder.field(1)), - builder.aggregateCall(SqlStdOperatorTable.COUNT, false, null, - "C"), - builder.aggregateCall(SqlStdOperatorTable.SUM, false, null, "S", + builder.aggregateCall(SqlStdOperatorTable.COUNT, false, false, + null, "C"), + builder.aggregateCall(SqlStdOperatorTable.SUM, false, false, + null, "S", builder.call(SqlStdOperatorTable.PLUS, builder.field(3), builder.literal(1)))) .build(); @@ -670,7 +671,7 @@ public class RelBuilderTest { builder.groupKey(ImmutableBitSet.of(7), ImmutableList.of(ImmutableBitSet.of(7), ImmutableBitSet.of())), - builder.aggregateCall(SqlStdOperatorTable.COUNT, false, + builder.aggregateCall(SqlStdOperatorTable.COUNT, false, false, builder.call(SqlStdOperatorTable.GREATER_THAN, builder.field("EMPNO"), builder.literal(100)), "C")) .build(); @@ -692,7 +693,7 @@ public class RelBuilderTest { builder.scan("EMP") .aggregate( builder.groupKey(builder.field("DEPTNO")), - builder.aggregateCall(SqlStdOperatorTable.SUM, false, + builder.aggregateCall(SqlStdOperatorTable.SUM, false, false, builder.field("COMM"), "C", builder.field("SAL"))) .build(); fail("expected error, got " + root); @@ -712,7 +713,7 @@ public class RelBuilderTest { builder.scan("EMP") .aggregate( builder.groupKey(builder.field("DEPTNO")), - builder.aggregateCall(SqlStdOperatorTable.SUM, false, + builder.aggregateCall(SqlStdOperatorTable.SUM, false, false, builder.call(SqlStdOperatorTable.LESS_THAN, builder.field("COMM"), builder.literal(100)), "C", builder.field("SAL"))) @@ -819,8 +820,8 @@ public class RelBuilderTest { RelNode root = builder.scan("EMP") .aggregate(builder.groupKey(6, 7), - builder.aggregateCall(SqlStdOperatorTable.GROUPING, false, null, - "g", builder.field("DEPTNO"))) + builder.aggregateCall(SqlStdOperatorTable.GROUPING, false, + false, null, "g", builder.field("DEPTNO"))) .build(); final String expected = "" + "LogicalAggregate(group=[{6, 7}], g=[GROUPING($7)])\n" @@ -834,8 +835,8 @@ public class RelBuilderTest { RelNode root = builder.scan("EMP") .aggregate(builder.groupKey(6, 7), - builder.aggregateCall(SqlStdOperatorTable.GROUPING, true, null, - "g", builder.field("DEPTNO"))) + builder.aggregateCall(SqlStdOperatorTable.GROUPING, true, + false, null, "g", builder.field("DEPTNO"))) .build(); fail("expected error, got " + root); } catch (IllegalArgumentException e) { @@ -850,7 +851,8 @@ public class RelBuilderTest { builder.scan("EMP") .aggregate(builder.groupKey(6, 7), builder.aggregateCall(SqlStdOperatorTable.GROUPING, false, - builder.literal(true), "g", builder.field("DEPTNO"))) + false, builder.literal(true), "g", + builder.field("DEPTNO"))) .build(); fail("expected error, got " + root); } catch (IllegalArgumentException e) { @@ -1339,8 +1341,8 @@ public class RelBuilderTest { .project(builder.field("DEPTNO"), builder.literal(20)) .aggregate(builder.groupKey(builder.field("EMP_alias", "DEPTNO")), - builder.aggregateCall(SqlStdOperatorTable.SUM, false, null, - null, builder.field(1))) + builder.aggregateCall(SqlStdOperatorTable.SUM, false, false, + null, null, builder.field(1))) .project(builder.alias(builder.field(1), "sum"), builder.field("EMP_alias", "DEPTNO")) .build(); http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index f2bed82..a09de16 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -1283,8 +1283,8 @@ public class RelMetadataTest extends SqlToRelTestBase { LogicalAggregate.create(join, ImmutableBitSet.of(2, 0), ImmutableList.<ImmutableBitSet>of(), ImmutableList.of( - AggregateCall.create( - SqlStdOperatorTable.COUNT, false, ImmutableIntList.of(), + AggregateCall.create(SqlStdOperatorTable.COUNT, + false, false, ImmutableIntList.of(), -1, 2, join, null, null))); rowSize = mq.getAverageRowSize(aggregate); columnSizes = mq.getAverageColumnSizes(aggregate); http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java ---------------------------------------------------------------------- diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java index 012d272..4cfee48 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java @@ -860,7 +860,7 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { switch (aggCall.getAggregation().getKind()) { case COUNT: if (aggCall.isDistinct()) { - if (config.approximateDistinctCount()) { + if (aggCall.isApproximate() || config.approximateDistinctCount()) { if (complexMetric == null) { aggregation = new JsonCardinalityAggregation("cardinality", name, list); } else { http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java ---------------------------------------------------------------------- diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index 0f4d8b7..baf5e41 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -142,8 +142,10 @@ public class DruidRules { node, query)) { return true; } - if ((config.approximateDistinctCount() && aggregateCall.isDistinct()) - || aggregateCall.getArgList().isEmpty()) { + if ((aggregateCall.isDistinct() + && (aggregateCall.isApproximate() + || config.approximateDistinctCount())) + || aggregateCall.getArgList().isEmpty()) { continue; } return true; http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java ---------------------------------------------------------------------- diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java index 1993789..3789a36 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java @@ -34,6 +34,7 @@ import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlSelectKeyword; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; import com.google.common.base.Preconditions; @@ -178,11 +179,12 @@ public class DruidTable extends AbstractTable implements TranslatableTable { assert isRolledUp(column); // Our rolled up columns are only allowed in COUNT(DISTINCT ...) aggregate functions. // We only allow this when approximate results are acceptable. - return config != null - && config.approximateDistinctCount() - && isCountDistinct(call) - && call.getOperandList().size() == 1 // for COUNT(a_1, a_2, ... a_n). n should be 1 - && isValidParentKind(parent); + return ((config != null + && config.approximateDistinctCount() + && isCountDistinct(call)) + || call.getOperator() == SqlStdOperatorTable.APPROX_COUNT_DISTINCT) + && call.getOperandList().size() == 1 // for COUNT(a_1, a_2, ... a_n). n should be 1 + && isValidParentKind(parent); } private boolean isValidParentKind(SqlNode node) { http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java ---------------------------------------------------------------------- diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java index 36c866e..30e1438 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java @@ -3137,19 +3137,28 @@ public class DruidAdapterIT { "month=November; avg$=34.51727744987063", "month=December; avg$=33.62788702774498"); - wikiApprox("select (count(distinct \"user_id\") + 100) - " - + "(count(distinct \"user_id\") * 2) from \"wiki\"") - .queryContains( - druidChecker("'aggregations':[{'type':'hyperUnique','name':'$f0'," - + "'fieldName':'user_unique'}],'postAggregations':[{'type':" - + "'arithmetic','name':'postagg#0','fn':'-','fields':[{'type':" - + "'arithmetic','name':'','fn':'+','fields':[{'type':" - + "'hyperUniqueCardinality','name':'','fieldName':'$f0'}," - + "{'type':'constant','name':'','value':100.0}]},{'type':" - + "'arithmetic','name':'','fn':'*','fields':[{'type':" - + "'hyperUniqueCardinality','name':'','fieldName':'$f0'}," - + "{'type':'constant','name':'','value':2.0}]}]}]")) - .returnsUnordered("EXPR$0=-10590"); + final String druid = "'aggregations':[{'type':'hyperUnique','name':'$f0'," + + "'fieldName':'user_unique'}],'postAggregations':[{'type':" + + "'arithmetic','name':'postagg#0','fn':'-','fields':[{'type':" + + "'arithmetic','name':'','fn':'+','fields':[{'type':" + + "'hyperUniqueCardinality','name':'','fieldName':'$f0'}," + + "{'type':'constant','name':'','value':100.0}]},{'type':" + + "'arithmetic','name':'','fn':'*','fields':[{'type':" + + "'hyperUniqueCardinality','name':'','fieldName':'$f0'}," + + "{'type':'constant','name':'','value':2.0}]}]}]"; + final String sql = "select (count(distinct \"user_id\") + 100) - " + + "(count(distinct \"user_id\") * 2) from \"wiki\""; + wikiApprox(sql) + .queryContains(druidChecker(druid)) + .returnsUnordered("EXPR$0=-10590"); + + // Change COUNT(DISTINCT ...) to APPROX_COUNT_DISTINCT(...) and get + // same result even if approximation is off by default. + final String sql2 = "select (approx_count_distinct(\"user_id\") + 100) - " + + "(approx_count_distinct(\"user_id\") * 2) from \"wiki\""; + sql(sql2, WIKI) + .queryContains(druidChecker(druid)) + .returnsUnordered("EXPR$0=-10590"); } /** http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/site/_docs/algebra.md ---------------------------------------------------------------------- diff --git a/site/_docs/algebra.md b/site/_docs/algebra.md index 527b2e1..920e71a 100644 --- a/site/_docs/algebra.md +++ b/site/_docs/algebra.md @@ -389,7 +389,7 @@ The following methods return an | Method | Description |:------------------- |:----------- -| `aggregateCall(op, distinct, filter, alias, expr...)`<br/>`aggregateCall(op, distinct, filter, alias, exprList)` | Creates a call to a given aggregate function, with an optional filter expression +| `aggregateCall(op, distinct, approximate, filter, alias, expr...)`<br/>`aggregateCall(op, distinct, approximate, filter, alias, exprList)` | Creates a call to a given aggregate function, with an optional filter expression | `count(distinct, alias, expr...)` | Creates a call to the COUNT aggregate function | `countStar(alias)` | Creates a call to the COUNT(*) aggregate function | `sum(distinct, alias, expr)` | Creates a call to the SUM aggregate function http://git-wip-us.apache.org/repos/asf/calcite/blob/fe3529d9/site/_docs/reference.md ---------------------------------------------------------------------- diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 8f0b4e8..ea6e983 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -1480,6 +1480,7 @@ passed to the aggregate function. | COLLECT( [ ALL | DISTINCT ] value) | Returns a multiset of the values | COUNT( [ ALL | DISTINCT ] value [, value ]*) | Returns the number of input rows for which *value* is not null (wholly not null if *value* is composite) | COUNT(*) | Returns the number of input rows +| APPROX_COUNT_DISTINCT(value [, value ]*) | Returns the approximate number of distinct values of *value*; the database is allowed to use an approximation but is not required to | AVG( [ ALL | DISTINCT ] numeric) | Returns the average (arithmetic mean) of *numeric* across all input values | SUM( [ ALL | DISTINCT ] numeric) | Returns the sum of *numeric* across all input values | MAX( [ ALL | DISTINCT ] value) | Returns the maximum value of *value* across all input values @@ -1507,7 +1508,7 @@ Not implemented: | Operator syntax | Description |:----------------------------------------- |:----------- -| COUNT(value [, value ]*) OVER window | Returns the number of rows in *window* for which *value* is not null (wholly not null if *value* is composite) +| COUNT(value [, value ]*) OVER window | Returns the number of rows in *window* for which *value* is not null (wholly not null if *value* is composite) | COUNT(*) OVER window | Returns the number of rows in *window* | AVG(numeric) OVER window | Returns the average (arithmetic mean) of *numeric* across all values in *window* | SUM(numeric) OVER window | Returns the sum of *numeric* across all values in *window* @@ -1524,7 +1525,8 @@ Not implemented: Not implemented: -* COUNT(DISTINCT value) OVER window +* COUNT(DISTINCT value [, value ]*) OVER window +* APPROX_COUNT_DISTINCT(value [, value ]*) OVER window * FIRST_VALUE(value) IGNORE NULLS OVER window * LAST_VALUE(value) IGNORE NULLS OVER window * PERCENT_RANK(value) OVER window
