Repository: calcite Updated Branches: refs/heads/master 0d996daf5 -> cdc1d960a
[CALCITE-1683] Druid Specific Filter Transpose Rules (Nishant Bangarwa) Close apache/calcite#393 Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/cdc1d960 Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/cdc1d960 Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/cdc1d960 Branch: refs/heads/master Commit: cdc1d960a2422d14dc7c25c856cb57103006f4a0 Parents: 0d996da Author: Nishant Bangarwa <[email protected]> Authored: Tue Mar 14 16:29:44 2017 +0000 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Tue Mar 14 16:39:54 2017 +0000 ---------------------------------------------------------------------- .../rel/rules/AggregateFilterTransposeRule.java | 13 +++- .../rel/rules/FilterAggregateTransposeRule.java | 10 ++- .../rel/rules/FilterProjectTransposeRule.java | 17 +++-- .../rel/rules/ProjectFilterTransposeRule.java | 14 ++-- .../calcite/adapter/druid/DruidQuery.java | 6 +- .../calcite/adapter/druid/DruidRules.java | 75 +++++++++++++++++++- .../org/apache/calcite/test/DruidAdapterIT.java | 12 +++- 7 files changed, 128 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java index b2ebf84..943f25a 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java @@ -18,16 +18,19 @@ package org.apache.calcite.rel.rules; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.SubstitutionVisitor; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.mapping.Mappings; @@ -57,9 +60,15 @@ public class AggregateFilterTransposeRule extends RelOptRule { new AggregateFilterTransposeRule(); private AggregateFilterTransposeRule() { - super( + this( operand(Aggregate.class, - operand(Filter.class, any()))); + operand(Filter.class, any())), + RelFactories.LOGICAL_BUILDER); + } + + protected AggregateFilterTransposeRule(RelOptRuleOperand operand, + RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, null); } public void onMatch(RelOptRuleCall call) { http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java index 6347cbf..118e637 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java @@ -19,6 +19,7 @@ package org.apache.calcite.rel.rules; import org.apache.calcite.plan.Contexts; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; @@ -64,10 +65,15 @@ public class FilterAggregateTransposeRule extends RelOptRule { Class<? extends Filter> filterClass, RelBuilderFactory builderFactory, Class<? extends Aggregate> aggregateClass) { - super( + this( operand(filterClass, operand(aggregateClass, any())), - builderFactory, null); + builderFactory); + } + + protected FilterAggregateTransposeRule(RelOptRuleOperand operand, + RelBuilderFactory builderFactory) { + super(operand, builderFactory, null); } @Deprecated // to be removed before 2.0 http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java index f37ca7b..2c0797b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java @@ -18,6 +18,7 @@ package org.apache.calcite.rel.rules; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; @@ -62,12 +63,10 @@ public class FilterProjectTransposeRule extends RelOptRule { Class<? extends Project> projectClass, boolean copyFilter, boolean copyProject, RelBuilderFactory relBuilderFactory) { - super( + this( operand(filterClass, operand(projectClass, any())), - relBuilderFactory, null); - this.copyFilter = copyFilter; - this.copyProject = copyProject; + copyFilter, copyProject, relBuilderFactory); } @Deprecated // to be removed before 2.0 @@ -81,6 +80,16 @@ public class FilterProjectTransposeRule extends RelOptRule { RelBuilder.proto(filterFactory, projectFactory)); } + protected FilterProjectTransposeRule( + RelOptRuleOperand operand, + boolean copyFilter, + boolean copyProject, + RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, null); + this.copyFilter = copyFilter; + this.copyProject = copyProject; + } + //~ Methods ---------------------------------------------------------------- public void onMatch(RelOptRuleCall call) { http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java index 516de3b..0c7cb5c 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java @@ -18,6 +18,7 @@ package org.apache.calcite.rel.rules; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Project; @@ -57,11 +58,17 @@ public class ProjectFilterTransposeRule extends RelOptRule { Class<? extends Filter> filterClass, RelBuilderFactory relBuilderFactory, PushProjector.ExprCondition preserveExprCondition) { - super( + this( operand( projectClass, operand(filterClass, any())), - relBuilderFactory, null); + preserveExprCondition, relBuilderFactory); + } + + protected ProjectFilterTransposeRule(RelOptRuleOperand operand, + PushProjector.ExprCondition preserveExprCondition, + RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, null); this.preserveExprCondition = preserveExprCondition; } @@ -71,8 +78,7 @@ public class ProjectFilterTransposeRule extends RelOptRule { public void onMatch(RelOptRuleCall call) { Project origProj; Filter filter; - - if (call.rels.length == 2) { + if (call.rels.length >= 2) { origProj = call.rel(0); filter = call.rel(1); } else { http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java ---------------------------------------------------------------------- diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java index 74c0cc0..061c52c 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java @@ -65,7 +65,6 @@ import org.apache.calcite.util.Util; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; - import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -275,8 +274,9 @@ public class DruidQuery extends AbstractRelNode implements BindableRel { /** Extends a DruidQuery. */ public static DruidQuery extendQuery(DruidQuery query, RelNode r) { final ImmutableList.Builder<RelNode> builder = ImmutableList.builder(); - return DruidQuery.create(query.getCluster(), r.getTraitSet(), query.getTable(), - query.druidTable, query.intervals, builder.addAll(query.rels).add(r).build()); + return DruidQuery.create(query.getCluster(), r.getTraitSet().replace(query.getConvention()), + query.getTable(), query.druidTable, query.intervals, + builder.addAll(query.rels).add(r).build()); } /** Extends a DruidQuery. */ http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java ---------------------------------------------------------------------- diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java index 0a81c47..91a29ab 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java @@ -27,8 +27,14 @@ import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.rules.AggregateFilterTransposeRule; +import org.apache.calcite.rel.rules.FilterAggregateTransposeRule; +import org.apache.calcite.rel.rules.FilterProjectTransposeRule; +import org.apache.calcite.rel.rules.ProjectFilterTransposeRule; import org.apache.calcite.rel.rules.ProjectSortTransposeRule; +import org.apache.calcite.rel.rules.PushProjector; import org.apache.calcite.rel.rules.SortProjectTransposeRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -72,8 +78,22 @@ public class DruidRules { public static final DruidProjectSortRule PROJECT_SORT = new DruidProjectSortRule(); public static final DruidSortProjectRule SORT_PROJECT = new DruidSortProjectRule(); - public static final List<RelOptRule> RULES = ImmutableList.of(FILTER, PROJECT_AGGREGATE, - PROJECT, AGGREGATE, PROJECT_SORT, SORT, SORT_PROJECT); + public static final DruidProjectFilterTransposeRule PROJECT_FILTER_TRANSPOSE_RULE = + new DruidProjectFilterTransposeRule(); + + public static final DruidFilterProjectTransposeRule FILTER_PROJECT_TRANSPOSE_RULE = + new DruidFilterProjectTransposeRule(); + + public static final DruidAggregateFilterTransposeRule AGGREGATE_FILTER_TRANSPOSE_RULE = + new DruidAggregateFilterTransposeRule(); + + public static final DruidFilterAggregateTransposeRule FILTER_AGGREGATE_TRANSPOSE_RULE = + new DruidFilterAggregateTransposeRule(); + + public static final List<RelOptRule> RULES = ImmutableList.of(FILTER, + PROJECT_FILTER_TRANSPOSE_RULE, AGGREGATE_FILTER_TRANSPOSE_RULE, PROJECT_AGGREGATE, + PROJECT, AGGREGATE, FILTER_AGGREGATE_TRANSPOSE_RULE, FILTER_PROJECT_TRANSPOSE_RULE, + PROJECT_SORT, SORT, SORT_PROJECT); /** Predicate that returns whether Druid can not handle an aggregate. */ private static final Predicate<Aggregate> BAD_AGG = @@ -555,6 +575,57 @@ public class DruidRules { return false; } + + /** + * Rule to push an {@link org.apache.calcite.rel.core.Project} past a + * {@link org.apache.calcite.rel.core.Filter} when filter is on top of {@link org.apache.calcite.adapter.druid.DruidQuery} + */ + private static class DruidProjectFilterTransposeRule extends ProjectFilterTransposeRule { + private DruidProjectFilterTransposeRule() { + super( + operand(Project.class, operand(Filter.class, any())), PushProjector.ExprCondition.FALSE, + RelFactories.LOGICAL_BUILDER); + } + } + + /** + * Rule to push an {@link org.apache.calcite.rel.core.Filter} past a + * {@link org.apache.calcite.rel.core.Project} when project is on top of {@link org.apache.calcite.adapter.druid.DruidQuery} + */ + private static class DruidFilterProjectTransposeRule extends FilterProjectTransposeRule { + private DruidFilterProjectTransposeRule() { + super( + operand(Filter.class, operand(Project.class, operand(DruidQuery.class, none()))), true, + true, RelFactories.LOGICAL_BUILDER + ); + } + } + + /** + * Rule to push an {@link org.apache.calcite.rel.core.Filter} past a + * {@link org.apache.calcite.rel.core.Project} when project is on top of {@link org.apache.calcite.adapter.druid.DruidQuery} + */ + private static class DruidAggregateFilterTransposeRule extends AggregateFilterTransposeRule { + private DruidAggregateFilterTransposeRule() { + super( + operand(Aggregate.class, operand(Filter.class, operand(DruidQuery.class, none()))), + RelFactories.LOGICAL_BUILDER); + } + } + + /** + * Rule to push an {@link org.apache.calcite.rel.core.Aggregate} past a + * {@link org.apache.calcite.rel.core.Filter} when filter is on top of {@link org.apache.calcite.adapter.druid.DruidQuery} + */ + private static class DruidFilterAggregateTransposeRule extends FilterAggregateTransposeRule { + private DruidFilterAggregateTransposeRule() { + super( + operand(Filter.class, operand(Aggregate.class, operand(DruidQuery.class, none()))), + RelFactories.LOGICAL_BUILDER + ); + } + } + } // End DruidRules.java http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java ---------------------------------------------------------------------- diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java index cd28362..099d0c9 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java @@ -1207,10 +1207,18 @@ public class DruidAdapterIT { + "EnumerableInterpreter\n" + " BindableSort(sort0=[$1], dir0=[DESC], fetch=[2])\n" + " BindableProject(state_province=[$0], CDC=[FLOOR($1)])\n" - + " BindableAggregate(group=[{1}], agg#0=[COUNT($0)])\n" - + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], groups=[{29, 30}], aggs=[[]])"; + + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], groups=[{30}], aggs=[[COUNT(DISTINCT $29)]])\n"; + + final String druidQuery = "{\"queryType\":\"groupBy\"," + + "\"dataSource\":\"foodmart\"," + + "\"granularity\":\"all\"," + + "\"dimensions\":[\"state_province\"]," + + "\"limitSpec\":{\"type\":\"default\"}," + + "\"aggregations\":[{\"type\":\"cardinality\",\"name\":\"$f1\",\"fieldNames\":[\"city\"]}]," + + "\"intervals\":[\"1900-01-09T00:00:00.000/2992-01-10T00:00:00.000\"]}"; sql(sql) .explainContains(explain) + .queryContains(druidChecker(druidQuery)) .returnsUnordered("state_province=CA; CDC=45", "state_province=WA; CDC=22"); }
