Repository: drill Updated Branches: refs/heads/master e85cdc54c -> 1c37185bf
DRILL-3410: rewrite OR and AND operators to have only 2 operands so partitoning pruning will work correctly Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1c37185b Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1c37185b Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1c37185b Branch: refs/heads/master Commit: 1c37185bfd6148e317d1d668759abd6b9fd6a53f Parents: e85cdc5 Author: Steven Phillips <[email protected]> Authored: Sat Jun 27 11:23:02 2015 -0700 Committer: Steven Phillips <[email protected]> Committed: Sat Jun 27 15:25:58 2015 -0700 ---------------------------------------------------------------------- .../logical/partition/PruneScanRule.java | 2 + .../partition/RewriteAsBinaryOperators.java | 103 +++++++++++++++++++ .../apache/drill/TestCTASPartitionFilter.java | 18 ++++ 3 files changed, 123 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/1c37185b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java index ae18331..869348a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java @@ -338,6 +338,8 @@ public abstract class PruneScanRule extends RelOptRule { condition = RelOptUtil.pushFilterPastProject(filterRel.getCondition(), projectRel); } + RewriteAsBinaryOperators visitor = new RewriteAsBinaryOperators(true, filterRel.getCluster().getRexBuilder()); + condition = condition.accept(visitor); Map<Integer, String> fieldNameMap = Maps.newHashMap(); List<String> fieldNames = scanRel.getRowType().getFieldNames(); http://git-wip-us.apache.org/repos/asf/drill/blob/1c37185b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/RewriteAsBinaryOperators.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/RewriteAsBinaryOperators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/RewriteAsBinaryOperators.java new file mode 100644 index 0000000..44b9a3a --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/RewriteAsBinaryOperators.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.logical.partition; + +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexRangeRef; +import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; + +import java.util.ArrayList; +import java.util.List; + +/** + * Rewrites an expression tree, replacing OR and AND operators with more than 2 operands with a chained operators + * each with only 2 operands. + * + * e.g. + * + * OR(A, B, C) ---> OR(A, OR(B, C)) + */ + public class RewriteAsBinaryOperators extends RexVisitorImpl<RexNode> { + + RexBuilder builder; + public RewriteAsBinaryOperators(boolean deep, RexBuilder builder) { + super(deep); + this.builder = builder; + } + + @Override + public RexNode visitInputRef(RexInputRef inputRef) { + return inputRef; + } + + @Override + public RexNode visitLiteral(RexLiteral literal) { + return literal; + } + + @Override + public RexNode visitOver(RexOver over) { + return over; + } + + @Override + public RexNode visitCorrelVariable(RexCorrelVariable correlVariable) { + return correlVariable; + } + + @Override + public RexNode visitCall(RexCall call) { + SqlOperator op = call.getOperator(); + SqlKind kind = op.getKind(); + if (kind == SqlKind.OR || kind == SqlKind.AND) { + if (call.getOperands().size() <= 2) { + return call; + } + List<RexNode> children = new ArrayList(call.getOperands()); + RexNode left = children.remove(0); + RexNode right = builder.makeCall(op, children).accept(this); + return builder.makeCall(op, left, right); + } + return call; + } + + @Override + public RexNode visitDynamicParam(RexDynamicParam dynamicParam) { + return dynamicParam; + } + + @Override + public RexNode visitRangeRef(RexRangeRef rangeRef) { + return rangeRef; + } + + @Override + public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { + return fieldAccess; + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/1c37185b/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java b/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java index 3943426..48d7ceb 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java @@ -39,6 +39,14 @@ public class TestCTASPartitionFilter extends PlanTestBase { testPlanMatchingPatterns(query, new String[]{numFilesPattern}, new String[]{excludedFilterPattern}); } + private static void testIncludeFilter(String query, int expectedNumFiles, + String includedFilterPattern, int expectedRowCount) throws Exception { + int actualRowCount = testSql(query); + assertEquals(expectedRowCount, actualRowCount); + String numFilesPattern = "numFiles=" + expectedNumFiles; + testPlanMatchingPatterns(query, new String[]{numFilesPattern, includedFilterPattern}, new String[]{}); + } + @Test public void withDistribution() throws Exception { test("alter session set `planner.slice_target` = 1"); @@ -58,4 +66,14 @@ public class TestCTASPartitionFilter extends PlanTestBase { String query = "select * from orders_no_distribution where o_orderpriority = '1-URGENT'"; testExcludeFilter(query, 2, "Filter", 24); } + + @Test + public void testDRILL3410() throws Exception { + test("alter session set `planner.slice_target` = 1"); + test("alter session set `store.partition.hash_distribute` = true"); + test("use dfs_test.tmp"); + test(String.format("create table drill_3410 partition by (o_orderpriority) as select * from dfs_test.`%s/multilevel/parquet`", TEST_RES_PATH)); + String query = "select * from drill_3410 where (o_orderpriority = '1-URGENT' and o_orderkey = 10) or (o_orderpriority = '2-HIGH' or o_orderkey = 11)"; + testIncludeFilter(query, 1, "Filter", 34); + } } \ No newline at end of file
