DRILL-6173: Support transitive closure during filter push down and partition pruning
closes #1216 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6fcaf426 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6fcaf426 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6fcaf426 Branch: refs/heads/master Commit: 6fcaf4268eddcb09010b5d9c5dfb3b3be5c3f903 Parents: 9173308 Author: Vitalii Diravka <[email protected]> Authored: Tue Apr 17 14:38:03 2018 +0300 Committer: Vitalii Diravka <[email protected]> Committed: Sun Apr 29 23:20:44 2018 +0300 ---------------------------------------------------------------------- .../drill/exec/TestHivePartitionPruning.java | 15 + .../apache/drill/exec/planner/PlannerPhase.java | 37 ++- .../apache/drill/exec/planner/RuleInstance.java | 10 + .../planner/sql/handlers/DefaultSqlHandler.java | 10 +- .../java/org/apache/drill/PlanTestBase.java | 7 + ...tFilterPushdownWithTransitivePredicates.java | 276 +++++++++++++++++++ .../transitiveClosure/first/0_0_1.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_10.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_11.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_12.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_13.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_14.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_15.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_16.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_2.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_3.parquet | Bin 0 -> 601 bytes .../transitiveClosure/first/0_0_4.parquet | Bin 0 -> 601 bytes .../transitiveClosure/first/0_0_5.parquet | Bin 0 -> 601 bytes .../transitiveClosure/first/0_0_6.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_7.parquet | Bin 0 -> 601 bytes .../transitiveClosure/first/0_0_8.parquet | Bin 0 -> 585 bytes .../transitiveClosure/first/0_0_9.parquet | Bin 0 -> 585 bytes .../transitiveClosure/second/0_0_1.parquet | Bin 0 -> 601 bytes .../transitiveClosure/second/0_0_2.parquet | Bin 0 -> 633 bytes .../transitiveClosure/second/0_0_3.parquet | Bin 0 -> 617 bytes .../transitiveClosure/second/0_0_4.parquet | Bin 0 -> 617 bytes .../transitiveClosure/second/0_0_5.parquet | Bin 0 -> 601 bytes .../transitiveClosure/second/0_0_6.parquet | Bin 0 -> 601 bytes .../transitiveClosure/second/0_0_7.parquet | Bin 0 -> 633 bytes .../transitiveClosure/third/0_0_1.parquet | Bin 0 -> 376 bytes .../transitiveClosure/third/0_0_2.parquet | Bin 0 -> 376 bytes .../transitiveClosure/third/0_0_3.parquet | Bin 0 -> 376 bytes .../transitiveClosure/third/0_0_4.parquet | Bin 0 -> 376 bytes .../transitiveClosure/third/0_0_5.parquet | Bin 0 -> 403 bytes .../transitiveClosure/third/0_0_6.parquet | Bin 0 -> 385 bytes .../transitiveClosure/third/0_0_7.parquet | Bin 0 -> 385 bytes 36 files changed, 347 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java index 7583f42..971349e 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java @@ -161,6 +161,21 @@ public class TestHivePartitionPruning extends HiveTestBase { assertEquals(-1, secondColumnIndex); } + @Test // DRILL-6173 + public void prunePartitionsBasedOnTransitivePredicates() throws Exception { + String query = String.format("SELECT * FROM hive.partition_pruning_test t1 " + + "JOIN hive.partition_with_few_schemas t2 ON t1.`d` = t2.`d` AND t1.`e` = t2.`e` " + + "WHERE t2.`e` IS NOT NULL AND t1.`d` = 1"); + + int actualRowCount = testSql(query); + int expectedRowCount = 450; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = + {"partition_with_few_schemas.*numPartitions=6", "partition_pruning_test.*numPartitions=6"}; + testPlanMatchingPatterns(query, expectedPlan); + } + @AfterClass public static void disableDecimalDataType() throws Exception { test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY)); http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java index 7e4c8c6..17fedc4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java @@ -181,9 +181,15 @@ public enum PlannerPhase { }, PRE_LOGICAL_PLANNING("Planning with Hep planner only for rules, which are failed for Volcano planner") { - public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) { + public RuleSet getRules (OptimizerRulesContext context, Collection<StoragePlugin> plugins) { return PlannerPhase.getSetOpTransposeRules(); } + }, + + TRANSITIVE_CLOSURE("Transitive closure") { + public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) { + return getJoinTransitiveClosureRules(); + } }; public final String description; @@ -399,7 +405,11 @@ public enum PlannerPhase { } - // Ruleset for join permutation, used only in VolcanoPlanner. + /** + * RuleSet for join permutation, used only in VolcanoPlanner. + * @param optimizerRulesContext shared state used during planning + * @return set of planning rules + */ static RuleSet getJoinPermRules(OptimizerRulesContext optimizerRulesContext) { return RuleSets.ofList(ImmutableSet.<RelOptRule> builder().add( RuleInstance.JOIN_PUSH_THROUGH_JOIN_RULE_RIGHT, @@ -496,20 +506,35 @@ public enum PlannerPhase { ).build(); } - /** * Get an immutable list of rules to transpose SetOp(Union) operator with other operators.<p> * Note: Used by Hep planner only (failed for Volcano planner - CALCITE-1271) * * @return SetOp(Union) transpose rules - **/ + */ private static RuleSet getSetOpTransposeRules() { - return RuleSets.ofList(ImmutableSet.<RelOptRule>builder() + return RuleSets.ofList(ImmutableSet.<RelOptRule> builder() .add( RuleInstance.FILTER_SET_OP_TRANSPOSE_RULE, RuleInstance.PROJECT_SET_OP_TRANSPOSE_RULE ).build()); } - + /** + * RuleSet for join transitive closure, used only in HepPlanner.<p> + * TODO: {@link RuleInstance#JOIN_PUSH_TRANSITIVE_PREDICATES_RULE} should be copied to #staticRuleSet, + * once CALCITE-1048 is solved. This still should be present in {@link #TRANSITIVE_CLOSURE} stage + * for applying additional filters before {@link #DIRECTORY_PRUNING}. + * + * @return set of planning rules + */ + static RuleSet getJoinTransitiveClosureRules() { + return RuleSets.ofList(ImmutableSet.<RelOptRule> builder() + .add( + DrillFilterJoinRules.DRILL_FILTER_ON_JOIN, + DrillFilterJoinRules.DRILL_JOIN, + RuleInstance.JOIN_PUSH_TRANSITIVE_PREDICATES_RULE, + RuleInstance.FILTER_MERGE_RULE + ).build()); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/main/java/org/apache/drill/exec/planner/RuleInstance.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/RuleInstance.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/RuleInstance.java index f8a394c..49feb41 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/RuleInstance.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/RuleInstance.java @@ -31,6 +31,7 @@ import org.apache.calcite.rel.rules.FilterMergeRule; import org.apache.calcite.rel.rules.FilterSetOpTransposeRule; import org.apache.calcite.rel.rules.JoinPushExpressionsRule; import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; +import org.apache.calcite.rel.rules.JoinPushTransitivePredicatesRule; import org.apache.calcite.rel.rules.ProjectRemoveRule; import org.apache.calcite.rel.rules.ProjectSetOpTransposeRule; import org.apache.calcite.rel.rules.ProjectToWindowRule; @@ -108,4 +109,13 @@ public interface RuleInstance { AbstractConverter.ExpandConversionRule EXPAND_CONVERSION_RULE = new AbstractConverter.ExpandConversionRule(DrillRelFactories.LOGICAL_BUILDER); + + /** + * Instance of the rule that infers predicates from on a + * {@link org.apache.calcite.rel.core.Join} and creates + * {@link org.apache.calcite.rel.core.Filter}s if those predicates can be pushed + * to its inputs. + */ + JoinPushTransitivePredicatesRule JOIN_PUSH_TRANSITIVE_PREDICATES_RULE = + new JoinPushTransitivePredicatesRule(Join.class, DrillRelFactories.LOGICAL_BUILDER); } http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java index 1867b46..e8dece6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java @@ -233,13 +233,19 @@ public class DefaultSqlHandler extends AbstractSqlHandler { } try { - final RelNode convertedRelNode; + // HEP for rules, which are failed at the LOGICAL_PLANNING stage for Volcano planner final RelNode setOpTransposeNode = transform(PlannerType.HEP, PlannerPhase.PRE_LOGICAL_PLANNING, relNode); + + // HEP Join Push Transitive Predicates + final RelNode transitiveClosureNode = + transform(PlannerType.HEP, PlannerPhase.TRANSITIVE_CLOSURE, setOpTransposeNode); + // HEP Directory pruning . - final RelNode pruned = transform(PlannerType.HEP_BOTTOM_UP, PlannerPhase.DIRECTORY_PRUNING, setOpTransposeNode); + final RelNode pruned = transform(PlannerType.HEP_BOTTOM_UP, PlannerPhase.DIRECTORY_PRUNING, transitiveClosureNode); final RelTraitSet logicalTraits = pruned.getTraitSet().plus(DrillRel.DRILL_LOGICAL); + final RelNode convertedRelNode; if (!context.getPlannerSettings().isHepOptEnabled()) { // hep is disabled, use volcano convertedRelNode = transform(PlannerType.VOLCANO, PlannerPhase.LOGICAL_PRUNE_AND_JOIN, pruned, logicalTraits); http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java index 553972a..b8847ff 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java @@ -104,6 +104,13 @@ public class PlanTestBase extends BaseTestQuery { } } + /** + * The same as above, but without excludedPatterns + */ + public static void testPlanMatchingPatterns(String query, String[] expectedPatterns) throws Exception { + testPlanMatchingPatterns(query, expectedPatterns, null); + } + private static Pattern[] stringsToPatterns(String[] strings) { if (strings == null) { http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushdownWithTransitivePredicates.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushdownWithTransitivePredicates.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushdownWithTransitivePredicates.java new file mode 100644 index 0000000..a58aebc --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushdownWithTransitivePredicates.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.parquet; + +import org.apache.drill.PlanTestBase; +import org.apache.drill.exec.util.StoragePluginTestUtils; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; + +public class TestParquetFilterPushdownWithTransitivePredicates extends PlanTestBase { + + private static final String TABLE_PATH = "parquetFilterPush/transitiveClosure"; + private static final String FIRST_TABLE_NAME = String.format("%s.`%s/%s`", + StoragePluginTestUtils.DFS_PLUGIN_NAME, TABLE_PATH, "first"); + private static final String SECOND_TABLE_NAME = String.format("%s.`%s/%s`", + StoragePluginTestUtils.DFS_PLUGIN_NAME, TABLE_PATH, "second"); + private static final String THIRD_TABLE_NAME = String.format("%s.`%s/%s`", + StoragePluginTestUtils.DFS_PLUGIN_NAME, TABLE_PATH, "third"); + + @BeforeClass + public static void copyData() { + dirTestWatcher.copyResourceToRoot(Paths.get(TABLE_PATH)); + } + + @Test + public void testForSeveralInnerJoins() throws Exception { + String query = String.format("SELECT * FROM %s t1 JOIN %s t2 ON t1.`month` = t2.`month` " + + "JOIN %s t3 ON t1.`period` = t3.`period` WHERE t2.`month` = 7 AND t1.`period` = 2", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 24; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=1", "second.*numRowGroups=1", "third.*numRowGroups=3"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test + public void testForFilterInJoinOperator() throws Exception { + String query = String.format("SELECT * FROM %s t1 JOIN %s t2 ON t1.`month` = t2.`month` AND t2.`month` = 7 " + + "JOIN %s t3 ON t1.`period` = t3.`period` AND t1.`period` = 2", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 24; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=1", "second.*numRowGroups=1", "third.*numRowGroups=3"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test + public void testForLeftAndRightJoins() throws Exception { + String query = String.format("SELECT * FROM %s t1 RIGHT JOIN %s t2 ON t1.`year` = t2.`year` " + + "LEFT JOIN %s t3 ON t1.`period` = t3.`period` WHERE t2.`year` = 1987 AND t1.`period` = 1", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 54; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=2", "second.*numRowGroups=2", "third.*numRowGroups=3"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test + public void testForCommaSeparatedJoins() throws Exception { + String query = String.format("SELECT * FROM %s t1, %s t2, %s t3 WHERE t1.`year` = t2.`year` " + + "AND t1.`period` = t3.`period` AND t2.`year` = 1990 AND t3.`period` = 1", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 24; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=2", "second.*numRowGroups=2", "third.*numRowGroups=3"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test + public void testForInAndNotOperators() throws Exception { + String query = String.format("SELECT * FROM %s t1 JOIN %s t2 " + + "ON t1.`year` = t2.`year` JOIN %s t3 ON t1.`period` = t3.`period` " + + "WHERE t2.`year` NOT IN (1987, 1988) AND t3.`period` IN (1, 2)", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + + int actualRowCount = testSql(query); + int expectedRowCount = 72; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=6", "second.*numRowGroups=3", "third.*numRowGroups=4"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test + public void testForBetweenOperator() throws Exception { + String query = String.format("SELECT * FROM %s t1 JOIN %s t2 " + + "ON t1.`year` = t2.`year` JOIN %s t3 ON t1.`period` = t3.`period` " + + "WHERE t2.`year` BETWEEN 1988 AND 1991 AND t3.`period` BETWEEN 2 AND 4 ", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 96; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=7", "second.*numRowGroups=5", "third.*numRowGroups=6"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test + public void testForGreaterThanAndLessThanOperators() throws Exception { + String query = String.format("SELECT * FROM %s t1 JOIN %s t2 " + + "ON t1.`year` = t2.`year` JOIN %s t3 ON t1.`period` = t3.`period` " + + "WHERE t2.`year` >= 1990 AND t3.`period` < 2", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 36; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=3", "second.*numRowGroups=3", "third.*numRowGroups=3"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + + @Test + public void testForSubQuery() throws Exception { + String query = String.format("SELECT * FROM %s t1 JOIN " + + "(SELECT `year`, `month` FROM %s WHERE `year` = 1987 AND `month` = 5) t2 " + + "ON t1.`year` = t2.`year` AND t1.`month` = t2.`month`", + FIRST_TABLE_NAME, SECOND_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 4; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=2", "second.*numRowGroups=1"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test + public void testForWithStatement() throws Exception { + String query = String.format("WITH `first_date` AS (SELECT `year`, `month` FROM %s WHERE `year` = 1987 and `month` = 5) " + + "SELECT t2.`year`, t2.`month` FROM %s t2 JOIN `first_date` ON t2.`year` = `first_date`.`year` AND t2.`month` = `first_date`.`month`", + FIRST_TABLE_NAME, SECOND_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 4; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=2", "second.*numRowGroups=1"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test // TODO: CALCITE-1048 + @Ignore // For now plan has "first.*numRowGroups=7". Replacing left join to inner should be made earlier. + public void testForTwoExists() throws Exception { + String query = String.format("SELECT * from %s t1 " + + " WHERE EXISTS (SELECT * FROM %s t2 WHERE t1.`year` = t2.`year` AND t2.`year` = 1988) " + + " AND EXISTS (SELECT * FROM %s t3 WHERE t1.`period` = t3.`period` AND t3.`period` = 2)", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 2; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=2", "second.*numRowGroups=2", "third.*numRowGroups=3"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test // TODO: CALCITE-1048 + @Ignore // For now plan has "first.*numRowGroups=16" + public void testForFilterInHaving() throws Exception { + String query = String.format("SELECT t1.`year`, t2.`year`, t1.`period`, t3.`period` FROM %s t1 " + + "JOIN %s t2 ON t1.`year` = t2.`year` " + + "JOIN %s t3 ON t1.`period` = t3.`period` " + + "GROUP BY t1.`year`, t2.`year`, t1.`period`, t3.`period` " + + "HAVING t2.`year` = 1987 AND t3.`period` = 1", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 1; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=2", "second.*numRowGroups=2", "third.*numRowGroups=3"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test // TODO: CALCITE-2241 + @Ignore // For now plan has "first.*numRowGroups=16", "second.*numRowGroups=7" + public void testForOrOperator() throws Exception { + String query = String.format("SELECT * FROM %s t1 " + + "JOIN %s t2 ON t1.`month` = t2.`month` " + + "WHERE t2.`month` = 4 OR t1.`month` = 11", + FIRST_TABLE_NAME, SECOND_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 13; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=4", "second.*numRowGroups=2"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test // TODO: CALCITE-2275 + @Ignore // For now plan has "first.*numRowGroups=14"" + public void testForInAndNotOperatorsInJoinCondition() throws Exception { + String query = String.format("SELECT * FROM %s t1 JOIN %s t2 " + + "ON t1.`year` = t2.`year` AND t2.`year` NOT IN (1987, 1988) JOIN %s t3 ON t1.`period` = t3.`period` " + + "WHERE t3.`period` IN (1, 2)", + FIRST_TABLE_NAME, SECOND_TABLE_NAME, THIRD_TABLE_NAME); + + + int actualRowCount = testSql(query); + int expectedRowCount = 72; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=6", "second.*numRowGroups=3", "third.*numRowGroups=4"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test // TODO: CALCITE-2274 + @Ignore // For now plan has "first.*numRowGroups=16"" + public void testForSubQueryAndDynamicStar() throws Exception { + String query = String.format("SELECT * FROM %s t1 JOIN " + + "(SELECT * FROM %s WHERE `year` = 1987 AND `month` = 5) t2 ON t1.`year` = t2.`year` AND t1.`month` = t2.`month`", + FIRST_TABLE_NAME, SECOND_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 4; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=2", "second.*numRowGroups=1"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + @Test // TODO: CALCITE-2274 + @Ignore // For now plan has "second.*numRowGroups=7" + public void testForWithStatementAndDynamicStar() throws Exception { + String query = String.format("WITH `first_date` AS (SELECT * FROM %s t1 WHERE t1.`year` = 1987 and t1.`month` = 5) " + + "SELECT * FROM %s t2 JOIN `first_date` ON t2.`year` = `first_date`.`year` AND t2.`month` = `first_date`.`month`", + FIRST_TABLE_NAME, SECOND_TABLE_NAME); + + int actualRowCount = testSql(query); + int expectedRowCount = 4; + assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount); + + final String[] expectedPlan = {"first.*numRowGroups=2", "second.*numRowGroups=1"}; + testPlanMatchingPatterns(query, expectedPlan); + } + + +} + http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_1.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_1.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_1.parquet new file mode 100644 index 0000000..42584c8 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_1.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_10.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_10.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_10.parquet new file mode 100644 index 0000000..896cd3e Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_10.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_11.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_11.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_11.parquet new file mode 100644 index 0000000..6cb20ed Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_11.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_12.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_12.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_12.parquet new file mode 100644 index 0000000..04034fd Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_12.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_13.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_13.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_13.parquet new file mode 100644 index 0000000..1c73e79 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_13.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_14.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_14.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_14.parquet new file mode 100644 index 0000000..a8bf1db Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_14.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_15.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_15.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_15.parquet new file mode 100644 index 0000000..39589b2 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_15.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_16.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_16.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_16.parquet new file mode 100644 index 0000000..4eb1159 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_16.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_2.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_2.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_2.parquet new file mode 100644 index 0000000..af55f4e Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_2.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_3.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_3.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_3.parquet new file mode 100644 index 0000000..fda90b5 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_3.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_4.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_4.parquet new file mode 100644 index 0000000..576321b Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_5.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_5.parquet new file mode 100644 index 0000000..69111e7 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_6.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_6.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_6.parquet new file mode 100644 index 0000000..f454df5 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_6.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_7.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_7.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_7.parquet new file mode 100644 index 0000000..7e2cf88 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_7.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_8.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_8.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_8.parquet new file mode 100644 index 0000000..cd13270 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_8.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_9.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_9.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_9.parquet new file mode 100644 index 0000000..a30bb83 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/first/0_0_9.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_1.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_1.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_1.parquet new file mode 100644 index 0000000..a9c4dfa Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_1.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_2.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_2.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_2.parquet new file mode 100644 index 0000000..0a38a86 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_2.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_3.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_3.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_3.parquet new file mode 100644 index 0000000..3065362 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_3.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_4.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_4.parquet new file mode 100644 index 0000000..32898ea Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_5.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_5.parquet new file mode 100644 index 0000000..0cac841 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_6.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_6.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_6.parquet new file mode 100644 index 0000000..3f52494 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_6.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_7.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_7.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_7.parquet new file mode 100644 index 0000000..925b9a9 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/second/0_0_7.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_1.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_1.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_1.parquet new file mode 100644 index 0000000..e286ef2 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_1.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_2.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_2.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_2.parquet new file mode 100644 index 0000000..d8ed9ad Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_2.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_3.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_3.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_3.parquet new file mode 100644 index 0000000..357dcad Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_3.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_4.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_4.parquet new file mode 100644 index 0000000..75a4c59 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_5.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_5.parquet new file mode 100644 index 0000000..67d14ba Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_6.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_6.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_6.parquet new file mode 100644 index 0000000..2828238 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_6.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/6fcaf426/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_7.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_7.parquet b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_7.parquet new file mode 100644 index 0000000..402adc6 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquetFilterPush/transitiveClosure/third/0_0_7.parquet differ
