Repository: tajo Updated Branches: refs/heads/branch-0.11.2 239370a8b -> 3f994bf2e
TAJO-2093: Partition Pruning doesn't handle Constant folding occasionally for BETWEEN clause. Closes #975 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3f994bf2 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3f994bf2 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3f994bf2 Branch: refs/heads/branch-0.11.2 Commit: 3f994bf2efa0bd9b5b2c5fddfbc218ddec25a8c2 Parents: 239370a Author: JaeHwa Jung <[email protected]> Authored: Tue Mar 22 10:32:59 2016 +0900 Committer: JaeHwa Jung <[email protected]> Committed: Tue Mar 22 10:32:59 2016 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../planner/TestPartitionedTableRewriter.java | 152 ++++++++++++++++++- .../apache/tajo/plan/expr/AlgebraicUtil.java | 2 +- .../plan/exprrewrite/rules/ConstantFolding.java | 16 ++ 4 files changed, 171 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/3f994bf2/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 71ba9d5..80c4a3f 100644 --- a/CHANGES +++ b/CHANGES @@ -24,6 +24,9 @@ Release 0.11.2 - unreleased BUG FIXES + TAJO-2093: Partition Pruning doesn't handle Constant folding occasionally + for BETWEEN clause. (jaehwa) + TAJO-2100: Add missing cancellation in defaultTaskScheduler when a worker is no respond. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/3f994bf2/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java index e8b1011..8005893 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java @@ -41,7 +41,9 @@ import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.KeyValueSet; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestName; import static org.junit.Assert.*; @@ -50,6 +52,10 @@ public class TestPartitionedTableRewriter extends QueryTestCaseBase { final static String PARTITION_TABLE_NAME = "tb_partition"; final static String MULTIPLE_PARTITION_TABLE_NAME = "tb_multiple_partition"; + // for getting a method name + @Rule + public TestName name = new TestName(); + @BeforeClass public static void setUp() throws Exception { FileSystem fs = FileSystem.get(conf); @@ -384,7 +390,7 @@ public class TestPartitionedTableRewriter extends QueryTestCaseBase { @Test public final void testPartitionPruningWitCTAS() throws Exception { - String tableName = "testPartitionPruningUsingDirectories".toLowerCase(); + String tableName = name.getMethodName().toLowerCase(); String canonicalTableName = CatalogUtil.getCanonicalTableName("\"TestPartitionedTableRewriter\"", tableName); executeString( @@ -429,4 +435,148 @@ public class TestPartitionedTableRewriter extends QueryTestCaseBase { executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); } + + + @Test + public void testConstantFoldingWithStringFunctions() throws Exception { + Expr expr = sqlParser.parse("SELECT * FROM " + MULTIPLE_PARTITION_TABLE_NAME + + " WHERE key1 between lower('PART123') and lower('PART125') order by n_nationkey"); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SORT, projNode.getChild().getType()); + SortNode sortNode = projNode.getChild(); + + assertEquals(NodeType.SELECTION, sortNode.getChild().getType()); + SelectionNode selNode = sortNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(2, filteredPaths.length); + + assertEquals("key3=1", filteredPaths[0].getName()); + assertEquals("key2=supp123", filteredPaths[0].getParent().getName()); + assertEquals("key1=part123", filteredPaths[0].getParent().getParent().getName()); + + assertEquals("key3=2", filteredPaths[1].getName()); + assertEquals("key2=supp123", filteredPaths[1].getParent().getName()); + assertEquals("key1=part123", filteredPaths[1].getParent().getParent().getName()); + } + + @Test + public final void testConstantFoldingWithExpression() throws Exception { + String tableName = name.getMethodName().toLowerCase(); + String canonicalTableName = CatalogUtil.getCanonicalTableName("\"TestPartitionedTableRewriter\"", tableName); + + executeString( + "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key float8) " + + " as select l_orderkey, l_partkey, l_quantity from default.lineitem"); + + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), tableName); + assertNotNull(tableDesc); + + Expr expr = sqlParser.parse("SELECT * FROM " + canonicalTableName + + " WHERE key between round(abs(35.0 + 1.0)) and round(abs(37.0 + 1.0)) ORDER BY key"); + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SORT, projNode.getChild().getType()); + SortNode sortNode = projNode.getChild(); + + assertEquals(NodeType.SELECTION, sortNode.getChild().getType()); + SelectionNode selNode = sortNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(2, filteredPaths.length); + assertEquals("key=36.0", filteredPaths[0].getName()); + assertEquals("key=38.0", filteredPaths[1].getName()); + + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); + } + + @Test + public final void testConstantFoldingWithDateFunctions() throws Exception { + String tableName = name.getMethodName().toLowerCase(); + String canonicalTableName = CatalogUtil.getCanonicalTableName("\"TestPartitionedTableRewriter\"", tableName); + + String[] partitionKeys = new String[] {"20160315", "20160316", "20160317"}; + + executeString( + "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(reg_date text) " + + " as select l_orderkey, l_partkey" + + ", case " + + " when l_orderkey = 2 then '20160315' " + + " when l_orderkey = 3 then '20160316' " + + " else '20160317' end as reg_date" + + " from default.lineitem"); + + TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), tableName); + assertNotNull(tableDesc); + + Expr expr = sqlParser.parse("SELECT * FROM " + canonicalTableName + + " WHERE reg_date between TO_CHAR( ADD_DAYS(TO_DATE('2016-03-14','YYYY-MM-DD'), -1) , 'YYYYMMDD') " + + " AND TO_CHAR( ADD_DAYS(TO_DATE('2016-03-14','YYYY-MM-DD'), 1) , 'YYYYMMDD') ORDER BY reg_date"); + + QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration()); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); + LogicalNode plan = newPlan.getRootBlock().getRoot(); + + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + + ProjectionNode projNode = root.getChild(); + + assertEquals(NodeType.SORT, projNode.getChild().getType()); + SortNode sortNode = projNode.getChild(); + + assertEquals(NodeType.SELECTION, sortNode.getChild().getType()); + SelectionNode selNode = sortNode.getChild(); + assertTrue(selNode.hasQual()); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + scanNode.setQual(selNode.getQual()); + + PartitionedTableRewriter rewriter = new PartitionedTableRewriter(); + OverridableConf conf = CommonTestingUtil.getSessionVarsForTest(); + + Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode); + assertNotNull(filteredPaths); + + assertEquals(1, filteredPaths.length); + assertEquals("reg_date=" + partitionKeys[0], filteredPaths[0].getName()); + + executeString("DROP TABLE " + canonicalTableName + " PURGE").close(); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/3f994bf2/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java index 19d5d16..e5f8596 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java @@ -192,7 +192,7 @@ public class AlgebraicUtil { } private final static AlgebraicOptimizer algebraicOptimizer = new AlgebraicOptimizer(); - + /** * Simplify the given expr. That is, all subexprs consisting of only constants * are calculated immediately. http://git-wip-us.apache.org/repos/asf/tajo/blob/3f994bf2/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java index 37b77fd..23ed78e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java @@ -78,6 +78,22 @@ public class ConstantFolding extends SimpleEvalNodeVisitor<LogicalPlanner.PlanCo return unaryEval; } + @Override + protected EvalNode visitBetween(LogicalPlanner.PlanContext context, BetweenPredicateEval evalNode, + Stack<EvalNode> stack) { + stack.push(evalNode); + + EvalNode predicand = visit(context, evalNode.getPredicand(), stack); + EvalNode begin = visit(context, evalNode.getBegin(), stack); + EvalNode end = visit(context, evalNode.getEnd(), stack); + + evalNode.setPredicand(predicand); + evalNode.setBegin(begin); + evalNode.setEnd(end); + + return evalNode; + } + // exceptional func names not to use constant folding private static final Set<String> NON_CONSTANT_FUNC_NAMES = new HashSet<String>(Arrays.asList("sleep", "random"));
