Repository: tajo Updated Branches: refs/heads/master 80218d03d -> 158bf1df9
TAJO-2005: Add TableStatUpdateRewriter. Closes #899 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/158bf1df Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/158bf1df Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/158bf1df Branch: refs/heads/master Commit: 158bf1df931d92fc937e53af96ea382055424643 Parents: 80218d0 Author: Hyunsik Choi <[email protected]> Authored: Fri Dec 4 00:22:27 2015 -0800 Committer: Hyunsik Choi <[email protected]> Committed: Fri Dec 4 00:22:27 2015 -0800 ---------------------------------------------------------------------- CHANGES | 2 + .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../planner/TestEvalNodeToExprConverter.java | 12 +-- .../engine/planner/TestJoinOrderAlgorithm.java | 2 +- .../engine/planner/TestLogicalOptimizer.java | 4 +- .../tajo/engine/planner/TestLogicalPlanner.java | 8 +- .../planner/physical/TestHashAntiJoinExec.java | 2 +- .../planner/physical/TestHashSemiJoinExec.java | 2 +- .../planner/physical/TestPhysicalPlanner.java | 2 +- .../engine/planner/physical/TestSortExec.java | 2 +- .../planner/physical/TestSortIntersectExec.java | 2 +- .../tajo/master/TestExecutionBlockCursor.java | 2 +- .../apache/tajo/querymaster/TestKillQuery.java | 4 +- .../org/apache/tajo/master/GlobalEngine.java | 2 +- .../tajo/querymaster/QueryMasterTask.java | 2 +- .../org/apache/tajo/plan/LogicalOptimizer.java | 16 ++-- .../org/apache/tajo/plan/LogicalPlanner.java | 17 ---- .../rewrite/BaseLogicalPlanRewriteEngine.java | 7 ++ .../BaseLogicalPlanRewriteRuleProvider.java | 3 + .../rewrite/LogicalPlanRewriteRuleContext.java | 30 +++---- .../plan/rewrite/TableStatUpdateRewriter.java | 89 ++++++++++++++++++++ 21 files changed, 147 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 1796ff6..75ce90e 100644 --- a/CHANGES +++ b/CHANGES @@ -8,6 +8,8 @@ Release 0.12.0 - unreleased IMPROVEMENT + TAJO-2005: Add TableStatUpdateRewriter. (hyunsik) + TAJO-1948: Change GroupbyNode::setAggFunctions and getAggFunctions to set and get List. (Dongkyu Hwangbo via hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 69f92f4..600a45f 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -103,7 +103,7 @@ public class ExprTestBase { analyzer = new SQLAnalyzer(); preLogicalPlanVerifier = new PreLogicalPlanVerifier(cat); planner = new LogicalPlanner(cat, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(util.getConfiguration(), cat); + optimizer = new LogicalOptimizer(util.getConfiguration(), cat, TablespaceManager.getInstance()); annotatedPlanVerifier = new LogicalPlanVerifier(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java index 018fa0e..1cf4cf2 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java @@ -119,7 +119,7 @@ public class TestEvalNodeToExprConverter { Expr expr = sqlAnalyzer.parse(QUERIES[0]); LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode node = plan.getRootBlock().getRoot(); @@ -149,7 +149,7 @@ public class TestEvalNodeToExprConverter { Expr expr = sqlAnalyzer.parse(QUERIES[1]); LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode node = plan.getRootBlock().getRoot(); @@ -178,7 +178,7 @@ public class TestEvalNodeToExprConverter { Expr expr = sqlAnalyzer.parse(QUERIES[2]); LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode node = plan.getRootBlock().getRoot(); @@ -236,7 +236,7 @@ public class TestEvalNodeToExprConverter { Expr expr = sqlAnalyzer.parse(QUERIES[3]); LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode node = plan.getRootBlock().getRoot(); @@ -291,7 +291,7 @@ public class TestEvalNodeToExprConverter { Expr expr = sqlAnalyzer.parse(QUERIES[4]); LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode node = plan.getRootBlock().getRoot(); @@ -346,7 +346,7 @@ public class TestEvalNodeToExprConverter { Expr expr = sqlAnalyzer.parse(QUERIES[5]); LogicalPlan plan = planner.createPlan(qc, expr); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode node = plan.getRootBlock().getRoot(); http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java index cf5699f..4ae3e5a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java @@ -125,7 +125,7 @@ public class TestJoinOrderAlgorithm { sqlAnalyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); defaultContext = LocalTajoTestingUtility.createDummyContext(util.getConfiguration()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java index b0c5ad1..1b911a1 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java @@ -104,10 +104,10 @@ public class TestLogicalOptimizer { catalog.createFunction(funcDesc); sqlAnalyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); defaultContext = LocalTajoTestingUtility.createDummyContext(util.getConfiguration()); - optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); } @AfterClass http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index d291c09..98874ba 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -511,7 +511,7 @@ public class TestLogicalPlanner { Schema expected = tpch.getOutSchema("q2"); assertSchema(expected, node.getOutSchema()); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode[] nodes = PlannerUtil.findAllNodes(node, NodeType.JOIN); @@ -550,7 +550,7 @@ public class TestLogicalPlanner { LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode[] nodes = PlannerUtil.findAllNodes(node, NodeType.SCAN); @@ -591,7 +591,7 @@ public class TestLogicalPlanner { LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode[] nodes = PlannerUtil.findAllNodes(node, NodeType.SCAN); @@ -638,7 +638,7 @@ public class TestLogicalPlanner { LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); Map<BinaryEval, Boolean> scanMap = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 103d2b1..aeda300 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -133,7 +133,7 @@ public class TestHashAntiJoinExec { catalog.createTable(people); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(conf, catalog); + optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance()); } @After http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index f99dc9e..2b50310 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -137,7 +137,7 @@ public class TestHashSemiJoinExec { catalog.createTable(people); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(conf, catalog); + optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance()); } @After http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index adeb250..53fbf0a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -166,7 +166,7 @@ public class TestPhysicalPlanner { catalog.createTable(score); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(conf, catalog); + optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance()); masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null); createLargeScoreTable(); http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 30a63be..872815d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -113,7 +113,7 @@ public class TestSortExec { queryContext = new QueryContext(conf); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(conf, catalog); + optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance()); } public static String[] QUERIES = { http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java index 66d38b7..df78667 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java @@ -141,7 +141,7 @@ public class TestSortIntersectExec { analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(conf, catalog); + optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance()); } @After http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 578b278..7c6d892 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -80,7 +80,7 @@ public class TestExecutionBlockCursor { analyzer = new SQLAnalyzer(); logicalPlanner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(conf, catalog); + optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance()); dispatcher = new AsyncDispatcher(); dispatcher.init(conf); http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java index f907a2b..ac5efd9 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java @@ -96,7 +96,7 @@ public class TestKillQuery { CatalogService catalog = cluster.getMaster().getCatalog(); LogicalPlanner planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance()); Expr expr = analyzer.parse(queryStr); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -160,7 +160,7 @@ public class TestKillQuery { CatalogService catalog = cluster.getMaster().getCatalog(); LogicalPlanner planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance()); Expr expr = analyzer.parse(queryStr); LogicalPlan plan = planner.createPlan(defaultContext, expr); http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 7f18fd4..c088a8b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -94,7 +94,7 @@ public class GlobalEngine extends AbstractService { preVerifier = new PreLogicalPlanVerifier(context.getCatalog()); planner = new LogicalPlanner(context.getCatalog(), TablespaceManager.getInstance()); // Access path rewriter is enabled only in QueryMasterTask - optimizer = new LogicalOptimizer(context.getConf(), context.getCatalog()); + optimizer = new LogicalOptimizer(context.getConf(), context.getCatalog(), TablespaceManager.getInstance()); annotatedPlanVerifier = new LogicalPlanVerifier(); postLogicalPlanVerifier = new PostLogicalPlanVerifier(); } catch (Throwable t) { http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index d6744c3..995a8e5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -318,7 +318,7 @@ public class QueryMasterTask extends CompositeService { LOG.info(SessionVars.INDEX_ENABLED.keyname() + " : " + queryContext.getBool(SessionVars.INDEX_ENABLED)); CatalogService catalog = getQueryTaskContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalPlanner planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - LogicalOptimizer optimizer = new LogicalOptimizer(systemConf, catalog); + LogicalOptimizer optimizer = new LogicalOptimizer(systemConf, catalog, TablespaceManager.getInstance()); Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class); jsonExpr = null; // remove the possible OOM http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 404e971..e9e6ab2 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -39,6 +39,7 @@ import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleProvider; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.storage.StorageService; import org.apache.tajo.util.ReflectionUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.graph.DirectedGraphCursor; @@ -55,21 +56,24 @@ import static org.apache.tajo.plan.joinorder.GreedyHeuristicJoinOrderAlgorithm.g public class LogicalOptimizer { private static final Log LOG = LogFactory.getLog(LogicalOptimizer.class.getName()); - private CatalogService catalog; + private final CatalogService catalog; + private final StorageService storage; private BaseLogicalPlanRewriteEngine rulesBeforeJoinOpt; private BaseLogicalPlanRewriteEngine rulesAfterToJoinOpt; private JoinOrderAlgorithm joinOrderAlgorithm = new GreedyHeuristicJoinOrderAlgorithm(); - public LogicalOptimizer(TajoConf conf, CatalogService catalog) { + public LogicalOptimizer(TajoConf conf, CatalogService catalog, StorageService storage) { this.catalog = catalog; + this.storage = storage; + // TODO: set the catalog instance to FilterPushdownRule Class clazz = conf.getClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS); LogicalPlanRewriteRuleProvider provider = (LogicalPlanRewriteRuleProvider) ReflectionUtil.newInstance(clazz, conf); - rulesBeforeJoinOpt = new BaseLogicalPlanRewriteEngine(); + rulesBeforeJoinOpt = new BaseLogicalPlanRewriteEngine(storage); rulesBeforeJoinOpt.addRewriteRule(provider.getPreRules()); - rulesAfterToJoinOpt = new BaseLogicalPlanRewriteEngine(); + rulesAfterToJoinOpt = new BaseLogicalPlanRewriteEngine(storage); rulesAfterToJoinOpt.addRewriteRule(provider.getPostRules()); } @@ -81,7 +85,7 @@ public class LogicalOptimizer { } public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws TajoException { - rulesBeforeJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog)); + rulesBeforeJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog, storage)); DirectedGraphCursor<String, BlockEdge> blockCursor = new DirectedGraphCursor<>(plan.getQueryBlockGraph(), plan.getRootBlock().getName()); @@ -94,7 +98,7 @@ public class LogicalOptimizer { } else { LOG.info("Skip join order optimization"); } - rulesAfterToJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog)); + rulesAfterToJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog, storage)); return plan.getRootBlock().getRoot(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 3f2b043..6c38c02 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1334,7 +1334,6 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex QueryBlock block = context.queryBlock; ScanNode scanNode = block.getNodeFromExpr(expr); - updatePhysicalInfo(scanNode.getTableDesc()); // Find expression which can be evaluated at this relation node. // Except for column references, additional expressions used in select list, where clause, order-by clauses @@ -1393,22 +1392,6 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return targets; } - private void updatePhysicalInfo(TableDesc desc) { - - // FAKEFILE is used for test - if (!desc.getMeta().getDataFormat().equals("SYSTEM") && !desc.getMeta().getDataFormat().equals("FAKEFILE")) { - try { - if (desc.getStats() != null) { - desc.getStats().setNumBytes(storage.getTableVolumn(desc.getUri())); - } - } catch (UnsupportedException t) { - LOG.warn(desc.getName() + " does not support Tablespace::getTableVolume()"); - // -1 means unknown volume size. - desc.getStats().setNumBytes(-1); - } - } - } - @Override public TableSubQueryNode visitTableSubQuery(PlanContext context, Stack<Expr> stack, TablePrimarySubQuery expr) throws TajoException { http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java index b39575e..f38b202 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.storage.StorageService; import java.util.LinkedHashMap; import java.util.Map; @@ -35,9 +36,15 @@ public class BaseLogicalPlanRewriteEngine implements LogicalPlanRewriteEngine { /** class logger */ private Log LOG = LogFactory.getLog(BaseLogicalPlanRewriteEngine.class); + private final StorageService storage; + /** a map for query rewrite rules */ private Map<String, LogicalPlanRewriteRule> rewriteRules = new LinkedHashMap<>(); + public BaseLogicalPlanRewriteEngine(StorageService storage) { + this.storage = storage; + } + /** * Add a query rewrite rule to this engine. * http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java index 120529c..3776f62 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java @@ -47,6 +47,9 @@ public class BaseLogicalPlanRewriteRuleProvider extends LogicalPlanRewriteRulePr rules.add(FilterPushDownRule.class); } + // for updating table stats + rules.add(TableStatUpdateRewriter.class); + return rules; } http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java index 6c43112..a899961 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java @@ -21,26 +21,24 @@ package org.apache.tajo.plan.rewrite; import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.storage.StorageService; public class LogicalPlanRewriteRuleContext { - private OverridableConf queryContext; - private LogicalPlan plan; + private final OverridableConf queryContext; + private final LogicalPlan plan; private CatalogService catalog; + private StorageService storage; - public LogicalPlanRewriteRuleContext(OverridableConf queryContext, LogicalPlan plan) { - setQueryContext(queryContext); - setPlan(plan); - } - - public LogicalPlanRewriteRuleContext(OverridableConf queryContext, LogicalPlan plan, CatalogService catalog) { - setQueryContext(queryContext); - setPlan(plan); - setCatalog(catalog); + public LogicalPlanRewriteRuleContext(OverridableConf context, LogicalPlan plan) { + this.queryContext = context; + this.plan = plan; } - public void setCatalog(CatalogService catalog) { + public LogicalPlanRewriteRuleContext(OverridableConf context, LogicalPlan plan, CatalogService catalog, StorageService storage) { + this(context, plan); this.catalog = catalog; + this.storage = storage; } public CatalogService getCatalog() { @@ -51,15 +49,11 @@ public class LogicalPlanRewriteRuleContext { return queryContext; } - public void setQueryContext(OverridableConf queryContext) { - this.queryContext = queryContext; - } - public LogicalPlan getPlan() { return plan; } - public void setPlan(LogicalPlan plan) { - this.plan = plan; + public StorageService getStorage() { + return storage; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/158bf1df/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java new file mode 100644 index 0000000..126b10e --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.storage.StorageService; + +import java.util.Stack; + +public class TableStatUpdateRewriter implements LogicalPlanRewriteRule { + private static final Log LOG = LogFactory.getLog(TableStatUpdateRewriter.class); + + private static final String NAME = "Table Stat Updater"; + + @Override + public String getName() { + return NAME; + } + + @Override + public boolean isEligible(LogicalPlanRewriteRuleContext context) { + return true; + } + + @Override + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException { + LogicalPlan plan = context.getPlan(); + LogicalPlan.QueryBlock rootBlock = plan.getRootBlock(); + + Rewriter r = new Rewriter(context.getStorage()); + r.visit(rootBlock, plan, rootBlock, rootBlock.getRoot(), new Stack<>()); + return plan; + } + + private final class Rewriter extends BasicLogicalPlanVisitor<Object, Object> { + private final StorageService storage; + + + private Rewriter(StorageService storage) { + this.storage = storage; + } + + @Override + public Object visitScan(Object object, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode, + Stack<LogicalNode> stack) throws TajoException { + updatePhysicalInfo(scanNode.getTableDesc()); + return scanNode; + } + + private void updatePhysicalInfo(TableDesc desc) { + // FAKEFILE is used for test + if (!desc.getMeta().getDataFormat().equals("SYSTEM") && !desc.getMeta().getDataFormat().equals("FAKEFILE")) { + try { + if (desc.getStats() != null) { + desc.getStats().setNumBytes(storage.getTableVolumn(desc.getUri())); + } + } catch (UnsupportedException t) { + LOG.warn(desc.getName() + " does not support Tablespace::getTableVolume()"); + // -1 means unknown volume size. + desc.getStats().setNumBytes(-1); + } + } + } + } +}
