Repository: tajo Updated Branches: refs/heads/master 18b898ffb -> 97e61e6f4
TAJO-1713: Change the type of edge cache in JoinGraphContext from HashMap to LRUMap. Closes #654 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/97e61e6f Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/97e61e6f Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/97e61e6f Branch: refs/heads/master Commit: 97e61e6f4e9ff1ebeac388dd22e139e1fd486e99 Parents: 18b898f Author: Jihoon Son <[email protected]> Authored: Thu Jul 30 10:24:52 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Thu Jul 30 10:24:52 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 +++ .../tajo/util/graph/SimpleDirectedGraph.java | 11 +++++++++ .../planner/physical/TestBSTIndexExec.java | 1 + .../engine/planner/physical/TestSortExec.java | 1 + .../org/apache/tajo/plan/LogicalOptimizer.java | 4 +-- .../apache/tajo/plan/joinorder/JoinGraph.java | 1 - .../tajo/plan/joinorder/JoinGraphContext.java | 26 +++++++++++++++++--- 7 files changed, 40 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 4632ddf..2b9ea92 100644 --- a/CHANGES +++ b/CHANGES @@ -373,6 +373,9 @@ Release 0.11.0 - unreleased TASKS + TAJO-1713: Change the type of edge cache in JoinGraphContext from + HashMap to LRUMap. (jihoon) + TAJO-1273: Merge DirectRawFile to master branch. (jinho) TAJO-1628: Add a documentation for join operation. (jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java index d72a2dd..045add5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java @@ -37,6 +37,17 @@ public class SimpleDirectedGraph<V, E> implements DirectedGraph<V,E> { /** map: parent -> child */ protected Map<V, Map<V, E>> reversedEdges = TUtil.newLinkedHashMap(); + public void clear() { + for (Map<V, E> eachEdge : directedEdges.values()) { + eachEdge.clear(); + } + for (Map<V, E> eachEdge : reversedEdges.values()) { + eachEdge.clear(); + } + directedEdges.clear(); + reversedEdges.clear(); + } + @Override public int getVertexSize() { return directedEdges.size(); http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index 036ca78..6fb7a45 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -83,6 +83,7 @@ public class TestBSTIndexExec { public void setup() throws Exception { this.randomValues = new HashMap<Integer, Integer>(); this.conf = new TajoConf(); + conf.set(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE); util = new TajoTestingCluster(); util.startCatalogCluster(); catalog = util.getMiniCatalogCluster().getCatalog(); http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 743c51d..1c55d10 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -72,6 +72,7 @@ public class TestSortExec { @BeforeClass public static void setUp() throws Exception { conf = new TajoConf(); + conf.set(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE); util = TpchTestBase.getInstance().getTestingCluster(); catalog = util.getMaster().getCatalog(); workDir = CommonTestingUtil.getTestDir(TEST_PATH); http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 2bfa4a7..fe49994 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -32,7 +32,6 @@ import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.expr.EvalTreeUtil; -import org.apache.tajo.plan.expr.EvalType; import org.apache.tajo.plan.joinorder.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.BaseLogicalPlanRewriteEngine; @@ -60,7 +59,6 @@ public class LogicalOptimizer { private JoinOrderAlgorithm joinOrderAlgorithm = new GreedyHeuristicJoinOrderAlgorithm(); public LogicalOptimizer(TajoConf conf) { - Class clazz = conf.getClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS); LogicalPlanRewriteRuleProvider provider = (LogicalPlanRewriteRuleProvider) ReflectionUtil.newInstance(clazz, conf); @@ -129,6 +127,8 @@ public class LogicalOptimizer { String optimizedOrder = JoinOrderStringBuilder.buildJoinOrderString(plan, block); block.addPlanHistory("Non-optimized join order: " + originalOrder + " (cost: " + nonOptimizedJoinCost + ")"); block.addPlanHistory("Optimized join order : " + optimizedOrder + " (cost: " + order.getCost() + ")"); + + joinGraphContext.clear(); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java index 7687474..8fcdce7 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java @@ -18,7 +18,6 @@ package org.apache.tajo.plan.joinorder; -import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.JoinSpec; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.util.graph.SimpleUndirectedGraph; http://git-wip-us.apache.org/repos/asf/tajo/blob/97e61e6f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java index 97cd569..a3b529c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java @@ -18,13 +18,13 @@ package org.apache.tajo.plan.joinorder; +import org.apache.commons.collections.map.LRUMap; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.JoinSpec; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import java.util.Collection; -import java.util.Map; import java.util.Set; public class JoinGraphContext { @@ -32,8 +32,8 @@ public class JoinGraphContext { private JoinGraph joinGraph = new JoinGraph(); // New join edges are frequently created during join order optimization. - // This cache is to reduce the overhead of join edge creation. - private Map<Pair<JoinVertex,JoinVertex>, JoinEdge> edgeCache = TUtil.newHashMap(); + // This cache is to reduce such overhead. + private LRUMap edgeCache = new LRUMap(10000); // candidate predicates contain the predicates which are not pushed to any join nodes yet. // evaluated predicates contain the predicates which are already pushed to some join nodes. @@ -129,9 +129,27 @@ public class JoinGraphContext { public JoinEdge getCachedOrNewJoinEdge(JoinSpec joinSpec, JoinVertex left, JoinVertex right) { Pair<JoinVertex,JoinVertex> cacheKey = new Pair<JoinVertex, JoinVertex>(left, right); if (edgeCache.containsKey(cacheKey)) { - return edgeCache.get(cacheKey); + return (JoinEdge) edgeCache.get(cacheKey); } else { return cacheEdge(new JoinEdge(joinSpec, left, right)); } } + + public void clear() { + rootVertexes.clear(); + candidateJoinConditions.clear(); + candidateJoinFilters.clear(); + evaluatedJoinConditions.clear(); + evaluatedJoinFilters.clear(); + edgeCache.clear(); + joinGraph.clear(); + + rootVertexes = null; + candidateJoinConditions = null; + candidateJoinFilters = null; + evaluatedJoinConditions = null; + evaluatedJoinFilters = null; + edgeCache = null; + joinGraph = null; + } } \ No newline at end of file
