Repository: tajo Updated Branches: refs/heads/master 690bc0974 -> 3ca900d7a
TAJO-1906: Decrease the default size of hash map and array list for testing. Closes #799 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3ca900d7 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3ca900d7 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3ca900d7 Branch: refs/heads/master Commit: 3ca900d7a01b60f689d8ecd411e30b7e36620d80 Parents: 690bc09 Author: Jihoon Son <[email protected]> Authored: Fri Oct 30 18:02:32 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Oct 30 18:02:32 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../main/java/org/apache/tajo/SessionVars.java | 3 + .../java/org/apache/tajo/conf/TajoConf.java | 3 + .../planner/physical/TestExternalSortExec.java | 11 --- .../planner/physical/TestPhysicalPlanner.java | 56 +------------ .../physical/TestProgressExternalSortExec.java | 18 +---- .../engine/planner/physical/TestSortExec.java | 2 + .../tajo/engine/query/TestGroupByQuery.java | 16 ++-- .../apache/tajo/engine/query/TestJoinQuery.java | 2 + .../apache/tajo/engine/query/TestSortQuery.java | 6 ++ .../TestGroupByQuery/testGroupBy2.result | 2 +- .../TestGroupByQuery/testPythonUdaf2.result | 2 +- .../TestTajoCli/testHelpSessionVars.result | 3 + .../engine/planner/PhysicalPlannerImpl.java | 16 ---- .../physical/BasicPhysicalExecutorVisitor.java | 8 -- .../planner/physical/CommonHashJoinExec.java | 3 +- .../DistinctGroupbyHashAggregationExec.java | 3 +- .../planner/physical/ExternalSortExec.java | 2 +- .../planner/physical/HashAggregateExec.java | 3 +- .../engine/planner/physical/MemSortExec.java | 83 -------------------- .../physical/MergeFullOuterJoinExec.java | 5 +- .../engine/planner/physical/MergeJoinExec.java | 5 +- .../physical/PhysicalExecutorVisitor.java | 3 - .../physical/RightOuterMergeJoinExec.java | 5 +- 24 files changed, 53 insertions(+), 210 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 5b2ff4e..53658b1 100644 --- a/CHANGES +++ b/CHANGES @@ -37,6 +37,9 @@ Release 0.12.0 - unreleased TASKS + TAJO-1906: Decrease the default size of hash map and array list for testing. + (jihoon) + TAJO-1934: Test failures in TestPgSQLQueryTests due to sort order. (Contributed by Dongkyu Hwangbo, committed by jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-common/src/main/java/org/apache/tajo/SessionVars.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index 7e419f0..2834e14 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -130,6 +130,9 @@ public enum SessionVars implements ConfigKey { Long.class, Validators.min("0")), NULL_CHAR(ConfVars.$TEXT_NULL, "null char of text file output", DEFAULT), CODEGEN(ConfVars.$CODEGEN, "Runtime code generation enabled (experiment)", DEFAULT), + AGG_HASH_TABLE_SIZE(ConfVars.$AGG_HASH_TABLE_SIZE, "Aggregation hash table size", DEFAULT), + SORT_HASH_TABLE_SIZE(ConfVars.$SORT_HASH_TABLE_SIZE, "Sort hash table size", DEFAULT), + JOIN_HASH_TABLE_SIZE(ConfVars.$JOIN_HASH_TABLE_SIZE, "Join hash table size", DEFAULT), // for index INDEX_ENABLED(ConfVars.$INDEX_ENABLED, "index scan enabled", DEFAULT), http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 2abd996..e84da0f 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -344,6 +344,9 @@ public class TajoConf extends Configuration { Validators.min("0")), $MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite $CODEGEN("tajo.executor.codegen.enabled", false), // Runtime code generation (todo this is broken) + $AGG_HASH_TABLE_SIZE("tajo.executor.aggregate.hash-table.size", 10000), + $SORT_HASH_TABLE_SIZE("tajo.executor.sort.hash-table.size", 100000), + $JOIN_HASH_TABLE_SIZE("tajo.executor.join.hash-table.size", 100000), // for index $INDEX_ENABLED("tajo.query.index.enabled", false), http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index e796bad..4ee2c9c 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -138,17 +138,6 @@ public class TestExternalSortExec { PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); ProjectionExec proj = (ProjectionExec) exec; - - // TODO - should be planed with user's optimization hint - ExternalSortExec extSort; - if (!(proj.getChild() instanceof ExternalSortExec)) { - UnaryPhysicalExec sortExec = proj.getChild(); - SeqScanExec scan = sortExec.getChild(); - - extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan); - proj.setChild(extSort); - } - Tuple tuple; Tuple preVal = null; Tuple curVal; http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 70c19b2..d3f1649 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -34,14 +34,14 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.engine.function.FunctionLoader; -import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.exception.*; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.plan.LogicalOptimizer; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; @@ -70,7 +70,6 @@ import java.util.Set; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; import static org.apache.tajo.plan.serder.PlanProto.ColumnPartitionEnforcer.ColumnPartitionAlgorithm; -import static org.apache.tajo.plan.serder.PlanProto.SortEnforce.SortAlgorithm; import static org.junit.Assert.*; public class TestPhysicalPlanner { @@ -1042,57 +1041,6 @@ public class TestPhysicalPlanner { }; @Test - public final void testSortEnforcer() throws IOException, TajoException { - FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), - new Path(employee.getUri()), Integer.MAX_VALUE); - - Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testSortEnforcer"); - Expr context = analyzer.parse(SORT_QUERY[0]); - LogicalPlan plan = planner.createPlan(defaultContext, context); - optimizer.optimize(plan); - LogicalNode rootNode = plan.getRootBlock().getRoot(); - - SortNode sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT); - - Enforcer enforcer = new Enforcer(); - enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.IN_MEMORY_SORT); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), - LocalTajoTestingUtility.newTaskAttemptId(masterPlan), - new FileFragment[] {frags[0]}, workDir); - ctx.setEnforcer(enforcer); - - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); - PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); - exec.init(); - exec.next(); - exec.close(); - - assertTrue(exec instanceof MemSortExec); - - context = analyzer.parse(SORT_QUERY[0]); - plan = planner.createPlan(defaultContext, context); - optimizer.optimize(plan); - rootNode = plan.getRootBlock().getRoot(); - - sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT); - - enforcer = new Enforcer(); - enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.MERGE_SORT); - ctx = new TaskAttemptContext(new QueryContext(conf), - LocalTajoTestingUtility.newTaskAttemptId(masterPlan), - new FileFragment[] {frags[0]}, workDir); - ctx.setEnforcer(enforcer); - - phyPlanner = new PhysicalPlannerImpl(conf); - exec = phyPlanner.createPlan(ctx, rootNode); - exec.init(); - exec.next(); - exec.close(); - - assertTrue(exec instanceof ExternalSortExec); - } - - @Test public final void testGroupByEnforcer() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index ec41b0d..349aec0 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -32,14 +32,14 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.parser.sql.SQLAnalyzer; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.parser.sql.SQLAnalyzer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -153,16 +153,6 @@ public class TestProgressExternalSortExec { PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); ProjectionExec proj = (ProjectionExec) exec; - - // TODO - should be planed with user's optimization hint - if (!(proj.getChild() instanceof ExternalSortExec)) { - UnaryPhysicalExec sortExec = proj.getChild(); - SeqScanExec scan = sortExec.getChild(); - - ExternalSortExec extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan); - proj.setChild(extSort); - } - Tuple tuple; Tuple preVal = null; Tuple curVal; http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 58c2a98..15708b1 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -27,6 +27,7 @@ import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.parser.sql.SQLAnalyzer; @@ -74,6 +75,7 @@ public class TestSortExec { public static void setUp() throws Exception { conf = new TajoConf(); conf.setBoolVar(TajoConf.ConfVars.$TEST_MODE, true); + conf.setIntVar(ConfVars.$SORT_HASH_TABLE_SIZE, 100); util = TpchTestBase.getInstance().getTestingCluster(); catalog = util.getMaster().getCatalog(); workDir = CommonTestingUtil.getTestDir(TEST_PATH); http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index dd321ae..e523a19 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -18,8 +18,6 @@ package org.apache.tajo.engine.query; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.tajo.*; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes.Type; @@ -44,7 +42,6 @@ import static org.junit.Assert.assertEquals; @Category(IntegrationTest.class) @RunWith(Parameterized.class) public class TestGroupByQuery extends QueryTestCaseBase { - private static final Log LOG = LogFactory.getLog(TestGroupByQuery.class); public TestGroupByQuery(String groupByOption) throws Exception { super(TajoConstants.DEFAULT_DATABASE_NAME); @@ -55,6 +52,7 @@ public class TestGroupByQuery extends QueryTestCaseBase { } else { variables.put(SessionVars.GROUPBY_MULTI_LEVEL_ENABLED.keyname(), "false"); } + variables.put(SessionVars.AGG_HASH_TABLE_SIZE.keyname(), "100"); client.updateSessionVariables(variables); } @@ -80,11 +78,11 @@ public class TestGroupByQuery extends QueryTestCaseBase { } @Test + @Option(sort = true) + @SimpleTest public final void testGroupBy2() throws Exception { // select count(1) as unique_key from lineitem group by l_linenumber; - ResultSet res = executeQuery(); - assertResultSet(res); - cleanupQuery(res); + runSimpleTests(); } @Test @@ -797,10 +795,10 @@ public class TestGroupByQuery extends QueryTestCaseBase { } @Test + @Option(sort = true) + @SimpleTest public final void testPythonUdaf2() throws Exception { - ResultSet res = executeQuery(); - assertResultSet(res); - cleanupQuery(res); + runSimpleTests(); } @Test http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 2372a11..7acb73a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -68,6 +68,8 @@ public class TestJoinQuery extends QueryTestCaseBase { ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$JOIN_HASH_TABLE_SIZE.keyname(), "100"); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java index 6d6a44c..6022672 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java @@ -28,6 +28,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -36,6 +38,10 @@ public class TestSortQuery extends QueryTestCaseBase { public TestSortQuery() { super(TajoConstants.DEFAULT_DATABASE_NAME); + + Map<String, String> variables = new HashMap<>(); + variables.put(SessionVars.SORT_HASH_TABLE_SIZE.keyname(), "100"); + client.updateSessionVariables(variables); } @Test http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testGroupBy2.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testGroupBy2.result b/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testGroupBy2.result index 6afdd23..1fda90c 100644 --- a/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testGroupBy2.result +++ b/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testGroupBy2.result @@ -1,4 +1,4 @@ unique_key ------------------------------- 2 -3 \ No newline at end of file +3 http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testPythonUdaf2.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testPythonUdaf2.result b/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testPythonUdaf2.result index 2852167..c2a8888 100644 --- a/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testPythonUdaf2.result +++ b/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testPythonUdaf2.result @@ -1,4 +1,4 @@ ?countpy,?count_1 ------------------------------- 2,2 -3,3 \ No newline at end of file +3,3 http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result b/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result index 2d87b56..4d4cc2b 100644 --- a/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result +++ b/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result @@ -36,6 +36,9 @@ Available Session Variables: \set MAX_OUTPUT_FILE_SIZE [int value] - Maximum per-output file size (mb). 0 means infinite. \set NULL_CHAR [text value] - null char of text file output \set CODEGEN [true or false] - Runtime code generation enabled (experiment) +\set AGG_HASH_TABLE_SIZE [int value] - Aggregation hash table size +\set SORT_HASH_TABLE_SIZE [int value] - Sort hash table size +\set JOIN_HASH_TABLE_SIZE [int value] - Join hash table size \set INDEX_ENABLED [true or false] - index scan enabled \set INDEX_SELECTIVITY_THRESHOLD [real value] - the selectivity threshold for index scan \set PARTITION_NO_RESULT_OVERWRITE_ENABLED [true or false] - If True, a partitioned table is overwritten even if a sub query leads to no result. Otherwise, the table data will be kept if there is no result http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 52e3b89..0cfcee5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -1165,22 +1165,6 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { } } - Enforcer enforcer = context.getEnforcer(); - EnforceProperty property = getAlgorithmEnforceProperty(enforcer, sortNode); - if (property != null) { - SortEnforce.SortAlgorithm algorithm = property.getSort().getAlgorithm(); - if (algorithm == SortEnforce.SortAlgorithm.IN_MEMORY_SORT) { - return new MemSortExec(context, sortNode, child); - } else { - return new ExternalSortExec(context, sortNode, child); - } - } - - return createBestSortPlan(context, sortNode, child); - } - - public SortExec createBestSortPlan(TaskAttemptContext context, SortNode sortNode, - PhysicalExec child) throws IOException { return new ExternalSortExec(context, sortNode, child); } http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BasicPhysicalExecutorVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BasicPhysicalExecutorVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BasicPhysicalExecutorVisitor.java index 62af4e1..192709e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BasicPhysicalExecutorVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BasicPhysicalExecutorVisitor.java @@ -55,8 +55,6 @@ public class BasicPhysicalExecutorVisitor<CONTEXT, RESULT> implements PhysicalEx return visitHaving(context, (HavingExec) exec, stack); } else if (exec instanceof LimitExec) { return visitLimit(context, (LimitExec) exec, stack); - } else if (exec instanceof MemSortExec) { - return visitMemSort(context, (MemSortExec) exec, stack); } else if (exec instanceof MergeFullOuterJoinExec) { return visitMergeFullOuterJoin(context, (MergeFullOuterJoinExec) exec, stack); } else if (exec instanceof MergeJoinExec) { @@ -180,12 +178,6 @@ public class BasicPhysicalExecutorVisitor<CONTEXT, RESULT> implements PhysicalEx } @Override - public RESULT visitMemSort(CONTEXT context, MemSortExec exec, Stack<PhysicalExec> stack) throws - PhysicalPlanningException { - return visitUnaryExecutor(context, exec, stack); - } - - @Override public RESULT visitMergeFullOuterJoin(CONTEXT context, MergeFullOuterJoinExec exec, Stack<PhysicalExec> stack) throws PhysicalPlanningException { return visitBinaryExecutor(context, exec, stack); http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java index 59239a6..92a68bd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner.physical; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.engine.planner.KeyProjector; @@ -158,7 +159,7 @@ public abstract class CommonHashJoinExec<T> extends CommonJoinExec { protected TupleMap<TupleList> buildRightToHashTableForNonCrossJoin() throws IOException { Tuple tuple; - TupleMap<TupleList> map = new TupleMap<>(100000); + TupleMap<TupleList> map = new TupleMap<>(context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE)); KeyProjector keyProjector = new KeyProjector(rightSchema, rightKeyList); while (!context.isStopped() && (tuple = rightChild.next()) != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java index 418d0fc..aaade21 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner.physical; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.statistics.TableStats; @@ -359,7 +360,7 @@ public class DistinctGroupbyHashAggregationExec extends UnaryPhysicalExec { public HashAggregator(GroupbyNode groupbyNode, Schema schema) throws IOException { - hashTable = new TupleMap<>(10000); + hashTable = new TupleMap<>(context.getQueryContext().getInt(SessionVars.AGG_HASH_TABLE_SIZE)); List<Column> groupingKeyColumnList = new ArrayList<>(distinctGroupingKeyColumnSet); http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java index 42d99bb..6ed22c8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java @@ -118,7 +118,7 @@ public class ExternalSortExec extends SortExec { this.sortBufferBytesNum = context.getQueryContext().getLong(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB; this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM); this.executorService = Executors.newFixedThreadPool(this.allocatedCoreNum); - this.inMemoryTable = new TupleList(100000); + this.inMemoryTable = new TupleList(context.getQueryContext().getInt(SessionVars.SORT_HASH_TABLE_SIZE)); this.sortTmpDir = getExecutorTmpDir(); localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java index e3f566a..9741982 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner.physical; +import org.apache.tajo.SessionVars; import org.apache.tajo.engine.planner.KeyProjector; import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.plan.logical.GroupbyNode; @@ -42,7 +43,7 @@ public class HashAggregateExec extends AggregationExec { public HashAggregateExec(TaskAttemptContext ctx, GroupbyNode plan, PhysicalExec subOp) throws IOException { super(ctx, plan, subOp); hashKeyProjector = new KeyProjector(inSchema, plan.getGroupingColumns()); - hashTable = new TupleMap<>(10000); + hashTable = new TupleMap<>(ctx.getQueryContext().getInt(SessionVars.AGG_HASH_TABLE_SIZE)); this.tuple = new VTuple(plan.getOutSchema().size()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java deleted file mode 100644 index 029592a..0000000 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.planner.physical; - -import org.apache.tajo.plan.logical.SortNode; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.worker.TaskAttemptContext; - -import java.io.IOException; -import java.util.Iterator; - -public class MemSortExec extends SortExec { - private SortNode plan; - private TupleList tupleSlots; - private boolean sorted = false; - private Iterator<Tuple> iterator; - - public MemSortExec(final TaskAttemptContext context, - SortNode plan, PhysicalExec child) { - super(context, plan.getInSchema(), plan.getOutSchema(), child, plan.getSortKeys()); - this.plan = plan; - } - - public void init() throws IOException { - super.init(); - this.tupleSlots = new TupleList(10000); - } - - @Override - public Tuple next() throws IOException { - - if (!sorted) { - Tuple tuple; - while (!context.isStopped() && (tuple = child.next()) != null) { - tupleSlots.add(tuple); - } - iterator = getSorter(tupleSlots).sort().iterator(); - sorted = true; - } - - if (iterator.hasNext()) { - return this.iterator.next(); - } else { - return null; - } - } - - @Override - public void rescan() throws IOException { - super.rescan(); - this.iterator = tupleSlots.iterator(); - sorted = true; - } - - @Override - public void close() throws IOException { - super.close(); - tupleSlots.clear(); - tupleSlots = null; - iterator = null; - plan = null; - } - - public SortNode getPlan() { - return this.plan; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java index 41e3648..824fb0e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.planner.physical; import com.google.common.base.Preconditions; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.NullTuple; @@ -45,8 +46,6 @@ public class MergeFullOuterJoinExec extends CommonJoinExec { private JoinTupleComparator joincomparator = null; private TupleComparator[] tupleComparator = null; - private final static int INITIAL_TUPLE_SLOT = 10000; - private boolean end = false; private int rightNumCols; @@ -64,6 +63,8 @@ public class MergeFullOuterJoinExec extends CommonJoinExec { super(context, plan, leftChild, rightChild); Preconditions.checkArgument(plan.hasJoinQual(), "Sort-merge join is only used for the equi-join, " + "but there is no join condition"); + + final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE); this.leftTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); this.rightTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); SortSpec[][] sortSpecs = new SortSpec[2][]; http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java index 3d8c108..ab831b5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.planner.physical; import com.google.common.base.Preconditions; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.Tuple; @@ -46,8 +47,6 @@ public class MergeJoinExec extends CommonJoinExec { private JoinTupleComparator joincomparator = null; private TupleComparator [] tupleComparator = null; - private final static int INITIAL_TUPLE_SLOT = 10000; - private boolean end = false; public MergeJoinExec(TaskAttemptContext context, JoinNode plan, PhysicalExec outer, @@ -56,6 +55,8 @@ public class MergeJoinExec extends CommonJoinExec { Preconditions.checkArgument(plan.hasJoinQual(), "Sort-merge join is only used for the equi-join, " + "but there is no join condition"); + final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE); + this.outerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); this.innerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); SortSpec[][] sortSpecs = new SortSpec[2][]; http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExecutorVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExecutorVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExecutorVisitor.java index 554c31e..fe86539 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExecutorVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExecutorVisitor.java @@ -64,9 +64,6 @@ public interface PhysicalExecutorVisitor<CONTEXT, RESULT> { RESULT visitLimit(CONTEXT context, LimitExec exec, Stack<PhysicalExec> stack) throws PhysicalPlanningException; - RESULT visitMemSort(CONTEXT context, MemSortExec exec, Stack<PhysicalExec> stack) - throws PhysicalPlanningException; - RESULT visitMergeFullOuterJoin(CONTEXT context, MergeFullOuterJoinExec exec, Stack<PhysicalExec> stack) throws PhysicalPlanningException; http://git-wip-us.apache.org/repos/asf/tajo/blob/3ca900d7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java index 239c6ab..706ec3e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.planner.physical; import com.google.common.base.Preconditions; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.NullTuple; @@ -44,8 +45,6 @@ public class RightOuterMergeJoinExec extends CommonJoinExec { private JoinTupleComparator joinComparator = null; private TupleComparator [] tupleComparator = null; - private final static int INITIAL_TUPLE_SLOT = 10000; - private boolean end = false; private int leftNumCols; @@ -59,6 +58,8 @@ public class RightOuterMergeJoinExec extends CommonJoinExec { super(context, plan, outer, inner); Preconditions.checkArgument(plan.hasJoinQual(), "Sort-merge join is only used for the equi-join, " + "but there is no join condition"); + + final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE); this.leftTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); this.innerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); SortSpec[][] sortSpecs = new SortSpec[2][];
