TAJO-1766: Improve the performance of cross join. Closes #706
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/1a7c353c Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/1a7c353c Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/1a7c353c Branch: refs/heads/master Commit: 1a7c353c203b10b28f028a3fad44eacb2010dae1 Parents: cb6fe80 Author: Jihoon Son <[email protected]> Authored: Tue Aug 25 18:20:12 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Tue Aug 25 18:20:12 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/algebra/Aggregation.java | 2 +- .../org/apache/tajo/catalog/SchemaUtil.java | 8 +- .../java/org/apache/tajo/QueryTestCaseBase.java | 6 +- .../main/java/org/apache/tajo/SessionVars.java | 7 +- .../java/org/apache/tajo/conf/TajoConf.java | 5 +- .../apache/tajo/exception/ErrorMessages.java | 5 + .../apache/tajo/exception/ExceptionUtil.java | 3 + .../exception/InvalidInputsForCrossJoin.java | 37 +++ .../TooLargeInputForCrossJoinException.java | 38 +++ tajo-common/src/main/proto/errors.proto | 22 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 5 +- .../planner/physical/TestBNLJoinExec.java | 224 ----------------- .../engine/planner/physical/TestNLJoinExec.java | 208 ---------------- .../apache/tajo/engine/query/TestCrossJoin.java | 163 ++++++++++++ .../tajo/engine/query/TestInnerJoinQuery.java | 67 ----- .../apache/tajo/engine/query/TestJoinQuery.java | 12 +- .../TestCrossJoin/testCrossJoinAndCaseWhen.sql | 18 ++ .../testCrossJoinWithAsterisk1.sql | 1 + .../testCrossJoinWithAsterisk2.sql | 1 + .../testCrossJoinWithAsterisk3.sql | 1 + .../testCrossJoinWithAsterisk4.sql | 1 + .../testCrossJoinAndCaseWhen.sql | 18 -- .../testCrossJoinWithAsterisk1.sql | 1 - .../testCrossJoinWithAsterisk2.sql | 1 - .../testCrossJoinWithAsterisk3.sql | 1 - .../testCrossJoinWithAsterisk4.sql | 1 - .../TestCrossJoin/testCrossJoin.1.Hash.plan | 76 ++++++ .../testCrossJoin.1.Hash_NoBroadcast.plan | 103 ++++++++ .../TestCrossJoin/testCrossJoin.1.Sort.plan | 76 ++++++ .../testCrossJoin.1.Sort_NoBroadcast.plan | 103 ++++++++ .../TestCrossJoin/testCrossJoin.1.result | 127 ++++++++++ .../TestCrossJoin/testCrossJoin.2.Hash.plan | 51 ++++ .../testCrossJoin.2.Hash_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.2.Sort.plan | 51 ++++ .../testCrossJoin.2.Sort_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.2.result | 27 ++ .../TestCrossJoin/testCrossJoin.3.Hash.plan | 51 ++++ .../testCrossJoin.3.Hash_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.3.Sort.plan | 51 ++++ .../testCrossJoin.3.Sort_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.3.result | 27 ++ .../TestCrossJoin/testCrossJoin.4.Hash.plan | 51 ++++ .../testCrossJoin.4.Hash_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.4.Sort.plan | 51 ++++ .../testCrossJoin.4.Sort_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.4.result | 27 ++ .../TestCrossJoin/testCrossJoin.5.Hash.plan | 76 ++++++ .../testCrossJoin.5.Hash_NoBroadcast.plan | 103 ++++++++ .../TestCrossJoin/testCrossJoin.5.Sort.plan | 76 ++++++ .../testCrossJoin.5.Sort_NoBroadcast.plan | 103 ++++++++ .../TestCrossJoin/testCrossJoin.5.result | 27 ++ .../testCrossJoinAndCaseWhen.Hash.plan | 78 ++++++ ...stCrossJoinAndCaseWhen.Hash_NoBroadcast.plan | 105 ++++++++ .../testCrossJoinAndCaseWhen.Sort.plan | 78 ++++++ ...stCrossJoinAndCaseWhen.Sort_NoBroadcast.plan | 105 ++++++++ .../testCrossJoinAndCaseWhen.result | 27 ++ .../testCrossJoinOfOneSmallTable.1.Hash.plan | 51 ++++ ...sJoinOfOneSmallTable.1.Hash_NoBroadcast.plan | 78 ++++++ .../testCrossJoinOfOneSmallTable.1.Sort.plan | 51 ++++ ...sJoinOfOneSmallTable.1.Sort_NoBroadcast.plan | 78 ++++++ .../testCrossJoinOfOneSmallTable.1.result | 127 ++++++++++ ...tCrossJoinOfOneSmallTableAndJoin.1.Hash.plan | 89 +++++++ ...OneSmallTableAndJoin.1.Hash_NoBroadcast.plan | 174 +++++++++++++ ...tCrossJoinOfOneSmallTableAndJoin.1.Sort.plan | 89 +++++++ ...OneSmallTableAndJoin.1.Sort_NoBroadcast.plan | 174 +++++++++++++ ...testCrossJoinOfOneSmallTableAndJoin.1.result | 2 + .../testCrossJoinOftwoSmallTables.1.Hash.plan | 51 ++++ ...JoinOftwoSmallTables.1.Hash_NoBroadcast.plan | 78 ++++++ .../testCrossJoinOftwoSmallTables.1.Sort.plan | 51 ++++ ...JoinOftwoSmallTables.1.Sort_NoBroadcast.plan | 78 ++++++ .../testCrossJoinOftwoSmallTables.1.result | 27 ++ .../testCrossJoinWithAsterisk1.Hash.plan | 76 ++++++ ...CrossJoinWithAsterisk1.Hash_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk1.Sort.plan | 76 ++++++ ...CrossJoinWithAsterisk1.Sort_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk1.result | 27 ++ .../testCrossJoinWithAsterisk2.Hash.plan | 76 ++++++ ...CrossJoinWithAsterisk2.Hash_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk2.Sort.plan | 76 ++++++ ...CrossJoinWithAsterisk2.Sort_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk2.result | 27 ++ .../testCrossJoinWithAsterisk3.Hash.plan | 76 ++++++ ...CrossJoinWithAsterisk3.Hash_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk3.Sort.plan | 76 ++++++ ...CrossJoinWithAsterisk3.Sort_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk3.result | 27 ++ .../testCrossJoinWithAsterisk4.Hash.plan | 76 ++++++ ...CrossJoinWithAsterisk4.Hash_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk4.Sort.plan | 76 ++++++ ...CrossJoinWithAsterisk4.Sort_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk4.result | 27 ++ .../testCrossJoinWithEmptyTable1.Hash.plan | 78 ++++++ ...ossJoinWithEmptyTable1.Hash_NoBroadcast.plan | 105 ++++++++ .../testCrossJoinWithEmptyTable1.Sort.plan | 78 ++++++ ...ossJoinWithEmptyTable1.Sort_NoBroadcast.plan | 105 ++++++++ .../testCrossJoinWithEmptyTable1.result | 2 + ...sJoinWithThetaJoinConditionInWhere.Hash.plan | 63 +++++ ...taJoinConditionInWhere.Hash_NoBroadcast.plan | 90 +++++++ ...sJoinWithThetaJoinConditionInWhere.Sort.plan | 63 +++++ ...taJoinConditionInWhere.Sort_NoBroadcast.plan | 90 +++++++ ...rossJoinWithThetaJoinConditionInWhere.result | 12 + .../testCrossJoin.1.Hash.plan | 76 ------ .../testCrossJoin.1.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoin.1.Sort.plan | 76 ------ .../testCrossJoin.1.Sort_NoBroadcast.plan | 103 -------- .../TestInnerJoinQuery/testCrossJoin.1.result | 127 ---------- .../testCrossJoin.2.Hash.plan | 51 ---- .../testCrossJoin.2.Hash_NoBroadcast.plan | 78 ------ .../testCrossJoin.2.Sort.plan | 51 ---- .../testCrossJoin.2.Sort_NoBroadcast.plan | 78 ------ .../TestInnerJoinQuery/testCrossJoin.2.result | 27 -- .../testCrossJoin.3.Hash.plan | 51 ---- .../testCrossJoin.3.Hash_NoBroadcast.plan | 78 ------ .../testCrossJoin.3.Sort.plan | 51 ---- .../testCrossJoin.3.Sort_NoBroadcast.plan | 78 ------ .../TestInnerJoinQuery/testCrossJoin.3.result | 27 -- .../testCrossJoin.4.Hash.plan | 51 ---- .../testCrossJoin.4.Hash_NoBroadcast.plan | 78 ------ .../testCrossJoin.4.Sort.plan | 51 ---- .../testCrossJoin.4.Sort_NoBroadcast.plan | 78 ------ .../TestInnerJoinQuery/testCrossJoin.4.result | 27 -- .../testCrossJoin.5.Hash.plan | 76 ------ .../testCrossJoin.5.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoin.5.Sort.plan | 76 ------ .../testCrossJoin.5.Sort_NoBroadcast.plan | 103 -------- .../TestInnerJoinQuery/testCrossJoin.5.result | 27 -- .../testCrossJoinAndCaseWhen.Hash.plan | 78 ------ ...stCrossJoinAndCaseWhen.Hash_NoBroadcast.plan | 105 -------- .../testCrossJoinAndCaseWhen.Sort.plan | 78 ------ ...stCrossJoinAndCaseWhen.Sort_NoBroadcast.plan | 105 -------- .../testCrossJoinAndCaseWhen.result | 27 -- .../testCrossJoinWithAsterisk1.Hash.plan | 76 ------ ...CrossJoinWithAsterisk1.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk1.Sort.plan | 76 ------ ...CrossJoinWithAsterisk1.Sort_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk1.result | 27 -- .../testCrossJoinWithAsterisk2.Hash.plan | 76 ------ ...CrossJoinWithAsterisk2.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk2.Sort.plan | 76 ------ ...CrossJoinWithAsterisk2.Sort_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk2.result | 27 -- .../testCrossJoinWithAsterisk3.Hash.plan | 76 ------ ...CrossJoinWithAsterisk3.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk3.Sort.plan | 76 ------ ...CrossJoinWithAsterisk3.Sort_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk3.result | 27 -- .../testCrossJoinWithAsterisk4.Hash.plan | 76 ------ ...CrossJoinWithAsterisk4.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk4.Sort.plan | 76 ------ ...CrossJoinWithAsterisk4.Sort_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk4.result | 27 -- .../testCrossJoinWithEmptyTable1.Hash.plan | 78 ------ ...ossJoinWithEmptyTable1.Hash_NoBroadcast.plan | 105 -------- .../testCrossJoinWithEmptyTable1.Sort.plan | 78 ------ ...ossJoinWithEmptyTable1.Sort_NoBroadcast.plan | 105 -------- .../testCrossJoinWithEmptyTable1.result | 2 - ...sJoinWithThetaJoinConditionInWhere.Hash.plan | 63 ----- ...taJoinConditionInWhere.Hash_NoBroadcast.plan | 90 ------- ...sJoinWithThetaJoinConditionInWhere.Sort.plan | 63 ----- ...taJoinConditionInWhere.Sort_NoBroadcast.plan | 90 ------- ...rossJoinWithThetaJoinConditionInWhere.result | 12 - .../testJoinWithMultipleJoinQual1.Hash.plan | 4 +- ...nWithMultipleJoinQual1.Hash_NoBroadcast.plan | 4 +- .../testJoinWithMultipleJoinQual1.Sort.plan | 4 +- ...nWithMultipleJoinQual1.Sort_NoBroadcast.plan | 4 +- .../testJoinWithOrPredicates.Hash.plan | 4 +- ...stJoinWithOrPredicates.Hash_NoBroadcast.plan | 4 +- .../testJoinWithOrPredicates.Sort.plan | 4 +- ...stJoinWithOrPredicates.Sort_NoBroadcast.plan | 4 +- .../testNaturalJoin.Hash.plan | 4 +- .../testNaturalJoin.Hash_NoBroadcast.plan | 4 +- .../testNaturalJoin.Sort.plan | 4 +- .../testNaturalJoin.Sort_NoBroadcast.plan | 4 +- ...testJoinFilterOfRowPreservedTable1.Hash.plan | 4 +- ...erOfRowPreservedTable1.Hash_NoBroadcast.plan | 4 +- ...testJoinFilterOfRowPreservedTable1.Sort.plan | 4 +- ...erOfRowPreservedTable1.Sort_NoBroadcast.plan | 4 +- .../testLeftOuterJoin2.Hash.plan | 56 ++--- .../testLeftOuterJoin2.Hash_NoBroadcast.plan | 118 ++++----- .../testLeftOuterJoin2.Sort.plan | 56 ++--- .../testLeftOuterJoin2.Sort_NoBroadcast.plan | 118 ++++----- .../testLeftOuterJoin3.Hash.plan | 73 ++---- .../testLeftOuterJoin3.Sort.plan | 73 ++---- ...tOuterJoinPredicationCaseByCase1.1.Hash.plan | 4 +- ...edicationCaseByCase1.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase1.1.Sort.plan | 4 +- ...edicationCaseByCase1.1.Sort_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase2.1.Hash.plan | 67 ++--- ...tOuterJoinPredicationCaseByCase2.1.Sort.plan | 67 ++--- ...tOuterJoinPredicationCaseByCase3.1.Hash.plan | 4 +- ...edicationCaseByCase3.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase3.1.Sort.plan | 4 +- ...edicationCaseByCase3.1.Sort_NoBroadcast.plan | 4 +- .../testLeftOuterJoinWithNull2.Hash.plan | 4 +- ...LeftOuterJoinWithNull2.Hash_NoBroadcast.plan | 4 +- .../testLeftOuterJoinWithNull2.Sort.plan | 4 +- ...LeftOuterJoinWithNull2.Sort_NoBroadcast.plan | 4 +- .../testOuterJoinAndCaseWhen1.Hash.plan | 73 ++---- .../testOuterJoinAndCaseWhen1.Sort.plan | 73 ++---- ...tOuterJoinPredicationCaseByCase1.1.Hash.plan | 71 ++---- ...edicationCaseByCase1.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase1.1.Sort.plan | 71 ++---- ...edicationCaseByCase1.1.Sort_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase2.1.Hash.plan | 4 +- ...edicationCaseByCase2.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase2.1.Sort.plan | 4 +- ...edicationCaseByCase2.1.Sort_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase3.1.Hash.plan | 69 ++---- ...edicationCaseByCase3.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase3.1.Sort.plan | 69 ++---- ...edicationCaseByCase3.1.Sort_NoBroadcast.plan | 4 +- .../TestTajoCli/testHelpSessionVars.result | 5 +- .../engine/planner/PhysicalPlannerImpl.java | 22 +- .../tajo/engine/planner/enforce/Enforcer.java | 4 - .../engine/planner/global/ExecutionBlock.java | 4 +- .../rewriter/rules/BroadcastJoinRule.java | 248 +++++++++++++++---- .../rewriter/rules/GlobalPlanRewriteUtil.java | 2 +- .../engine/planner/physical/BNLJoinExec.java | 183 -------------- .../planner/physical/BSTIndexScanExec.java | 26 +- .../physical/BasicPhysicalExecutorVisitor.java | 29 +-- .../planner/physical/CommonHashJoinExec.java | 91 +++++-- .../planner/physical/HashFullOuterJoinExec.java | 2 +- .../engine/planner/physical/HashJoinExec.java | 7 +- .../planner/physical/IndexExecutorUtil.java | 33 +++ .../engine/planner/physical/NLJoinExec.java | 75 ------ .../physical/PartitionMergeScanExec.java | 5 + .../physical/PhysicalExecutorVisitor.java | 9 +- .../tajo/engine/planner/physical/ScanExec.java | 3 + .../engine/planner/physical/SeqScanExec.java | 5 + .../engine/planner/physical/StoreIndexExec.java | 10 +- .../apache/tajo/engine/utils/CacheHolder.java | 26 +- .../org/apache/tajo/master/GlobalEngine.java | 15 +- .../tajo/master/QueryCoordinatorService.java | 6 +- .../org/apache/tajo/master/QueryInProgress.java | 7 +- .../apache/tajo/master/exec/QueryExecutor.java | 33 ++- .../apache/tajo/querymaster/QueryMaster.java | 6 + .../tajo/querymaster/QueryMasterTask.java | 36 +-- .../tajo/worker/ExecutionBlockContext.java | 2 +- .../apache/tajo/worker/TaskAttemptContext.java | 11 - tajo-core/src/main/proto/ResourceProtos.proto | 4 +- .../org/apache/tajo/plan/LogicalPlanner.java | 1 + .../GreedyHeuristicJoinOrderAlgorithm.java | 20 +- .../plan/rewrite/rules/FilterPushDownRule.java | 3 +- .../org/apache/tajo/plan/util/IndexUtil.java | 72 ------ .../org/apache/tajo/plan/util/PlannerUtil.java | 6 + .../plan/verifier/PostLogicalPlanVerifier.java | 148 +++++++++++ tajo-plan/src/main/proto/Plan.proto | 2 - 248 files changed, 6929 insertions(+), 6174 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index b3ed451..c6a85ad 100644 --- a/CHANGES +++ b/CHANGES @@ -34,6 +34,8 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1766: Improve the performance of cross join. (jihoon) + TAJO-1792: tajo-cluster-tests is not available when it is used as an external maven module. (hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java index 8bcd7e2..c44e1bb 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java @@ -145,7 +145,7 @@ public class Aggregation extends UnaryOperator { } } - public static enum GroupType { + public enum GroupType { OrdinaryGroup(""), Cube("Cube"), Rollup("Rollup"), http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java index 09a2e45..4a8e419 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java @@ -20,6 +20,8 @@ package org.apache.tajo.catalog; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedDataTypeException; import org.apache.tajo.util.TUtil; import java.util.HashMap; @@ -226,7 +228,7 @@ public class SchemaUtil { case BOOLEAN: return 1; case CHAR: - return 1; + return col.getDataType().getLength(); case BIT: return 1; case INT2: @@ -242,7 +244,7 @@ public class SchemaUtil { case INET4: return 4; case INET6: - return 32; + return 16; case TEXT: return 256; case BLOB: @@ -254,7 +256,7 @@ public class SchemaUtil { case TIMESTAMP: return 8; default: - return 0; + throw new TajoRuntimeException(new UnsupportedDataTypeException(col.getDataType().toString())); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 8200cec..0f277f0 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -578,8 +578,8 @@ public class QueryTestCaseBase { // plan test if (prefix.length() > 0) { - String planResultName = methodName + (fromFile ? "" : "" + (i + 1)) + - ((option.parameterized() && testParameter != null) ? "" + testParameter : "") + ".plan"; + String planResultName = methodName + (fromFile ? "" : "." + (i + 1)) + + ((option.parameterized() && testParameter != null) ? "." + testParameter : "") + ".plan"; Path resultPath = StorageUtil.concatPath(currentResultPath, planResultName); if (currentResultFS.exists(resultPath)) { assertEquals("Plan Verification for: " + (i + 1) + " th test", @@ -596,7 +596,7 @@ public class QueryTestCaseBase { ResultSet result = client.executeQueryAndGetResult(spec.value()); // result test - String fileName = methodName + (fromFile ? "" : "" + (i + 1)) + ".result"; + String fileName = methodName + (fromFile ? "" : "." + (i + 1)) + ".result"; Path resultPath = StorageUtil.concatPath(currentResultPath, fileName); if (currentResultFS.exists(resultPath)) { assertEquals("Result Verification for: " + (i + 1) + " th test", http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-common/src/main/java/org/apache/tajo/SessionVars.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index 6c7d075..7d47c23 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -89,8 +89,11 @@ public enum SessionVars implements ConfigKey { // Query and Optimization --------------------------------------------------- // for distributed query strategies - BROADCAST_TABLE_SIZE_LIMIT(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, "limited size (bytes) of broadcast table", - DEFAULT, Long.class, Validators.min("0")), + BROADCAST_NON_CROSS_JOIN_THRESHOLD(ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD, + "restriction for the total bytes of broadcasted table for non-cross join", DEFAULT, Long.class, + Validators.min("0")), + BROADCAST_CROSS_JOIN_THRESHOLD(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD, + "restriction for the total bytes of broadcasted table for cross join", DEFAULT, Long.class, Validators.min("0")), JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size (mb) ", DEFAULT, Integer.class, Validators.min("1")), http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 1f7f2fa..75826e6 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -310,7 +310,10 @@ public class TajoConf extends Configuration { // Query and Optimization --------------------------------------------------- // for distributed query strategies - $DIST_QUERY_BROADCAST_JOIN_THRESHOLD("tajo.dist-query.join.broadcast.threshold-bytes", (long)5 * 1048576), + $DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD("tajo.dist-query.broadcast.non-cross-join.threshold-bytes", + (long)5 * 1048576), // 5 MB + $DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD("tajo.dist-query.broadcast.cross-join.threshold-bytes", + (long)1 * 1048576), // 1 MB $DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 128), $DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128), http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index e617efa..edae2d7 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -108,6 +108,11 @@ public class ErrorMessages { ADD_MESSAGE(AMBIGUOUS_PARTITION_DIRECTORY, "There is a directory which is assumed to be a partitioned directory" + " : '%s'", 1); + ADD_MESSAGE(TOO_LARGE_INPUT_FOR_CROSS_JOIN, "Cross join of large tables is not allowed: (%s). " + + "To execute cross join, please increase BROADCAST_CROSS_JOIN_THRESHOLD " + + "which is currently set to %s.", 2); + ADD_MESSAGE(INVALID_INPUTS_FOR_CROSS_JOIN, "At least one of both inputs for the cross join must be a simple " + + "relation."); ADD_MESSAGE(CLIENT_CONNECTION_EXCEPTION, "Client connection to '%s' has error: %s", 2); ADD_MESSAGE(CLIENT_UNABLE_TO_ESTABLISH_CONNECTION, "Client is unable to establish connection to '%s'", 1); http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java index 2fd49ac..6e36f94 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java @@ -80,6 +80,9 @@ public class ExceptionUtil { ADD_EXCEPTION(INVALID_DATATYPE, InvalidDataTypeException.class); ADD_EXCEPTION(INVALID_TABLE_PROPERTY, InvalidTablePropertyException.class); ADD_EXCEPTION(MISSING_TABLE_PROPERTY, MissingTablePropertyException.class); + + ADD_EXCEPTION(TOO_LARGE_INPUT_FOR_CROSS_JOIN, TooLargeInputForCrossJoinException.class); + ADD_EXCEPTION(INVALID_INPUTS_FOR_CROSS_JOIN, InvalidInputsForCrossJoin.class); } private static void ADD_EXCEPTION(Errors.ResultCode code, Class<? extends DefaultTajoException> cls) { http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-common/src/main/java/org/apache/tajo/exception/InvalidInputsForCrossJoin.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/InvalidInputsForCrossJoin.java b/tajo-common/src/main/java/org/apache/tajo/exception/InvalidInputsForCrossJoin.java new file mode 100644 index 0000000..3fafd39 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/exception/InvalidInputsForCrossJoin.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.exception; + +import org.apache.tajo.error.Errors.ResultCode; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; + +/** + * + * This exception occurs when both inputs of a cross join are not the simple relation. + */ +public class InvalidInputsForCrossJoin extends TajoException { + + public InvalidInputsForCrossJoin(ReturnState e) { + super(e); + } + + public InvalidInputsForCrossJoin() { + super(ResultCode.INVALID_INPUTS_FOR_CROSS_JOIN); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-common/src/main/java/org/apache/tajo/exception/TooLargeInputForCrossJoinException.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/TooLargeInputForCrossJoinException.java b/tajo-common/src/main/java/org/apache/tajo/exception/TooLargeInputForCrossJoinException.java new file mode 100644 index 0000000..32cd44e --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/exception/TooLargeInputForCrossJoinException.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.exception; + +import org.apache.tajo.error.Errors.ResultCode; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; +import org.apache.tajo.util.StringUtils; + +/** + * + * This exception occurs when the cross join cannot be executed with the broadcast join. + */ +public class TooLargeInputForCrossJoinException extends TajoException { + + public TooLargeInputForCrossJoinException(ReturnState e) { + super(e); + } + + public TooLargeInputForCrossJoinException(String[] relations, long currentBroadcastThreshold) { + super(ResultCode.TOO_LARGE_INPUT_FOR_CROSS_JOIN, StringUtils.join(relations), "" + currentBroadcastThreshold); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-common/src/main/proto/errors.proto ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto index 4057456..b947a95 100644 --- a/tajo-common/src/main/proto/errors.proto +++ b/tajo-common/src/main/proto/errors.proto @@ -173,25 +173,27 @@ enum ResultCode { CLIENT_PROTOCOL_PROTOCOL_VIOLATION = 1103; // SQLState: ? // 53 - Invalid Operand or Inconsistent Specification - INSUFFICIENT_RESOURCE = 53000; - DISK_FULL = 53100; - OUT_OF_MEMORY = 53200; + INSUFFICIENT_RESOURCE = 53000; + DISK_FULL = 53100; + OUT_OF_MEMORY = 53200; // 54 - SQL or Product Limit Exceeded - PROGRAM_LIMIT_EXCEEDED = 54000; - STATEMENT_TOO_COMPLEX = 54001; - STRING_CONSTANT_TOOL_LONG = 54002; + PROGRAM_LIMIT_EXCEEDED = 54000; + STATEMENT_TOO_COMPLEX = 54001; + STRING_CONSTANT_TOOL_LONG = 54002; + TOO_LARGE_INPUT_FOR_CROSS_JOIN = 54003; + INVALID_INPUTS_FOR_CROSS_JOIN = 54004; - TOO_MANY_TABLES = 54004; - TOO_MANY_COLUMNS = 54011; - TOO_MANY_ARGUMENTS = 54023; + TOO_MANY_TABLES = 54006; + TOO_MANY_COLUMNS = 54011; + TOO_MANY_ARGUMENTS = 54023; // 55 - Object Not in Prerequisite State // 56 - Miscellaneous SQL or Product Error // 57 - Resource Not Available or Operator Intervention // 58 - System Error - IO_ERROR = 58030; + IO_ERROR = 58030; // underlying system errors based on errno.h. http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index e744631..5005670 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -35,7 +35,10 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.FileUtil; -import org.junit.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; import org.junit.rules.TestName; import java.io.*; http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java deleted file mode 100644 index 71bad20..0000000 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ /dev/null @@ -1,224 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.planner.physical; - -import org.apache.hadoop.fs.Path; -import org.apache.tajo.LocalTajoTestingUtility; -import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.algebra.Expr; -import org.apache.tajo.catalog.*; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.PhysicalPlanner; -import org.apache.tajo.engine.planner.PhysicalPlannerImpl; -import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.exception.TajoException; -import org.apache.tajo.plan.LogicalPlanner; -import org.apache.tajo.plan.logical.JoinNode; -import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.NodeType; -import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.util.CommonTestingUtil; -import org.apache.tajo.util.TUtil; -import org.apache.tajo.worker.TaskAttemptContext; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; - -import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; -import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; -import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TestBNLJoinExec { - private TajoConf conf; - private final String TEST_PATH = TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestBNLJoinExec"; - private TajoTestingCluster util; - private CatalogService catalog; - private SQLAnalyzer analyzer; - private LogicalPlanner planner; - private Path testDir; - - private static int OUTER_TUPLE_NUM = 1000; - private static int INNER_TUPLE_NUM = 1000; - - private TableDesc employee; - private TableDesc people; - - @Before - public void setUp() throws Exception { - util = new TajoTestingCluster(); - catalog = util.startCatalogCluster().getCatalog(); - testDir = CommonTestingUtil.getTestDir(TEST_PATH); - catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); - catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - conf = util.getConfiguration(); - - Schema schema = new Schema(); - schema.addColumn("managerid", Type.INT4); - schema.addColumn("empid", Type.INT4); - schema.addColumn("memid", Type.INT4); - schema.addColumn("deptname", Type.TEXT); - - TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); - Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) - .getAppender(employeeMeta, schema, employeePath); - appender.init(); - VTuple tuple = new VTuple(schema.size()); - for (int i = 0; i < OUTER_TUPLE_NUM; i++) { - tuple.put(new Datum[] { DatumFactory.createInt4(i), - DatumFactory.createInt4(i), DatumFactory.createInt4(10 + i), - DatumFactory.createText("dept_" + i) }); - appender.addTuple(tuple); - } - appender.flush(); - appender.close(); - employee = CatalogUtil.newTableDesc("default.employee", schema, employeeMeta, employeePath); - catalog.createTable(employee); - - Schema peopleSchema = new Schema(); - peopleSchema.addColumn("empid", Type.INT4); - peopleSchema.addColumn("fk_memid", Type.INT4); - peopleSchema.addColumn("name", Type.TEXT); - peopleSchema.addColumn("age", Type.INT4); - TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT"); - Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(peopleMeta, peopleSchema, peoplePath); - appender.init(); - tuple = new VTuple(peopleSchema.size()); - for (int i = 1; i < INNER_TUPLE_NUM; i += 2) { - tuple.put(new Datum[] { DatumFactory.createInt4(i), - DatumFactory.createInt4(10 + i), - DatumFactory.createText("name_" + i), - DatumFactory.createInt4(30 + i) }); - appender.addTuple(tuple); - } - appender.flush(); - appender.close(); - - people = CatalogUtil.newTableDesc("default.people", peopleSchema, peopleMeta, peoplePath); - catalog.createTable(people); - analyzer = new SQLAnalyzer(); - planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - } - - @After - public void tearDown() throws Exception { - util.shutdownCatalogCluster(); - } - - // employee (managerId, empId, memId, deptName) - // people (empId, fk_memId, name, age) - String[] QUERIES = { - "select managerId, e.empId, deptName, e.memId from employee as e, people p", - "select managerId, e.empId, deptName, e.memId from employee as e " + - "inner join people as p on e.empId = p.empId and e.memId = p.fk_memId" }; - - @Test - public final void testBNLCrossJoin() throws IOException, TajoException { - Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), - expr).getRootBlock().getRoot(); - JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); - Enforcer enforcer = new Enforcer(); - enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN); - - FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), - new Path(employee.getUri()), - Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), - new Path(people.getUri()), - Integer.MAX_VALUE); - FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); - Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testBNLCrossJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), - LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir); - ctx.setEnforcer(enforcer); - - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); - PhysicalExec exec = phyPlanner.createPlan(ctx, plan); - - ProjectionExec proj = (ProjectionExec) exec; - assertTrue(proj.getChild() instanceof BNLJoinExec); - - int i = 0; - exec.init(); - while (exec.next() != null) { - i++; - } - exec.close(); - assertEquals(OUTER_TUPLE_NUM * INNER_TUPLE_NUM / 2, i); // expected 10 * 5 - } - - @Test - public final void testBNLInnerJoin() throws IOException, TajoException { - Expr context = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), - context).getRootBlock().getRoot(); - - FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), - new Path(employee.getUri()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), - new Path(people.getUri()), Integer.MAX_VALUE); - FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); - - - JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); - Enforcer enforcer = new Enforcer(); - enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN); - - Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testBNLInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), - LocalTajoTestingUtility.newTaskAttemptId(), - merged, workDir); - ctx.setEnforcer(enforcer); - - - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); - PhysicalExec exec = phyPlanner.createPlan(ctx, plan); - - ProjectionExec proj = (ProjectionExec) exec; - assertTrue(proj.getChild() instanceof BNLJoinExec); - - Tuple tuple; - int i = 1; - int count = 0; - exec.init(); - while ((tuple = exec.next()) != null) { - count++; - assertTrue(i == tuple.getInt4(0)); - assertTrue(i == tuple.getInt4(1)); - assertTrue(("dept_" + i).equals(tuple.getText(2))); - assertTrue(10 + i == tuple.getInt4(3)); - i += 2; - } - exec.close(); - assertEquals(INNER_TUPLE_NUM / 2, count); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java deleted file mode 100644 index d7968fe..0000000 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ /dev/null @@ -1,208 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.planner.physical; - -import org.apache.hadoop.fs.Path; -import org.apache.tajo.LocalTajoTestingUtility; -import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.algebra.Expr; -import org.apache.tajo.catalog.*; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.PhysicalPlanner; -import org.apache.tajo.engine.planner.PhysicalPlannerImpl; -import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.exception.TajoException; -import org.apache.tajo.plan.LogicalPlanner; -import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.util.CommonTestingUtil; -import org.apache.tajo.util.TUtil; -import org.apache.tajo.worker.TaskAttemptContext; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; - -import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; -import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TestNLJoinExec { - private TajoConf conf; - private final String TEST_PATH = TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestNLJoinExec"; - private TajoTestingCluster util; - private CatalogService catalog; - private SQLAnalyzer analyzer; - private LogicalPlanner planner; - private Path testDir; - - private TableDesc employee; - private TableDesc people; - - private MasterPlan masterPlan; - - @Before - public void setUp() throws Exception { - util = new TajoTestingCluster(); - catalog = util.startCatalogCluster().getCatalog(); - testDir = CommonTestingUtil.getTestDir(TEST_PATH); - catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); - catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - conf = util.getConfiguration(); - - Schema schema = new Schema(); - schema.addColumn("managerid", Type.INT4); - schema.addColumn("empid", Type.INT4); - schema.addColumn("memid", Type.INT4); - schema.addColumn("deptname", Type.TEXT); - - TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT"); - Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) - .getAppender(employeeMeta, schema, employeePath); - appender.init(); - VTuple tuple = new VTuple(schema.size()); - for (int i = 0; i < 50; i++) { - tuple.put(new Datum[] { - DatumFactory.createInt4(i), - DatumFactory.createInt4(i), - DatumFactory.createInt4(10 + i), - DatumFactory.createText("dept_" + i)}); - appender.addTuple(tuple); - } - appender.flush(); - appender.close(); - employee = CatalogUtil.newTableDesc("default.employee", schema, employeeMeta, employeePath); - catalog.createTable(employee); - - Schema peopleSchema = new Schema(); - peopleSchema.addColumn("empid", Type.INT4); - peopleSchema.addColumn("fk_memid", Type.INT4); - peopleSchema.addColumn("name", Type.TEXT); - peopleSchema.addColumn("age", Type.INT4); - TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT"); - Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileTablespace) TablespaceManager.getLocalFs()) - .getAppender(peopleMeta, peopleSchema, peoplePath); - appender.init(); - tuple = new VTuple(peopleSchema.size()); - for (int i = 1; i < 50; i += 2) { - tuple.put(new Datum[] { - DatumFactory.createInt4(i), - DatumFactory.createInt4(10 + i), - DatumFactory.createText("name_" + i), - DatumFactory.createInt4(30 + i)}); - appender.addTuple(tuple); - } - appender.flush(); - appender.close(); - - people = CatalogUtil.newTableDesc("default.people", peopleSchema, peopleMeta, peoplePath); - catalog.createTable(people); - analyzer = new SQLAnalyzer(); - planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); - - masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null); - } - - @After - public void tearDown() throws Exception { - util.shutdownCatalogCluster(); - } - - String[] QUERIES = { - "select managerId, e.empId, deptName, e.memId from employee as e, people p", - "select managerId, e.empId, deptName, e.memId from employee as e inner join people as p on " + - "e.empId = p.empId and e.memId = p.fk_memId" - }; - - @Test - public final void testNLCrossJoin() throws IOException, TajoException { - FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), - new Path(employee.getUri()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), - new Path(people.getUri()), Integer.MAX_VALUE); - - FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); - - Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testNLCrossJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), - LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir); - ctx.setEnforcer(new Enforcer()); - Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), - expr).getRootBlock().getRoot(); - - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); - PhysicalExec exec = phyPlanner.createPlan(ctx, plan); - - int i = 0; - exec.init(); - while (exec.next() != null) { - i++; - } - exec.close(); - assertEquals(50*50/2, i); // expected 10 * 5 - } - - @Test - public final void testNLInnerJoin() throws IOException, TajoException { - FileFragment[] empFrags = FileTablespace.splitNG(conf, "default.e", employee.getMeta(), - new Path(employee.getUri()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = FileTablespace.splitNG(conf, "default.p", people.getMeta(), - new Path(people.getUri()), Integer.MAX_VALUE); - - FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); - - Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testNLInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), - LocalTajoTestingUtility.newTaskAttemptId(masterPlan), merged, workDir); - ctx.setEnforcer(new Enforcer()); - Expr expr = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), - expr).getRootBlock().getRoot(); - - PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); - PhysicalExec exec = phyPlanner.createPlan(ctx, plan); - - Tuple tuple; - int i = 1; - int count = 0; - exec.init(); - while ((tuple = exec.next()) != null) { - count++; - assertTrue(i == tuple.getInt4(0)); - assertTrue(i == tuple.getInt4(1)); - assertTrue(("dept_" + i).equals(tuple.getText(2))); - assertTrue(10 + i == tuple.getInt4(3)); - i += 2; - } - exec.close(); - assertEquals(50 / 2, count); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCrossJoin.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCrossJoin.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCrossJoin.java new file mode 100644 index 0000000..7990ee7 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCrossJoin.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import org.apache.tajo.IntegrationTest; +import org.apache.tajo.NamedTest; +import org.apache.tajo.exception.InvalidInputsForCrossJoin; +import org.apache.tajo.exception.TooLargeInputForCrossJoinException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.sql.SQLException; + +@Category(IntegrationTest.class) +@RunWith(Parameterized.class) +@NamedTest("TestJoinQuery") +public class TestCrossJoin extends TestJoinQuery { + + public TestCrossJoin(String joinOption) throws Exception { + super(joinOption); + } + + @BeforeClass + public static void setup() throws Exception { + TestJoinQuery.setup(); + } + + @AfterClass + public static void classTearDown() throws SQLException { + TestJoinQuery.classTearDown(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true, sort = true) + @SimpleTest(queries = { + @QuerySpec("select n_name, r_name, n_regionkey, r_regionkey from nation, region order by n_name, r_name"), + // testCrossJoinWithAsterisk + @QuerySpec("select region.*, customer.* from region, customer"), + @QuerySpec("select region.*, customer.* from customer, region"), + @QuerySpec("select * from customer, region"), + @QuerySpec("select length(r_comment) as len, *, c_custkey*10 from customer, region order by len,r_regionkey,r_name") + }) + public final void testCrossJoin() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true) + @SimpleTest() + public final void testCrossJoinWithEmptyTable1() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true) + @SimpleTest() + public void testCrossJoinAndCaseWhen() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true) + @SimpleTest() + public void testCrossJoinWithAsterisk1() throws Exception { + // select region.*, customer.* from region, customer; + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true) + @SimpleTest() + public void testCrossJoinWithAsterisk2() throws Exception { + // select region.*, customer.* from customer, region; + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true) + @SimpleTest() + public void testCrossJoinWithAsterisk3() throws Exception { + // select * from customer, region + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true) + @SimpleTest() + public void testCrossJoinWithAsterisk4() throws Exception { + // select length(r_regionkey), *, c_custkey*10 from customer, region + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true, sort = true) + @SimpleTest() + public final void testCrossJoinWithThetaJoinConditionInWhere() throws Exception { + runSimpleTests(); + } + + @Test (expected = TooLargeInputForCrossJoinException.class) + public final void testCrossJoinOfOneLargeTableAndJoin() throws Exception { + executeString("select * from nation cross join region left outer join lineitem on r_regionkey = l_orderkey inner join supplier on l_suppkey = s_suppkey"); + } + + @Test (expected = TooLargeInputForCrossJoinException.class) + public final void testCrossJoinOfTwoLargeTables() throws Exception { + executeString("select * from nation n1 cross join nation n2"); + } + + @Test (expected = InvalidInputsForCrossJoin.class) + public final void testCrossJoinOfSubqueries() throws Exception { + executeString("select * from (select * from nation, region where n_regionkey = r_regionkey) t1 " + + "cross join (select * from orders, lineitem where l_orderkey = o_orderkey) t2"); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true) + @SimpleTest (queries = { + @QuerySpec("select * from nation cross join region") + }) + public final void testCrossJoinOfOneSmallTable() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true) + @SimpleTest (queries = { + @QuerySpec("select * from orders cross join region left outer join lineitem on r_regionkey = l_orderkey " + + "inner join supplier on l_suppkey = s_suppkey") + }) + public final void testCrossJoinOfOneSmallTableAndJoin() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true, withExplainGlobal = true, parameterized = true) + @SimpleTest (queries = { + @QuerySpec("select * from lineitem cross join region") + }) + public final void testCrossJoinOftwoSmallTables() throws Exception { + runSimpleTests(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestInnerJoinQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestInnerJoinQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestInnerJoinQuery.java index 88b3548..6ec828c 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestInnerJoinQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestInnerJoinQuery.java @@ -50,27 +50,6 @@ public class TestInnerJoinQuery extends TestJoinQuery { } @Test - @Option(withExplain = true, withExplainGlobal = true, parameterized = true, sort = true) - @SimpleTest(queries = { - @QuerySpec("select n_name, r_name, n_regionkey, r_regionkey from nation, region order by n_name, r_name"), - // testCrossJoinWithAsterisk - @QuerySpec("select region.*, customer.* from region, customer"), - @QuerySpec("select region.*, customer.* from customer, region"), - @QuerySpec("select * from customer, region"), - @QuerySpec("select length(r_comment) as len, *, c_custkey*10 from customer, region order by len,r_regionkey,r_name") - }) - public final void testCrossJoin() throws Exception { - runSimpleTests(); - } - - @Test - @Option(withExplain = true, withExplainGlobal = true, parameterized = true, sort = true) - @SimpleTest() - public final void testCrossJoinWithThetaJoinConditionInWhere() throws Exception { - runSimpleTests(); - } - - @Test @Option(withExplain = true, withExplainGlobal = true, parameterized = true) @SimpleTest() public final void testInnerJoinWithThetaJoinConditionInWhere() throws Exception { @@ -171,13 +150,6 @@ public class TestInnerJoinQuery extends TestJoinQuery { @Test @Option(withExplain = true, withExplainGlobal = true, parameterized = true) - @SimpleTest() - public final void testCrossJoinWithEmptyTable1() throws Exception { - runSimpleTests(); - } - - @Test - @Option(withExplain = true, withExplainGlobal = true, parameterized = true) @SimpleTest(prepare = { "CREATE DATABASE JOINS", "CREATE TABLE JOINS.part_ as SELECT * FROM part", @@ -290,45 +262,6 @@ public class TestInnerJoinQuery extends TestJoinQuery { @Test @Option(withExplain = true, withExplainGlobal = true, parameterized = true) @SimpleTest() - public void testCrossJoinAndCaseWhen() throws Exception { - runSimpleTests(); - } - - @Test - @Option(withExplain = true, withExplainGlobal = true, parameterized = true) - @SimpleTest() - public void testCrossJoinWithAsterisk1() throws Exception { - // select region.*, customer.* from region, customer; - runSimpleTests(); - } - - @Test - @Option(withExplain = true, withExplainGlobal = true, parameterized = true) - @SimpleTest() - public void testCrossJoinWithAsterisk2() throws Exception { - // select region.*, customer.* from customer, region; - runSimpleTests(); - } - - @Test - @Option(withExplain = true, withExplainGlobal = true, parameterized = true) - @SimpleTest() - public void testCrossJoinWithAsterisk3() throws Exception { - // select * from customer, region - runSimpleTests(); - } - - @Test - @Option(withExplain = true, withExplainGlobal = true, parameterized = true) - @SimpleTest() - public void testCrossJoinWithAsterisk4() throws Exception { - // select length(r_regionkey), *, c_custkey*10 from customer, region - runSimpleTests(); - } - - @Test - @Option(withExplain = true, withExplainGlobal = true, parameterized = true) - @SimpleTest() public final void testBroadcastTwoPartJoin() throws Exception { runSimpleTests(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index e5b9b98..706f201 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -59,7 +59,10 @@ public class TestJoinQuery extends QueryTestCaseBase { super(TajoConstants.DEFAULT_DATABASE_NAME, joinOption); testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "true"); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "" + (5 * 1024)); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.varname, + "" + (5 * 1024)); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname, + "" + (2 * 1024)); testingCluster.setAllTajoDaemonConfValue( ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, @@ -72,7 +75,6 @@ public class TestJoinQuery extends QueryTestCaseBase { if (joinOption.indexOf("NoBroadcast") >= 0) { testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false"); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "-1"); } if (joinOption.indexOf("Hash") >= 0) { @@ -112,8 +114,10 @@ public class TestJoinQuery extends QueryTestCaseBase { public static void classTearDown() throws SQLException { testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, ConfVars.$TEST_BROADCAST_JOIN_ENABLED.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, - ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.varname, + ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname, + ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.defaultVal); testingCluster.setAllTajoDaemonConfValue( ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinAndCaseWhen.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinAndCaseWhen.sql b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinAndCaseWhen.sql new file mode 100644 index 0000000..d058aba --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinAndCaseWhen.sql @@ -0,0 +1,18 @@ +select + r_regionkey, + n_regionkey, + case + when r_regionkey = 1 then 'one' + when r_regionkey = 2 then 'two' + when r_regionkey = 3 then 'three' + when r_regionkey = 4 then 'four' + else 'zero' + end as cond +from + region, + nation +where + r_regionkey = n_regionkey +order by + r_regionkey, + n_regionkey \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk1.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk1.sql b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk1.sql new file mode 100644 index 0000000..111a371 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk1.sql @@ -0,0 +1 @@ +select region.*, customer.* from region, customer order by r_regionkey,r_name,c_custkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk2.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk2.sql b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk2.sql new file mode 100644 index 0000000..ca1672e --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk2.sql @@ -0,0 +1 @@ +select region.*, customer.* from customer, region order by r_regionkey,r_name,c_custkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk3.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk3.sql b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk3.sql new file mode 100644 index 0000000..fd44916 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk3.sql @@ -0,0 +1 @@ +select * from customer, region order by c_custkey,c_name,r_regionkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk4.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk4.sql b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk4.sql new file mode 100644 index 0000000..fc5b1c3 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCrossJoin/testCrossJoinWithAsterisk4.sql @@ -0,0 +1 @@ +select length(r_comment) as len, *, c_custkey*10 from customer, region order by len,r_regionkey,r_name,c_custkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinAndCaseWhen.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinAndCaseWhen.sql b/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinAndCaseWhen.sql deleted file mode 100644 index d058aba..0000000 --- a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinAndCaseWhen.sql +++ /dev/null @@ -1,18 +0,0 @@ -select - r_regionkey, - n_regionkey, - case - when r_regionkey = 1 then 'one' - when r_regionkey = 2 then 'two' - when r_regionkey = 3 then 'three' - when r_regionkey = 4 then 'four' - else 'zero' - end as cond -from - region, - nation -where - r_regionkey = n_regionkey -order by - r_regionkey, - n_regionkey \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk1.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk1.sql b/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk1.sql deleted file mode 100644 index 111a371..0000000 --- a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk1.sql +++ /dev/null @@ -1 +0,0 @@ -select region.*, customer.* from region, customer order by r_regionkey,r_name,c_custkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk2.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk2.sql b/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk2.sql deleted file mode 100644 index ca1672e..0000000 --- a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk2.sql +++ /dev/null @@ -1 +0,0 @@ -select region.*, customer.* from customer, region order by r_regionkey,r_name,c_custkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk3.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk3.sql b/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk3.sql deleted file mode 100644 index fd44916..0000000 --- a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk3.sql +++ /dev/null @@ -1 +0,0 @@ -select * from customer, region order by c_custkey,c_name,r_regionkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk4.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk4.sql b/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk4.sql deleted file mode 100644 index fc5b1c3..0000000 --- a/tajo-core-tests/src/test/resources/queries/TestInnerJoinQuery/testCrossJoinWithAsterisk4.sql +++ /dev/null @@ -1 +0,0 @@ -select length(r_comment) as len, *, c_custkey*10 from customer, region order by len,r_regionkey,r_name,c_custkey; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Hash.plan ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Hash.plan b/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Hash.plan new file mode 100644 index 0000000..0fb22b3 --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Hash.plan @@ -0,0 +1,76 @@ +explain +------------------------------- +SORT(2) + => Sort Keys: default.nation.n_name (TEXT) (asc),default.region.r_name (TEXT) (asc) + JOIN(6)(CROSS) + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(1) on default.region + => target list: default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(2) default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(3) default.region.r_comment (TEXT), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(0) on default.nation + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4) + => out schema: {(2) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4)} + => in schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} +explain +------------------------------- +------------------------------------------------------------------------------- +Execution Block Graph (TERMINAL - eb_0000000000000_0000_000005) +------------------------------------------------------------------------------- +|-eb_0000000000000_0000_000005 + |-eb_0000000000000_0000_000004 + |-eb_0000000000000_0000_000003 +------------------------------------------------------------------------------- +Order of Execution +------------------------------------------------------------------------------- +1: eb_0000000000000_0000_000003 +2: eb_0000000000000_0000_000004 +3: eb_0000000000000_0000_000005 +------------------------------------------------------------------------------- + +======================================================= +Block Id: eb_0000000000000_0000_000003 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 3 => 4 (type=RANGE_SHUFFLE, key=default.nation.n_name (TEXT), default.region.r_name (TEXT), num=32) + +[Enforcers] + 0: type=Broadcast, tables=default.region + +SORT(10) + => Sort Keys: default.nation.n_name (TEXT) (asc),default.region.r_name (TEXT) (asc) + JOIN(6)(CROSS) + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(1) on default.region + => target list: default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(2) default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(3) default.region.r_comment (TEXT), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(0) on default.nation + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4) + => out schema: {(2) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4)} + => in schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000004 [ROOT] +======================================================= + +[Incoming] +[q_0000000000000_0000] 3 => 4 (type=RANGE_SHUFFLE, key=default.nation.n_name (TEXT), default.region.r_name (TEXT), num=32) + +[Enforcers] + 0: sorted input=eb_0000000000000_0000_000003 + +SORT(2) + => Sort Keys: default.nation.n_name (TEXT) (asc),default.region.r_name (TEXT) (asc) + SCAN(11) on eb_0000000000000_0000_000003 + => out schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000005 [TERMINAL] +======================================================= http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Hash_NoBroadcast.plan ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Hash_NoBroadcast.plan b/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Hash_NoBroadcast.plan new file mode 100644 index 0000000..4278681 --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Hash_NoBroadcast.plan @@ -0,0 +1,103 @@ +explain +------------------------------- +SORT(2) + => Sort Keys: default.nation.n_name (TEXT) (asc),default.region.r_name (TEXT) (asc) + JOIN(6)(CROSS) + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(1) on default.region + => target list: default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(2) default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(3) default.region.r_comment (TEXT), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(0) on default.nation + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4) + => out schema: {(2) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4)} + => in schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} +explain +------------------------------- +------------------------------------------------------------------------------- +Execution Block Graph (TERMINAL - eb_0000000000000_0000_000005) +------------------------------------------------------------------------------- +|-eb_0000000000000_0000_000005 + |-eb_0000000000000_0000_000004 + |-eb_0000000000000_0000_000003 + |-eb_0000000000000_0000_000002 + |-eb_0000000000000_0000_000001 +------------------------------------------------------------------------------- +Order of Execution +------------------------------------------------------------------------------- +1: eb_0000000000000_0000_000001 +2: eb_0000000000000_0000_000002 +3: eb_0000000000000_0000_000003 +4: eb_0000000000000_0000_000004 +5: eb_0000000000000_0000_000005 +------------------------------------------------------------------------------- + +======================================================= +Block Id: eb_0000000000000_0000_000001 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 1 => 3 (type=HASH_SHUFFLE) + +SCAN(0) on default.nation + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4) + => out schema: {(2) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4)} + => in schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000002 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE) + +SCAN(1) on default.region + => target list: default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(2) default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(3) default.region.r_comment (TEXT), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000003 [INTERMEDIATE] +======================================================= + +[Incoming] +[q_0000000000000_0000] 1 => 3 (type=HASH_SHUFFLE) +[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE) + +[Outgoing] +[q_0000000000000_0000] 3 => 4 (type=RANGE_SHUFFLE, key=default.nation.n_name (TEXT), default.region.r_name (TEXT), num=32) + +SORT(10) + => Sort Keys: default.nation.n_name (TEXT) (asc),default.region.r_name (TEXT) (asc) + JOIN(6)(CROSS) + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(9) on eb_0000000000000_0000_000002 + => out schema: {(2) default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(2) default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(8) on eb_0000000000000_0000_000001 + => out schema: {(2) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4)} + => in schema: {(2) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000004 [ROOT] +======================================================= + +[Incoming] +[q_0000000000000_0000] 3 => 4 (type=RANGE_SHUFFLE, key=default.nation.n_name (TEXT), default.region.r_name (TEXT), num=32) + +[Enforcers] + 0: sorted input=eb_0000000000000_0000_000003 + +SORT(2) + => Sort Keys: default.nation.n_name (TEXT) (asc),default.region.r_name (TEXT) (asc) + SCAN(11) on eb_0000000000000_0000_000003 + => out schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000005 [TERMINAL] +======================================================= http://git-wip-us.apache.org/repos/asf/tajo/blob/1a7c353c/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Sort.plan ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Sort.plan b/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Sort.plan new file mode 100644 index 0000000..0fb22b3 --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestCrossJoin/testCrossJoin.1.Sort.plan @@ -0,0 +1,76 @@ +explain +------------------------------- +SORT(2) + => Sort Keys: default.nation.n_name (TEXT) (asc),default.region.r_name (TEXT) (asc) + JOIN(6)(CROSS) + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(1) on default.region + => target list: default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(2) default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(3) default.region.r_comment (TEXT), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(0) on default.nation + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4) + => out schema: {(2) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4)} + => in schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} +explain +------------------------------- +------------------------------------------------------------------------------- +Execution Block Graph (TERMINAL - eb_0000000000000_0000_000005) +------------------------------------------------------------------------------- +|-eb_0000000000000_0000_000005 + |-eb_0000000000000_0000_000004 + |-eb_0000000000000_0000_000003 +------------------------------------------------------------------------------- +Order of Execution +------------------------------------------------------------------------------- +1: eb_0000000000000_0000_000003 +2: eb_0000000000000_0000_000004 +3: eb_0000000000000_0000_000005 +------------------------------------------------------------------------------- + +======================================================= +Block Id: eb_0000000000000_0000_000003 [LEAF] +======================================================= + +[Outgoing] +[q_0000000000000_0000] 3 => 4 (type=RANGE_SHUFFLE, key=default.nation.n_name (TEXT), default.region.r_name (TEXT), num=32) + +[Enforcers] + 0: type=Broadcast, tables=default.region + +SORT(10) + => Sort Keys: default.nation.n_name (TEXT) (asc),default.region.r_name (TEXT) (asc) + JOIN(6)(CROSS) + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(1) on default.region + => target list: default.region.r_name (TEXT), default.region.r_regionkey (INT4) + => out schema: {(2) default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(3) default.region.r_comment (TEXT), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + SCAN(0) on default.nation + => target list: default.nation.n_name (TEXT), default.nation.n_regionkey (INT4) + => out schema: {(2) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4)} + => in schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000004 [ROOT] +======================================================= + +[Incoming] +[q_0000000000000_0000] 3 => 4 (type=RANGE_SHUFFLE, key=default.nation.n_name (TEXT), default.region.r_name (TEXT), num=32) + +[Enforcers] + 0: sorted input=eb_0000000000000_0000_000003 + +SORT(2) + => Sort Keys: default.nation.n_name (TEXT) (asc),default.region.r_name (TEXT) (asc) + SCAN(11) on eb_0000000000000_0000_000003 + => out schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + => in schema: {(4) default.nation.n_name (TEXT), default.nation.n_regionkey (INT4), default.region.r_name (TEXT), default.region.r_regionkey (INT4)} + +======================================================= +Block Id: eb_0000000000000_0000_000005 [TERMINAL] +=======================================================
