Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into TAJO-1730
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/e3a92621 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/e3a92621 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/e3a92621 Branch: refs/heads/TAJO-1730 Commit: e3a9262178c6ab3755df42ca99c8cfdaa3d5f108 Parents: fa81988 848a8c3 Author: Hyunsik Choi <[email protected]> Authored: Tue Sep 1 12:23:24 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Tue Sep 1 12:23:24 2015 +0900 ---------------------------------------------------------------------- CHANGES | 11 +- .../org/apache/tajo/algebra/Aggregation.java | 2 +- .../org/apache/tajo/catalog/SchemaUtil.java | 8 +- .../java/org/apache/tajo/QueryTestCaseBase.java | 6 +- .../src/test/resources/tpch/customer.tbl | 10 +- .../src/test/resources/tpch/lineitem.tbl | 10 +- .../src/test/resources/tpch/nation.tbl | 50 ++-- .../src/test/resources/tpch/orders.tbl | 6 +- .../src/test/resources/tpch/partsupp.tbl | 6 +- .../src/test/resources/tpch/region.tbl | 10 +- .../src/test/resources/tpch/supplier.tbl | 6 +- .../main/java/org/apache/tajo/SessionVars.java | 7 +- .../java/org/apache/tajo/conf/TajoConf.java | 5 +- .../apache/tajo/exception/ErrorMessages.java | 5 + .../apache/tajo/exception/ExceptionUtil.java | 3 + .../exception/InvalidInputsForCrossJoin.java | 37 +++ .../TooLargeInputForCrossJoinException.java | 38 +++ .../java/org/apache/tajo/util/StringUtils.java | 2 +- tajo-common/src/main/proto/errors.proto | 22 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 5 +- .../planner/physical/TestBNLJoinExec.java | 224 ----------------- .../engine/planner/physical/TestNLJoinExec.java | 208 ---------------- .../apache/tajo/engine/query/TestCrossJoin.java | 163 ++++++++++++ .../tajo/engine/query/TestInnerJoinQuery.java | 67 ----- .../apache/tajo/engine/query/TestJoinQuery.java | 12 +- .../tajo/engine/query/TestTablePartitions.java | 46 +++- .../TestCrossJoin/testCrossJoinAndCaseWhen.sql | 18 ++ .../testCrossJoinWithAsterisk1.sql | 1 + .../testCrossJoinWithAsterisk2.sql | 1 + .../testCrossJoinWithAsterisk3.sql | 1 + .../testCrossJoinWithAsterisk4.sql | 1 + .../testCrossJoinAndCaseWhen.sql | 18 -- .../testCrossJoinWithAsterisk1.sql | 1 - .../testCrossJoinWithAsterisk2.sql | 1 - .../testCrossJoinWithAsterisk3.sql | 1 - .../testCrossJoinWithAsterisk4.sql | 1 - .../TestCrossJoin/testCrossJoin.1.Hash.plan | 76 ++++++ .../testCrossJoin.1.Hash_NoBroadcast.plan | 103 ++++++++ .../TestCrossJoin/testCrossJoin.1.Sort.plan | 76 ++++++ .../testCrossJoin.1.Sort_NoBroadcast.plan | 103 ++++++++ .../TestCrossJoin/testCrossJoin.1.result | 127 ++++++++++ .../TestCrossJoin/testCrossJoin.2.Hash.plan | 51 ++++ .../testCrossJoin.2.Hash_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.2.Sort.plan | 51 ++++ .../testCrossJoin.2.Sort_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.2.result | 27 ++ .../TestCrossJoin/testCrossJoin.3.Hash.plan | 51 ++++ .../testCrossJoin.3.Hash_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.3.Sort.plan | 51 ++++ .../testCrossJoin.3.Sort_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.3.result | 27 ++ .../TestCrossJoin/testCrossJoin.4.Hash.plan | 51 ++++ .../testCrossJoin.4.Hash_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.4.Sort.plan | 51 ++++ .../testCrossJoin.4.Sort_NoBroadcast.plan | 78 ++++++ .../TestCrossJoin/testCrossJoin.4.result | 27 ++ .../TestCrossJoin/testCrossJoin.5.Hash.plan | 76 ++++++ .../testCrossJoin.5.Hash_NoBroadcast.plan | 103 ++++++++ .../TestCrossJoin/testCrossJoin.5.Sort.plan | 76 ++++++ .../testCrossJoin.5.Sort_NoBroadcast.plan | 103 ++++++++ .../TestCrossJoin/testCrossJoin.5.result | 27 ++ .../testCrossJoinAndCaseWhen.Hash.plan | 78 ++++++ ...stCrossJoinAndCaseWhen.Hash_NoBroadcast.plan | 105 ++++++++ .../testCrossJoinAndCaseWhen.Sort.plan | 78 ++++++ ...stCrossJoinAndCaseWhen.Sort_NoBroadcast.plan | 105 ++++++++ .../testCrossJoinAndCaseWhen.result | 27 ++ .../testCrossJoinOfOneSmallTable.1.Hash.plan | 51 ++++ ...sJoinOfOneSmallTable.1.Hash_NoBroadcast.plan | 78 ++++++ .../testCrossJoinOfOneSmallTable.1.Sort.plan | 51 ++++ ...sJoinOfOneSmallTable.1.Sort_NoBroadcast.plan | 78 ++++++ .../testCrossJoinOfOneSmallTable.1.result | 127 ++++++++++ ...tCrossJoinOfOneSmallTableAndJoin.1.Hash.plan | 89 +++++++ ...OneSmallTableAndJoin.1.Hash_NoBroadcast.plan | 174 +++++++++++++ ...tCrossJoinOfOneSmallTableAndJoin.1.Sort.plan | 89 +++++++ ...OneSmallTableAndJoin.1.Sort_NoBroadcast.plan | 174 +++++++++++++ ...testCrossJoinOfOneSmallTableAndJoin.1.result | 2 + .../testCrossJoinOftwoSmallTables.1.Hash.plan | 51 ++++ ...JoinOftwoSmallTables.1.Hash_NoBroadcast.plan | 78 ++++++ .../testCrossJoinOftwoSmallTables.1.Sort.plan | 51 ++++ ...JoinOftwoSmallTables.1.Sort_NoBroadcast.plan | 78 ++++++ .../testCrossJoinOftwoSmallTables.1.result | 27 ++ .../testCrossJoinWithAsterisk1.Hash.plan | 76 ++++++ ...CrossJoinWithAsterisk1.Hash_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk1.Sort.plan | 76 ++++++ ...CrossJoinWithAsterisk1.Sort_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk1.result | 27 ++ .../testCrossJoinWithAsterisk2.Hash.plan | 76 ++++++ ...CrossJoinWithAsterisk2.Hash_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk2.Sort.plan | 76 ++++++ ...CrossJoinWithAsterisk2.Sort_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk2.result | 27 ++ .../testCrossJoinWithAsterisk3.Hash.plan | 76 ++++++ ...CrossJoinWithAsterisk3.Hash_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk3.Sort.plan | 76 ++++++ ...CrossJoinWithAsterisk3.Sort_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk3.result | 27 ++ .../testCrossJoinWithAsterisk4.Hash.plan | 76 ++++++ ...CrossJoinWithAsterisk4.Hash_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk4.Sort.plan | 76 ++++++ ...CrossJoinWithAsterisk4.Sort_NoBroadcast.plan | 103 ++++++++ .../testCrossJoinWithAsterisk4.result | 27 ++ .../testCrossJoinWithEmptyTable1.Hash.plan | 78 ++++++ ...ossJoinWithEmptyTable1.Hash_NoBroadcast.plan | 105 ++++++++ .../testCrossJoinWithEmptyTable1.Sort.plan | 78 ++++++ ...ossJoinWithEmptyTable1.Sort_NoBroadcast.plan | 105 ++++++++ .../testCrossJoinWithEmptyTable1.result | 2 + ...sJoinWithThetaJoinConditionInWhere.Hash.plan | 63 +++++ ...taJoinConditionInWhere.Hash_NoBroadcast.plan | 90 +++++++ ...sJoinWithThetaJoinConditionInWhere.Sort.plan | 63 +++++ ...taJoinConditionInWhere.Sort_NoBroadcast.plan | 90 +++++++ ...rossJoinWithThetaJoinConditionInWhere.result | 12 + .../testCrossJoin.1.Hash.plan | 76 ------ .../testCrossJoin.1.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoin.1.Sort.plan | 76 ------ .../testCrossJoin.1.Sort_NoBroadcast.plan | 103 -------- .../TestInnerJoinQuery/testCrossJoin.1.result | 127 ---------- .../testCrossJoin.2.Hash.plan | 51 ---- .../testCrossJoin.2.Hash_NoBroadcast.plan | 78 ------ .../testCrossJoin.2.Sort.plan | 51 ---- .../testCrossJoin.2.Sort_NoBroadcast.plan | 78 ------ .../TestInnerJoinQuery/testCrossJoin.2.result | 27 -- .../testCrossJoin.3.Hash.plan | 51 ---- .../testCrossJoin.3.Hash_NoBroadcast.plan | 78 ------ .../testCrossJoin.3.Sort.plan | 51 ---- .../testCrossJoin.3.Sort_NoBroadcast.plan | 78 ------ .../TestInnerJoinQuery/testCrossJoin.3.result | 27 -- .../testCrossJoin.4.Hash.plan | 51 ---- .../testCrossJoin.4.Hash_NoBroadcast.plan | 78 ------ .../testCrossJoin.4.Sort.plan | 51 ---- .../testCrossJoin.4.Sort_NoBroadcast.plan | 78 ------ .../TestInnerJoinQuery/testCrossJoin.4.result | 27 -- .../testCrossJoin.5.Hash.plan | 76 ------ .../testCrossJoin.5.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoin.5.Sort.plan | 76 ------ .../testCrossJoin.5.Sort_NoBroadcast.plan | 103 -------- .../TestInnerJoinQuery/testCrossJoin.5.result | 27 -- .../testCrossJoinAndCaseWhen.Hash.plan | 78 ------ ...stCrossJoinAndCaseWhen.Hash_NoBroadcast.plan | 105 -------- .../testCrossJoinAndCaseWhen.Sort.plan | 78 ------ ...stCrossJoinAndCaseWhen.Sort_NoBroadcast.plan | 105 -------- .../testCrossJoinAndCaseWhen.result | 27 -- .../testCrossJoinWithAsterisk1.Hash.plan | 76 ------ ...CrossJoinWithAsterisk1.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk1.Sort.plan | 76 ------ ...CrossJoinWithAsterisk1.Sort_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk1.result | 27 -- .../testCrossJoinWithAsterisk2.Hash.plan | 76 ------ ...CrossJoinWithAsterisk2.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk2.Sort.plan | 76 ------ ...CrossJoinWithAsterisk2.Sort_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk2.result | 27 -- .../testCrossJoinWithAsterisk3.Hash.plan | 76 ------ ...CrossJoinWithAsterisk3.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk3.Sort.plan | 76 ------ ...CrossJoinWithAsterisk3.Sort_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk3.result | 27 -- .../testCrossJoinWithAsterisk4.Hash.plan | 76 ------ ...CrossJoinWithAsterisk4.Hash_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk4.Sort.plan | 76 ------ ...CrossJoinWithAsterisk4.Sort_NoBroadcast.plan | 103 -------- .../testCrossJoinWithAsterisk4.result | 27 -- .../testCrossJoinWithEmptyTable1.Hash.plan | 78 ------ ...ossJoinWithEmptyTable1.Hash_NoBroadcast.plan | 105 -------- .../testCrossJoinWithEmptyTable1.Sort.plan | 78 ------ ...ossJoinWithEmptyTable1.Sort_NoBroadcast.plan | 105 -------- .../testCrossJoinWithEmptyTable1.result | 2 - ...sJoinWithThetaJoinConditionInWhere.Hash.plan | 63 ----- ...taJoinConditionInWhere.Hash_NoBroadcast.plan | 90 ------- ...sJoinWithThetaJoinConditionInWhere.Sort.plan | 63 ----- ...taJoinConditionInWhere.Sort_NoBroadcast.plan | 90 ------- ...rossJoinWithThetaJoinConditionInWhere.result | 12 - .../testJoinWithMultipleJoinQual1.Hash.plan | 4 +- ...nWithMultipleJoinQual1.Hash_NoBroadcast.plan | 4 +- .../testJoinWithMultipleJoinQual1.Sort.plan | 4 +- ...nWithMultipleJoinQual1.Sort_NoBroadcast.plan | 4 +- .../testJoinWithOrPredicates.Hash.plan | 4 +- ...stJoinWithOrPredicates.Hash_NoBroadcast.plan | 4 +- .../testJoinWithOrPredicates.Sort.plan | 4 +- ...stJoinWithOrPredicates.Sort_NoBroadcast.plan | 4 +- .../testNaturalJoin.Hash.plan | 4 +- .../testNaturalJoin.Hash_NoBroadcast.plan | 4 +- .../testNaturalJoin.Sort.plan | 4 +- .../testNaturalJoin.Sort_NoBroadcast.plan | 4 +- ...testJoinFilterOfRowPreservedTable1.Hash.plan | 4 +- ...erOfRowPreservedTable1.Hash_NoBroadcast.plan | 4 +- ...testJoinFilterOfRowPreservedTable1.Sort.plan | 4 +- ...erOfRowPreservedTable1.Sort_NoBroadcast.plan | 4 +- .../testLeftOuterJoin2.Hash.plan | 56 ++--- .../testLeftOuterJoin2.Hash_NoBroadcast.plan | 118 ++++----- .../testLeftOuterJoin2.Sort.plan | 56 ++--- .../testLeftOuterJoin2.Sort_NoBroadcast.plan | 118 ++++----- .../testLeftOuterJoin3.Hash.plan | 73 ++---- .../testLeftOuterJoin3.Sort.plan | 73 ++---- ...tOuterJoinPredicationCaseByCase1.1.Hash.plan | 4 +- ...edicationCaseByCase1.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase1.1.Sort.plan | 4 +- ...edicationCaseByCase1.1.Sort_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase2.1.Hash.plan | 67 ++--- ...tOuterJoinPredicationCaseByCase2.1.Sort.plan | 67 ++--- ...tOuterJoinPredicationCaseByCase3.1.Hash.plan | 4 +- ...edicationCaseByCase3.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase3.1.Sort.plan | 4 +- ...edicationCaseByCase3.1.Sort_NoBroadcast.plan | 4 +- .../testLeftOuterJoinWithNull2.Hash.plan | 4 +- ...LeftOuterJoinWithNull2.Hash_NoBroadcast.plan | 4 +- .../testLeftOuterJoinWithNull2.Sort.plan | 4 +- ...LeftOuterJoinWithNull2.Sort_NoBroadcast.plan | 4 +- .../testOuterJoinAndCaseWhen1.Hash.plan | 73 ++---- .../testOuterJoinAndCaseWhen1.Sort.plan | 73 ++---- ...tOuterJoinPredicationCaseByCase1.1.Hash.plan | 71 ++---- ...edicationCaseByCase1.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase1.1.Sort.plan | 71 ++---- ...edicationCaseByCase1.1.Sort_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase2.1.Hash.plan | 4 +- ...edicationCaseByCase2.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase2.1.Sort.plan | 4 +- ...edicationCaseByCase2.1.Sort_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase3.1.Hash.plan | 69 ++---- ...edicationCaseByCase3.1.Hash_NoBroadcast.plan | 4 +- ...tOuterJoinPredicationCaseByCase3.1.Sort.plan | 69 ++---- ...edicationCaseByCase3.1.Sort_NoBroadcast.plan | 4 +- .../TestTajoCli/testHelpSessionVars.result | 5 +- .../engine/planner/PhysicalPlannerImpl.java | 22 +- .../tajo/engine/planner/enforce/Enforcer.java | 4 - .../engine/planner/global/ExecutionBlock.java | 4 +- .../rewriter/rules/BroadcastJoinRule.java | 248 +++++++++++++++---- .../rewriter/rules/GlobalPlanRewriteUtil.java | 2 +- .../engine/planner/physical/BNLJoinExec.java | 183 -------------- .../planner/physical/BSTIndexScanExec.java | 26 +- .../physical/BasicPhysicalExecutorVisitor.java | 29 +-- .../planner/physical/CommonHashJoinExec.java | 91 +++++-- .../planner/physical/HashFullOuterJoinExec.java | 2 +- .../engine/planner/physical/HashJoinExec.java | 7 +- .../planner/physical/IndexExecutorUtil.java | 33 +++ .../engine/planner/physical/NLJoinExec.java | 75 ------ .../physical/PartitionMergeScanExec.java | 5 + .../physical/PhysicalExecutorVisitor.java | 9 +- .../tajo/engine/planner/physical/ScanExec.java | 3 + .../engine/planner/physical/SeqScanExec.java | 5 + .../engine/planner/physical/StoreIndexExec.java | 10 +- .../apache/tajo/engine/utils/CacheHolder.java | 26 +- .../org/apache/tajo/master/GlobalEngine.java | 15 +- .../tajo/master/QueryCoordinatorService.java | 6 +- .../org/apache/tajo/master/QueryInProgress.java | 7 +- .../apache/tajo/master/exec/QueryExecutor.java | 33 ++- .../java/org/apache/tajo/querymaster/Query.java | 68 ++--- .../apache/tajo/querymaster/QueryMaster.java | 6 + .../tajo/querymaster/QueryMasterTask.java | 57 +++-- .../apache/tajo/querymaster/Repartitioner.java | 6 +- .../java/org/apache/tajo/querymaster/Stage.java | 27 +- .../apache/tajo/querymaster/TaskAttempt.java | 17 +- .../tajo/worker/ExecutionBlockContext.java | 2 +- .../apache/tajo/worker/TaskAttemptContext.java | 11 - tajo-core/src/main/proto/ResourceProtos.proto | 4 +- .../org/apache/tajo/plan/LogicalPlanner.java | 1 + .../GreedyHeuristicJoinOrderAlgorithm.java | 20 +- .../plan/rewrite/rules/FilterPushDownRule.java | 3 +- .../org/apache/tajo/plan/util/IndexUtil.java | 72 ------ .../org/apache/tajo/plan/util/PlannerUtil.java | 6 + .../plan/verifier/PostLogicalPlanVerifier.java | 148 +++++++++++ tajo-plan/src/main/proto/Plan.proto | 2 - .../tajo/storage/pgsql/EmbedPgSQLServer.java | 45 +++- .../storage/pgsql/TestPgSQLJdbcTableSpace.java | 10 +- .../pgsql/TestPgSQLMetadataProvider.java | 2 +- .../pgsql/TestPgSQLSimpleQueryTests.java | 1 + .../testSelectAll.result | 2 +- .../testSelectLimit.result | 2 +- 267 files changed, 7149 insertions(+), 6310 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java ---------------------------------------------------------------------- diff --cc tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java index aa426c8,7a9f0c3..2d57c0a --- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java @@@ -406,17 -405,7 +406,17 @@@ public class StringUtils * @return A joined string */ public static String join(Object[] objects) { - return join(objects, ","); - return join(objects, ", ", 0, objects.length); ++ return join(objects, ", "); + } + + /** + * Concatenate all objects' string with the delimiter ", " + * + * @param objects Iterable objects + * @return A joined string + */ + public static String join(Object[] objects, String delimiter) { + return join(objects, delimiter, 0, objects.length); } /** http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 6dd5a7f,d9fb218..8defa99 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@@ -968,13 -993,9 +968,19 @@@ public class PlannerUtil return inSubqueries; } + public static int [] getTargetIds(Schema inputSchema, Column...targets) { + int [] targetIds = new int[targets.length]; + for (int i = 0; i < targetIds.length; i++) { + targetIds[i] = inputSchema.getColumnId(targets[i].getQualifiedName()); + } + Arrays.sort(targetIds); + + return targetIds; + } ++ + public static List<EvalNode> getAllEqualEvals(EvalNode qual) { + EvalTreeUtil.EvalFinder finder = new EvalTreeUtil.EvalFinder(EvalType.EQUAL); + finder.visit(null, qual, new Stack<EvalNode>()); + return finder.getEvalNodes(); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-plan/src/main/proto/Plan.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java ---------------------------------------------------------------------- diff --cc tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java index a30a1cd,0000000..f9425ad mode 100644,000000..100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/EmbedPgSQLServer.java @@@ -1,135 -1,0 +1,174 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.pgsql; + +import io.airlift.testing.postgresql.TestingPostgreSqlServer; +import net.minidev.json.JSONObject; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.FileUtil; +import org.apache.tajo.util.JavaResourceUtil; + +import java.io.IOException; +import java.net.URI; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; + +public class EmbedPgSQLServer { + private static final Log LOG = LogFactory.getLog(EmbedPgSQLServer.class); + + private static EmbedPgSQLServer instance; + + public static final String [] TPCH_TABLES = { + "customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier" + }; + + public static final String SPACENAME = "pgsql_cluster"; + public static final String DATABASE_NAME = "tpch"; + - private TestingPostgreSqlServer server; ++ private final TestingPostgreSqlServer server; + + static { + try { + instance = new EmbedPgSQLServer(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static EmbedPgSQLServer getInstance() { + return instance; + } + + private EmbedPgSQLServer() throws Exception { + server = new TestingPostgreSqlServer("testuser", + "tpch" + ); + ++ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { ++ @Override ++ public void run() { ++ try { ++ server.close(); ++ } catch (IOException e) { ++ e.printStackTrace(); ++ } ++ } ++ })); ++ + loadTPCHTables(); + registerTablespace(); + } + + private void loadTPCHTables() throws SQLException, IOException { + Path testPath = CommonTestingUtil.getTestDir(); + + try (Connection connection = DriverManager.getConnection(getJdbcUrlForAdmin(), "postgres", null)) { + connection.setCatalog("tpch"); + + try (Statement statement = connection.createStatement()) { + + for (String tableName : TPCH_TABLES) { + String sql = JavaResourceUtil.readTextFromResource("tpch/pgsql/" + tableName + ".sql"); + statement.addBatch(sql); + } + + statement.executeBatch(); + + for (String tableName : TPCH_TABLES) { - String table = JavaResourceUtil.readTextFromResource("tpch/" + tableName + ".tbl"); ++ String csvTable = JavaResourceUtil.readTextFromResource("tpch/" + tableName + ".tbl"); ++ String fixedCsvTable = fixExtraColumn(csvTable); + Path filePath = new Path(testPath, tableName + ".tbl"); - FileUtil.writeTextToFile(table, filePath); ++ FileUtil.writeTextToFile(fixedCsvTable, filePath); + + String copyCommand = + "COPY " + tableName + " FROM '" + filePath.toUri().getPath() + "' WITH (FORMAT csv, DELIMITER '|');"; + statement.executeUpdate(copyCommand); + } + + } catch (Throwable t) { + t.printStackTrace(); + throw t; + } + } + } + ++ private String fixExtraColumn(String csvTable) { ++ final String [] lines = csvTable.split("\n"); ++ final StringBuilder rewritten = new StringBuilder(); ++ ++ for (String l : lines) { ++ if (l.charAt(l.length() - 1) == '|') { ++ rewritten.append(l.substring(0, l.length() - 1)); ++ } else { ++ rewritten.append(l.substring(0, l.length())); ++ } ++ rewritten.append("\n"); ++ } ++ ++ return rewritten.toString(); ++ } ++ + private void registerTablespace() throws IOException { + JSONObject configElements = new JSONObject(); + configElements.put("database", DATABASE_NAME); + + Map<String, JSONObject> configMap = new HashMap<>(); + configMap.put(TablespaceManager.TABLESPACE_SPEC_CONFIGS_KEY, configElements); + JSONObject config = new JSONObject(configMap); + + PgSQLTablespace tablespace = new PgSQLTablespace(SPACENAME, URI.create(getJdbcUrlForAdmin()), config); + tablespace.init(new TajoConf()); + + TablespaceManager.addTableSpaceForTest(tablespace); + } + ++ /** ++ * get JDBC URL for a created user ++ * ++ * @return JDBC URL for the created user ++ */ + public String getJdbcUrl() { + return server.getJdbcUrl() + "&connectTimeout=5&socketTimeout=5"; + } + ++ /** ++ * get JDBC URL for the Admin user ++ * ++ * @return JDBC URL for the Admin user ++ */ + public String getJdbcUrlForAdmin() { ++ // replace 'user' by postgres (admin) + String url = server.getJdbcUrl().split("\\?")[0]; + url += "?user=postgres&connectTimeout=5&socketTimeout=5"; + return url; + } + + public TestingPostgreSqlServer getServer() { + return server; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java ---------------------------------------------------------------------- diff --cc tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java index 179eb33,0000000..f3b2d26 mode 100644,000000..100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java @@@ -1,75 -1,0 +1,75 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.pgsql; + +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.storage.fragment.Fragment; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class TestPgSQLJdbcTableSpace { - static String jdbcUrl = EmbedPgSQLServer.getInstance().getJdbcUrl(); ++ static String jdbcUrl = EmbedPgSQLServer.getInstance().getJdbcUrlForAdmin(); + - @Test ++ @Test(timeout = 5000) + public void testTablespaceHandler() throws Exception { + assertTrue((TablespaceManager.getByName("pgsql_cluster").get()) instanceof PgSQLTablespace); + assertEquals("pgsql_cluster", (TablespaceManager.getByName("pgsql_cluster").get().getName())); + + assertTrue((TablespaceManager.get(jdbcUrl).get()) instanceof PgSQLTablespace); + assertTrue((TablespaceManager.get(jdbcUrl + "&table=tb1").get()) instanceof PgSQLTablespace); + + assertEquals(jdbcUrl, TablespaceManager.get(jdbcUrl).get().getUri().toASCIIString()); + assertTrue(TablespaceManager.get(jdbcUrl).get().getMetadataProvider() instanceof PgSQLMetadataProvider); + } + - @Test(expected = TajoRuntimeException.class) ++ @Test(timeout = 5000, expected = TajoRuntimeException.class) + public void testCreateTable() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName("pgsql_cluster").get(); + space.createTable(null, false); + } + - @Test(expected = TajoRuntimeException.class) ++ @Test(timeout = 5000, expected = TajoRuntimeException.class) + public void testDropTable() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName("pgsql_cluster").get(); + space.purgeTable(null); + } + - @Test ++ @Test(timeout = 5000) + public void testGetSplits() throws IOException, TajoException { + Tablespace space = TablespaceManager.getByName("pgsql_cluster").get(); + MetadataProvider provider = space.getMetadataProvider(); + TableDesc table = provider.getTableDescriptor(null, "lineitem"); + List<Fragment> fragments = space.getSplits("lineitem", table, null); + assertNotNull(fragments); + assertEquals(1, fragments.size()); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java ---------------------------------------------------------------------- diff --cc tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java index aac25f7,0000000..7650f86 mode 100644,000000..100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java @@@ -1,81 -1,0 +1,81 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.pgsql; + +import com.google.common.collect.Sets; +import org.apache.tajo.catalog.MetadataProvider; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TablespaceManager; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Set; + +import static org.junit.Assert.*; + +public class TestPgSQLMetadataProvider { - static final String jdbcUrl = EmbedPgSQLServer.getInstance().getJdbcUrl(); ++ static final String jdbcUrl = EmbedPgSQLServer.getInstance().getJdbcUrlForAdmin(); + + @BeforeClass + public static void setUp() throws Exception { + } + + @Test + public void testGetTablespaceName() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals("pgsql_cluster", provider.getTablespaceName()); + } + + @Test + public void testGetDatabaseName() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertEquals("tpch", provider.getDatabaseName()); + } + + @Test + public void testGetSchemas() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + assertTrue(provider.getSchemas().isEmpty()); + } + + @Test + public void testGetTables() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + + Set<String> found = Sets.newHashSet(provider.getTables(null, null)); + assertEquals(Sets.newHashSet(EmbedPgSQLServer.TPCH_TABLES), found); + } + + @Test + public void testGetTableDescriptor() throws Exception { + Tablespace tablespace = TablespaceManager.get(jdbcUrl).get(); + MetadataProvider provider = tablespace.getMetadataProvider(); + + for (String tableName : EmbedPgSQLServer.TPCH_TABLES) { + TableDesc table = provider.getTableDescriptor(null, tableName); + assertEquals("tpch." + tableName, table.getName()); + assertEquals(jdbcUrl + "&table=" + tableName, table.getUri().toASCIIString()); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLSimpleQueryTests.java ---------------------------------------------------------------------- diff --cc tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLSimpleQueryTests.java index 3971bbc,0000000..2968831 mode 100644,000000..100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLSimpleQueryTests.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLSimpleQueryTests.java @@@ -1,47 -1,0 +1,48 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage.pgsql; + +import org.apache.tajo.QueryTestCaseBase; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestPgSQLSimpleQueryTests extends QueryTestCaseBase { ++ private static final String jdbcUrl = EmbedPgSQLServer.getInstance().getJdbcUrl(); + + public TestPgSQLSimpleQueryTests() { + super(EmbedPgSQLServer.DATABASE_NAME); + } + + @BeforeClass + public static void setUp() { + QueryTestCaseBase.testingCluster.getMaster().refresh(); + } + + @SimpleTest + @Test + public void testSelectAll() throws Exception { + runSimpleTests(); + } + + @SimpleTest + @Test + public void testSelectLimit() throws Exception { + runSimpleTests(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectAll.result ---------------------------------------------------------------------- diff --cc tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectAll.result index 8d66d30,0000000..6649940 mode 100644,000000..100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectAll.result +++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectAll.result @@@ -1,7 -1,0 +1,7 @@@ +l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment +------------------------------- +1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,0096-02-03,0096-01-01,0096-02-05,DELIVER IN PERSON ,TRUCK ,egular courts above the - 1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,0096-03-05,0096-01-03,0096-03-06,TAKE BACK RETURN ,MAIL ,ly final dependencies: slyly bold ++1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,0096-03-05,0096-01-03,0096-03-06,TAKE BACK RETURN ,MAIL ,ly final dependencies: slyly bold +2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,0096-12-02,0096-12-02,0096-12-31,TAKE BACK RETURN ,RAIL ,ven requests. deposits breach a +3,2,1798,1,45.0,54058.05,0.06,0.0,R,F,0094-01-03,0093-12-02,0094-01-03,NONE ,AIR ,ongside of the furiously brave acco +3,3,6540,2,49.0,46796.47,0.1,0.0,R,F,0093-10-02,0093-11-01,0093-10-03,TAKE BACK RETURN ,RAIL , unusual accounts. eve http://git-wip-us.apache.org/repos/asf/tajo/blob/e3a92621/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectLimit.result ---------------------------------------------------------------------- diff --cc tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectLimit.result index 52c87f6,0000000..88116c8 mode 100644,000000..100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectLimit.result +++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectLimit.result @@@ -1,5 -1,0 +1,5 @@@ +l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment +------------------------------- +1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,0096-02-03,0096-01-01,0096-02-05,DELIVER IN PERSON ,TRUCK ,egular courts above the - 1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,0096-03-05,0096-01-03,0096-03-06,TAKE BACK RETURN ,MAIL ,ly final dependencies: slyly bold ++1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,0096-03-05,0096-01-03,0096-03-06,TAKE BACK RETURN ,MAIL ,ly final dependencies: slyly bold +2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,0096-12-02,0096-12-02,0096-12-31,TAKE BACK RETURN ,RAIL ,ven requests. deposits breach a
