Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/2cbc1b9c Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/2cbc1b9c Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/2cbc1b9c Branch: refs/heads/index_support Commit: 2cbc1b9c7dc1ec04c4f6c8d7c8f746225d02421d Parents: 6f44096 5491f0e Author: Jihoon Son <[email protected]> Authored: Sat May 16 21:05:31 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Sat May 16 21:05:31 2015 +0900 ---------------------------------------------------------------------- CHANGES | 15 + .../java/org/apache/tajo/catalog/Schema.java | 16 + .../java/org/apache/tajo/catalog/TableMeta.java | 6 +- .../org/apache/tajo/catalog/TestTableMeta.java | 27 + .../org/apache/tajo/cli/tools/TajoAdmin.java | 6 +- .../org/apache/tajo/cli/tools/TajoHAAdmin.java | 9 +- .../cli/tsql/commands/DescTableCommand.java | 1 + .../apache/tajo/client/DummyServiceTracker.java | 19 +- .../org/apache/tajo/storage/RowStoreUtil.java | 24 +- .../java/org/apache/tajo/conf/TajoConf.java | 2 + .../ValueTooLongForTypeCharactersException.java | 27 + .../java/org/apache/tajo/ha/HAConstants.java | 1 + .../java/org/apache/tajo/ha/HAServiceUtil.java | 253 -------- .../apache/tajo/service/BaseServiceTracker.java | 31 +- .../apache/tajo/service/HAServiceTracker.java | 25 +- .../org/apache/tajo/service/ServiceTracker.java | 28 +- .../org/apache/tajo/storage/EmptyTuple.java | 140 +--- .../java/org/apache/tajo/storage/NullTuple.java | 175 +++++ .../java/org/apache/tajo/storage/VTuple.java | 20 +- .../engine/planner/PhysicalPlannerImpl.java | 25 +- .../planner/physical/BSTIndexScanExec.java | 4 +- .../physical/BasicPhysicalExecutorVisitor.java | 8 - .../planner/physical/ColPartitionStoreExec.java | 7 +- .../planner/physical/CommonHashJoinExec.java | 191 ++++++ .../engine/planner/physical/CommonJoinExec.java | 172 ++++- .../planner/physical/HashFullOuterJoinExec.java | 247 +++---- .../engine/planner/physical/HashJoinExec.java | 212 +----- .../planner/physical/HashLeftAntiJoinExec.java | 59 +- .../planner/physical/HashLeftOuterJoinExec.java | 292 +-------- .../planner/physical/HashLeftSemiJoinExec.java | 48 +- .../planner/physical/NLLeftOuterJoinExec.java | 101 --- .../physical/PhysicalExecutorVisitor.java | 3 - .../physical/RangeShuffleFileWriteExec.java | 3 +- .../physical/RightOuterMergeJoinExec.java | 40 +- .../engine/planner/physical/SeqScanExec.java | 2 +- .../engine/planner/physical/StoreTableExec.java | 9 +- .../apache/tajo/engine/utils/CacheHolder.java | 3 +- .../org/apache/tajo/ha/HdfsServiceTracker.java | 322 +++++---- .../org/apache/tajo/master/GlobalEngine.java | 4 +- .../java/org/apache/tajo/master/TajoMaster.java | 24 +- .../apache/tajo/master/exec/DDLExecutor.java | 5 +- .../exec/NonForwardQueryResultFileScanner.java | 7 +- .../apache/tajo/master/exec/QueryExecutor.java | 4 +- .../java/org/apache/tajo/querymaster/Query.java | 7 +- .../tajo/querymaster/QueryMasterTask.java | 4 +- .../apache/tajo/querymaster/Repartitioner.java | 24 +- .../java/org/apache/tajo/querymaster/Stage.java | 6 +- .../main/java/org/apache/tajo/util/JSPUtil.java | 2 +- .../java/org/apache/tajo/worker/TajoWorker.java | 6 +- .../main/java/org/apache/tajo/worker/Task.java | 3 +- .../tajo/worker/WorkerHeartbeatService.java | 2 +- .../resources/webapps/admin/catalogview.jsp | 11 +- .../main/resources/webapps/admin/cluster.jsp | 10 +- .../src/main/resources/webapps/admin/index.jsp | 10 +- .../src/main/resources/webapps/admin/query.jsp | 7 +- .../resources/webapps/admin/query_executor.jsp | 9 +- .../org/apache/tajo/BackendTestingUtil.java | 3 +- .../planner/global/TestBroadcastJoinPlan.java | 3 +- .../planner/physical/TestBNLJoinExec.java | 5 +- .../planner/physical/TestExternalSortExec.java | 3 +- .../physical/TestFullOuterHashJoinExec.java | 9 +- .../physical/TestFullOuterMergeJoinExec.java | 10 +- .../planner/physical/TestHashAntiJoinExec.java | 5 +- .../planner/physical/TestHashJoinExec.java | 5 +- .../planner/physical/TestHashSemiJoinExec.java | 13 +- .../physical/TestLeftOuterHashJoinExec.java | 113 ++-- .../physical/TestLeftOuterNLJoinExec.java | 474 -------------- .../planner/physical/TestMergeJoinExec.java | 5 +- .../engine/planner/physical/TestNLJoinExec.java | 5 +- .../planner/physical/TestPhysicalPlanner.java | 11 +- .../physical/TestProgressExternalSortExec.java | 3 +- .../physical/TestRightOuterHashJoinExec.java | 7 +- .../physical/TestRightOuterMergeJoinExec.java | 11 +- .../engine/planner/physical/TestSortExec.java | 5 +- .../tajo/engine/query/TestHBaseTable.java | 12 +- .../tajo/engine/query/TestInsertQuery.java | 45 +- .../tajo/engine/query/TestJoinBroadcast.java | 2 +- .../apache/tajo/engine/util/TestTupleUtil.java | 18 + .../apache/tajo/ha/TestHAServiceHDFSImpl.java | 28 +- .../org/apache/tajo/jdbc/TestResultSet.java | 2 +- .../tajo/master/TestExecutionBlockCursor.java | 4 +- .../org/apache/tajo/storage/TestRowFile.java | 5 +- .../queries/TestInsertQuery/test1_ddl.sql | 1 + .../TestInsertQuery/test1_nolength_ddl.sql | 1 + .../testInsertIntoSelectWithFixedSizeChar.sql | 4 + ...tIntoSelectWithFixedSizeCharWithNoLength.sql | 2 + .../testJoinFilterOfRowPreservedTable1.sql | 2 +- .../testJoinFilterOfRowPreservedTable1.result | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 4 + .../plan/expr/AggregationFunctionCallEval.java | 4 +- .../apache/tajo/plan/expr/AlgebraicUtil.java | 5 + .../org/apache/tajo/plan/expr/EvalNode.java | 39 +- .../java/org/apache/tajo/plan/expr/InEval.java | 2 +- .../plan/expr/PatternMatchPredicateEval.java | 2 +- .../tajo/plan/expr/WindowFunctionEval.java | 2 +- .../stream/TextFieldSerializerDeserializer.java | 8 +- .../storage/BinarySerializerDeserializer.java | 10 + .../org/apache/tajo/storage/FrameTuple.java | 14 +- .../org/apache/tajo/storage/MergeScanner.java | 2 +- .../org/apache/tajo/storage/RowStoreUtil.java | 20 +- .../org/apache/tajo/storage/StorageManager.java | 645 +------------------ .../org/apache/tajo/storage/TableSpace.java | 74 +++ .../apache/tajo/storage/TableSpaceManager.java | 254 ++++++++ .../storage/TextSerializerDeserializer.java | 10 +- .../tajo/storage/hbase/HBasePutAppender.java | 4 +- .../apache/tajo/storage/hbase/HBaseScanner.java | 7 +- .../tajo/storage/hbase/HBaseStorageManager.java | 40 +- .../storage/hbase/TestHBaseStorageManager.java | 5 +- .../org/apache/tajo/storage/FileAppender.java | 2 +- .../apache/tajo/storage/FileStorageManager.java | 377 ++++++++++- .../storage/HashShuffleAppenderManager.java | 4 +- .../tajo/storage/parquet/TajoWriteSupport.java | 7 + .../text/TextFieldSerializerDeserializer.java | 8 +- .../tajo/storage/TestCompressionStorages.java | 5 +- .../tajo/storage/TestDelimitedTextFile.java | 9 +- .../tajo/storage/TestFileStorageManager.java | 11 +- .../apache/tajo/storage/TestFileSystems.java | 3 +- .../org/apache/tajo/storage/TestLineReader.java | 9 +- .../apache/tajo/storage/TestMergeScanner.java | 7 +- .../org/apache/tajo/storage/TestStorages.java | 94 ++- .../apache/tajo/storage/index/TestBSTIndex.java | 53 +- .../index/TestSingleCSVFileBSTIndex.java | 5 +- .../apache/tajo/storage/json/TestJsonSerDe.java | 8 +- 123 files changed, 2389 insertions(+), 2900 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/CHANGES ---------------------------------------------------------------------- diff --cc CHANGES index 42d9b36,21f5e5a..4372e04 --- a/CHANGES +++ b/CHANGES @@@ -123,8 -128,16 +128,18 @@@ Release 0.11.0 - unrelease BUG FIXES + TAJO-1594: Catalog schema is invalid for some databases. (jihoon) + + TAJO-1605: Fix master build failure on jdk 1.6. (jinho) + + TAJO-1485: Datum 'Char' returned only 1byte. + (Contributed by DaeMyung Kang, Committed by jihoon) + + TAJO-1586: TajoMaster HA startup failure on Yarn. (jaehwa) + + TAJO-1598: TableMeta should change equals mechanism. + (Contributed by DaeMyung Kang, Committed by jihoon) + TAJO-1593: Add missing stop condition to Taskrunner. (jinho) TAJO-1556: "insert into select" with reordered column list does not work. http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java ---------------------------------------------------------------------- diff --cc tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java index 31dfb13,6df26b7..3341c68 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java @@@ -21,6 -21,6 +21,7 @@@ package org.apache.tajo.cli.tsql.comman import org.apache.commons.lang.CharUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.tajo.TajoConstants; ++import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 2dd7fa8,ff42d5f..04315eb --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@@ -46,11 -47,8 +46,12 @@@ import org.apache.tajo.ipc.TajoWorkerPr import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.serder.LogicalNodeDeserializer; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.*; +import org.apache.tajo.storage.FileStorageManager; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.storage.StorageManager; ++import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index 211d438,4612d45..7df48a4 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@@ -68,21 -54,10 +68,21 @@@ public class BSTIndexScanExec extends P super(context, scanNode.getInSchema(), scanNode.getOutSchema()); this.scanNode = scanNode; this.qual = scanNode.getQual(); - this.datum = datum; + + SortSpec[] keySortSpecs = new SortSpec[predicates.length]; + values = new Datum[predicates.length]; + for (int i = 0; i < predicates.length; i++) { + keySortSpecs[i] = predicates[i].getKeySortSpec(); + values[i] = predicates[i].getValue(); + } + + TupleComparator comparator = new BaseTupleComparator(keySchema, + keySortSpecs); + + Schema fileScanOutSchema = mergeSubSchemas(inSchema, keySchema, scanNode.getTargets(), qual); - this.fileScanner = StorageManager.getSeekableScanner(context.getConf(), + this.fileScanner = TableSpaceManager.getSeekableScanner(context.getConf(), - scanNode.getTableDesc().getMeta(), scanNode.getInSchema(), fragment, outSchema); + scanNode.getTableDesc().getMeta(), inSchema, fragment, fileScanOutSchema); this.fileScanner.init(); this.projector = new Projector(context, inSchema, outSchema, scanNode.getTargets()); @@@ -92,23 -67,6 +92,23 @@@ this.reader.open(); } + private static Schema mergeSubSchemas(Schema originalSchema, Schema subSchema, Target[] targets, EvalNode qual) { + Schema mergedSchema = new Schema(); + Set<Column> qualAndTargets = TUtil.newHashSet(); + qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(qual)); + for (Target target : targets) { + qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(target.getEvalTree())); + } - for (Column column : originalSchema.getColumns()) { ++ for (Column column : originalSchema.getRootColumns()) { + if (subSchema.contains(column) + || qualAndTargets.contains(column) + || qualAndTargets.contains(column)) { + mergedSchema.addColumn(column); + } + } + return mergedSchema; + } + @Override public void init() throws IOException { super.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 991cb25,aeb4166..fc9212c --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@@ -55,8 -54,8 +54,9 @@@ import org.apache.tajo.plan.verifier.Pr import org.apache.tajo.plan.verifier.VerificationState; import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.storage.StorageManager; + import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.IPCUtil; import java.io.IOException; import java.sql.SQLException; http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 12f4b79,d77001c..640ec60 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@@ -36,14 -36,7 +36,13 @@@ import org.apache.tajo.algebra.Expr import org.apache.tajo.algebra.JsonHelper; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.TableDesc; - import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.logical.LogicalRootNode; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.UnimplementedException; http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/main/java/org/apache/tajo/worker/Task.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java ---------------------------------------------------------------------- diff --cc tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index ab94aac,5efdede..35cd612 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@@ -80,9 -80,9 +80,9 @@@ public class TestExecutionBlockCursor analyzer = new SQLAnalyzer(); logicalPlanner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(conf); + optimizer = new LogicalOptimizer(conf, catalog); - StorageManager sm = StorageManager.getFileStorageManager(conf); + StorageManager sm = TableSpaceManager.getFileStorageManager(conf); dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/2cbc1b9c/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java ---------------------------------------------------------------------- diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java index ed6a570,c6b7354..7517e3a --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java @@@ -18,12 -18,10 +18,13 @@@ package org.apache.tajo.plan.expr; +import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.Column; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import java.util.ArrayList; + import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Stack;
