Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/94faf6ec Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/94faf6ec Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/94faf6ec Branch: refs/heads/index_support Commit: 94faf6ec86ff596150c3c826813688f9f246c11a Parents: 5893d2c 3413107 Author: Jihoon Son <[email protected]> Authored: Fri Dec 19 21:49:03 2014 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Dec 19 21:49:03 2014 +0900 ---------------------------------------------------------------------- CHANGES | 19 + .../tajo/catalog/AbstractCatalogClient.java | 2 +- .../org/apache/tajo/catalog/CatalogUtil.java | 24 +- .../org/apache/tajo/catalog/DDLBuilder.java | 6 +- .../apache/tajo/catalog/store/HCatalogUtil.java | 3 +- .../org/apache/tajo/cli/tools/TajoAdmin.java | 2 +- .../org/apache/tajo/cli/tools/TajoHAAdmin.java | 2 +- .../apache/tajo/client/SessionConnection.java | 2 +- .../org/apache/tajo/client/TajoClientImpl.java | 5 +- .../apache/tajo/client/TajoHAClientUtil.java | 20 +- .../java/org/apache/tajo/QueryIdFactory.java | 12 +- .../org/apache/tajo/QueryUnitAttemptId.java | 94 -- .../main/java/org/apache/tajo/QueryUnitId.java | 90 -- .../java/org/apache/tajo/TaskAttemptId.java | 94 ++ .../src/main/java/org/apache/tajo/TaskId.java | 90 ++ .../java/org/apache/tajo/ha/HAConstants.java | 27 + .../java/org/apache/tajo/ha/HAServiceUtil.java | 292 ++++++ .../org/apache/tajo/util/HAServiceUtil.java | 293 ------ .../java/org/apache/tajo/util/TajoIdUtils.java | 8 +- .../apache/tajo/validation/PathValidator.java | 2 +- tajo-common/src/main/proto/TajoIdProtos.proto | 6 +- .../apache/tajo/validation/TestValidators.java | 12 + .../physical/HashShuffleFileWriteExec.java | 3 +- .../engine/planner/physical/PhysicalExec.java | 2 +- .../engine/planner/physical/SeqScanExec.java | 2 +- .../tajo/engine/query/QueryUnitRequest.java | 51 -- .../tajo/engine/query/QueryUnitRequestImpl.java | 328 ------- .../apache/tajo/engine/query/TaskRequest.java | 51 ++ .../tajo/engine/query/TaskRequestImpl.java | 328 +++++++ .../tajo/master/DefaultTaskScheduler.java | 162 ++-- .../org/apache/tajo/master/GlobalEngine.java | 8 +- .../apache/tajo/master/LazyTaskScheduler.java | 46 +- .../master/NonForwardQueryResultScanner.java | 6 +- .../apache/tajo/master/TajoContainerProxy.java | 6 +- .../event/ContainerAllocatorEventType.java | 2 +- .../tajo/master/event/LocalTaskEvent.java | 8 +- .../event/QueryUnitAttemptScheduleEvent.java | 87 -- .../tajo/master/event/SubQueryEventType.java | 2 +- .../tajo/master/event/SubQueryTaskEvent.java | 8 +- .../master/event/TaskAttemptAssignedEvent.java | 4 +- .../tajo/master/event/TaskAttemptEvent.java | 8 +- .../master/event/TaskAttemptScheduleEvent.java | 4 +- .../event/TaskAttemptStatusUpdateEvent.java | 4 +- .../event/TaskAttemptToSchedulerEvent.java | 87 ++ .../tajo/master/event/TaskCompletionEvent.java | 4 +- .../org/apache/tajo/master/event/TaskEvent.java | 8 +- .../tajo/master/event/TaskFatalErrorEvent.java | 6 +- .../tajo/master/event/TaskRequestEvent.java | 9 +- .../tajo/master/event/TaskTAttemptEvent.java | 10 +- .../tajo/master/ha/HAServiceHDFSImpl.java | 100 +- .../apache/tajo/master/querymaster/Query.java | 2 +- .../master/querymaster/QueryJobManager.java | 5 +- .../tajo/master/querymaster/QueryMaster.java | 2 +- .../querymaster/QueryMasterManagerService.java | 22 +- .../master/querymaster/QueryMasterTask.java | 16 +- .../tajo/master/querymaster/QueryUnit.java | 907 ------------------- .../master/querymaster/QueryUnitAttempt.java | 443 --------- .../tajo/master/querymaster/Repartitioner.java | 14 +- .../tajo/master/querymaster/SubQuery.java | 72 +- .../apache/tajo/master/querymaster/Task.java | 907 +++++++++++++++++++ .../tajo/master/querymaster/TaskAttempt.java | 443 +++++++++ .../main/java/org/apache/tajo/util/JSPUtil.java | 84 +- .../apache/tajo/util/history/HistoryReader.java | 22 +- .../apache/tajo/util/history/HistoryWriter.java | 4 +- .../tajo/util/history/QueryUnitHistory.java | 167 ---- .../tajo/util/history/SubQueryHistory.java | 22 +- .../apache/tajo/util/history/TaskHistory.java | 167 ++++ .../tajo/worker/ExecutionBlockContext.java | 12 +- .../java/org/apache/tajo/worker/FetchImpl.java | 19 +- .../apache/tajo/worker/InterDataRetriever.java | 8 +- .../tajo/worker/TajoResourceAllocator.java | 2 +- .../java/org/apache/tajo/worker/TajoWorker.java | 1 + .../tajo/worker/TajoWorkerManagerService.java | 8 +- .../main/java/org/apache/tajo/worker/Task.java | 31 +- .../apache/tajo/worker/TaskAttemptContext.java | 12 +- .../org/apache/tajo/worker/TaskHistory.java | 18 +- .../java/org/apache/tajo/worker/TaskRunner.java | 18 +- .../apache/tajo/worker/TaskRunnerHistory.java | 16 +- .../apache/tajo/worker/TaskRunnerManager.java | 10 +- .../tajo/worker/WorkerHeartbeatService.java | 2 +- .../retriever/AdvancedDataRetriever.java | 14 +- .../ConnectivityCheckerRuleForTajoWorker.java | 2 +- .../src/main/proto/QueryMasterProtocol.proto | 2 +- .../src/main/proto/TajoWorkerProtocol.proto | 22 +- .../main/resources/webapps/admin/querytasks.jsp | 70 +- .../main/resources/webapps/admin/queryunit.jsp | 134 --- .../src/main/resources/webapps/admin/task.jsp | 134 +++ .../resources/webapps/worker/querytasks.jsp | 70 +- .../main/resources/webapps/worker/queryunit.jsp | 175 ---- .../src/main/resources/webapps/worker/task.jsp | 174 ++++ .../resources/webapps/worker/taskdetail.jsp | 14 +- .../resources/webapps/worker/taskhistory.jsp | 6 +- .../src/main/resources/webapps/worker/tasks.jsp | 14 +- .../apache/tajo/LocalTajoTestingUtility.java | 10 +- .../org/apache/tajo/TajoTestingCluster.java | 3 + .../org/apache/tajo/TestQueryIdFactory.java | 6 +- .../test/java/org/apache/tajo/TestTajoIds.java | 32 +- .../org/apache/tajo/client/TestTajoClient.java | 12 +- .../planner/physical/TestBNLJoinExec.java | 4 +- .../planner/physical/TestBSTIndexExec.java | 2 +- .../planner/physical/TestExternalSortExec.java | 2 +- .../physical/TestFullOuterHashJoinExec.java | 8 +- .../physical/TestFullOuterMergeJoinExec.java | 12 +- .../planner/physical/TestHashAntiJoinExec.java | 2 +- .../planner/physical/TestHashJoinExec.java | 4 +- .../planner/physical/TestHashSemiJoinExec.java | 2 +- .../physical/TestLeftOuterHashJoinExec.java | 10 +- .../physical/TestLeftOuterNLJoinExec.java | 10 +- .../planner/physical/TestMergeJoinExec.java | 2 +- .../engine/planner/physical/TestNLJoinExec.java | 4 +- .../planner/physical/TestPhysicalPlanner.java | 60 +- .../physical/TestProgressExternalSortExec.java | 2 +- .../physical/TestRightOuterHashJoinExec.java | 6 +- .../physical/TestRightOuterMergeJoinExec.java | 12 +- .../engine/planner/physical/TestSortExec.java | 2 +- .../tajo/engine/query/TestGroupByQuery.java | 25 +- .../apache/tajo/master/TestRepartitioner.java | 21 +- .../tajo/master/ha/TestHAServiceHDFSImpl.java | 115 ++- .../querymaster/TestIntermediateEntry.java | 2 +- .../querymaster/TestQueryUnitStatusUpdate.java | 194 ---- .../querymaster/TestTaskStatusUpdate.java | 194 ++++ .../java/org/apache/tajo/util/TestJSPUtil.java | 54 +- .../util/history/TestHistoryWriterReader.java | 39 +- .../org/apache/tajo/worker/TestHistory.java | 8 +- .../tajo/worker/TestRangeRetrieverHandler.java | 4 +- .../worker/dataserver/TestHttpDataServer.java | 12 +- .../TestGroupByQuery/testGroupbyWithLimit1.sql | 1 + .../TestGroupByQuery/testGroupbyWithLimit2.sql | 1 + .../TestGroupByQuery/testGroupbyWithLimit3.sql | 1 + .../queries/TestQueryUnitStatusUpdate/case1.sql | 1 - .../queries/TestQueryUnitStatusUpdate/case2.sql | 5 - .../queries/TestQueryUnitStatusUpdate/case3.sql | 10 - .../TestSelectQuery/datetime_table_ddl.sql | 2 +- .../datetime_table_timezoned_ddl.sql | 2 +- .../queries/TestTaskStatusUpdate/case1.sql | 1 + .../queries/TestTaskStatusUpdate/case2.sql | 5 + .../queries/TestTaskStatusUpdate/case3.sql | 10 + .../testGroupbyWithLimit1.result | 3 + .../testGroupbyWithLimit2.result | 3 + .../testGroupbyWithLimit3.result | 3 + tajo-dist/pom.xml | 4 +- .../configuration/catalog_configuration.rst | 4 +- .../src/main/sphinx/table_management/csv.rst | 4 +- .../sphinx/table_management/table_overview.rst | 4 +- tajo-docs/src/main/sphinx/time_zone.rst | 6 +- .../org/apache/tajo/plan/LogicalPlanner.java | 7 + .../tajo/pullserver/HttpDataServerHandler.java | 2 +- .../retriever/AdvancedDataRetriever.java | 16 +- .../org/apache/tajo/storage/StorageManager.java | 6 +- .../org/apache/tajo/storage/StorageUtil.java | 4 +- .../storage/hbase/AbstractHBaseAppender.java | 6 +- .../tajo/storage/hbase/HBasePutAppender.java | 4 +- .../tajo/storage/hbase/HBaseStorageManager.java | 2 +- .../tajo/storage/hbase/HFileAppender.java | 8 +- .../java/org/apache/tajo/storage/CSVFile.java | 4 +- .../org/apache/tajo/storage/FileAppender.java | 6 +- .../apache/tajo/storage/FileStorageManager.java | 8 +- .../tajo/storage/HashShuffleAppender.java | 12 +- .../storage/HashShuffleAppenderManager.java | 6 +- .../java/org/apache/tajo/storage/RawFile.java | 4 +- .../java/org/apache/tajo/storage/RowFile.java | 4 +- .../apache/tajo/storage/avro/AvroAppender.java | 4 +- .../tajo/storage/parquet/ParquetAppender.java | 4 +- .../org/apache/tajo/storage/rcfile/RCFile.java | 4 +- .../sequencefile/SequenceFileAppender.java | 4 +- .../tajo/storage/text/DelimitedTextFile.java | 5 +- 166 files changed, 3991 insertions(+), 3810 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/94faf6ec/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/94faf6ec/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index ac00c15,6db76ae..a4e49f7 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@@ -987,15 -987,9 +987,15 @@@ public class TestPhysicalPlanner FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); + Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee"); + if (sm.getFileSystem().exists(indexPath)) { + sm.getFileSystem().delete(indexPath, true); + } + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), - LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); + ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(createIndexStmt[0]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); http://git-wip-us.apache.org/repos/asf/tajo/blob/94faf6ec/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ----------------------------------------------------------------------
