Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/829fcd96 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/829fcd96 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/829fcd96 Branch: refs/heads/index_support Commit: 829fcd96377926e77c3161639037ff3e88f62203 Parents: 5893d2c 3c833e2 Author: Jihoon Son <[email protected]> Authored: Mon Dec 22 17:10:25 2014 +0900 Committer: Jihoon Son <[email protected]> Committed: Mon Dec 22 17:10:25 2014 +0900 ---------------------------------------------------------------------- CHANGES | 27 + .../tajo/catalog/AbstractCatalogClient.java | 2 +- .../org/apache/tajo/catalog/CatalogUtil.java | 24 +- .../org/apache/tajo/catalog/DDLBuilder.java | 6 +- .../apache/tajo/catalog/store/HCatalogUtil.java | 3 +- .../org/apache/tajo/cli/tools/TajoAdmin.java | 2 +- .../org/apache/tajo/cli/tools/TajoHAAdmin.java | 2 +- .../apache/tajo/client/SessionConnection.java | 2 +- .../org/apache/tajo/client/TajoClientImpl.java | 5 +- .../apache/tajo/client/TajoHAClientUtil.java | 20 +- tajo-client/src/main/proto/ClientProtos.proto | 4 +- .../java/org/apache/tajo/QueryIdFactory.java | 12 +- .../org/apache/tajo/QueryUnitAttemptId.java | 94 -- .../main/java/org/apache/tajo/QueryUnitId.java | 90 -- .../java/org/apache/tajo/TaskAttemptId.java | 94 ++ .../src/main/java/org/apache/tajo/TaskId.java | 90 ++ .../java/org/apache/tajo/ha/HAConstants.java | 27 + .../java/org/apache/tajo/ha/HAServiceUtil.java | 292 ++++ .../org/apache/tajo/util/HAServiceUtil.java | 293 ---- .../java/org/apache/tajo/util/TajoIdUtils.java | 8 +- .../apache/tajo/validation/PathValidator.java | 2 +- tajo-common/src/main/proto/TajoIdProtos.proto | 6 +- .../apache/tajo/validation/TestValidators.java | 12 + .../physical/HashShuffleFileWriteExec.java | 3 +- .../engine/planner/physical/PhysicalExec.java | 2 +- .../engine/planner/physical/SeqScanExec.java | 2 +- .../tajo/engine/query/QueryUnitRequest.java | 51 - .../tajo/engine/query/QueryUnitRequestImpl.java | 328 ----- .../apache/tajo/engine/query/TaskRequest.java | 51 + .../tajo/engine/query/TaskRequestImpl.java | 328 +++++ .../tajo/master/DefaultTaskScheduler.java | 188 +-- .../org/apache/tajo/master/GlobalEngine.java | 789 +--------- .../apache/tajo/master/LazyTaskScheduler.java | 70 +- .../master/NonForwardQueryResultScanner.java | 6 +- .../apache/tajo/master/TajoContainerProxy.java | 6 +- .../java/org/apache/tajo/master/TajoMaster.java | 2 +- .../tajo/master/TajoMasterClientService.java | 9 +- .../tajo/master/TaskSchedulerFactory.java | 12 +- .../event/ContainerAllocatorEventType.java | 2 +- .../tajo/master/event/LocalTaskEvent.java | 8 +- .../tajo/master/event/QueryCompletedEvent.java | 8 +- .../tajo/master/event/QueryEventType.java | 4 +- .../tajo/master/event/QuerySubQueryEvent.java | 35 - .../event/QueryUnitAttemptScheduleEvent.java | 87 -- .../tajo/master/event/StageCompletedEvent.java | 42 + .../event/StageContainerAllocationEvent.java | 38 + .../event/StageDiagnosticsUpdateEvent.java | 34 + .../apache/tajo/master/event/StageEvent.java | 35 + .../tajo/master/event/StageEventType.java | 43 + .../tajo/master/event/StageTaskEvent.java | 43 + .../master/event/SubQueryCompletedEvent.java | 42 - .../event/SubQueryContainerAllocationEvent.java | 38 - .../event/SubQueryDiagnosticsUpdateEvent.java | 34 - .../apache/tajo/master/event/SubQueryEvent.java | 35 - .../tajo/master/event/SubQueryEventType.java | 43 - .../tajo/master/event/SubQueryTaskEvent.java | 43 - .../master/event/TaskAttemptAssignedEvent.java | 4 +- .../tajo/master/event/TaskAttemptEvent.java | 8 +- .../master/event/TaskAttemptScheduleEvent.java | 4 +- .../event/TaskAttemptStatusUpdateEvent.java | 4 +- .../event/TaskAttemptToSchedulerEvent.java | 87 ++ .../tajo/master/event/TaskCompletionEvent.java | 4 +- .../org/apache/tajo/master/event/TaskEvent.java | 8 +- .../apache/tajo/master/event/TaskEventType.java | 4 +- .../tajo/master/event/TaskFatalErrorEvent.java | 6 +- .../tajo/master/event/TaskRequestEvent.java | 9 +- .../tajo/master/event/TaskTAttemptEvent.java | 10 +- .../apache/tajo/master/exec/DDLExecutor.java | 434 ++++++ .../apache/tajo/master/exec/QueryExecutor.java | 391 +++++ .../master/exec/prehook/CreateTableHook.java | 53 + .../exec/prehook/DistributedQueryHook.java | 27 + .../prehook/DistributedQueryHookManager.java | 45 + .../master/exec/prehook/InsertIntoHook.java | 61 + .../tajo/master/ha/HAServiceHDFSImpl.java | 100 +- .../apache/tajo/master/querymaster/Query.java | 164 +-- .../master/querymaster/QueryJobManager.java | 5 +- .../tajo/master/querymaster/QueryMaster.java | 2 +- .../querymaster/QueryMasterManagerService.java | 24 +- .../master/querymaster/QueryMasterTask.java | 44 +- .../tajo/master/querymaster/QueryUnit.java | 907 ------------ .../master/querymaster/QueryUnitAttempt.java | 443 ------ .../tajo/master/querymaster/Repartitioner.java | 186 +-- .../apache/tajo/master/querymaster/Stage.java | 1342 +++++++++++++++++ .../tajo/master/querymaster/StageState.java | 30 + .../tajo/master/querymaster/SubQuery.java | 1345 ------------------ .../tajo/master/querymaster/SubQueryState.java | 30 - .../apache/tajo/master/querymaster/Task.java | 907 ++++++++++++ .../tajo/master/querymaster/TaskAttempt.java | 443 ++++++ .../main/java/org/apache/tajo/util/JSPUtil.java | 120 +- .../apache/tajo/util/history/HistoryReader.java | 22 +- .../apache/tajo/util/history/HistoryWriter.java | 14 +- .../apache/tajo/util/history/QueryHistory.java | 23 +- .../tajo/util/history/QueryUnitHistory.java | 167 --- .../apache/tajo/util/history/StageHistory.java | 270 ++++ .../tajo/util/history/SubQueryHistory.java | 270 ---- .../apache/tajo/util/history/TaskHistory.java | 167 +++ .../tajo/worker/ExecutionBlockContext.java | 12 +- .../java/org/apache/tajo/worker/FetchImpl.java | 19 +- .../apache/tajo/worker/InterDataRetriever.java | 8 +- .../tajo/worker/TajoResourceAllocator.java | 16 +- .../java/org/apache/tajo/worker/TajoWorker.java | 1 + .../tajo/worker/TajoWorkerManagerService.java | 8 +- .../main/java/org/apache/tajo/worker/Task.java | 51 +- .../apache/tajo/worker/TaskAttemptContext.java | 14 +- .../org/apache/tajo/worker/TaskHistory.java | 18 +- .../java/org/apache/tajo/worker/TaskRunner.java | 18 +- .../apache/tajo/worker/TaskRunnerHistory.java | 16 +- .../apache/tajo/worker/TaskRunnerManager.java | 10 +- .../tajo/worker/WorkerHeartbeatService.java | 2 +- .../retriever/AdvancedDataRetriever.java | 14 +- .../ConnectivityCheckerRuleForTajoWorker.java | 2 +- .../src/main/proto/QueryMasterProtocol.proto | 2 +- .../src/main/proto/TajoWorkerProtocol.proto | 22 +- .../resources/webapps/admin/querydetail.jsp | 32 +- .../main/resources/webapps/admin/querytasks.jsp | 106 +- .../main/resources/webapps/admin/queryunit.jsp | 134 -- .../src/main/resources/webapps/admin/task.jsp | 134 ++ .../resources/webapps/worker/querydetail.jsp | 30 +- .../main/resources/webapps/worker/queryplan.jsp | 52 +- .../resources/webapps/worker/querytasks.jsp | 86 +- .../main/resources/webapps/worker/queryunit.jsp | 175 --- .../src/main/resources/webapps/worker/task.jsp | 174 +++ .../resources/webapps/worker/taskdetail.jsp | 14 +- .../resources/webapps/worker/taskhistory.jsp | 6 +- .../src/main/resources/webapps/worker/tasks.jsp | 14 +- .../apache/tajo/LocalTajoTestingUtility.java | 10 +- .../org/apache/tajo/TajoTestingCluster.java | 11 +- .../org/apache/tajo/TestQueryIdFactory.java | 14 +- .../test/java/org/apache/tajo/TestTajoIds.java | 32 +- .../org/apache/tajo/client/TestTajoClient.java | 20 +- .../planner/physical/TestBNLJoinExec.java | 4 +- .../planner/physical/TestBSTIndexExec.java | 2 +- .../planner/physical/TestExternalSortExec.java | 2 +- .../physical/TestFullOuterHashJoinExec.java | 8 +- .../physical/TestFullOuterMergeJoinExec.java | 12 +- .../planner/physical/TestHashAntiJoinExec.java | 2 +- .../planner/physical/TestHashJoinExec.java | 4 +- .../planner/physical/TestHashSemiJoinExec.java | 2 +- .../physical/TestLeftOuterHashJoinExec.java | 10 +- .../physical/TestLeftOuterNLJoinExec.java | 10 +- .../planner/physical/TestMergeJoinExec.java | 2 +- .../engine/planner/physical/TestNLJoinExec.java | 4 +- .../planner/physical/TestPhysicalPlanner.java | 60 +- .../physical/TestProgressExternalSortExec.java | 2 +- .../physical/TestRightOuterHashJoinExec.java | 6 +- .../physical/TestRightOuterMergeJoinExec.java | 12 +- .../engine/planner/physical/TestSortExec.java | 2 +- .../tajo/engine/query/TestGroupByQuery.java | 39 +- .../tajo/engine/query/TestTablePartitions.java | 2 +- .../tajo/engine/query/TestUnionQuery.java | 6 +- .../apache/tajo/master/TestRepartitioner.java | 21 +- .../tajo/master/ha/TestHAServiceHDFSImpl.java | 115 +- .../querymaster/TestIntermediateEntry.java | 2 +- .../tajo/master/querymaster/TestKillQuery.java | 8 +- .../querymaster/TestQueryUnitStatusUpdate.java | 194 --- .../querymaster/TestTaskStatusUpdate.java | 194 +++ .../java/org/apache/tajo/util/TestJSPUtil.java | 54 +- .../util/history/TestHistoryWriterReader.java | 59 +- .../org/apache/tajo/worker/TestHistory.java | 8 +- .../tajo/worker/TestRangeRetrieverHandler.java | 4 +- .../worker/dataserver/TestHttpDataServer.java | 12 +- .../TestGroupByQuery/testGroupbyWithLimit1.sql | 1 + .../TestGroupByQuery/testGroupbyWithLimit2.sql | 1 + .../TestGroupByQuery/testGroupbyWithLimit3.sql | 1 + .../queries/TestQueryUnitStatusUpdate/case1.sql | 1 - .../queries/TestQueryUnitStatusUpdate/case2.sql | 5 - .../queries/TestQueryUnitStatusUpdate/case3.sql | 10 - .../TestSelectQuery/datetime_table_ddl.sql | 2 +- .../datetime_table_timezoned_ddl.sql | 2 +- .../queries/TestTaskStatusUpdate/case1.sql | 1 + .../queries/TestTaskStatusUpdate/case2.sql | 5 + .../queries/TestTaskStatusUpdate/case3.sql | 10 + .../testGroupbyWithLimit1.result | 3 + .../testGroupbyWithLimit2.result | 3 + .../testGroupbyWithLimit3.result | 3 + tajo-dist/pom.xml | 6 +- .../configuration/catalog_configuration.rst | 4 +- .../src/main/sphinx/table_management/csv.rst | 4 +- .../sphinx/table_management/table_overview.rst | 4 +- tajo-docs/src/main/sphinx/time_zone.rst | 6 +- .../org/apache/tajo/plan/LogicalPlanner.java | 7 + .../tajo/pullserver/HttpDataServerHandler.java | 2 +- .../tajo/pullserver/PullServerAuxService.java | 16 +- .../tajo/pullserver/TajoPullServerService.java | 16 +- .../retriever/AdvancedDataRetriever.java | 16 +- .../org/apache/tajo/storage/StorageManager.java | 19 +- .../org/apache/tajo/storage/StorageUtil.java | 4 +- .../src/main/resources/storage-default.xml | 22 + .../src/test/resources/storage-default.xml | 22 + .../storage/hbase/AbstractHBaseAppender.java | 6 +- .../tajo/storage/hbase/HBasePutAppender.java | 4 +- .../tajo/storage/hbase/HBaseStorageManager.java | 2 +- .../tajo/storage/hbase/HFileAppender.java | 8 +- .../java/org/apache/tajo/storage/CSVFile.java | 4 +- .../org/apache/tajo/storage/FileAppender.java | 6 +- .../apache/tajo/storage/FileStorageManager.java | 12 +- .../tajo/storage/HashShuffleAppender.java | 12 +- .../storage/HashShuffleAppenderManager.java | 6 +- .../java/org/apache/tajo/storage/RawFile.java | 24 +- .../java/org/apache/tajo/storage/RowFile.java | 4 +- .../apache/tajo/storage/avro/AvroAppender.java | 4 +- .../tajo/storage/parquet/ParquetAppender.java | 4 +- .../org/apache/tajo/storage/rcfile/RCFile.java | 4 +- .../sequencefile/SequenceFileAppender.java | 4 +- .../tajo/storage/text/DelimitedLineReader.java | 14 +- .../tajo/storage/text/DelimitedTextFile.java | 21 +- .../org/apache/tajo/storage/TestStorages.java | 77 + .../src/test/resources/storage-default.xml | 22 + 208 files changed, 7475 insertions(+), 6860 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/829fcd96/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/829fcd96/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index ac00c15,6db76ae..a4e49f7 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@@ -987,15 -987,9 +987,15 @@@ public class TestPhysicalPlanner FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); + Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee"); + if (sm.getFileSystem().exists(indexPath)) { + sm.getFileSystem().delete(indexPath, true); + } + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), - LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), + LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); + ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(createIndexStmt[0]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); http://git-wip-us.apache.org/repos/asf/tajo/blob/829fcd96/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ----------------------------------------------------------------------
