Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/708453cd Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/708453cd Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/708453cd Branch: refs/heads/index_support Commit: 708453cd8d65880ad081e75c511187d9354d4415 Parents: 1e9c38d b4adc18 Author: Jihoon Son <[email protected]> Authored: Tue Dec 2 12:56:06 2014 +0900 Committer: Jihoon Son <[email protected]> Committed: Tue Dec 2 12:56:06 2014 +0900 ---------------------------------------------------------------------- CHANGES | 26 ++ .../org/apache/tajo/catalog/CatalogUtil.java | 11 +- .../org/apache/tajo/catalog/SchemaUtil.java | 8 + .../java/org/apache/tajo/catalog/TableDesc.java | 21 +- .../src/main/proto/CatalogProtos.proto | 1 + .../org/apache/tajo/catalog/TestTableDesc.java | 10 +- .../tajo/catalog/store/HCatalogStore.java | 11 +- .../tajo/catalog/store/TestHCatalogStore.java | 22 +- .../org/apache/tajo/catalog/TestCatalog.java | 16 +- .../org/apache/tajo/client/ResultSetUtil.java | 8 + .../org/apache/tajo/jdbc/TajoResultSet.java | 4 +- .../java/org/apache/tajo/conf/TajoConf.java | 12 +- .../apache/tajo/storage/StorageConstants.java | 3 +- .../java/org/apache/tajo/storage/VTuple.java | 4 +- .../org/apache/tajo/util/ReflectionUtil.java | 4 +- .../apache/tajo/datum/TestTimestampDatum.java | 3 +- tajo-core/pom.xml | 1 + .../engine/planner/global/GlobalPlanner.java | 2 +- .../planner/physical/PhysicalPlanUtil.java | 7 +- .../org/apache/tajo/master/ContainerProxy.java | 10 +- .../tajo/master/DefaultTaskScheduler.java | 18 +- .../org/apache/tajo/master/GlobalEngine.java | 17 +- .../tajo/master/LaunchTaskRunnersEvent.java | 5 +- .../apache/tajo/master/LazyTaskScheduler.java | 9 +- .../apache/tajo/master/TajoContainerProxy.java | 22 +- .../java/org/apache/tajo/master/TajoMaster.java | 2 +- .../apache/tajo/master/TajoMasterService.java | 5 +- .../tajo/master/TaskRunnerGroupEvent.java | 8 +- .../apache/tajo/master/cluster/ServerName.java | 123 -------- .../tajo/master/container/TajoContainer.java | 173 ++++++++++ .../tajo/master/container/TajoContainerId.java | 172 ++++++++++ .../master/container/TajoConverterUtils.java | 263 ++++++++++++++++ .../master/container/TajoRecordFactory.java | 31 ++ .../container/TajoRecordFactoryPBImpl.java | 104 ++++++ .../container/TajoRecordFactoryProvider.java | 70 +++++ .../tajo/master/container/TajoRecords.java | 39 +++ .../impl/pb/TajoContainerIdPBImpl.java | 100 ++++++ .../tajo/master/event/LocalTaskEvent.java | 9 +- .../event/QueryUnitAttemptScheduleEvent.java | 10 +- .../event/SubQueryContainerAllocationEvent.java | 8 +- .../master/event/TaskAttemptAssignedEvent.java | 8 +- .../tajo/master/event/TaskRequestEvent.java | 8 +- .../apache/tajo/master/querymaster/Query.java | 80 +++-- .../master/querymaster/QueryInProgress.java | 3 +- .../querymaster/QueryMasterManagerService.java | 4 +- .../master/querymaster/QueryMasterTask.java | 23 +- .../master/querymaster/QueryUnitAttempt.java | 6 +- .../tajo/master/querymaster/Repartitioner.java | 6 +- .../tajo/master/querymaster/SubQuery.java | 19 +- .../apache/tajo/master/rm/TajoRMContext.java | 8 +- .../tajo/master/rm/TajoWorkerContainer.java | 15 +- .../tajo/master/rm/TajoWorkerContainerId.java | 53 ++-- .../master/rm/TajoWorkerResourceManager.java | 63 ++-- .../tajo/master/rm/WorkerResourceManager.java | 4 +- .../tajo/webapp/QueryExecutorServlet.java | 16 + .../tajo/worker/AbstractResourceAllocator.java | 14 +- .../apache/tajo/worker/ResourceAllocator.java | 6 +- .../tajo/worker/TajoResourceAllocator.java | 101 +++--- .../java/org/apache/tajo/worker/TaskRunner.java | 16 +- .../apache/tajo/worker/TaskRunnerHistory.java | 14 +- .../src/main/proto/ContainerProtocol.proto | 48 +++ .../src/main/proto/QueryMasterProtocol.proto | 3 + .../main/proto/ResourceTrackerProtocol.proto | 3 + .../src/main/proto/TajoMasterProtocol.proto | 7 +- .../src/main/proto/TajoWorkerProtocol.proto | 5 +- .../src/main/resources/webapps/admin/index.jsp | 2 +- .../resources/webapps/admin/query_executor.jsp | 4 +- .../apache/tajo/LocalTajoTestingUtility.java | 2 +- .../java/org/apache/tajo/QueryTestCaseBase.java | 4 +- .../apache/tajo/cli/tools/TestDDLBuilder.java | 8 +- .../org/apache/tajo/client/TestTajoClient.java | 18 +- .../org/apache/tajo/cluster/TestServerName.java | 102 ------ .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 2 +- .../engine/planner/TestLogicalOptimizer.java | 6 +- .../tajo/engine/planner/TestLogicalPlanner.java | 6 +- .../tajo/engine/planner/TestPlannerUtil.java | 8 +- .../planner/physical/TestBNLJoinExec.java | 14 +- .../planner/physical/TestBSTIndexExec.java | 2 +- .../planner/physical/TestExternalSortExec.java | 6 +- .../physical/TestFullOuterHashJoinExec.java | 23 +- .../physical/TestFullOuterMergeJoinExec.java | 36 ++- .../planner/physical/TestHashAntiJoinExec.java | 8 +- .../planner/physical/TestHashJoinExec.java | 14 +- .../planner/physical/TestHashSemiJoinExec.java | 18 +- .../physical/TestLeftOuterHashJoinExec.java | 38 +-- .../physical/TestLeftOuterNLJoinExec.java | 20 +- .../planner/physical/TestMergeJoinExec.java | 6 +- .../engine/planner/physical/TestNLJoinExec.java | 16 +- .../planner/physical/TestPhysicalPlanner.java | 65 ++-- .../physical/TestProgressExternalSortExec.java | 6 +- .../physical/TestRightOuterHashJoinExec.java | 23 +- .../physical/TestRightOuterMergeJoinExec.java | 26 +- .../engine/planner/physical/TestSortExec.java | 2 +- .../apache/tajo/engine/query/TestCTASQuery.java | 9 +- .../tajo/engine/query/TestCreateTable.java | 9 +- .../tajo/engine/query/TestInsertQuery.java | 20 +- .../tajo/engine/query/TestJoinBroadcast.java | 9 +- .../tajo/engine/query/TestTablePartitions.java | 26 +- .../org/apache/tajo/jdbc/TestResultSet.java | 2 +- .../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 181 ++++++++--- .../tajo/master/rm/TestTajoResourceManager.java | 16 +- .../tajo/worker/TestRangeRetrieverHandler.java | 4 +- .../dataset/TestTajoJdbc/table1/table1.tbl | 5 + .../TestTajoJdbc/create_table_with_date_ddl.sql | 10 + .../TestTajoJdbc/testSortWithDateTime.result | 7 + .../src/main/sphinx/hcatalog_integration.rst | 13 + .../org/apache/tajo/jdbc/MetaDataTuple.java | 4 +- .../org/apache/tajo/plan/LogicalPlanner.java | 9 +- .../apache/tajo/plan/logical/InsertNode.java | 2 +- .../rewrite/rules/PartitionedTableRewriter.java | 4 +- tajo-storage/pom.xml | 7 +- .../org/apache/tajo/storage/StorageManager.java | 1 + .../tajo/storage/json/JsonLineDeserializer.java | 220 +++++++++++++ .../apache/tajo/storage/json/JsonLineSerDe.java | 37 +++ .../tajo/storage/json/JsonLineSerializer.java | 134 ++++++++ .../tajo/storage/text/ByteBufLineReader.java | 22 +- .../tajo/storage/text/CSVLineDeserializer.java | 96 ++++++ .../apache/tajo/storage/text/CSVLineSerDe.java | 41 +++ .../tajo/storage/text/CSVLineSerializer.java | 71 +++++ .../tajo/storage/text/DelimitedTextFile.java | 163 ++++------ .../text/TextFieldSerializerDeserializer.java | 2 +- .../tajo/storage/text/TextLineDeserializer.java | 60 ++++ .../apache/tajo/storage/text/TextLineSerDe.java | 65 ++++ .../tajo/storage/text/TextLineSerializer.java | 45 +++ .../src/main/resources/storage-default.xml | 16 +- .../apache/tajo/storage/TestFileSystems.java | 267 ++++++++-------- .../org/apache/tajo/storage/TestLineReader.java | 32 +- .../org/apache/tajo/storage/TestStorages.java | 107 ++++--- .../apache/tajo/storage/avro/TestAvroUtil.java | 2 +- .../apache/tajo/storage/json/TestJsonSerDe.java | 101 ++++++ .../java/org/apache/tajo/storage/s3/INode.java | 124 -------- .../storage/s3/InMemoryFileSystemStore.java | 175 ----------- .../apache/tajo/storage/s3/S3OutputStream.java | 234 -------------- .../tajo/storage/s3/SmallBlockS3FileSystem.java | 314 ------------------- .../dataset/TestJsonSerDe/testVariousType.json | 1 + .../src/test/resources/dataset/testLineText.txt | 2 + .../resources/dataset/testVariousTypes.avsc | 20 ++ .../src/test/resources/storage-default.xml | 16 +- .../src/test/resources/testVariousTypes.avsc | 21 -- 140 files changed, 3157 insertions(+), 1961 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 9aaaf3d,d507b97..ae208a8 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@@ -977,19 -980,14 +980,19 @@@ public class TestPhysicalPlanner } public final String [] createIndexStmt = { - "create index idx_employee on employee using bst (name null first, empId desc)" + "create index idx_employee on employee using TWO_LEVEL_BIN_TREE (name null first, empId desc)" }; - //@Test + @Test public final void testCreateIndex() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), - Integer.MAX_VALUE); + FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); + Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee"); + if (sm.getFileSystem().exists(indexPath)) { + sm.getFileSystem().delete(indexPath, true); + } + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ----------------------------------------------------------------------
