Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/e05cd298 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/e05cd298 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/e05cd298 Branch: refs/heads/index_support Commit: e05cd29886f9a9b177bf5b955dda494eebeb12ca Parents: 28151a9 bee0fd8 Author: Jihoon Son <[email protected]> Authored: Fri Dec 12 17:21:37 2014 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Dec 12 17:21:37 2014 +0900 ---------------------------------------------------------------------- BUILDING | 2 + CHANGES | 30 + .../java/org/apache/tajo/algebra/OpType.java | 3 + .../org/apache/tajo/algebra/SetSession.java | 68 + .../org/apache/tajo/catalog/CatalogUtil.java | 9 + .../java/org/apache/tajo/catalog/Schema.java | 1 - .../src/main/proto/CatalogProtos.proto | 13 +- .../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 2 +- tajo-client/pom.xml | 6 +- .../java/org/apache/tajo/cli/tsql/TajoCli.java | 2 +- .../org/apache/tajo/client/QueryClient.java | 4 +- .../org/apache/tajo/client/QueryClientImpl.java | 18 +- .../apache/tajo/client/SessionConnection.java | 88 +- .../org/apache/tajo/client/TajoClientImpl.java | 19 + .../org/apache/tajo/client/TajoClientUtil.java | 4 +- .../org/apache/tajo/jdbc/TajoResultSet.java | 12 +- .../org/apache/tajo/jdbc/TajoResultSetBase.java | 7 +- tajo-client/src/main/proto/ClientProtos.proto | 22 +- .../main/proto/TajoMasterClientProtocol.proto | 2 +- tajo-common/pom.xml | 1 + .../main/java/org/apache/tajo/QueryVars.java | 58 + .../main/java/org/apache/tajo/SessionVars.java | 2 +- .../java/org/apache/tajo/TajoConstants.java | 3 +- .../java/org/apache/tajo/conf/TajoConf.java | 85 +- .../org/apache/tajo/rule/EvaluationContext.java | 55 + .../tajo/rule/EvaluationFailedException.java | 40 + .../org/apache/tajo/rule/EvaluationResult.java | 56 + .../org/apache/tajo/rule/SelfDiagnosisRule.java | 25 + .../tajo/rule/SelfDiagnosisRuleDefinition.java | 37 + .../tajo/rule/SelfDiagnosisRuleEngine.java | 162 ++ .../tajo/rule/SelfDiagnosisRuleProvider.java | 32 + .../tajo/rule/SelfDiagnosisRuleSession.java | 134 ++ .../tajo/rule/SelfDiagnosisRuleVisibility.java | 38 + .../apache/tajo/rule/base/BaseRuleProvider.java | 53 + .../base/CheckHadoopRuntimeVersionRule.java | 112 ++ .../tajo/rule/base/TajoConfValidationRule.java | 90 + .../java/org/apache/tajo/util/KeyValueSet.java | 5 + .../java/org/apache/tajo/util/ProtoUtil.java | 13 + .../tajo/validation/PathListValidator.java | 63 + .../apache/tajo/validation/PathValidator.java | 4 +- .../org/apache/tajo/validation/Validators.java | 4 + ...g.apache.tajo.rule.SelfDiagnosisRuleProvider | 2 + .../org/apache/tajo/rule/TestRuleEngine.java | 146 ++ .../org/apache/tajo/rule/TestRuleSession.java | 465 +++++ .../apache/tajo/validation/TestValidators.java | 30 + tajo-core/pom.xml | 41 +- .../org/apache/tajo/engine/parser/SQLLexer.g4 | 7 +- .../org/apache/tajo/engine/parser/SQLParser.g4 | 25 +- .../engine/function/datetime/CurrentDate.java | 2 +- .../function/datetime/DatePartFromTime.java | 2 +- .../datetime/DatePartFromTimestamp.java | 2 +- .../function/datetime/ToCharTimestamp.java | 2 +- .../function/datetime/ToTimestampText.java | 2 +- .../tajo/engine/function/string/ToCharLong.java | 55 + .../apache/tajo/engine/json/CoreGsonHelper.java | 2 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 57 +- .../engine/planner/PhysicalPlannerImpl.java | 69 +- .../engine/planner/global/ExecutionBlock.java | 2 + .../planner/physical/BSTIndexScanExec.java | 7 +- .../planner/physical/ColPartitionStoreExec.java | 5 +- .../planner/physical/ExternalSortExec.java | 18 +- .../physical/HashShuffleFileWriteExec.java | 2 +- .../physical/PartitionMergeScanExec.java | 7 +- .../planner/physical/PhysicalPlanUtil.java | 9 +- .../physical/RangeShuffleFileWriteExec.java | 6 +- .../engine/planner/physical/SeqScanExec.java | 32 +- .../engine/planner/physical/StoreTableExec.java | 38 +- .../apache/tajo/engine/query/QueryContext.java | 45 +- .../org/apache/tajo/engine/utils/TupleUtil.java | 7 +- .../DefaultFragmentScheduleAlgorithm.java | 8 +- .../tajo/master/DefaultTaskScheduler.java | 5 +- .../org/apache/tajo/master/FragmentPair.java | 14 +- .../org/apache/tajo/master/GlobalEngine.java | 141 +- .../master/GreedyFragmentScheduleAlgorithm.java | 20 +- .../apache/tajo/master/LazyTaskScheduler.java | 26 +- .../master/NonForwardQueryResultScanner.java | 22 +- .../java/org/apache/tajo/master/TajoMaster.java | 22 +- .../tajo/master/TajoMasterClientService.java | 36 +- .../master/event/FragmentScheduleEvent.java | 16 +- .../apache/tajo/master/querymaster/Query.java | 385 +--- .../tajo/master/querymaster/QueryMaster.java | 9 - .../master/querymaster/QueryMasterTask.java | 52 +- .../tajo/master/querymaster/QueryUnit.java | 25 +- .../tajo/master/querymaster/Repartitioner.java | 141 +- .../tajo/master/querymaster/SubQuery.java | 49 +- .../master/rm/TajoWorkerResourceManager.java | 1 + .../apache/tajo/master/rule/FileSystemRule.java | 110 ++ .../tajo/master/rule/MasterRuleProvider.java | 53 + .../java/org/apache/tajo/util/IndexUtil.java | 2 +- .../org/apache/tajo/worker/TajoQueryEngine.java | 4 +- .../java/org/apache/tajo/worker/TajoWorker.java | 17 + .../main/java/org/apache/tajo/worker/Task.java | 48 +- .../ConnectivityCheckerRuleForTajoWorker.java | 92 + .../tajo/worker/rule/WorkerRuleProvider.java | 53 + ...g.apache.tajo.rule.SelfDiagnosisRuleProvider | 3 + .../main/resources/webapps/worker/queryunit.jsp | 5 +- .../org/apache/tajo/BackendTestingUtil.java | 2 +- .../org/apache/tajo/HBaseTestClusterUtil.java | 182 ++ .../java/org/apache/tajo/QueryTestCaseBase.java | 16 +- .../org/apache/tajo/TajoTestingCluster.java | 16 +- .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../tajo/engine/eval/TestSQLExpression.java | 5 +- .../engine/function/TestDateTimeFunctions.java | 5 +- .../tajo/engine/parser/TestSQLAnalyzer.java | 60 + .../tajo/engine/planner/TestPlannerUtil.java | 4 +- .../planner/global/TestBroadcastJoinPlan.java | 9 +- .../planner/physical/TestBNLJoinExec.java | 20 +- .../planner/physical/TestBSTIndexExec.java | 20 +- .../planner/physical/TestExternalSortExec.java | 14 +- .../physical/TestFullOuterHashJoinExec.java | 41 +- .../physical/TestFullOuterMergeJoinExec.java | 64 +- .../planner/physical/TestHashAntiJoinExec.java | 15 +- .../planner/physical/TestHashJoinExec.java | 21 +- .../planner/physical/TestHashSemiJoinExec.java | 20 +- .../physical/TestLeftOuterHashJoinExec.java | 45 +- .../physical/TestLeftOuterNLJoinExec.java | 48 +- .../planner/physical/TestMergeJoinExec.java | 15 +- .../engine/planner/physical/TestNLJoinExec.java | 20 +- .../planner/physical/TestPhysicalPlanner.java | 135 +- .../physical/TestProgressExternalSortExec.java | 12 +- .../physical/TestRightOuterHashJoinExec.java | 34 +- .../physical/TestRightOuterMergeJoinExec.java | 54 +- .../engine/planner/physical/TestSortExec.java | 11 +- .../tajo/engine/query/TestHBaseTable.java | 1469 ++++++++++++++ .../tajo/engine/query/TestJoinBroadcast.java | 4 +- .../tajo/engine/query/TestSelectQuery.java | 47 +- .../tajo/engine/query/TestSetSessionQuery.java | 81 + .../apache/tajo/engine/query/TestSortQuery.java | 39 + .../org/apache/tajo/jdbc/TestResultSet.java | 7 +- .../tajo/master/TestExecutionBlockCursor.java | 2 +- .../tajo/master/rule/TestMasterRules.java | 156 ++ .../apache/tajo/storage/TestFileFragment.java | 4 +- .../org/apache/tajo/storage/TestRowFile.java | 7 +- .../tajo/worker/TestRangeRetrieverHandler.java | 22 +- .../dataset/TestHBaseTable/splits.data | 4 + .../queries/TestSQLAnalyzer/setcatalog1.sql | 1 + .../queries/TestSQLAnalyzer/setcatalog2.sql | 1 + .../queries/TestSQLAnalyzer/setsession1.sql | 1 + .../queries/TestSQLAnalyzer/setsession2.sql | 1 + .../queries/TestSQLAnalyzer/setsession3.sql | 1 + .../queries/TestSQLAnalyzer/setsession4.sql | 1 + .../queries/TestSQLAnalyzer/setsession5.sql | 1 + .../queries/TestSQLAnalyzer/setsession6.sql | 1 + .../queries/TestSQLAnalyzer/setsession7.sql | 1 + .../queries/TestSQLAnalyzer/settimezone1.sql | 1 + .../queries/TestSQLAnalyzer/settimezone2.sql | 1 + .../queries/TestSQLAnalyzer/settimezone3.sql | 1 + .../TestSelectQuery/testTimezonedTable2.sql | 1 + .../TestSelectQuery/testTimezonedTable4.sql | 2 + .../TestSelectQuery/testTimezonedTable5.sql | 2 + .../TestHBaseTable/testBinaryMappedQuery.result | 81 + .../results/TestHBaseTable/testCATS.result | 100 + .../testColumnKeyValueSelectQuery.result | 12 + .../TestHBaseTable/testIndexPredication.result | 38 + .../TestHBaseTable/testInsertInto.result | 3 + .../testInsertIntoBinaryMultiRegion.result | 100 + .../testInsertIntoColumnKeyValue.result | 21 + .../testInsertIntoMultiRegion.result | 100 + .../testInsertIntoMultiRegion2.result | 100 + ...stInsertIntoMultiRegionMultiRowFields.result | 100 + ...estInsertIntoMultiRegionWithSplitFile.result | 100 + .../testInsertIntoRowField.result | 4 + .../testInsertIntoUsingPut.result | 3 + .../results/TestHBaseTable/testJoin.result | 7 + .../TestHBaseTable/testNonForwardQuery.result | 102 + .../testRowFieldSelectQuery.result | 88 + .../TestHBaseTable/testSimpleSelectQuery.result | 88 + .../results/TestSQLAnalyzer/setcatalog1.result | 5 + .../results/TestSQLAnalyzer/setcatalog2.result | 5 + .../results/TestSQLAnalyzer/setsession1.result | 5 + .../results/TestSQLAnalyzer/setsession2.result | 5 + .../results/TestSQLAnalyzer/setsession3.result | 5 + .../results/TestSQLAnalyzer/setsession4.result | 5 + .../results/TestSQLAnalyzer/setsession5.result | 5 + .../results/TestSQLAnalyzer/setsession6.result | 5 + .../results/TestSQLAnalyzer/setsession7.result | 5 + .../results/TestSQLAnalyzer/settimezone1.result | 5 + .../results/TestSQLAnalyzer/settimezone2.result | 5 + .../results/TestSQLAnalyzer/settimezone3.result | 4 + .../TestTajoCli/testHelpSessionVars.result | 2 +- .../results/TestTajoCli/testSetTimezone1.result | 4 + tajo-dist/pom.xml | 7 +- tajo-dist/src/main/bin/tajo | 15 + .../src/main/conf/catalog-site.xml.template | 76 + tajo-dist/src/main/conf/tajo-env.sh | 3 + tajo-dist/src/main/conf/tajo-site.xml.template | 90 + tajo-docs/src/main/sphinx/configuration.rst | 4 +- .../sphinx/configuration/catalog-site-xml.rst | 3 + .../main/sphinx/configuration/cluster_setup.rst | 2 +- .../configuration/configuration_defaults.rst | 28 - .../configuration/service_config_defaults.rst | 28 + .../main/sphinx/configuration/tajo-site-xml.rst | 24 + tajo-docs/src/main/sphinx/index.rst | 1 + .../sphinx/table_management/table_overview.rst | 67 +- tajo-docs/src/main/sphinx/time_zone.rst | 129 ++ tajo-jdbc/pom.xml | 6 +- .../org/apache/tajo/plan/ExprAnnotator.java | 12 +- .../org/apache/tajo/plan/LogicalOptimizer.java | 7 + .../tajo/plan/LogicalPlanPreprocessor.java | 7 + .../org/apache/tajo/plan/LogicalPlanner.java | 15 +- .../tajo/plan/algebra/AlgebraVisitor.java | 3 + .../tajo/plan/algebra/BaseAlgebraVisitor.java | 8 + .../org/apache/tajo/plan/expr/CastEval.java | 5 +- .../apache/tajo/plan/logical/InsertNode.java | 4 +- .../org/apache/tajo/plan/logical/NodeType.java | 2 + .../tajo/plan/logical/SetSessionNode.java | 68 + .../org/apache/tajo/plan/logical/SortNode.java | 20 +- .../rewrite/rules/PartitionedTableRewriter.java | 2 +- .../org/apache/tajo/plan/util/PlannerUtil.java | 143 +- .../plan/verifier/PreLogicalPlanVerifier.java | 20 + .../plan/visitor/BasicLogicalPlanVisitor.java | 9 + .../tajo/plan/visitor/LogicalPlanVisitor.java | 4 + tajo-project/pom.xml | 17 + tajo-project/src/site/site.xml | 4 +- tajo-pullserver/pom.xml | 7 +- tajo-storage/pom.xml | 310 +-- .../java/org/apache/tajo/storage/Appender.java | 41 - .../tajo/storage/BaseTupleComparator.java | 206 -- .../storage/BinarySerializerDeserializer.java | 258 --- .../org/apache/tajo/storage/BufferPool.java | 74 - .../tajo/storage/ByteBufInputChannel.java | 72 - .../java/org/apache/tajo/storage/CSVFile.java | 586 ------ .../tajo/storage/CompressedSplitLineReader.java | 182 -- .../org/apache/tajo/storage/DataLocation.java | 45 - .../org/apache/tajo/storage/DiskDeviceInfo.java | 62 - .../java/org/apache/tajo/storage/DiskInfo.java | 75 - .../org/apache/tajo/storage/DiskMountInfo.java | 101 - .../java/org/apache/tajo/storage/DiskUtil.java | 207 -- .../storage/FieldSerializerDeserializer.java | 37 - .../org/apache/tajo/storage/FileAppender.java | 65 - .../org/apache/tajo/storage/FileScanner.java | 123 -- .../org/apache/tajo/storage/FrameTuple.java | 225 --- .../tajo/storage/HashShuffleAppender.java | 209 -- .../storage/HashShuffleAppenderManager.java | 226 --- .../java/org/apache/tajo/storage/LazyTuple.java | 270 --- .../org/apache/tajo/storage/LineReader.java | 559 ------ .../org/apache/tajo/storage/MemoryUtil.java | 163 -- .../org/apache/tajo/storage/MergeScanner.java | 198 -- .../org/apache/tajo/storage/NullScanner.java | 62 - .../tajo/storage/NumericPathComparator.java | 34 - .../java/org/apache/tajo/storage/RawFile.java | 772 -------- .../java/org/apache/tajo/storage/RowFile.java | 496 ----- .../org/apache/tajo/storage/RowStoreUtil.java | 377 ---- .../java/org/apache/tajo/storage/Scanner.java | 103 - .../apache/tajo/storage/SeekableScanner.java | 28 - .../tajo/storage/SerializerDeserializer.java | 34 - .../apache/tajo/storage/SplitLineReader.java | 39 - .../java/org/apache/tajo/storage/Storage.java | 45 - .../org/apache/tajo/storage/StorageManager.java | 812 -------- .../org/apache/tajo/storage/StorageUtil.java | 223 --- .../apache/tajo/storage/TableStatistics.java | 129 -- .../storage/TextSerializerDeserializer.java | 225 --- .../apache/tajo/storage/TupleComparator.java | 32 - .../org/apache/tajo/storage/TupleRange.java | 112 -- .../storage/annotation/ForSplitableStore.java | 29 - .../apache/tajo/storage/avro/AvroAppender.java | 219 --- .../apache/tajo/storage/avro/AvroScanner.java | 286 --- .../org/apache/tajo/storage/avro/AvroUtil.java | 77 - .../apache/tajo/storage/avro/package-info.java | 85 - .../apache/tajo/storage/compress/CodecPool.java | 185 -- .../AlreadyExistsStorageException.java | 39 - .../exception/UnknownCodecException.java | 32 - .../exception/UnknownDataTypeException.java | 32 - .../exception/UnsupportedFileTypeException.java | 36 - .../tajo/storage/fragment/FileFragment.java | 224 --- .../apache/tajo/storage/fragment/Fragment.java | 31 - .../storage/fragment/FragmentConvertor.java | 132 -- .../apache/tajo/storage/index/IndexMethod.java | 33 - .../apache/tajo/storage/index/IndexReader.java | 35 - .../apache/tajo/storage/index/IndexWriter.java | 33 - .../tajo/storage/index/OrderIndexReader.java | 45 - .../apache/tajo/storage/index/bst/BSTIndex.java | 623 ------ .../tajo/storage/json/JsonLineDeserializer.java | 225 --- .../apache/tajo/storage/json/JsonLineSerDe.java | 37 - .../tajo/storage/json/JsonLineSerializer.java | 134 -- .../tajo/storage/parquet/ParquetAppender.java | 150 -- .../tajo/storage/parquet/ParquetScanner.java | 119 -- .../tajo/storage/parquet/TajoParquetReader.java | 85 - .../tajo/storage/parquet/TajoParquetWriter.java | 104 - .../tajo/storage/parquet/TajoReadSupport.java | 101 - .../storage/parquet/TajoRecordConverter.java | 380 ---- .../storage/parquet/TajoRecordMaterializer.java | 78 - .../storage/parquet/TajoSchemaConverter.java | 206 -- .../tajo/storage/parquet/TajoWriteSupport.java | 148 -- .../tajo/storage/parquet/package-info.java | 96 - .../storage/rcfile/BytesRefArrayWritable.java | 261 --- .../tajo/storage/rcfile/BytesRefWritable.java | 248 --- .../storage/rcfile/ColumnProjectionUtils.java | 117 -- .../rcfile/LazyDecompressionCallback.java | 32 - .../rcfile/NonSyncByteArrayInputStream.java | 113 -- .../rcfile/NonSyncByteArrayOutputStream.java | 144 -- .../storage/rcfile/NonSyncDataInputBuffer.java | 507 ----- .../storage/rcfile/NonSyncDataOutputBuffer.java | 91 - .../org/apache/tajo/storage/rcfile/RCFile.java | 1805 ----------------- .../SchemaAwareCompressionInputStream.java | 43 - .../SchemaAwareCompressionOutputStream.java | 44 - .../sequencefile/SequenceFileAppender.java | 272 --- .../sequencefile/SequenceFileScanner.java | 336 ---- .../tajo/storage/text/ByteBufLineReader.java | 182 -- .../tajo/storage/text/CSVLineDeserializer.java | 96 - .../apache/tajo/storage/text/CSVLineSerDe.java | 41 - .../tajo/storage/text/CSVLineSerializer.java | 71 - .../tajo/storage/text/DelimitedLineReader.java | 156 -- .../tajo/storage/text/DelimitedTextFile.java | 478 ----- .../tajo/storage/text/FieldSplitProcessor.java | 38 - .../tajo/storage/text/LineSplitProcessor.java | 45 - .../text/TextFieldSerializerDeserializer.java | 253 --- .../tajo/storage/text/TextLineDeserializer.java | 60 - .../tajo/storage/text/TextLineParsingError.java | 31 - .../apache/tajo/storage/text/TextLineSerDe.java | 65 - .../tajo/storage/text/TextLineSerializer.java | 45 - .../thirdparty/parquet/CodecFactory.java | 196 -- .../parquet/ColumnChunkPageWriteStore.java | 211 -- .../parquet/InternalParquetRecordReader.java | 187 -- .../parquet/InternalParquetRecordWriter.java | 160 -- .../thirdparty/parquet/ParquetFileWriter.java | 504 ----- .../thirdparty/parquet/ParquetReader.java | 151 -- .../thirdparty/parquet/ParquetWriter.java | 224 --- .../org/apache/tajo/tuple/BaseTupleBuilder.java | 112 -- .../org/apache/tajo/tuple/RowBlockReader.java | 33 - .../org/apache/tajo/tuple/TupleBuilder.java | 26 - .../tajo/tuple/offheap/DirectBufTuple.java | 41 - .../tajo/tuple/offheap/FixedSizeLimitSpec.java | 32 - .../apache/tajo/tuple/offheap/HeapTuple.java | 272 --- .../tajo/tuple/offheap/OffHeapMemory.java | 102 - .../tajo/tuple/offheap/OffHeapRowBlock.java | 176 -- .../tuple/offheap/OffHeapRowBlockReader.java | 63 - .../tuple/offheap/OffHeapRowBlockUtils.java | 54 - .../tuple/offheap/OffHeapRowBlockWriter.java | 58 - .../tajo/tuple/offheap/OffHeapRowWriter.java | 232 --- .../tajo/tuple/offheap/ResizableLimitSpec.java | 142 -- .../apache/tajo/tuple/offheap/RowWriter.java | 73 - .../apache/tajo/tuple/offheap/UnSafeTuple.java | 311 --- .../offheap/UnSafeTupleBytesComparator.java | 99 - .../tajo/tuple/offheap/ZeroCopyTuple.java | 35 - tajo-storage/src/main/proto/IndexProtos.proto | 31 - .../src/main/resources/storage-default.xml | 175 -- .../java/org/apache/tajo/HttpFileServer.java | 84 - .../org/apache/tajo/HttpFileServerHandler.java | 184 -- .../tajo/HttpFileServerPipelineFactory.java | 54 - .../tajo/storage/TestCompressionStorages.java | 185 -- .../tajo/storage/TestDelimitedTextFile.java | 164 -- .../apache/tajo/storage/TestFileSystems.java | 138 -- .../org/apache/tajo/storage/TestFrameTuple.java | 84 - .../org/apache/tajo/storage/TestLazyTuple.java | 258 --- .../org/apache/tajo/storage/TestLineReader.java | 220 --- .../apache/tajo/storage/TestMergeScanner.java | 201 -- .../apache/tajo/storage/TestSplitProcessor.java | 72 - .../apache/tajo/storage/TestStorageManager.java | 202 -- .../org/apache/tajo/storage/TestStorages.java | 868 --------- .../tajo/storage/TestTupleComparator.java | 77 - .../org/apache/tajo/storage/TestVTuple.java | 160 -- .../apache/tajo/storage/avro/TestAvroUtil.java | 108 -- .../apache/tajo/storage/index/TestBSTIndex.java | 946 --------- .../index/TestSingleCSVFileBSTIndex.java | 248 --- .../apache/tajo/storage/json/TestJsonSerDe.java | 101 - .../tajo/storage/parquet/TestReadWrite.java | 114 -- .../storage/parquet/TestSchemaConverter.java | 132 -- .../apache/tajo/tuple/TestBaseTupleBuilder.java | 76 - .../tajo/tuple/offheap/TestHeapTuple.java | 45 - .../tajo/tuple/offheap/TestOffHeapRowBlock.java | 577 ------ .../tajo/tuple/offheap/TestResizableSpec.java | 59 - .../testErrorTolerance1.json | 6 - .../testErrorTolerance2.json | 4 - .../testErrorTolerance3.json | 1 + .../dataset/TestJsonSerDe/testVariousType.json | 1 - .../src/test/resources/dataset/testLineText.txt | 2 - .../resources/dataset/testVariousTypes.avsc | 20 - .../src/test/resources/storage-default.xml | 168 -- tajo-storage/tajo-storage-common/pom.xml | 337 ++++ .../java/org/apache/tajo/storage/Appender.java | 41 + .../tajo/storage/BaseTupleComparator.java | 206 ++ .../storage/BinarySerializerDeserializer.java | 258 +++ .../org/apache/tajo/storage/BufferPool.java | 74 + .../tajo/storage/ByteBufInputChannel.java | 72 + .../org/apache/tajo/storage/DataLocation.java | 45 + .../org/apache/tajo/storage/DiskDeviceInfo.java | 62 + .../java/org/apache/tajo/storage/DiskInfo.java | 75 + .../org/apache/tajo/storage/DiskMountInfo.java | 101 + .../java/org/apache/tajo/storage/DiskUtil.java | 207 ++ .../org/apache/tajo/storage/FrameTuple.java | 225 +++ .../java/org/apache/tajo/storage/LazyTuple.java | 270 +++ .../org/apache/tajo/storage/MemoryUtil.java | 163 ++ .../org/apache/tajo/storage/MergeScanner.java | 201 ++ .../org/apache/tajo/storage/NullScanner.java | 109 ++ .../tajo/storage/NumericPathComparator.java | 34 + .../org/apache/tajo/storage/RowStoreUtil.java | 377 ++++ .../java/org/apache/tajo/storage/Scanner.java | 103 + .../apache/tajo/storage/SeekableScanner.java | 28 + .../tajo/storage/SerializerDeserializer.java | 34 + .../org/apache/tajo/storage/StorageManager.java | 979 ++++++++++ .../apache/tajo/storage/StorageProperty.java | 40 + .../org/apache/tajo/storage/StorageUtil.java | 220 +++ .../apache/tajo/storage/TableStatistics.java | 129 ++ .../storage/TextSerializerDeserializer.java | 227 +++ .../apache/tajo/storage/TupleComparator.java | 32 + .../org/apache/tajo/storage/TupleRange.java | 112 ++ .../storage/annotation/ForSplitableStore.java | 29 + .../apache/tajo/storage/compress/CodecPool.java | 185 ++ .../AlreadyExistsStorageException.java | 39 + .../exception/UnknownCodecException.java | 32 + .../exception/UnknownDataTypeException.java | 32 + .../exception/UnsupportedFileTypeException.java | 36 + .../apache/tajo/storage/fragment/Fragment.java | 39 + .../storage/fragment/FragmentConvertor.java | 129 ++ .../org/apache/tajo/tuple/BaseTupleBuilder.java | 112 ++ .../org/apache/tajo/tuple/RowBlockReader.java | 33 + .../org/apache/tajo/tuple/TupleBuilder.java | 26 + .../tajo/tuple/offheap/DirectBufTuple.java | 41 + .../tajo/tuple/offheap/FixedSizeLimitSpec.java | 32 + .../apache/tajo/tuple/offheap/HeapTuple.java | 272 +++ .../tajo/tuple/offheap/OffHeapMemory.java | 102 + .../tajo/tuple/offheap/OffHeapRowBlock.java | 176 ++ .../tuple/offheap/OffHeapRowBlockReader.java | 63 + .../tuple/offheap/OffHeapRowBlockUtils.java | 54 + .../tuple/offheap/OffHeapRowBlockWriter.java | 58 + .../tajo/tuple/offheap/OffHeapRowWriter.java | 232 +++ .../tajo/tuple/offheap/ResizableLimitSpec.java | 142 ++ .../apache/tajo/tuple/offheap/RowWriter.java | 73 + .../apache/tajo/tuple/offheap/UnSafeTuple.java | 311 +++ .../offheap/UnSafeTupleBytesComparator.java | 99 + .../tajo/tuple/offheap/ZeroCopyTuple.java | 35 + .../src/main/proto/IndexProtos.proto | 31 + .../src/main/resources/storage-default.xml | 198 ++ .../org/apache/tajo/storage/TestFrameTuple.java | 84 + .../org/apache/tajo/storage/TestLazyTuple.java | 258 +++ .../tajo/storage/TestTupleComparator.java | 162 ++ .../org/apache/tajo/storage/TestVTuple.java | 160 ++ .../apache/tajo/tuple/TestBaseTupleBuilder.java | 76 + .../tajo/tuple/offheap/TestHeapTuple.java | 45 + .../tajo/tuple/offheap/TestOffHeapRowBlock.java | 577 ++++++ .../tajo/tuple/offheap/TestResizableSpec.java | 59 + .../src/test/resources/storage-default.xml | 164 ++ tajo-storage/tajo-storage-hbase/pom.xml | 349 ++++ .../storage/hbase/AbstractHBaseAppender.java | 223 +++ .../storage/hbase/AddSortForInsertRewriter.java | 90 + .../tajo/storage/hbase/ColumnMapping.java | 236 +++ .../HBaseBinarySerializerDeserializer.java | 97 + .../tajo/storage/hbase/HBaseFragment.java | 198 ++ .../tajo/storage/hbase/HBasePutAppender.java | 120 ++ .../apache/tajo/storage/hbase/HBaseScanner.java | 449 +++++ .../storage/hbase/HBaseStorageConstants.java | 33 + .../tajo/storage/hbase/HBaseStorageManager.java | 1135 +++++++++++ .../hbase/HBaseTextSerializerDeserializer.java | 71 + .../tajo/storage/hbase/HFileAppender.java | 167 ++ .../tajo/storage/hbase/IndexPredication.java | 61 + .../tajo/storage/hbase/RowKeyMapping.java | 40 + .../src/main/proto/StorageFragmentProtos.proto | 35 + .../tajo/storage/hbase/TestColumnMapping.java | 93 + .../storage/hbase/TestHBaseStorageManager.java | 109 ++ tajo-storage/tajo-storage-hdfs/pom.xml | 385 ++++ .../java/org/apache/tajo/storage/CSVFile.java | 587 ++++++ .../tajo/storage/CompressedSplitLineReader.java | 182 ++ .../storage/FieldSerializerDeserializer.java | 37 + .../org/apache/tajo/storage/FileAppender.java | 87 + .../org/apache/tajo/storage/FileScanner.java | 124 ++ .../apache/tajo/storage/FileStorageManager.java | 882 +++++++++ .../tajo/storage/HashShuffleAppender.java | 209 ++ .../storage/HashShuffleAppenderManager.java | 225 +++ .../org/apache/tajo/storage/LineReader.java | 559 ++++++ .../java/org/apache/tajo/storage/RawFile.java | 773 ++++++++ .../java/org/apache/tajo/storage/RowFile.java | 498 +++++ .../apache/tajo/storage/SplitLineReader.java | 39 + .../apache/tajo/storage/avro/AvroAppender.java | 220 +++ .../apache/tajo/storage/avro/AvroScanner.java | 286 +++ .../org/apache/tajo/storage/avro/AvroUtil.java | 77 + .../apache/tajo/storage/avro/package-info.java | 85 + .../tajo/storage/fragment/FileFragment.java | 237 +++ .../apache/tajo/storage/index/IndexMethod.java | 32 + .../apache/tajo/storage/index/IndexReader.java | 35 + .../apache/tajo/storage/index/IndexWriter.java | 33 + .../tajo/storage/index/OrderIndexReader.java | 45 + .../apache/tajo/storage/index/bst/BSTIndex.java | 623 ++++++ .../tajo/storage/json/JsonLineDeserializer.java | 226 +++ .../apache/tajo/storage/json/JsonLineSerDe.java | 37 + .../tajo/storage/json/JsonLineSerializer.java | 130 ++ .../tajo/storage/parquet/ParquetAppender.java | 151 ++ .../tajo/storage/parquet/ParquetScanner.java | 119 ++ .../tajo/storage/parquet/TajoParquetReader.java | 85 + .../tajo/storage/parquet/TajoParquetWriter.java | 104 + .../tajo/storage/parquet/TajoReadSupport.java | 99 + .../storage/parquet/TajoRecordConverter.java | 380 ++++ .../storage/parquet/TajoRecordMaterializer.java | 77 + .../storage/parquet/TajoSchemaConverter.java | 206 ++ .../tajo/storage/parquet/TajoWriteSupport.java | 148 ++ .../tajo/storage/parquet/package-info.java | 96 + .../storage/rcfile/BytesRefArrayWritable.java | 261 +++ .../tajo/storage/rcfile/BytesRefWritable.java | 248 +++ .../storage/rcfile/ColumnProjectionUtils.java | 117 ++ .../rcfile/LazyDecompressionCallback.java | 32 + .../rcfile/NonSyncByteArrayInputStream.java | 113 ++ .../rcfile/NonSyncByteArrayOutputStream.java | 144 ++ .../storage/rcfile/NonSyncDataInputBuffer.java | 507 +++++ .../storage/rcfile/NonSyncDataOutputBuffer.java | 91 + .../org/apache/tajo/storage/rcfile/RCFile.java | 1807 ++++++++++++++++++ .../SchemaAwareCompressionInputStream.java | 43 + .../SchemaAwareCompressionOutputStream.java | 44 + .../sequencefile/SequenceFileAppender.java | 274 +++ .../sequencefile/SequenceFileScanner.java | 336 ++++ .../tajo/storage/text/ByteBufLineReader.java | 182 ++ .../tajo/storage/text/CSVLineDeserializer.java | 96 + .../apache/tajo/storage/text/CSVLineSerDe.java | 41 + .../tajo/storage/text/CSVLineSerializer.java | 70 + .../tajo/storage/text/DelimitedLineReader.java | 156 ++ .../tajo/storage/text/DelimitedTextFile.java | 481 +++++ .../tajo/storage/text/FieldSplitProcessor.java | 38 + .../tajo/storage/text/LineSplitProcessor.java | 45 + .../text/TextFieldSerializerDeserializer.java | 253 +++ .../tajo/storage/text/TextLineDeserializer.java | 60 + .../tajo/storage/text/TextLineParsingError.java | 31 + .../apache/tajo/storage/text/TextLineSerDe.java | 65 + .../tajo/storage/text/TextLineSerializer.java | 45 + .../thirdparty/parquet/CodecFactory.java | 190 ++ .../parquet/ColumnChunkPageWriteStore.java | 206 ++ .../parquet/InternalParquetRecordReader.java | 188 ++ .../parquet/InternalParquetRecordWriter.java | 160 ++ .../thirdparty/parquet/ParquetFileWriter.java | 492 +++++ .../thirdparty/parquet/ParquetReader.java | 146 ++ .../thirdparty/parquet/ParquetWriter.java | 224 +++ .../src/main/proto/StorageFragmentProtos.proto | 34 + .../java/org/apache/tajo/HttpFileServer.java | 84 + .../org/apache/tajo/HttpFileServerHandler.java | 184 ++ .../tajo/HttpFileServerPipelineFactory.java | 54 + .../tajo/storage/TestCompressionStorages.java | 185 ++ .../tajo/storage/TestDelimitedTextFile.java | 180 ++ .../tajo/storage/TestFileStorageManager.java | 203 ++ .../apache/tajo/storage/TestFileSystems.java | 137 ++ .../org/apache/tajo/storage/TestLineReader.java | 220 +++ .../apache/tajo/storage/TestMergeScanner.java | 202 ++ .../apache/tajo/storage/TestSplitProcessor.java | 72 + .../org/apache/tajo/storage/TestStorages.java | 878 +++++++++ .../apache/tajo/storage/avro/TestAvroUtil.java | 106 + .../apache/tajo/storage/index/TestBSTIndex.java | 947 +++++++++ .../index/TestSingleCSVFileBSTIndex.java | 248 +++ .../apache/tajo/storage/json/TestJsonSerDe.java | 101 + .../tajo/storage/parquet/TestReadWrite.java | 109 ++ .../storage/parquet/TestSchemaConverter.java | 130 ++ .../testErrorTolerance1.json | 6 + .../testErrorTolerance2.json | 4 + .../dataset/TestJsonSerDe/testVariousType.json | 1 + .../src/test/resources/dataset/testLineText.txt | 2 + .../resources/dataset/testVariousTypes.avsc | 20 + .../src/test/resources/storage-default.xml | 178 ++ 543 files changed, 39250 insertions(+), 28022 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --cc tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 9cab012,90230c9..e08f9af --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@@ -24,13 -24,13 +24,15 @@@ import org.antlr.v4.runtime.ANTLRInputS import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.tree.TerminalNode; + import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.*; import org.apache.tajo.algebra.Aggregation.GroupType; +import org.apache.tajo.algebra.CreateIndex.IndexMethodSpec; import org.apache.tajo.algebra.LiteralValue.LiteralType; +import org.apache.tajo.algebra.Sort.SortSpec; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.engine.parser.SQLParser.*; + import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.StringUtils; http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index ae208a8,cce4ba7..ac00c15 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@@ -980,19 -979,14 +979,19 @@@ public class TestPhysicalPlanner } public final String [] createIndexStmt = { - "create index idx_employee on employee using bst (name null first, empId desc)" + "create index idx_employee on employee using TWO_LEVEL_BIN_TREE (name null first, empId desc)" }; - //@Test + @Test public final void testCreateIndex() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), + FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); + Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee"); + if (sm.getFileSystem().exists(indexPath)) { + sm.getFileSystem().delete(indexPath, true); + } + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/e05cd298/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java ----------------------------------------------------------------------
