TAJO-1122: Refactor the tajo-storage project structure. Closes #247
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/dfd7f996 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/dfd7f996 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/dfd7f996 Branch: refs/heads/hbase_storage Commit: dfd7f996d3c4a3e095dcd2ddc62fd7d6d793640d Parents: 87c957e Author: HyoungJun Kim <[email protected]> Authored: Thu Nov 27 10:43:34 2014 +0900 Committer: HyoungJun Kim <[email protected]> Committed: Thu Nov 27 10:43:34 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../src/main/proto/CatalogProtos.proto | 9 - .../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 2 +- tajo-client/pom.xml | 6 +- .../org/apache/tajo/jdbc/TajoResultSet.java | 5 +- .../main/java/org/apache/tajo/util/TUtil.java | 9 + tajo-core/pom.xml | 13 +- .../engine/planner/PhysicalPlannerImpl.java | 12 +- .../planner/physical/ColPartitionStoreExec.java | 3 +- .../planner/physical/PhysicalPlanUtil.java | 6 +- .../physical/RangeShuffleFileWriteExec.java | 4 +- .../engine/planner/physical/StoreTableExec.java | 4 +- .../org/apache/tajo/master/GlobalEngine.java | 4 +- .../apache/tajo/master/LazyTaskScheduler.java | 1 + .../java/org/apache/tajo/master/TajoMaster.java | 2 +- .../tajo/master/querymaster/Repartitioner.java | 14 +- .../tajo/master/querymaster/SubQuery.java | 2 +- .../master/rm/TajoWorkerResourceManager.java | 1 + .../main/java/org/apache/tajo/worker/Task.java | 10 +- .../org/apache/tajo/BackendTestingUtil.java | 2 +- .../tajo/engine/planner/TestPlannerUtil.java | 2 +- .../planner/global/TestBroadcastJoinPlan.java | 9 +- .../planner/physical/TestBNLJoinExec.java | 6 +- .../planner/physical/TestBSTIndexExec.java | 5 +- .../planner/physical/TestExternalSortExec.java | 5 +- .../physical/TestFullOuterHashJoinExec.java | 15 +- .../physical/TestFullOuterMergeJoinExec.java | 16 +- .../planner/physical/TestHashAntiJoinExec.java | 7 +- .../planner/physical/TestHashJoinExec.java | 7 +- .../planner/physical/TestHashSemiJoinExec.java | 7 +- .../physical/TestLeftOuterHashJoinExec.java | 13 +- .../physical/TestLeftOuterNLJoinExec.java | 13 +- .../planner/physical/TestMergeJoinExec.java | 7 +- .../engine/planner/physical/TestNLJoinExec.java | 6 +- .../planner/physical/TestPhysicalPlanner.java | 23 +- .../physical/TestProgressExternalSortExec.java | 3 +- .../physical/TestRightOuterHashJoinExec.java | 9 +- .../physical/TestRightOuterMergeJoinExec.java | 16 +- .../engine/planner/physical/TestSortExec.java | 5 +- .../tajo/engine/query/TestJoinBroadcast.java | 4 +- .../org/apache/tajo/jdbc/TestResultSet.java | 5 +- .../org/apache/tajo/storage/TestRowFile.java | 6 +- .../tajo/worker/TestRangeRetrieverHandler.java | 4 +- tajo-dist/pom.xml | 7 +- tajo-jdbc/pom.xml | 6 +- tajo-project/pom.xml | 16 + tajo-storage/pom.xml | 328 +--- .../java/org/apache/tajo/storage/Appender.java | 41 - .../tajo/storage/BaseTupleComparator.java | 206 -- .../storage/BinarySerializerDeserializer.java | 258 --- .../org/apache/tajo/storage/BufferPool.java | 74 - .../tajo/storage/ByteBufInputChannel.java | 76 - .../java/org/apache/tajo/storage/CSVFile.java | 588 ------ .../tajo/storage/CompressedSplitLineReader.java | 182 -- .../org/apache/tajo/storage/DataLocation.java | 45 - .../org/apache/tajo/storage/DiskDeviceInfo.java | 62 - .../java/org/apache/tajo/storage/DiskInfo.java | 75 - .../org/apache/tajo/storage/DiskMountInfo.java | 101 - .../java/org/apache/tajo/storage/DiskUtil.java | 207 -- .../storage/FieldSerializerDeserializer.java | 35 - .../org/apache/tajo/storage/FileAppender.java | 85 - .../org/apache/tajo/storage/FileScanner.java | 124 -- .../apache/tajo/storage/FileStorageManager.java | 832 -------- .../org/apache/tajo/storage/FrameTuple.java | 225 --- .../tajo/storage/HashShuffleAppender.java | 209 -- .../storage/HashShuffleAppenderManager.java | 225 --- .../java/org/apache/tajo/storage/LazyTuple.java | 270 --- .../org/apache/tajo/storage/LineReader.java | 559 ------ .../org/apache/tajo/storage/MemoryUtil.java | 163 -- .../org/apache/tajo/storage/MergeScanner.java | 201 -- .../org/apache/tajo/storage/NullScanner.java | 62 - .../tajo/storage/NumericPathComparator.java | 34 - .../java/org/apache/tajo/storage/RawFile.java | 774 -------- .../java/org/apache/tajo/storage/RowFile.java | 498 ----- .../org/apache/tajo/storage/RowStoreUtil.java | 377 ---- .../java/org/apache/tajo/storage/Scanner.java | 103 - .../apache/tajo/storage/SeekableScanner.java | 28 - .../tajo/storage/SerializerDeserializer.java | 34 - .../apache/tajo/storage/SplitLineReader.java | 39 - .../java/org/apache/tajo/storage/Storage.java | 45 - .../org/apache/tajo/storage/StorageManager.java | 926 --------- .../apache/tajo/storage/StorageProperty.java | 40 - .../org/apache/tajo/storage/StorageUtil.java | 248 --- .../apache/tajo/storage/TableStatistics.java | 129 -- .../storage/TextSerializerDeserializer.java | 226 --- .../apache/tajo/storage/TupleComparator.java | 32 - .../org/apache/tajo/storage/TupleRange.java | 112 -- .../storage/annotation/ForSplitableStore.java | 29 - .../apache/tajo/storage/avro/AvroAppender.java | 221 --- .../apache/tajo/storage/avro/AvroScanner.java | 287 --- .../org/apache/tajo/storage/avro/AvroUtil.java | 77 - .../apache/tajo/storage/avro/package-info.java | 85 - .../apache/tajo/storage/compress/CodecPool.java | 185 -- .../AlreadyExistsStorageException.java | 39 - .../exception/UnknownCodecException.java | 32 - .../exception/UnknownDataTypeException.java | 32 - .../exception/UnsupportedFileTypeException.java | 36 - .../tajo/storage/fragment/FileFragment.java | 237 --- .../apache/tajo/storage/fragment/Fragment.java | 39 - .../storage/fragment/FragmentConvertor.java | 129 -- .../storage/hbase/AbstractHBaseAppender.java | 223 --- .../storage/hbase/AddSortForInsertRewriter.java | 87 - .../tajo/storage/hbase/ColumnMapping.java | 236 --- .../HBaseBinarySerializerDeserializer.java | 97 - .../tajo/storage/hbase/HBaseFragment.java | 198 -- .../tajo/storage/hbase/HBasePutAppender.java | 120 -- .../apache/tajo/storage/hbase/HBaseScanner.java | 445 ----- .../storage/hbase/HBaseStorageConstants.java | 33 - .../tajo/storage/hbase/HBaseStorageManager.java | 1126 ----------- .../hbase/HBaseTextSerializerDeserializer.java | 71 - .../tajo/storage/hbase/HFileAppender.java | 167 -- .../tajo/storage/hbase/IndexPredication.java | 61 - .../tajo/storage/hbase/RowKeyMapping.java | 40 - .../apache/tajo/storage/index/IndexMethod.java | 33 - .../apache/tajo/storage/index/IndexReader.java | 35 - .../apache/tajo/storage/index/IndexWriter.java | 33 - .../tajo/storage/index/OrderIndexReader.java | 45 - .../apache/tajo/storage/index/bst/BSTIndex.java | 623 ------ .../tajo/storage/parquet/ParquetAppender.java | 151 -- .../tajo/storage/parquet/ParquetScanner.java | 119 -- .../tajo/storage/parquet/TajoParquetReader.java | 85 - .../tajo/storage/parquet/TajoParquetWriter.java | 104 - .../tajo/storage/parquet/TajoReadSupport.java | 101 - .../storage/parquet/TajoRecordConverter.java | 386 ---- .../storage/parquet/TajoRecordMaterializer.java | 78 - .../storage/parquet/TajoSchemaConverter.java | 206 -- .../tajo/storage/parquet/TajoWriteSupport.java | 149 -- .../tajo/storage/parquet/package-info.java | 96 - .../storage/rcfile/BytesRefArrayWritable.java | 261 --- .../tajo/storage/rcfile/BytesRefWritable.java | 248 --- .../storage/rcfile/ColumnProjectionUtils.java | 117 -- .../rcfile/LazyDecompressionCallback.java | 32 - .../rcfile/NonSyncByteArrayInputStream.java | 113 -- .../rcfile/NonSyncByteArrayOutputStream.java | 144 -- .../storage/rcfile/NonSyncDataInputBuffer.java | 507 ----- .../storage/rcfile/NonSyncDataOutputBuffer.java | 91 - .../org/apache/tajo/storage/rcfile/RCFile.java | 1808 ------------------ .../SchemaAwareCompressionInputStream.java | 43 - .../SchemaAwareCompressionOutputStream.java | 44 - .../sequencefile/SequenceFileAppender.java | 274 --- .../sequencefile/SequenceFileScanner.java | 336 ---- .../tajo/storage/text/ByteBufLineReader.java | 154 -- .../tajo/storage/text/DelimitedLineReader.java | 157 -- .../tajo/storage/text/DelimitedTextFile.java | 468 ----- .../tajo/storage/text/FieldSplitProcessor.java | 38 - .../tajo/storage/text/LineSplitProcessor.java | 45 - .../text/TextFieldSerializerDeserializer.java | 223 --- .../thirdparty/parquet/CodecFactory.java | 196 -- .../parquet/ColumnChunkPageWriteStore.java | 211 -- .../parquet/InternalParquetRecordReader.java | 187 -- .../parquet/InternalParquetRecordWriter.java | 160 -- .../thirdparty/parquet/ParquetFileWriter.java | 504 ----- .../thirdparty/parquet/ParquetReader.java | 151 -- .../thirdparty/parquet/ParquetWriter.java | 224 --- .../org/apache/tajo/tuple/BaseTupleBuilder.java | 112 -- .../org/apache/tajo/tuple/RowBlockReader.java | 33 - .../org/apache/tajo/tuple/TupleBuilder.java | 26 - .../tajo/tuple/offheap/DirectBufTuple.java | 41 - .../tajo/tuple/offheap/FixedSizeLimitSpec.java | 32 - .../apache/tajo/tuple/offheap/HeapTuple.java | 272 --- .../tajo/tuple/offheap/OffHeapMemory.java | 102 - .../tajo/tuple/offheap/OffHeapRowBlock.java | 176 -- .../tuple/offheap/OffHeapRowBlockReader.java | 63 - .../tuple/offheap/OffHeapRowBlockUtils.java | 54 - .../tuple/offheap/OffHeapRowBlockWriter.java | 58 - .../tajo/tuple/offheap/OffHeapRowWriter.java | 232 --- .../tajo/tuple/offheap/ResizableLimitSpec.java | 142 -- .../apache/tajo/tuple/offheap/RowWriter.java | 73 - .../apache/tajo/tuple/offheap/UnSafeTuple.java | 311 --- .../offheap/UnSafeTupleBytesComparator.java | 99 - .../tajo/tuple/offheap/ZeroCopyTuple.java | 35 - tajo-storage/src/main/proto/IndexProtos.proto | 31 - .../src/main/proto/StorageFragmentProtos.proto | 35 - .../src/main/resources/storage-default.xml | 180 -- .../java/org/apache/tajo/HttpFileServer.java | 84 - .../org/apache/tajo/HttpFileServerHandler.java | 184 -- .../tajo/HttpFileServerPipelineFactory.java | 54 - .../tajo/storage/TestCompressionStorages.java | 185 -- .../apache/tajo/storage/TestFileSystems.java | 130 -- .../org/apache/tajo/storage/TestFrameTuple.java | 84 - .../org/apache/tajo/storage/TestLazyTuple.java | 258 --- .../org/apache/tajo/storage/TestLineReader.java | 163 -- .../apache/tajo/storage/TestMergeScanner.java | 202 -- .../apache/tajo/storage/TestSplitProcessor.java | 72 - .../apache/tajo/storage/TestStorageManager.java | 203 -- .../org/apache/tajo/storage/TestStorages.java | 857 --------- .../tajo/storage/TestTupleComparator.java | 77 - .../org/apache/tajo/storage/TestVTuple.java | 160 -- .../apache/tajo/storage/avro/TestAvroUtil.java | 108 -- .../tajo/storage/hbase/TestColumnMapping.java | 95 - .../storage/hbase/TestHBaseStorageManager.java | 109 -- .../apache/tajo/storage/index/TestBSTIndex.java | 946 --------- .../index/TestSingleCSVFileBSTIndex.java | 248 --- .../tajo/storage/parquet/TestReadWrite.java | 114 -- .../storage/parquet/TestSchemaConverter.java | 132 -- .../java/org/apache/tajo/storage/s3/INode.java | 124 -- .../storage/s3/InMemoryFileSystemStore.java | 175 -- .../apache/tajo/storage/s3/S3OutputStream.java | 234 --- .../tajo/storage/s3/SmallBlockS3FileSystem.java | 314 --- .../apache/tajo/tuple/TestBaseTupleBuilder.java | 76 - .../tajo/tuple/offheap/TestHeapTuple.java | 45 - .../tajo/tuple/offheap/TestOffHeapRowBlock.java | 577 ------ .../tajo/tuple/offheap/TestResizableSpec.java | 59 - .../src/test/resources/storage-default.xml | 154 -- .../src/test/resources/testVariousTypes.avsc | 21 - tajo-storage/tajo-storage-common/pom.xml | 337 ++++ .../java/org/apache/tajo/storage/Appender.java | 41 + .../tajo/storage/BaseTupleComparator.java | 206 ++ .../storage/BinarySerializerDeserializer.java | 258 +++ .../org/apache/tajo/storage/BufferPool.java | 74 + .../tajo/storage/ByteBufInputChannel.java | 76 + .../org/apache/tajo/storage/DataLocation.java | 45 + .../org/apache/tajo/storage/DiskDeviceInfo.java | 62 + .../java/org/apache/tajo/storage/DiskInfo.java | 75 + .../org/apache/tajo/storage/DiskMountInfo.java | 101 + .../java/org/apache/tajo/storage/DiskUtil.java | 207 ++ .../storage/FieldSerializerDeserializer.java | 35 + .../org/apache/tajo/storage/FrameTuple.java | 225 +++ .../java/org/apache/tajo/storage/LazyTuple.java | 270 +++ .../org/apache/tajo/storage/MemoryUtil.java | 163 ++ .../org/apache/tajo/storage/MergeScanner.java | 201 ++ .../org/apache/tajo/storage/NullScanner.java | 109 ++ .../tajo/storage/NumericPathComparator.java | 34 + .../org/apache/tajo/storage/RowStoreUtil.java | 377 ++++ .../java/org/apache/tajo/storage/Scanner.java | 103 + .../apache/tajo/storage/SeekableScanner.java | 28 + .../tajo/storage/SerializerDeserializer.java | 34 + .../org/apache/tajo/storage/StorageManager.java | 933 +++++++++ .../apache/tajo/storage/StorageProperty.java | 40 + .../org/apache/tajo/storage/StorageUtil.java | 221 +++ .../apache/tajo/storage/TableStatistics.java | 129 ++ .../storage/TextSerializerDeserializer.java | 226 +++ .../apache/tajo/storage/TupleComparator.java | 32 + .../org/apache/tajo/storage/TupleRange.java | 112 ++ .../storage/annotation/ForSplitableStore.java | 29 + .../apache/tajo/storage/compress/CodecPool.java | 185 ++ .../AlreadyExistsStorageException.java | 39 + .../exception/UnknownCodecException.java | 32 + .../exception/UnknownDataTypeException.java | 32 + .../exception/UnsupportedFileTypeException.java | 36 + .../apache/tajo/storage/fragment/Fragment.java | 39 + .../storage/fragment/FragmentConvertor.java | 129 ++ .../org/apache/tajo/tuple/BaseTupleBuilder.java | 112 ++ .../org/apache/tajo/tuple/RowBlockReader.java | 33 + .../org/apache/tajo/tuple/TupleBuilder.java | 26 + .../tajo/tuple/offheap/DirectBufTuple.java | 41 + .../tajo/tuple/offheap/FixedSizeLimitSpec.java | 32 + .../apache/tajo/tuple/offheap/HeapTuple.java | 272 +++ .../tajo/tuple/offheap/OffHeapMemory.java | 102 + .../tajo/tuple/offheap/OffHeapRowBlock.java | 176 ++ .../tuple/offheap/OffHeapRowBlockReader.java | 63 + .../tuple/offheap/OffHeapRowBlockUtils.java | 54 + .../tuple/offheap/OffHeapRowBlockWriter.java | 58 + .../tajo/tuple/offheap/OffHeapRowWriter.java | 232 +++ .../tajo/tuple/offheap/ResizableLimitSpec.java | 142 ++ .../apache/tajo/tuple/offheap/RowWriter.java | 73 + .../apache/tajo/tuple/offheap/UnSafeTuple.java | 311 +++ .../offheap/UnSafeTupleBytesComparator.java | 99 + .../tajo/tuple/offheap/ZeroCopyTuple.java | 35 + .../src/main/proto/IndexProtos.proto | 31 + .../src/main/resources/storage-default.xml | 184 ++ .../org/apache/tajo/storage/TestFrameTuple.java | 84 + .../org/apache/tajo/storage/TestLazyTuple.java | 258 +++ .../tajo/storage/TestTupleComparator.java | 77 + .../org/apache/tajo/storage/TestVTuple.java | 160 ++ .../src/test/resources/storage-default.xml | 164 ++ tajo-storage/tajo-storage-hbase/pom.xml | 349 ++++ .../storage/hbase/AbstractHBaseAppender.java | 223 +++ .../storage/hbase/AddSortForInsertRewriter.java | 90 + .../tajo/storage/hbase/ColumnMapping.java | 236 +++ .../HBaseBinarySerializerDeserializer.java | 97 + .../tajo/storage/hbase/HBaseFragment.java | 198 ++ .../tajo/storage/hbase/HBasePutAppender.java | 120 ++ .../apache/tajo/storage/hbase/HBaseScanner.java | 449 +++++ .../storage/hbase/HBaseStorageConstants.java | 33 + .../tajo/storage/hbase/HBaseStorageManager.java | 1135 +++++++++++ .../hbase/HBaseTextSerializerDeserializer.java | 71 + .../tajo/storage/hbase/HFileAppender.java | 167 ++ .../tajo/storage/hbase/IndexPredication.java | 61 + .../tajo/storage/hbase/RowKeyMapping.java | 40 + .../src/main/proto/StorageFragmentProtos.proto | 35 + .../tajo/storage/hbase/TestColumnMapping.java | 93 + .../storage/hbase/TestHBaseStorageManager.java | 109 ++ tajo-storage/tajo-storage-hdfs/pom.xml | 380 ++++ .../java/org/apache/tajo/storage/CSVFile.java | 587 ++++++ .../tajo/storage/CompressedSplitLineReader.java | 182 ++ .../org/apache/tajo/storage/FileAppender.java | 87 + .../org/apache/tajo/storage/FileScanner.java | 124 ++ .../apache/tajo/storage/FileStorageManager.java | 854 +++++++++ .../tajo/storage/HashShuffleAppender.java | 209 ++ .../storage/HashShuffleAppenderManager.java | 225 +++ .../org/apache/tajo/storage/LineReader.java | 559 ++++++ .../java/org/apache/tajo/storage/RawFile.java | 773 ++++++++ .../java/org/apache/tajo/storage/RowFile.java | 498 +++++ .../apache/tajo/storage/SplitLineReader.java | 39 + .../apache/tajo/storage/avro/AvroAppender.java | 219 +++ .../apache/tajo/storage/avro/AvroScanner.java | 286 +++ .../org/apache/tajo/storage/avro/AvroUtil.java | 77 + .../apache/tajo/storage/avro/package-info.java | 85 + .../tajo/storage/fragment/FileFragment.java | 237 +++ .../apache/tajo/storage/index/IndexMethod.java | 32 + .../apache/tajo/storage/index/IndexReader.java | 35 + .../apache/tajo/storage/index/IndexWriter.java | 33 + .../tajo/storage/index/OrderIndexReader.java | 45 + .../apache/tajo/storage/index/bst/BSTIndex.java | 623 ++++++ .../tajo/storage/parquet/ParquetAppender.java | 151 ++ .../tajo/storage/parquet/ParquetScanner.java | 119 ++ .../tajo/storage/parquet/TajoParquetReader.java | 85 + .../tajo/storage/parquet/TajoParquetWriter.java | 104 + .../tajo/storage/parquet/TajoReadSupport.java | 99 + .../storage/parquet/TajoRecordConverter.java | 380 ++++ .../storage/parquet/TajoRecordMaterializer.java | 77 + .../storage/parquet/TajoSchemaConverter.java | 206 ++ .../tajo/storage/parquet/TajoWriteSupport.java | 148 ++ .../tajo/storage/parquet/package-info.java | 96 + .../storage/rcfile/BytesRefArrayWritable.java | 261 +++ .../tajo/storage/rcfile/BytesRefWritable.java | 248 +++ .../storage/rcfile/ColumnProjectionUtils.java | 117 ++ .../rcfile/LazyDecompressionCallback.java | 32 + .../rcfile/NonSyncByteArrayInputStream.java | 113 ++ .../rcfile/NonSyncByteArrayOutputStream.java | 144 ++ .../storage/rcfile/NonSyncDataInputBuffer.java | 507 +++++ .../storage/rcfile/NonSyncDataOutputBuffer.java | 91 + .../org/apache/tajo/storage/rcfile/RCFile.java | 1807 +++++++++++++++++ .../SchemaAwareCompressionInputStream.java | 43 + .../SchemaAwareCompressionOutputStream.java | 44 + .../sequencefile/SequenceFileAppender.java | 274 +++ .../sequencefile/SequenceFileScanner.java | 336 ++++ .../tajo/storage/text/ByteBufLineReader.java | 154 ++ .../tajo/storage/text/DelimitedLineReader.java | 157 ++ .../tajo/storage/text/DelimitedTextFile.java | 468 +++++ .../tajo/storage/text/FieldSplitProcessor.java | 38 + .../tajo/storage/text/LineSplitProcessor.java | 45 + .../text/TextFieldSerializerDeserializer.java | 223 +++ .../thirdparty/parquet/CodecFactory.java | 190 ++ .../parquet/ColumnChunkPageWriteStore.java | 206 ++ .../parquet/InternalParquetRecordReader.java | 188 ++ .../parquet/InternalParquetRecordWriter.java | 160 ++ .../thirdparty/parquet/ParquetFileWriter.java | 492 +++++ .../thirdparty/parquet/ParquetReader.java | 146 ++ .../thirdparty/parquet/ParquetWriter.java | 224 +++ .../src/main/proto/StorageFragmentProtos.proto | 34 + .../java/org/apache/tajo/HttpFileServer.java | 84 + .../org/apache/tajo/HttpFileServerHandler.java | 184 ++ .../tajo/HttpFileServerPipelineFactory.java | 54 + .../tajo/storage/TestCompressionStorages.java | 185 ++ .../tajo/storage/TestFileStorageManager.java | 203 ++ .../apache/tajo/storage/TestFileSystems.java | 129 ++ .../apache/tajo/storage/TestMergeScanner.java | 202 ++ .../org/apache/tajo/storage/TestStorages.java | 867 +++++++++ .../apache/tajo/storage/avro/TestAvroUtil.java | 106 + .../apache/tajo/storage/index/TestBSTIndex.java | 947 +++++++++ .../index/TestSingleCSVFileBSTIndex.java | 248 +++ .../tajo/storage/parquet/TestReadWrite.java | 109 ++ .../storage/parquet/TestSchemaConverter.java | 130 ++ .../java/org/apache/tajo/storage/s3/INode.java | 124 ++ .../storage/s3/InMemoryFileSystemStore.java | 176 ++ .../apache/tajo/storage/s3/S3OutputStream.java | 234 +++ .../tajo/storage/s3/SmallBlockS3FileSystem.java | 314 +++ .../src/test/resources/storage-default.xml | 164 ++ .../src/test/resources/testVariousTypes.avsc | 21 + tajo-yarn-pullserver/pom.xml | 7 +- 362 files changed, 30704 insertions(+), 30655 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 235b374..87ef9a6 100644 --- a/CHANGES +++ b/CHANGES @@ -17,6 +17,9 @@ Release 0.9.1 - unreleased IMPROVEMENT + TAJO-1122: Refactor the tajo-storage project structure. + (Hyoungjun Kim) + TAJO-1195: Remove unused CachedDNSResolver Class. (DaeMyung Kang via jaehwa) TAJO-1184: Upgrade netty-buffer to 4.0.24.Final. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index b3ae9be..92437af 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -73,15 +73,6 @@ message FragmentProto { required bytes contents = 3; } -message FileFragmentProto { - required string id = 1; - required string path = 2; - required int64 startOffset = 3; - required int64 length = 4; - repeated string hosts = 7; - repeated int32 diskIds = 8; -} - message TableProto { required StoreType storeType = 1; optional KeyValueSetProto params = 2; http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml index 2950a96..4187150 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml @@ -113,7 +113,7 @@ </dependency> <dependency> <groupId>org.apache.tajo</groupId> - <artifactId>tajo-storage</artifactId> + <artifactId>tajo-storage-common</artifactId> </dependency> <dependency> <groupId>junit</groupId> http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-client/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-client/pom.xml b/tajo-client/pom.xml index 7fc8c74..c023db8 100644 --- a/tajo-client/pom.xml +++ b/tajo-client/pom.xml @@ -195,7 +195,11 @@ </dependency> <dependency> <groupId>org.apache.tajo</groupId> - <artifactId>tajo-storage</artifactId> + <artifactId>tajo-storage-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hdfs</artifactId> </dependency> <dependency> <groupId>org.apache.tajo</groupId> http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java index c0b01e2..b6e47c7 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java @@ -30,10 +30,7 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryClient; import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.FileScanner; -import org.apache.tajo.storage.MergeScanner; -import org.apache.tajo.storage.Scanner; -import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java index 0ceb2b2..dcf5c4e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java @@ -119,6 +119,15 @@ public class TUtil { return new ArrayList<T>(); } + public static List<Integer> newIntList(int...items) { + List<Integer> list = new ArrayList<Integer>(); + for (int item: items) { + list.add(item); + } + + return list; + } + public static <T> List<T> newList(T...items) { List<T> list = new ArrayList<T>(); for (T t : items) { http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index b58ae89..361b342 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -245,7 +245,16 @@ </dependency> <dependency> <groupId>org.apache.tajo</groupId> - <artifactId>tajo-storage</artifactId> + <artifactId>tajo-storage-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hdfs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hbase</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.apache.tajo</groupId> @@ -482,8 +491,8 @@ <dependency> <groupId>org.apache.tajo</groupId> <artifactId>tajo-hcatalog</artifactId> - <scope>test</scope> <version>${tajo.version}</version> + <scope>test</scope> <exclusions> <exclusion> <groupId>com.google.protobuf</groupId> http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 98a621e..3880bcb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -49,11 +49,7 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecAr import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.BaseTupleComparator; -import org.apache.tajo.storage.FileStorageManager; -import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.TupleComparator; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -921,7 +917,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { if (broadcastFlag) { PartitionedTableScanNode partitionedTableScanNode = (PartitionedTableScanNode) scanNode; List<Fragment> fileFragments = TUtil.newList(); - FileStorageManager fileStorageManager = StorageManager.getFileStorageManager(ctx.getConf()); + FileStorageManager fileStorageManager = (FileStorageManager)StorageManager.getFileStorageManager(ctx.getConf()); for (Path path : partitionedTableScanNode.getInputPaths()) { fileFragments.addAll(TUtil.newList(fileStorageManager.split(scanNode.getCanonicalName(), path))); } @@ -1186,8 +1182,8 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { FragmentConvertor.convert(ctx.getConf(), fragmentProtos); String indexName = IndexUtil.getIndexNameOfFrag(fragments.get(0), annotation.getSortKeys()); - Path indexPath = new Path( - StorageManager.getFileStorageManager(ctx.getConf()).getTablePath(annotation.getTableName()), "index"); + FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(ctx.getConf()); + Path indexPath = new Path(sm.getTablePath(annotation.getTableName()), "index"); TupleComparator comp = new BaseTupleComparator(annotation.getKeySchema(), annotation.getSortKeys()); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index c5df5f9..8ee4e2f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -35,6 +35,7 @@ import org.apache.tajo.plan.logical.InsertNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.StoreTableNode; import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.unit.StorageUnit; @@ -161,7 +162,7 @@ public abstract class ColPartitionStoreExec extends UnaryPhysicalExec { actualFilePath = new Path(lastFileName + "_" + suffixId); } - appender = StorageManager.getFileStorageManager(context.getConf()) + appender = ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 818bd2e..f38ae3c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -34,11 +34,7 @@ import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.PersistentStoreNode; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.FileStorageManager; -import org.apache.tajo.storage.BaseTupleComparator; -import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.TupleComparator; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java index 568c6ec..585d73a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java @@ -80,8 +80,8 @@ public class RangeShuffleFileWriteExec extends UnaryPhysicalExec { context.getDataChannel().getStoreType() : CatalogProtos.StoreType.RAW); FileSystem fs = new RawLocalFileSystem(); fs.mkdirs(storeTablePath); - this.appender = (FileAppender) StorageManager.getFileStorageManager(context.getConf()).getAppender(meta, - outSchema, new Path(storeTablePath, "output")); + this.appender = (FileAppender) ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) + .getAppender(meta, outSchema, new Path(storeTablePath, "output")); this.appender.enableStats(); this.appender.init(); this.indexWriter = bst.getIndexWriter(new Path(storeTablePath, "index"), http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index a5e0b5d..3d3da5c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -31,6 +31,7 @@ import org.apache.tajo.plan.logical.InsertNode; import org.apache.tajo.plan.logical.PersistentStoreNode; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.unit.StorageUnit; @@ -92,7 +93,8 @@ public class StoreTableExec extends UnaryPhysicalExec { lastFileName = new Path(lastFileName + "_" + suffixId); } - appender = StorageManager.getFileStorageManager(context.getConf()).getAppender(meta, appenderSchema, lastFileName); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) + .getAppender(meta, appenderSchema, lastFileName); if (suffixId > 0) { LOG.info(prevFile + " exceeds " + SessionVars.MAX_OUTPUT_FILE_SIZE.keyname() + " (" + maxPerFileSize + " MB), " + http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 0d3ae19..87646de 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -324,6 +324,7 @@ public class GlobalEngine extends AbstractService { responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR); responseBuilder.setErrorMessage("Fail starting QueryMaster."); + LOG.error("Fail starting QueryMaster: " + sql); } else { responseBuilder.setIsForwarded(true); responseBuilder.setQueryId(queryInfo.getQueryId().getProto()); @@ -332,7 +333,8 @@ public class GlobalEngine extends AbstractService { responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost()); } responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort()); - LOG.info("Query is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort()); + LOG.info("Query " + queryInfo.getQueryId().toString() + "," + queryInfo.getSql() + "," + + " is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort()); } } SubmitQueryResponse response = responseBuilder.build(); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java index aff4b7d..038c334 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java @@ -42,6 +42,7 @@ import org.apache.tajo.master.querymaster.QueryUnitAttempt; import org.apache.tajo.master.querymaster.SubQuery; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.NetUtils; import org.apache.tajo.worker.FetchImpl; http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index d015efe..49a6d92 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -167,7 +167,7 @@ public class TajoMaster extends CompositeService { // check the system directory and create if they are not created. checkAndInitializeSystemDirectories(); - this.storeManager = StorageManager.getFileStorageManager(systemConf, null); + this.storeManager = (FileStorageManager)StorageManager.getFileStorageManager(systemConf, null); catalogServer = new CatalogServer(FunctionLoader.load()); addIfService(catalogServer); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index 8164869..a240ace 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -97,7 +97,8 @@ public class Repartitioner { for (int i = 0; i < scans.length; i++) { TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName()); if (tableDesc == null) { // if it is a real table stored on storage - FileStorageManager storageManager = StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + FileStorageManager storageManager = + (FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf()); tablePath = storageManager.getTablePath(scans[i].getTableName()); if (execBlock.getUnionScanMap() != null && !execBlock.getUnionScanMap().isEmpty()) { @@ -410,7 +411,8 @@ public class Repartitioner { Path[] partitionScanPaths = null; TableDesc tableDesc = masterContext.getTableDescMap().get(eachScan.getCanonicalName()); if (eachScan.getType() == NodeType.PARTITIONS_SCAN) { - FileStorageManager storageManager = StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + FileStorageManager storageManager = + (FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf()); PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)eachScan; partitionScanPaths = partitionScan.getInputPaths(); @@ -535,7 +537,7 @@ public class Repartitioner { partitionScanPaths = partitionScan.getInputPaths(); // set null to inputPaths in getFragmentsFromPartitionedTable() FileStorageManager storageManager = - StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + (FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf()); scanFragments = getFragmentsFromPartitionedTable(storageManager, scan, desc); } else { StorageManager storageManager = @@ -644,7 +646,8 @@ public class Repartitioner { ExecutionBlock execBlock = subQuery.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = StorageManager.getFileStorageManager(subQuery.getContext().getConf()).getTablePath(scan.getTableName()); + tablePath = ((FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf())) + .getTablePath(scan.getTableName()); ExecutionBlock sampleChildBlock = masterPlan.getChild(subQuery.getId(), 0); SortNode sortNode = PlannerUtil.findTopNode(sampleChildBlock.getPlan(), NodeType.SORT); @@ -809,7 +812,8 @@ public class Repartitioner { ExecutionBlock execBlock = subQuery.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = StorageManager.getFileStorageManager(subQuery.getContext().getConf()).getTablePath(scan.getTableName()); + tablePath = ((FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf())) + .getTablePath(scan.getTableName()); Fragment frag = new FileFragment(scan.getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST}); List<Fragment> fragments = new ArrayList<Fragment>(); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index 6676072..745456a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -1051,7 +1051,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> { if (scan.getType() == NodeType.PARTITIONS_SCAN) { // After calling this method, partition paths are removed from the physical plan. FileStorageManager storageManager = - StorageManager.getFileStorageManager(subQuery.getContext().getConf()); + (FileStorageManager)StorageManager.getFileStorageManager(subQuery.getContext().getConf()); fragments = Repartitioner.getFragmentsFromPartitionedTable(storageManager, scan, table); } else { StorageManager storageManager = http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java index 0e3ccad..5d7472f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java @@ -38,6 +38,7 @@ import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.querymaster.QueryInProgress; import org.apache.tajo.rpc.CallFuture; import org.apache.tajo.util.ApplicationIdUtils; +import org.apache.tajo.util.StringUtils; import java.io.IOException; import java.util.*; http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/main/java/org/apache/tajo/worker/Task.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 0657e66..00eabcc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -52,11 +52,7 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.pullserver.TajoPullServerService; import org.apache.tajo.pullserver.retriever.FileChunk; import org.apache.tajo.rpc.NullCallback; -import org.apache.tajo.storage.BaseTupleComparator; -import org.apache.tajo.storage.HashShuffleAppenderManager; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.StorageUtil; -import org.apache.tajo.storage.TupleComparator; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.NetUtils; import org.jboss.netty.channel.socket.ClientSocketChannelFactory; @@ -157,8 +153,8 @@ public class Task { this.sortComp = new BaseTupleComparator(finalSchema, sortNode.getSortKeys()); } } else { - Path outFilePath = StorageManager.getFileStorageManager(systemConf).getAppenderFilePath( - taskId, queryContext.getStagingDir()); + Path outFilePath = ((FileStorageManager)StorageManager.getFileStorageManager(systemConf)) + .getAppenderFilePath(taskId, queryContext.getStagingDir()); LOG.info("Output File Path: " + outFilePath); context.setOutputPath(outFilePath); } http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java index 0959f2e..fb98be2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java @@ -47,7 +47,7 @@ public class BackendTestingUtil { public static void writeTmpTable(TajoConf conf, Path tablePath) throws IOException { - FileStorageManager sm = StorageManager.getFileStorageManager(conf, tablePath); + FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, tablePath); FileSystem fs = sm.getFileSystem(); Appender appender; http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index 211f1fe..9ce7b5b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -40,10 +40,10 @@ import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.VTuple; -import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index 64d7cee..3803c7a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -42,10 +42,7 @@ import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.plan.logical.*; -import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.*; import org.apache.tajo.util.CommonTestingUtil; import org.junit.After; import org.junit.Before; @@ -140,8 +137,8 @@ public class TestBroadcastJoinPlan { contentsData += j; } } - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(tableMeta, schema, - dataPath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(tableMeta, schema, dataPath); appender.init(); Tuple tuple = new VTuple(schema.size()); int writtenSize = 0; http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index 9517e1b..6a6aafb 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -89,7 +89,8 @@ public class TestBNLJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < OUTER_TUPLE_NUM; i++) { @@ -110,7 +111,8 @@ public class TestBNLJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < INNER_TUPLE_NUM; i += 2) { http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index 61f32da..dc3c28d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -91,7 +91,7 @@ public class TestBSTIndexExec { Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = StorageManager.getFileStorageManager(conf, workDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, workDir); idxPath = new Path(workDir, "test.idx"); @@ -117,8 +117,7 @@ public class TestBSTIndexExec { fs = tablePath.getFileSystem(conf); fs.mkdirs(tablePath.getParent()); - FileAppender appender = (FileAppender)StorageManager.getFileStorageManager(conf).getAppender(meta, schema, - tablePath); + FileAppender appender = (FileAppender)sm.getAppender(meta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 10000; i++) { http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 00d023e..c0bf6ce 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -76,7 +76,7 @@ public class TestExternalSortExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); - sm = StorageManager.getFileStorageManager(conf, testDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -85,7 +85,8 @@ public class TestExternalSortExec { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 3a4e963..ecd1c23 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -84,7 +84,7 @@ public class TestFullOuterHashJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -107,7 +107,8 @@ public class TestFullOuterHashJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -136,7 +137,8 @@ public class TestFullOuterHashJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -175,7 +177,8 @@ public class TestFullOuterHashJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -227,8 +230,8 @@ public class TestFullOuterHashJoinExec { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 6684002..a81979f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -112,7 +112,8 @@ public class TestFullOuterMergeJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -150,7 +151,8 @@ public class TestFullOuterMergeJoinExec { TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = StorageManager.getFileStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); + Appender appender4 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { @@ -181,7 +183,8 @@ public class TestFullOuterMergeJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -220,7 +223,8 @@ public class TestFullOuterMergeJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -272,8 +276,8 @@ public class TestFullOuterMergeJoinExec { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); appender5.close(); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 48c1cae..4fe6ff2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -86,8 +86,8 @@ public class TestHashAntiJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -112,7 +112,8 @@ public class TestHashAntiJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index dcd18cd..55e87d4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -88,8 +88,8 @@ public class TestHashJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); for (int i = 0; i < 10; i++) { @@ -111,7 +111,8 @@ public class TestHashJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 5cb1a7d..a2f1155 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -87,8 +87,8 @@ public class TestHashSemiJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -113,7 +113,8 @@ public class TestHashSemiJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); // make 27 tuples http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index d709554..0477771 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -108,7 +108,8 @@ public class TestLeftOuterHashJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -137,7 +138,8 @@ public class TestLeftOuterHashJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -176,7 +178,8 @@ public class TestLeftOuterHashJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -228,8 +231,8 @@ public class TestLeftOuterHashJoinExec { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index 89f5feb..36dd77e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -104,7 +104,8 @@ public class TestLeftOuterNLJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -133,7 +134,8 @@ public class TestLeftOuterNLJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -172,7 +174,8 @@ public class TestLeftOuterNLJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -224,8 +227,8 @@ public class TestLeftOuterNLJoinExec { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index 12509f1..10d4d33 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -89,8 +89,8 @@ public class TestMergeJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); for (int i = 0; i < 10; i++) { @@ -118,7 +118,8 @@ public class TestMergeJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index abace81..e976456 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -86,7 +86,8 @@ public class TestNLJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 50; i++) { @@ -109,7 +110,8 @@ public class TestNLJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManager.getFileStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 50; i += 2) { http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 6aad738..cce4ba7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -107,7 +107,7 @@ public class TestPhysicalPlanner { util.startCatalogCluster(); conf = util.getConfiguration(); testDir = CommonTestingUtil.getTestDir("target/test-data/TestPhysicalPlanner"); - sm = StorageManager.getFileStorageManager(conf, testDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir); catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); @@ -130,8 +130,7 @@ public class TestPhysicalPlanner { Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, employeeSchema, - employeePath); + Appender appender = sm.getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); for (int i = 0; i < 100; i++) { @@ -149,7 +148,7 @@ public class TestPhysicalPlanner { Path scorePath = new Path(testDir, "score"); TableMeta scoreMeta = CatalogUtil.newTableMeta(StoreType.CSV, new KeyValueSet()); - appender = StorageManager.getFileStorageManager(conf).getAppender(scoreMeta, scoreSchema, scorePath); + appender = sm.getAppender(scoreMeta, scoreSchema, scorePath); appender.init(); score = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"), scoreSchema, scoreMeta, @@ -190,8 +189,8 @@ public class TestPhysicalPlanner { Schema scoreSchmea = score.getSchema(); TableMeta scoreLargeMeta = CatalogUtil.newTableMeta(StoreType.RAW, new KeyValueSet()); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(scoreLargeMeta, scoreSchmea, - scoreLargePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(scoreLargeMeta, scoreSchmea, scoreLargePath); appender.enableStats(); appender.init(); largeScore = new TableDesc( @@ -452,8 +451,8 @@ public class TestPhysicalPlanner { exec.next(); exec.close(); - Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), - ctx.getOutputPath()); + Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; int i = 0; @@ -512,7 +511,7 @@ public class TestPhysicalPlanner { // checking the file contents long totalNum = 0; for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) { - Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner( + Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)).getFileScanner( CatalogUtil.newTableMeta(StoreType.CSV), rootNode.getOutSchema(), status.getPath()); @@ -549,8 +548,8 @@ public class TestPhysicalPlanner { exec.next(); exec.close(); - Scanner scanner = StorageManager.getFileStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), - ctx.getOutputPath()); + Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)).getFileScanner( + outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; int i = 0; @@ -1084,7 +1083,7 @@ public class TestPhysicalPlanner { Path outputPath = StorageUtil.concatPath(workDir, "output", "output"); TableMeta meta = CatalogUtil.newTableMeta(channel.getStoreType(), new KeyValueSet()); SeekableScanner scanner = - StorageManager.getSeekableScanner(conf, meta, exec.getSchema(), outputPath); + FileStorageManager.getSeekableScanner(conf, meta, exec.getSchema(), outputPath); scanner.init(); int cnt = 0; http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index f3936b1..3c78b12 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -89,7 +89,8 @@ public class TestProgressExternalSortExec { TableMeta employeeMeta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.RAW); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManager.getFileStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 400d00f..879ca21 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -103,7 +103,8 @@ public class TestRightOuterHashJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -132,7 +133,8 @@ public class TestRightOuterHashJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -171,7 +173,8 @@ public class TestRightOuterHashJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 2708273..8bc00cc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -110,7 +110,8 @@ public class TestRightOuterMergeJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManager.getFileStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -148,7 +149,8 @@ public class TestRightOuterMergeJoinExec { TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = StorageManager.getFileStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); + Appender appender4 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { @@ -179,7 +181,8 @@ public class TestRightOuterMergeJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManager.getFileStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -218,7 +221,8 @@ public class TestRightOuterMergeJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManager.getFileStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -270,8 +274,8 @@ public class TestRightOuterMergeJoinExec { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManager.getFileStorageManager(conf).getAppender(phone3Meta, phone3Schema, - phone3Path); + Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush();
