TAJO-1603: Refactor StorageManager. (hyunsik) Closes #570
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5491f0e7 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5491f0e7 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5491f0e7 Branch: refs/heads/index_support Commit: 5491f0e7507c7efa1b2306d4c1f1d25240e482a9 Parents: 36a703c Author: Hyunsik Choi <[email protected]> Authored: Fri May 15 01:48:09 2015 -0700 Committer: Hyunsik Choi <[email protected]> Committed: Fri May 15 01:48:09 2015 -0700 ---------------------------------------------------------------------- CHANGES | 2 + .../engine/planner/PhysicalPlannerImpl.java | 5 +- .../planner/physical/BSTIndexScanExec.java | 2 +- .../planner/physical/ColPartitionStoreExec.java | 7 +- .../physical/RangeShuffleFileWriteExec.java | 3 +- .../engine/planner/physical/SeqScanExec.java | 2 +- .../engine/planner/physical/StoreTableExec.java | 9 +- .../org/apache/tajo/master/GlobalEngine.java | 4 +- .../java/org/apache/tajo/master/TajoMaster.java | 3 +- .../apache/tajo/master/exec/DDLExecutor.java | 5 +- .../exec/NonForwardQueryResultFileScanner.java | 7 +- .../apache/tajo/master/exec/QueryExecutor.java | 4 +- .../java/org/apache/tajo/querymaster/Query.java | 7 +- .../tajo/querymaster/QueryMasterTask.java | 4 +- .../apache/tajo/querymaster/Repartitioner.java | 24 +- .../java/org/apache/tajo/querymaster/Stage.java | 6 +- .../java/org/apache/tajo/worker/TajoWorker.java | 4 +- .../main/java/org/apache/tajo/worker/Task.java | 3 +- .../org/apache/tajo/BackendTestingUtil.java | 3 +- .../planner/global/TestBroadcastJoinPlan.java | 3 +- .../planner/physical/TestBNLJoinExec.java | 5 +- .../planner/physical/TestBSTIndexExec.java | 3 +- .../planner/physical/TestExternalSortExec.java | 3 +- .../physical/TestFullOuterHashJoinExec.java | 9 +- .../physical/TestFullOuterMergeJoinExec.java | 10 +- .../planner/physical/TestHashAntiJoinExec.java | 5 +- .../planner/physical/TestHashJoinExec.java | 5 +- .../planner/physical/TestHashSemiJoinExec.java | 5 +- .../physical/TestLeftOuterHashJoinExec.java | 9 +- .../planner/physical/TestMergeJoinExec.java | 5 +- .../engine/planner/physical/TestNLJoinExec.java | 5 +- .../planner/physical/TestPhysicalPlanner.java | 11 +- .../physical/TestProgressExternalSortExec.java | 3 +- .../physical/TestRightOuterHashJoinExec.java | 7 +- .../physical/TestRightOuterMergeJoinExec.java | 11 +- .../engine/planner/physical/TestSortExec.java | 5 +- .../tajo/engine/query/TestHBaseTable.java | 12 +- .../tajo/engine/query/TestJoinBroadcast.java | 2 +- .../org/apache/tajo/jdbc/TestResultSet.java | 2 +- .../tajo/master/TestExecutionBlockCursor.java | 4 +- .../org/apache/tajo/storage/TestRowFile.java | 5 +- .../org/apache/tajo/storage/MergeScanner.java | 2 +- .../org/apache/tajo/storage/StorageManager.java | 645 +------------------ .../org/apache/tajo/storage/TableSpace.java | 74 +++ .../apache/tajo/storage/TableSpaceManager.java | 254 ++++++++ .../tajo/storage/hbase/HBasePutAppender.java | 4 +- .../apache/tajo/storage/hbase/HBaseScanner.java | 7 +- .../tajo/storage/hbase/HBaseStorageManager.java | 40 +- .../storage/hbase/TestHBaseStorageManager.java | 5 +- .../org/apache/tajo/storage/FileAppender.java | 2 +- .../apache/tajo/storage/FileStorageManager.java | 377 ++++++++++- .../storage/HashShuffleAppenderManager.java | 4 +- .../tajo/storage/TestCompressionStorages.java | 5 +- .../tajo/storage/TestDelimitedTextFile.java | 9 +- .../tajo/storage/TestFileStorageManager.java | 11 +- .../apache/tajo/storage/TestFileSystems.java | 3 +- .../org/apache/tajo/storage/TestLineReader.java | 9 +- .../apache/tajo/storage/TestMergeScanner.java | 7 +- .../org/apache/tajo/storage/TestStorages.java | 48 +- .../apache/tajo/storage/index/TestBSTIndex.java | 53 +- .../index/TestSingleCSVFileBSTIndex.java | 5 +- .../apache/tajo/storage/json/TestJsonSerDe.java | 8 +- 62 files changed, 934 insertions(+), 871 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 44ae4b4..21f5e5a 100644 --- a/CHANGES +++ b/CHANGES @@ -24,6 +24,8 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1603: Refactor StorageManager. (hyunsik) + TAJO-1542: Refactoring of HashJoinExecs. (Contributed Navis, Committed by hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 978dde8..ff42d5f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -30,7 +30,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; @@ -925,7 +924,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { if (broadcastFlag) { PartitionedTableScanNode partitionedTableScanNode = (PartitionedTableScanNode) scanNode; List<Fragment> fileFragments = TUtil.newList(); - FileStorageManager fileStorageManager = (FileStorageManager)StorageManager.getFileStorageManager(ctx.getConf()); + FileStorageManager fileStorageManager = (FileStorageManager) TableSpaceManager.getFileStorageManager(ctx.getConf()); for (Path path : partitionedTableScanNode.getInputPaths()) { fileFragments.addAll(TUtil.newList(fileStorageManager.split(scanNode.getCanonicalName(), path))); } @@ -1189,7 +1188,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { FragmentConvertor.convert(ctx.getConf(), fragmentProtos); String indexName = IndexUtil.getIndexNameOfFrag(fragments.get(0), annotation.getSortKeys()); - FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(ctx.getConf()); + FileStorageManager sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(ctx.getConf()); Path indexPath = new Path(sm.getTablePath(annotation.getTableName()), "index"); TupleComparator comp = new BaseTupleComparator(annotation.getKeySchema(), http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index 806d34c..4612d45 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -56,7 +56,7 @@ public class BSTIndexScanExec extends PhysicalExec { this.qual = scanNode.getQual(); this.datum = datum; - this.fileScanner = StorageManager.getSeekableScanner(context.getConf(), + this.fileScanner = TableSpaceManager.getSeekableScanner(context.getConf(), scanNode.getTableDesc().getMeta(), scanNode.getInSchema(), fragment, outSchema); this.fileScanner.init(); this.projector = new Projector(context, inSchema, outSchema, scanNode.getTargets()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index 4481569..8d53a6f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -34,10 +34,7 @@ import org.apache.tajo.plan.logical.CreateTableNode; import org.apache.tajo.plan.logical.InsertNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.StoreTableNode; -import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.FileStorageManager; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.storage.*; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.worker.TaskAttemptContext; @@ -168,7 +165,7 @@ public abstract class ColPartitionStoreExec extends UnaryPhysicalExec { actualFilePath = new Path(lastFileName + "_" + suffixId); } - appender = ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) + appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(context.getConf())) .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java index cb1fa05..6fd2ce4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.io.IOUtils; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.*; @@ -78,7 +77,7 @@ public class RangeShuffleFileWriteExec extends UnaryPhysicalExec { context.getDataChannel().getStoreType() : "RAW"); FileSystem fs = new RawLocalFileSystem(); fs.mkdirs(storeTablePath); - this.appender = (FileAppender) ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) + this.appender = (FileAppender) ((FileStorageManager) TableSpaceManager.getFileStorageManager(context.getConf())) .getAppender(meta, outSchema, new Path(storeTablePath, "output")); this.appender.enableStats(); this.appender.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index 3d95068..2225dae 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -202,7 +202,7 @@ public class SeqScanExec extends ScanExec { FragmentConvertor.convert(context.getConf(), fragments), projected ); } else { - StorageManager storageManager = StorageManager.getStorageManager( + StorageManager storageManager = TableSpaceManager.getStorageManager( context.getConf(), plan.getTableDesc().getMeta().getStoreType()); this.scanner = storageManager.getScanner(meta, plan.getPhysicalSchema(), fragments[0], projected); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index 5622699..b0263d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -30,10 +30,7 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.plan.logical.InsertNode; import org.apache.tajo.plan.logical.PersistentStoreNode; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.FileStorageManager; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.*; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.worker.TaskAttemptContext; @@ -93,7 +90,7 @@ public class StoreTableExec extends UnaryPhysicalExec { lastFileName = new Path(lastFileName + "_" + suffixId); } - appender = ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) + appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(context.getConf())) .getAppender(meta, appenderSchema, lastFileName); if (suffixId > 0) { @@ -101,7 +98,7 @@ public class StoreTableExec extends UnaryPhysicalExec { "The remain output will be written into " + lastFileName.toString()); } } else { - appender = StorageManager.getStorageManager(context.getConf(), meta.getStoreType()).getAppender( + appender = TableSpaceManager.getStorageManager(context.getConf(), meta.getStoreType()).getAppender( context.getQueryContext(), context.getTaskId(), meta, appenderSchema, context.getQueryContext().getStagingDir()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index adbc8a4..aeb4166 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -35,7 +35,6 @@ import org.apache.tajo.algebra.JsonHelper; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.parser.SQLSyntaxError; @@ -55,6 +54,7 @@ import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; import org.apache.tajo.plan.verifier.VerificationState; import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.CommonTestingUtil; import java.io.IOException; @@ -302,7 +302,7 @@ public class GlobalEngine extends AbstractService { InsertNode iNode = rootNode.getChild(); Schema outSchema = iNode.getChild().getOutSchema(); - StorageManager.getStorageManager(queryContext.getConf(), storeType) + TableSpaceManager.getStorageManager(queryContext.getConf(), storeType) .verifyInsertTableSchema(tableDesc, outSchema); } catch (Throwable t) { state.addVerification(t.getMessage()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index fb2a160..abf070e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -54,6 +54,7 @@ import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.session.SessionManager; import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.*; import org.apache.tajo.util.history.HistoryReader; import org.apache.tajo.util.history.HistoryWriter; @@ -182,7 +183,7 @@ public class TajoMaster extends CompositeService { // check the system directory and create if they are not created. checkAndInitializeSystemDirectories(); diagnoseTajoMaster(); - this.storeManager = StorageManager.getFileStorageManager(systemConf); + this.storeManager = TableSpaceManager.getFileStorageManager(systemConf); catalogServer = new CatalogServer(loadFunctions()); addIfService(catalogServer); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index c84b0c7..888194d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -38,6 +38,7 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.storage.TableSpaceManager; import java.io.IOException; import java.util.ArrayList; @@ -242,7 +243,7 @@ public class DDLExecutor { desc.setPartitionMethod(partitionDesc); } - StorageManager.getStorageManager(queryContext.getConf(), storeType).createTable(desc, ifNotExists); + TableSpaceManager.getStorageManager(queryContext.getConf(), storeType).createTable(desc, ifNotExists); if (catalog.createTable(desc)) { LOG.info("Table " + desc.getName() + " is created (" + desc.getStats().getNumBytes() + ")"); @@ -289,7 +290,7 @@ public class DDLExecutor { if (purge) { try { - StorageManager.getStorageManager(queryContext.getConf(), + TableSpaceManager.getStorageManager(queryContext.getConf(), tableDesc.getMeta().getStoreType()).purgeTable(tableDesc); } catch (IOException e) { throw new InternalError(e.getMessage()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java index 9c0bd48..d2022b5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java @@ -33,11 +33,8 @@ import org.apache.tajo.plan.expr.EvalTreeUtil; import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.engine.planner.physical.SeqScanExec; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.FileStorageManager; -import org.apache.tajo.storage.RowStoreUtil; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.StringUtils; @@ -104,7 +101,7 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc } private void initSeqScanExec() throws IOException { - StorageManager storageManager = StorageManager.getStorageManager(tajoConf, tableDesc.getMeta().getStoreType()); + StorageManager storageManager = TableSpaceManager.getStorageManager(tajoConf, tableDesc.getMeta().getStoreType()); List<Fragment> fragments = null; setPartition(storageManager); fragments = storageManager.getNonForwardSplit(tableDesc, currentFragmentIndex, MAX_FRAGMENT_NUM_PER_SCAN); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index 481bdbe..695c38e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -438,7 +438,7 @@ public class QueryExecutor { String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { - StorageManager sm = StorageManager.getStorageManager(context.getConf(), storeType); + StorageManager sm = TableSpaceManager.getStorageManager(context.getConf(), storeType); StorageProperty storageProperty = sm.getStorageProperty(); if (!storageProperty.isSupportsInsertInto()) { throw new VerifyException("Inserting into non-file storage is not supported."); @@ -476,7 +476,7 @@ public class QueryExecutor { String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { - StorageManager sm = StorageManager.getStorageManager(planner.getConf(), storeType); + StorageManager sm = TableSpaceManager.getStorageManager(planner.getConf(), storeType); StorageProperty storageProperty = sm.getStorageProperty(); if (storageProperty.isSortedInsert()) { String tableName = PlannerUtil.getStoreTableName(plan); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index a2e434b..362dfa6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -36,7 +36,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.ExecutionBlock; @@ -46,8 +45,8 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.event.*; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.QueryHistory; import org.apache.tajo.util.history.StageHistory; @@ -424,7 +423,7 @@ public class Query implements EventHandler<QueryEvent> { if (storeType != null) { LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); try { - StorageManager.getStorageManager(query.systemConf, storeType).rollbackOutputCommit(rootNode.getChild()); + TableSpaceManager.getStorageManager(query.systemConf, storeType).rollbackOutputCommit(rootNode.getChild()); } catch (IOException e) { LOG.warn(query.getId() + ", failed processing cleanup storage when query failed:" + e.getMessage(), e); } @@ -445,7 +444,7 @@ public class Query implements EventHandler<QueryEvent> { CatalogService catalog = lastStage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, rootNode.getChild()); - Path finalOutputDir = StorageManager.getStorageManager(query.systemConf, storeType) + Path finalOutputDir = TableSpaceManager.getStorageManager(query.systemConf, storeType) .commitOutputData(query.context.getQueryContext(), lastStage.getId(), lastStage.getMasterPlan().getLogicalPlan(), lastStage.getSchema(), tableDesc); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 8d20141..d77001c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -36,7 +36,6 @@ import org.apache.tajo.algebra.Expr; import org.apache.tajo.algebra.JsonHelper; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; @@ -59,6 +58,7 @@ import org.apache.tajo.session.Session; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageProperty; import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.metrics.TajoMetrics; import org.apache.tajo.util.metrics.reporter.MetricsConsoleReporter; import org.apache.tajo.worker.AbstractResourceAllocator; @@ -324,7 +324,7 @@ public class QueryMasterTask extends CompositeService { String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { - sm = StorageManager.getStorageManager(systemConf, storeType); + sm = TableSpaceManager.getStorageManager(systemConf, storeType); StorageProperty storageProperty = sm.getStorageProperty(); if (storageProperty.isSortedInsert()) { String tableName = PlannerUtil.getStoreTableName(plan); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 7160a2d..b43b506 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -27,7 +27,6 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf.ConfVars; @@ -48,10 +47,7 @@ import org.apache.tajo.plan.logical.SortNode.SortPurpose; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.*; -import org.apache.tajo.storage.FileStorageManager; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.RowStoreUtil; -import org.apache.tajo.storage.TupleRange; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Pair; @@ -96,7 +92,7 @@ public class Repartitioner { TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName()); if (tableDesc == null) { // if it is a real table stored on storage FileStorageManager storageManager = - (FileStorageManager)StorageManager.getFileStorageManager(stage.getContext().getConf()); + (FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); tablePath = storageManager.getTablePath(scans[i].getTableName()); if (execBlock.getUnionScanMap() != null && !execBlock.getUnionScanMap().isEmpty()) { @@ -117,7 +113,7 @@ public class Repartitioner { } StorageManager storageManager = - StorageManager.getStorageManager(stage.getContext().getConf(), tableDesc.getMeta().getStoreType()); + TableSpaceManager.getStorageManager(stage.getContext().getConf(), tableDesc.getMeta().getStoreType()); // if table has no data, storageManager will return empty FileFragment. // So, we need to handle FileFragment by its size. @@ -412,7 +408,7 @@ public class Repartitioner { TableDesc tableDesc = masterContext.getTableDescMap().get(eachScan.getCanonicalName()); if (eachScan.getType() == NodeType.PARTITIONS_SCAN) { FileStorageManager storageManager = - (FileStorageManager)StorageManager.getFileStorageManager(stage.getContext().getConf()); + (FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)eachScan; partitionScanPaths = partitionScan.getInputPaths(); @@ -420,7 +416,7 @@ public class Repartitioner { getFragmentsFromPartitionedTable(storageManager, eachScan, tableDesc); partitionScan.setInputPaths(partitionScanPaths); } else { - StorageManager storageManager = StorageManager.getStorageManager(stage.getContext().getConf(), + StorageManager storageManager = TableSpaceManager.getStorageManager(stage.getContext().getConf(), tableDesc.getMeta().getStoreType()); Collection<Fragment> scanFragments = storageManager.getSplits(eachScan.getCanonicalName(), tableDesc, eachScan); @@ -540,11 +536,11 @@ public class Repartitioner { partitionScanPaths = partitionScan.getInputPaths(); // set null to inputPaths in getFragmentsFromPartitionedTable() FileStorageManager storageManager = - (FileStorageManager)StorageManager.getFileStorageManager(stage.getContext().getConf()); + (FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); scanFragments = getFragmentsFromPartitionedTable(storageManager, scan, desc); } else { StorageManager storageManager = - StorageManager.getStorageManager(stage.getContext().getConf(), desc.getMeta().getStoreType()); + TableSpaceManager.getStorageManager(stage.getContext().getConf(), desc.getMeta().getStoreType()); scanFragments = storageManager.getSplits(scan.getCanonicalName(), desc, scan); } @@ -649,7 +645,7 @@ public class Repartitioner { ExecutionBlock execBlock = stage.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = ((FileStorageManager)StorageManager.getFileStorageManager(stage.getContext().getConf())) + tablePath = ((FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf())) .getTablePath(scan.getTableName()); ExecutionBlock sampleChildBlock = masterPlan.getChild(stage.getId(), 0); @@ -678,7 +674,7 @@ public class Repartitioner { throw new IOException("Can't get table meta data from catalog: " + PlannerUtil.getStoreTableName(masterPlan.getLogicalPlan())); } - ranges = StorageManager.getStorageManager(stage.getContext().getConf(), storeType) + ranges = TableSpaceManager.getStorageManager(stage.getContext().getConf(), storeType) .getInsertSortRanges(stage.getContext().getQueryContext(), tableDesc, sortNode.getInSchema(), sortSpecs, mergedRange); @@ -815,7 +811,7 @@ public class Repartitioner { ExecutionBlock execBlock = stage.getBlock(); ScanNode scan = execBlock.getScanNodes()[0]; Path tablePath; - tablePath = ((FileStorageManager)StorageManager.getFileStorageManager(stage.getContext().getConf())) + tablePath = ((FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf())) .getTablePath(scan.getTableName()); Fragment frag = new FileFragment(scan.getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST}); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index f03ee2f..548ea26 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -34,7 +34,6 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; @@ -62,6 +61,7 @@ import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.querymaster.Task.IntermediateEntry; import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; @@ -1090,11 +1090,11 @@ public class Stage implements EventHandler<StageEvent> { if (scan.getType() == NodeType.PARTITIONS_SCAN) { // After calling this method, partition paths are removed from the physical plan. FileStorageManager storageManager = - (FileStorageManager)StorageManager.getFileStorageManager(stage.getContext().getConf()); + (FileStorageManager) TableSpaceManager.getFileStorageManager(stage.getContext().getConf()); fragments = Repartitioner.getFragmentsFromPartitionedTable(storageManager, scan, table); } else { StorageManager storageManager = - StorageManager.getStorageManager(stage.getContext().getConf(), meta.getStoreType()); + TableSpaceManager.getStorageManager(stage.getContext().getConf(), meta.getStoreType()); fragments = storageManager.getSplits(scan.getCanonicalName(), table, scan); } http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index 0cecd73..4f07ca6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -56,7 +56,7 @@ import org.apache.tajo.rule.EvaluationFailedException; import org.apache.tajo.rule.SelfDiagnosisRuleEngine; import org.apache.tajo.rule.SelfDiagnosisRuleSession; import org.apache.tajo.storage.HashShuffleAppenderManager; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.*; import org.apache.tajo.util.history.HistoryReader; import org.apache.tajo.util.history.HistoryWriter; @@ -370,7 +370,7 @@ public class TajoWorker extends CompositeService { } try { - StorageManager.close(); + TableSpaceManager.shutdown(); } catch (IOException ie) { LOG.error(ie.getMessage(), ie); } http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/main/java/org/apache/tajo/worker/Task.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 53ed73e..01f56b8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -52,7 +52,6 @@ import org.apache.tajo.plan.serder.LogicalNodeDeserializer; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.pullserver.TajoPullServerService; import org.apache.tajo.pullserver.retriever.FileChunk; -import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.NullCallback; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; @@ -162,7 +161,7 @@ public class Task { this.sortComp = new BaseTupleComparator(finalSchema, sortNode.getSortKeys()); } } else { - Path outFilePath = ((FileStorageManager)StorageManager.getFileStorageManager(systemConf)) + Path outFilePath = ((FileStorageManager) TableSpaceManager.getFileStorageManager(systemConf)) .getAppenderFilePath(taskId, queryContext.getStagingDir()); LOG.info("Output File Path: " + outFilePath); context.setOutputPath(outFilePath); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java index 0774eff..a1c6011 100644 --- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; @@ -47,7 +46,7 @@ public class BackendTestingUtil { public static void writeTmpTable(TajoConf conf, Path tablePath) throws IOException { - FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); + FileStorageManager sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); Appender appender; Path filePath = new Path(tablePath, "table.csv"); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index 9ff360f..44a22ae 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -27,7 +27,6 @@ import org.apache.tajo.QueryIdFactory; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; @@ -137,7 +136,7 @@ public class TestBroadcastJoinPlan { contentsData += j; } } - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(tableMeta, schema, dataPath); appender.init(); Tuple tuple = new VTuple(schema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index a50d813..104e09b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -23,7 +23,6 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -87,7 +86,7 @@ public class TestBNLJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -109,7 +108,7 @@ public class TestBNLJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index ac8860f..a8597e9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -26,7 +26,6 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -91,7 +90,7 @@ public class TestBSTIndexExec { Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); + sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); idxPath = new Path(workDir, "test.idx"); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 0b93c76..221a622 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -24,7 +24,6 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -83,7 +82,7 @@ public class TestExternalSortExec { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index e5ba9b3..501557f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -23,7 +23,6 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -105,7 +104,7 @@ public class TestFullOuterHashJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -135,7 +134,7 @@ public class TestFullOuterHashJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -175,7 +174,7 @@ public class TestFullOuterHashJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -228,7 +227,7 @@ public class TestFullOuterHashJoinExec { TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender5 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 1bbaa66..4b8e1fa 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -109,7 +109,7 @@ public class TestFullOuterMergeJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -148,7 +148,7 @@ public class TestFullOuterMergeJoinExec { TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV"); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender4 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); @@ -180,7 +180,7 @@ public class TestFullOuterMergeJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -220,7 +220,7 @@ public class TestFullOuterMergeJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -273,7 +273,7 @@ public class TestFullOuterMergeJoinExec { TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender5 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 30b9b19..624aae0 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -23,7 +23,6 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -84,7 +83,7 @@ public class TestHashAntiJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -110,7 +109,7 @@ public class TestHashAntiJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index cd42d80..36d02b1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -25,7 +25,6 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -86,7 +85,7 @@ public class TestHashJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -109,7 +108,7 @@ public class TestHashJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 7210214..1a0151a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -23,7 +23,6 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -85,7 +84,7 @@ public class TestHashSemiJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -111,7 +110,7 @@ public class TestHashSemiJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 9afc51f..127d309 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -23,7 +23,6 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -106,7 +105,7 @@ public class TestLeftOuterHashJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -136,7 +135,7 @@ public class TestLeftOuterHashJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -176,7 +175,7 @@ public class TestLeftOuterHashJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -229,7 +228,7 @@ public class TestLeftOuterHashJoinExec { TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender5 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index 6e7b5dc..c83d436 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -24,7 +24,6 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -86,7 +85,7 @@ public class TestMergeJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -115,7 +114,7 @@ public class TestMergeJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 96c28ee..ff9fdae 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -23,7 +23,6 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -84,7 +83,7 @@ public class TestNLJoinExec { TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -108,7 +107,7 @@ public class TestNLJoinExec { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); - appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 3e0b231..8c72d39 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.*; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -99,7 +98,7 @@ public class TestPhysicalPlanner { util.startCatalogCluster(); conf = util.getConfiguration(); testDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestPhysicalPlanner"); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); + sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); @@ -181,7 +180,7 @@ public class TestPhysicalPlanner { Schema scoreSchmea = score.getSchema(); TableMeta scoreLargeMeta = CatalogUtil.newTableMeta("RAW", new KeyValueSet()); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(scoreLargeMeta, scoreSchmea, scoreLargePath); appender.enableStats(); appender.init(); @@ -443,7 +442,7 @@ public class TestPhysicalPlanner { exec.next(); exec.close(); - Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Scanner scanner = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; @@ -503,7 +502,7 @@ public class TestPhysicalPlanner { // checking the file contents long totalNum = 0; for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) { - Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)).getFileScanner( + Scanner scanner = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)).getFileScanner( CatalogUtil.newTableMeta("CSV"), rootNode.getOutSchema(), status.getPath()); @@ -540,7 +539,7 @@ public class TestPhysicalPlanner { exec.next(); exec.close(); - Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)).getFileScanner( + Scanner scanner = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)).getFileScanner( outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index d56c3b0..94ebe51 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -25,7 +25,6 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; @@ -87,7 +86,7 @@ public class TestProgressExternalSortExec { TableMeta employeeMeta = CatalogUtil.newTableMeta("RAW"); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 097d75b..3455cb3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -23,7 +23,6 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -101,7 +100,7 @@ public class TestRightOuterHashJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -131,7 +130,7 @@ public class TestRightOuterHashJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -171,7 +170,7 @@ public class TestRightOuterHashJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 181f70e..c6bf2ef 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -23,7 +23,6 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -108,7 +107,7 @@ public class TestRightOuterMergeJoinExec { TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender1 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); @@ -147,7 +146,7 @@ public class TestRightOuterMergeJoinExec { TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV"); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender4 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); @@ -179,7 +178,7 @@ public class TestRightOuterMergeJoinExec { TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender2 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); @@ -219,7 +218,7 @@ public class TestRightOuterMergeJoinExec { TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender3 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -272,7 +271,7 @@ public class TestRightOuterMergeJoinExec { TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender5 = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 32746f8..a350831 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -25,7 +25,6 @@ import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.TpchTestBase; import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; @@ -70,7 +69,7 @@ public class TestSortExec { util = TpchTestBase.getInstance().getTestingCluster(); catalog = util.getMaster().getCatalog(); workDir = CommonTestingUtil.getTestDir(TEST_PATH); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); + sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -82,7 +81,7 @@ public class TestSortExec { tablePath = StorageUtil.concatPath(workDir, "employee", "table1"); sm.getFileSystem().mkdirs(tablePath.getParent()); - Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + Appender appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 2bc16c6..d2faf7e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -32,13 +32,13 @@ import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.TextDatum; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.hbase.*; import org.apache.tajo.util.Bytes; @@ -205,7 +205,7 @@ public class TestHBaseTable extends QueryTestCaseBase { assertTableExists("external_hbase_mapped_table"); - HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, "HBASE")) + HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager(conf, "HBASE")) .getConnection(testingCluster.getHBaseUtil().getConf()); HTableInterface htable = hconn.getTable("external_hbase_table"); @@ -244,7 +244,7 @@ public class TestHBaseTable extends QueryTestCaseBase { assertTableExists("external_hbase_mapped_table"); - HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, "HBASE")) + HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager(conf, "HBASE")) .getConnection(testingCluster.getHBaseUtil().getConf()); HTableInterface htable = hconn.getTable("external_hbase_table"); @@ -297,7 +297,7 @@ public class TestHBaseTable extends QueryTestCaseBase { assertTableExists("external_hbase_mapped_table"); - HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, "HBASE")) + HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager(conf, "HBASE")) .getConnection(testingCluster.getHBaseUtil().getConf()); HTableInterface htable = hconn.getTable("external_hbase_table"); @@ -334,7 +334,7 @@ public class TestHBaseTable extends QueryTestCaseBase { assertTableExists("external_hbase_mapped_table"); - HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, "HBASE")) + HConnection hconn = ((HBaseStorageManager) TableSpaceManager.getStorageManager(conf, "HBASE")) .getConnection(testingCluster.getHBaseUtil().getConf()); HTableInterface htable = hconn.getTable("external_hbase_table"); @@ -469,7 +469,7 @@ public class TestHBaseTable extends QueryTestCaseBase { EvalNode evalNodeEq = new BinaryEval(EvalType.EQUAL, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")), new ConstEval(new TextDatum("021"))); scanNode.setQual(evalNodeEq); - StorageManager storageManager = StorageManager.getStorageManager(conf, "HBASE"); + StorageManager storageManager = TableSpaceManager.getStorageManager(conf, "HBASE"); List<Fragment> fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode); assertEquals(1, fragments.size()); assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStartRow())); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java index a1eceea..8387abd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java @@ -569,7 +569,7 @@ public class TestJoinBroadcast extends QueryTestCaseBase { } Path dataPath = new Path(table.getPath().toString(), fileIndex + ".csv"); fileIndex++; - appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) + appender = ((FileStorageManager) TableSpaceManager.getFileStorageManager(conf)) .getAppender(tableMeta, schema, dataPath); appender.init(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index 1e4543c..70d07c3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -66,7 +66,7 @@ public class TestResultSet { public static void setup() throws Exception { util = TpchTestBase.getInstance().getTestingCluster(); conf = util.getConfiguration(); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); + sm = (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); scoreSchema = new Schema(); scoreSchema.addColumn("deptname", Type.TEXT); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 3d23d16..5efdede 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -23,7 +23,6 @@ import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.parser.SQLAnalyzer; @@ -35,6 +34,7 @@ import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.TableSpaceManager; import org.apache.tajo.util.CommonTestingUtil; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -82,7 +82,7 @@ public class TestExecutionBlockCursor { logicalPlanner = new LogicalPlanner(catalog); optimizer = new LogicalOptimizer(conf); - StorageManager sm = StorageManager.getFileStorageManager(conf); + StorageManager sm = TableSpaceManager.getFileStorageManager(conf); dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java index 362cb24..52b59ea 100644 --- a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java +++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java @@ -27,7 +27,6 @@ import org.apache.tajo.TpchTestBase; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.proto.CatalogProtos.TableProto; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; @@ -69,7 +68,7 @@ public class TestRowFile { TableMeta meta = CatalogUtil.newTableMeta("ROWFILE"); FileStorageManager sm = - (FileStorageManager)StorageManager.getFileStorageManager(conf); + (FileStorageManager) TableSpaceManager.getFileStorageManager(conf); Path tablePath = new Path("/test"); Path metaPath = new Path(tablePath, ".meta"); @@ -110,7 +109,7 @@ public class TestRowFile { int tupleCnt = 0; start = System.currentTimeMillis(); - Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = TableSpaceManager.getFileStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); while ((tuple=scanner.next()) != null) { tupleCnt++; http://git-wip-us.apache.org/repos/asf/tajo/blob/5491f0e7/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java index 5423fd7..d007aea 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java @@ -131,7 +131,7 @@ public class MergeScanner implements Scanner { private Scanner getNextScanner() throws IOException { if (iterator.hasNext()) { currentFragment = iterator.next(); - currentScanner = StorageManager.getStorageManager((TajoConf)conf, meta.getStoreType()).getScanner(meta, schema, + currentScanner = TableSpaceManager.getStorageManager((TajoConf) conf, meta.getStoreType()).getScanner(meta, schema, currentFragment, target); currentScanner.init(); return currentScanner;
