Repository: tajo Updated Branches: refs/heads/index_support 291817c83 -> 1c1dfa0ed
TAJO-1324: Remove warehouse directory rewriting in Unit Test. (jinho) Closes #370 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/9ad117f0 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/9ad117f0 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/9ad117f0 Branch: refs/heads/index_support Commit: 9ad117f07b30874685e58e28b76f8d0a2650b0f5 Parents: e9cd25a Author: jhkim <[email protected]> Authored: Tue Feb 3 15:14:33 2015 +0900 Committer: jhkim <[email protected]> Committed: Tue Feb 3 15:14:33 2015 +0900 ---------------------------------------------------------------------- .travis.yml | 2 +- CHANGES | 2 ++ .../java/org/apache/tajo/master/TajoMaster.java | 9 ++------- .../java/org/apache/tajo/BackendTestingUtil.java | 5 ++--- .../org/apache/tajo/LocalTajoTestingUtility.java | 2 +- .../java/org/apache/tajo/TajoTestingCluster.java | 12 ++++++++---- .../engine/planner/physical/TestBNLJoinExec.java | 2 -- .../engine/planner/physical/TestBSTIndexExec.java | 2 +- .../planner/physical/TestExternalSortExec.java | 2 -- .../physical/TestFullOuterHashJoinExec.java | 2 -- .../physical/TestFullOuterMergeJoinExec.java | 2 -- .../planner/physical/TestHashAntiJoinExec.java | 2 -- .../engine/planner/physical/TestHashJoinExec.java | 2 -- .../planner/physical/TestHashSemiJoinExec.java | 2 -- .../physical/TestLeftOuterHashJoinExec.java | 2 -- .../planner/physical/TestLeftOuterNLJoinExec.java | 2 -- .../planner/physical/TestMergeJoinExec.java | 11 ++++------- .../engine/planner/physical/TestNLJoinExec.java | 2 -- .../planner/physical/TestPhysicalPlanner.java | 2 +- .../physical/TestProgressExternalSortExec.java | 2 -- .../physical/TestRightOuterHashJoinExec.java | 2 -- .../physical/TestRightOuterMergeJoinExec.java | 2 -- .../engine/planner/physical/TestSortExec.java | 2 +- .../apache/tajo/engine/query/TestCreateTable.java | 2 +- .../java/org/apache/tajo/storage/TestRowFile.java | 3 +-- .../org/apache/tajo/storage/StorageManager.java | 18 +----------------- .../tajo/storage/HashShuffleAppenderManager.java | 4 ++-- .../tajo/storage/TestFileStorageManager.java | 4 ++-- .../org/apache/tajo/storage/TestMergeScanner.java | 2 +- 29 files changed, 31 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 0a6eec0..ab16f1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,4 +37,4 @@ notifications: install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -Pparallel-test,hcatalog-0.12.0 -DLOG_LEVEL=WARN -Dmaven.fork.count=3 + mvn clean install -Pparallel-test,hcatalog-0.12.0 -DLOG_LEVEL=WARN -Dmaven.fork.count=2 http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 6b568e3..47eaf9c 100644 --- a/CHANGES +++ b/CHANGES @@ -176,6 +176,8 @@ Release 0.10.0 - unreleased BUG FIXES + TAJO-1324: Remove warehouse directory rewriting in Unit Test. (jinho) + TAJO-1321: Cli prints wrong response time. (jihoon) TAJO-1313: Tajo-dump creates DDLs for information_schema tables. http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index a11606f..aee8a74 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -52,7 +52,6 @@ import org.apache.tajo.rule.SelfDiagnosisRuleSession; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.session.SessionManager; -import org.apache.tajo.storage.FileStorageManager; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.util.*; import org.apache.tajo.util.history.HistoryReader; @@ -110,7 +109,7 @@ public class TajoMaster extends CompositeService { private CatalogServer catalogServer; private CatalogService catalog; - private FileStorageManager storeManager; + private StorageManager storeManager; private GlobalEngine globalEngine; private AsyncDispatcher dispatcher; private TajoMasterClientService tajoMasterClientService; @@ -171,7 +170,7 @@ public class TajoMaster extends CompositeService { // check the system directory and create if they are not created. checkAndInitializeSystemDirectories(); diagnoseTajoMaster(); - this.storeManager = (FileStorageManager)StorageManager.getFileStorageManager(systemConf, null); + this.storeManager = StorageManager.getFileStorageManager(systemConf); catalogServer = new CatalogServer(FunctionLoader.load()); addIfService(catalogServer); @@ -410,10 +409,6 @@ public class TajoMaster extends CompositeService { return this.catalogServer; } - public FileStorageManager getStorageManager() { - return this.storeManager; - } - public class MasterContext { private final TajoConf conf; http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java index fb98be2..308ed28 100644 --- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java @@ -47,12 +47,11 @@ public class BackendTestingUtil { public static void writeTmpTable(TajoConf conf, Path tablePath) throws IOException { - FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, tablePath); - FileSystem fs = sm.getFileSystem(); - + FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); Appender appender; Path filePath = new Path(tablePath, "table.csv"); + FileSystem fs = filePath.getFileSystem(conf); if (fs.exists(tablePath)) { fs.delete(tablePath, true); } http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java index e464446..7b5e610 100644 --- a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java +++ b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java @@ -106,7 +106,7 @@ public class LocalTajoTestingUtility { client = new TajoClientImpl(conf); FileSystem fs = util.getDefaultFileSystem(); - Path rootDir = util.getMaster().getStorageManager().getWarehouseDir(); + Path rootDir = TajoConf.getWarehouseDir(conf); fs.mkdirs(rootDir); for (int i = 0; i < tablepaths.length; i++) { Path localPath = new Path(tablepaths[i]); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index a93d7f0..43bd52a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -158,6 +158,12 @@ public class TajoTestingCluster { conf.setIntVar(ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM, 2); conf.setIntVar(ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM, 2); + // Resource allocator + conf.setIntVar(ConfVars.YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM, 2); + + // Memory cache termination + conf.setIntVar(ConfVars.WORKER_HISTORY_EXPIRE_PERIOD, 1); + this.standbyWorkerMode = conf.getVar(ConfVars.RESOURCE_MANAGER_CLASS) .indexOf(TajoWorkerResourceManager.class.getName()) >= 0; @@ -635,8 +641,7 @@ public class TajoTestingCluster { TajoTestingCluster util = TpchTestBase.getInstance().getTestingCluster(); FileSystem fs = util.getDefaultFileSystem(); - Path rootDir = util.getMaster(). - getStorageManager().getWarehouseDir(); + Path rootDir = TajoConf.getWarehouseDir(util.getConfiguration()); fs.mkdirs(rootDir); for (int i = 0; i < names.length; i++) { createTable(names[i], schemas[i], tableOption, tables[i]); @@ -688,8 +693,7 @@ public class TajoTestingCluster { TajoClient client = new TajoClientImpl(conf); try { FileSystem fs = util.getDefaultFileSystem(); - Path rootDir = util.getMaster(). - getStorageManager().getWarehouseDir(); + Path rootDir = TajoConf.getWarehouseDir(util.getConfiguration()); if (!fs.exists(rootDir)) { fs.mkdirs(rootDir); } http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index 0bfcc22..842a120 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -62,7 +62,6 @@ public class TestBNLJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; private Path testDir; private static int OUTER_TUPLE_NUM = 1000; @@ -79,7 +78,6 @@ public class TestBNLJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index 2ddb4ef..b22a87e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -91,7 +91,7 @@ public class TestBSTIndexExec { Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, workDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); idxPath = new Path(workDir, "test.idx"); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 2047e41..5d9d46d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -59,7 +59,6 @@ public class TestExternalSortExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private FileStorageManager sm; private Path testDir; private final int numTuple = 100000; @@ -76,7 +75,6 @@ public class TestExternalSortExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 04117c1..780e698 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -61,7 +61,6 @@ public class TestFullOuterHashJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private FileStorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -84,7 +83,6 @@ public class TestFullOuterHashJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 5d2991b..ef6f1ce 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -63,7 +63,6 @@ public class TestFullOuterMergeJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -89,7 +88,6 @@ public class TestFullOuterMergeJoinExec { catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 6085f3d..64efa28 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -61,7 +61,6 @@ public class TestHashAntiJoinExec { private SQLAnalyzer analyzer; private LogicalPlanner planner; private LogicalOptimizer optimizer; - private StorageManager sm; private Path testDir; private TableDesc employee; @@ -76,7 +75,6 @@ public class TestHashAntiJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index 50d6ca1..578b586 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -62,7 +62,6 @@ public class TestHashJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -78,7 +77,6 @@ public class TestHashJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 978f635..42a018b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -62,7 +62,6 @@ public class TestHashSemiJoinExec { private SQLAnalyzer analyzer; private LogicalPlanner planner; private LogicalOptimizer optimizer; - private StorageManager sm; private Path testDir; private TableDesc employee; @@ -77,7 +76,6 @@ public class TestHashSemiJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 593c28b..5747e58 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -62,7 +62,6 @@ public class TestLeftOuterHashJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -85,7 +84,6 @@ public class TestLeftOuterHashJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index 72f4ab6..ab4b881 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -60,7 +60,6 @@ public class TestLeftOuterNLJoinExec { private SQLAnalyzer analyzer; private LogicalPlanner planner; private QueryContext defaultContext; - private StorageManager sm; private Path testDir; private TableDesc dep3; @@ -81,7 +80,6 @@ public class TestLeftOuterNLJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index 65f7799..1cc4e63 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -18,7 +18,6 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoConstants; @@ -31,16 +30,17 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.*; +import org.apache.tajo.engine.planner.PhysicalPlanner; +import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; -import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; -import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -64,7 +64,6 @@ public class TestMergeJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; private TableDesc employee; private TableDesc people; @@ -78,8 +77,6 @@ public class TestMergeJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - FileSystem fs = testDir.getFileSystem(conf); - sm = StorageManager.getFileStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index ffbf34a..240e7fd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -60,7 +60,6 @@ public class TestNLJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; private Path testDir; private TableDesc employee; @@ -76,7 +75,6 @@ public class TestNLJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 0c8fa36..e2216b9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -99,7 +99,7 @@ public class TestPhysicalPlanner { util.startCatalogCluster(); conf = util.getConfiguration(); testDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestPhysicalPlanner"); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index 29fff7c..161b0d8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -62,7 +62,6 @@ public class TestProgressExternalSortExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; private Path testDir; private final int numTuple = 100000; @@ -80,7 +79,6 @@ public class TestProgressExternalSortExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); - sm = StorageManager.getFileStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", TajoDataTypes.Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 7578690..06ab6c8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -59,7 +59,6 @@ public class TestRightOuterHashJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -80,7 +79,6 @@ public class TestRightOuterHashJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 8a812ea..30fd999 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -61,7 +61,6 @@ public class TestRightOuterMergeJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -87,7 +86,6 @@ public class TestRightOuterMergeJoinExec { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManager.getFileStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index a42fbac..f2c2b1a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -70,7 +70,7 @@ public class TestSortExec { util = TpchTestBase.getInstance().getTestingCluster(); catalog = util.getMaster().getCatalog(); workDir = CommonTestingUtil.getTestDir(TEST_PATH); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, workDir); + sm = (FileStorageManager)StorageManager.getFileStorageManager(conf); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index 18f55ff..21b3910 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -105,8 +105,8 @@ public class TestCreateTable extends QueryTestCaseBase { // checking the existence of the table directory and validating the path - FileSystem fs = testingCluster.getMaster().getStorageManager().getFileSystem(); Path warehouseDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration()); + FileSystem fs = warehouseDir.getFileSystem(testingCluster.getConfiguration()); assertTrue(fs.exists(new Path(oldTableDesc.getPath()))); assertEquals(StorageUtil.concatPath(warehouseDir, databaseName, originalTableName), new Path(oldTableDesc.getPath())); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java index f36ff24..859a1f2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java +++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java @@ -32,7 +32,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos.TableProto; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.storage.fragment.FileFragment; @@ -70,7 +69,7 @@ public class TestRowFile { TableMeta meta = CatalogUtil.newTableMeta(StoreType.ROWFILE); FileStorageManager sm = - (FileStorageManager)StorageManager.getFileStorageManager(conf, new Path(conf.getVar(ConfVars.ROOT_DIR))); + (FileStorageManager)StorageManager.getFileStorageManager(conf); Path tablePath = new Path("/test"); Path metaPath = new Path(tablePath, ".meta"); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java index d929591..ce963c8 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -271,23 +271,7 @@ public abstract class StorageManager { * @throws java.io.IOException */ public static StorageManager getFileStorageManager(TajoConf tajoConf) throws IOException { - return getFileStorageManager(tajoConf, null); - } - - /** - * Returns FileStorageManager instance and sets WAREHOUSE_DIR property in tajoConf with warehousePath parameter. - * - * @param tajoConf Tajo system property. - * @param warehousePath The warehouse directory to be set in the tajoConf. - * @return - * @throws java.io.IOException - */ - public static StorageManager getFileStorageManager(TajoConf tajoConf, Path warehousePath) throws IOException { - TajoConf copiedConf = new TajoConf(tajoConf); - if (warehousePath != null) { - copiedConf.setVar(ConfVars.WAREHOUSE_DIR, warehousePath.toUri().toString()); - } - return getStorageManager(copiedConf, StoreType.CSV); + return getStorageManager(tajoConf, StoreType.CSV); } /** http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index dededc6..466c6c7 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -84,8 +84,8 @@ public class HashShuffleAppenderManager { if (!fs.exists(dataFile.getParent())) { fs.mkdirs(dataFile.getParent()); } - FileAppender appender = (FileAppender)((FileStorageManager)StorageManager.getFileStorageManager( - tajoConf, null)).getAppender(meta, outSchema, dataFile); + FileAppender appender = (FileAppender)((FileStorageManager)StorageManager.getFileStorageManager(tajoConf)) + .getAppender(meta, outSchema, dataFile); appender.enableStats(); appender.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java index c4df8d7..4b39cdd 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java @@ -128,7 +128,7 @@ public class TestFileStorageManager { } assertTrue(fs.exists(tablePath)); - FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(tajoConf, tablePath); + FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(tajoConf); assertEquals(fs.getUri(), sm.getFileSystem().getUri()); Schema schema = new Schema(); @@ -182,7 +182,7 @@ public class TestFileStorageManager { DFSTestUtil.createFile(fs, tmpFile, 10, (short) 2, 0xDEADDEADl); } assertTrue(fs.exists(tablePath)); - FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(tajoConf, tablePath); + FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(tajoConf); assertEquals(fs.getUri(), sm.getFileSystem().getUri()); Schema schema = new Schema(); http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index a0daa7d..561e2ef 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@ -95,7 +95,7 @@ public class TestMergeScanner { conf.setStrings("tajo.storage.projectable-scanner", "rcfile", "parquet", "avro"); testDir = CommonTestingUtil.getTestDir(TEST_PATH); fs = testDir.getFileSystem(conf); - sm = StorageManager.getFileStorageManager(conf, testDir); + sm = StorageManager.getFileStorageManager(conf); } @Test
