TAJO-1163: TableDesc should use URI instead of Path. Closes #235
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/f6e09a5a Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/f6e09a5a Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/f6e09a5a Branch: refs/heads/hbase_storage Commit: f6e09a5ae6f35dd6fe7f9ae895f0517f902b9b4d Parents: a56f8b5 Author: Hyunsik Choi <[email protected]> Authored: Tue Nov 25 15:32:13 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Tue Nov 25 15:32:13 2014 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/catalog/CatalogUtil.java | 2 +- .../java/org/apache/tajo/catalog/TableDesc.java | 21 ++++--- .../org/apache/tajo/catalog/TestTableDesc.java | 10 +-- .../tajo/catalog/store/HCatalogStore.java | 11 ++-- .../tajo/catalog/store/TestHCatalogStore.java | 22 +++---- .../org/apache/tajo/catalog/TestCatalog.java | 16 ++--- .../org/apache/tajo/jdbc/TajoResultSet.java | 4 +- .../engine/planner/global/GlobalPlanner.java | 2 +- .../planner/physical/PhysicalPlanUtil.java | 7 ++- .../org/apache/tajo/master/GlobalEngine.java | 10 +-- .../apache/tajo/master/querymaster/Query.java | 6 +- .../tajo/master/querymaster/Repartitioner.java | 6 +- .../tajo/master/querymaster/SubQuery.java | 2 +- .../apache/tajo/LocalTajoTestingUtility.java | 2 +- .../java/org/apache/tajo/QueryTestCaseBase.java | 4 +- .../apache/tajo/cli/tools/TestDDLBuilder.java | 8 +-- .../org/apache/tajo/client/TestTajoClient.java | 18 +++--- .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 2 +- .../engine/planner/TestLogicalOptimizer.java | 6 +- .../tajo/engine/planner/TestLogicalPlanner.java | 6 +- .../tajo/engine/planner/TestPlannerUtil.java | 8 +-- .../planner/physical/TestBNLJoinExec.java | 14 +++-- .../planner/physical/TestBSTIndexExec.java | 2 +- .../planner/physical/TestExternalSortExec.java | 6 +- .../physical/TestFullOuterHashJoinExec.java | 23 ++++--- .../physical/TestFullOuterMergeJoinExec.java | 36 ++++++----- .../planner/physical/TestHashAntiJoinExec.java | 8 +-- .../planner/physical/TestHashJoinExec.java | 14 +++-- .../planner/physical/TestHashSemiJoinExec.java | 18 +++--- .../physical/TestLeftOuterHashJoinExec.java | 38 ++++++------ .../physical/TestLeftOuterNLJoinExec.java | 20 +++--- .../planner/physical/TestMergeJoinExec.java | 6 +- .../engine/planner/physical/TestNLJoinExec.java | 16 ++--- .../planner/physical/TestPhysicalPlanner.java | 65 ++++++++++---------- .../physical/TestProgressExternalSortExec.java | 6 +- .../physical/TestRightOuterHashJoinExec.java | 23 ++++--- .../physical/TestRightOuterMergeJoinExec.java | 26 ++++---- .../engine/planner/physical/TestSortExec.java | 2 +- .../apache/tajo/engine/query/TestCTASQuery.java | 8 +-- .../tajo/engine/query/TestCreateTable.java | 9 +-- .../tajo/engine/query/TestInsertQuery.java | 6 +- .../tajo/engine/query/TestJoinBroadcast.java | 9 +-- .../tajo/engine/query/TestTablePartitions.java | 26 ++++---- .../org/apache/tajo/jdbc/TestResultSet.java | 2 +- .../tajo/worker/TestRangeRetrieverHandler.java | 4 +- .../org/apache/tajo/plan/LogicalPlanner.java | 9 +-- .../apache/tajo/plan/logical/InsertNode.java | 2 +- .../rewrite/rules/PartitionedTableRewriter.java | 4 +- 50 files changed, 309 insertions(+), 270 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 6b52026..06a0f47 100644 --- a/CHANGES +++ b/CHANGES @@ -60,6 +60,8 @@ Release 0.9.1 - unreleased TAJO-1172: Remove Trevni storage type and its related classes. (DaeMyung Kang via hyunsik) + TAJO-1163: TableDesc should use URI instead of Path. (hyunsik) + BUG FIXES http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 4171e80..737c9ae 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -292,7 +292,7 @@ public class CatalogUtil { } public static TableDesc newTableDesc(String tableName, Schema schema, TableMeta meta, Path path) { - return new TableDesc(tableName, schema, meta, path); + return new TableDesc(tableName, schema, meta, path.toUri()); } public static TableDesc newTableDesc(TableDescProto proto) { http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java index 64255d5..ce167e1 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java @@ -24,7 +24,6 @@ import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; @@ -35,14 +34,16 @@ import org.apache.tajo.json.GsonObject; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; +import java.net.URI; + public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Cloneable { private final Log LOG = LogFactory.getLog(TableDesc.class); @Expose protected String tableName; // required @Expose protected Schema schema; @Expose protected TableMeta meta; // required - /** uri is set if external flag is TRUE.*/ - @Expose protected Path uri; // optional + /** uri is set if external flag is TRUE. */ + @Expose protected URI uri; // optional @Expose protected TableStats stats; // optional /** the description of table partition */ @Expose protected PartitionMethodDesc partitionMethodDesc; // optional @@ -53,26 +54,26 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone } public TableDesc(String tableName, Schema schema, TableMeta meta, - Path path, boolean external) { + URI uri, boolean external) { this(); this.tableName = tableName; this.schema = schema; this.meta = meta; - this.uri = path; + this.uri = uri; this.external = external; } - public TableDesc(String tableName, Schema schema, TableMeta meta, Path path) { + public TableDesc(String tableName, Schema schema, TableMeta meta, URI path) { this(tableName, schema, meta, path, true); } - public TableDesc(String tableName, Schema schema, StoreType type, KeyValueSet options, Path path) { + public TableDesc(String tableName, Schema schema, StoreType type, KeyValueSet options, URI path) { this(tableName, schema, new TableMeta(type, options), path); } public TableDesc(TableDescProto proto) { this(proto.getTableName(), new Schema(proto.getSchema()), - new TableMeta(proto.getMeta()), proto.hasPath() ? new Path(proto.getPath()) : null, proto.getIsExternal()); + new TableMeta(proto.getMeta()), proto.hasPath() ? URI.create(proto.getPath()) : null, proto.getIsExternal()); if(proto.hasStats()) { this.stats = new TableStats(proto.getStats()); } @@ -89,11 +90,11 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone return this.tableName; } - public void setPath(Path uri) { + public void setPath(URI uri) { this.uri = uri; } - public Path getPath() { + public URI getPath() { return this.uri; } http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java index b7a9906..feeaadd 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java @@ -47,7 +47,7 @@ public class TestTableDesc { schema.addColumn("addr", Type.TEXT); info = CatalogUtil.newTableMeta(StoreType.CSV); path = new Path(CommonTestingUtil.getTestDir(), "table1"); - desc = new TableDesc("table1", schema, info, path); + desc = new TableDesc("table1", schema, info, path.toUri()); stats = new TableStats(); stats.setNumRows(957685); stats.setNumBytes(1023234); @@ -76,10 +76,10 @@ public class TestTableDesc { Path path = new Path(CommonTestingUtil.getTestDir(), "tajo"); - TableDesc desc = new TableDesc("table1", schema, info, path); + TableDesc desc = new TableDesc("table1", schema, info, path.toUri()); assertEquals("table1", desc.getName()); - assertEquals(path, desc.getPath()); + assertEquals(path.toUri(), desc.getPath()); assertEquals(info, desc.getMeta()); testClone(desc); } @@ -87,7 +87,7 @@ public class TestTableDesc { @Test public void testGetProto() throws CloneNotSupportedException, IOException { Path path = new Path(CommonTestingUtil.getTestDir(), "tajo"); - TableDesc desc = new TableDesc("table1", schema, info, path); + TableDesc desc = new TableDesc("table1", schema, info, path.toUri()); desc.setStats(stats); CatalogProtos.TableDescProto proto = desc.getProto(); @@ -98,7 +98,7 @@ public class TestTableDesc { @Test public void testToJson() throws CloneNotSupportedException, IOException { Path path = new Path(CommonTestingUtil.getTestDir(), "tajo"); - TableDesc desc = new TableDesc("table1", schema, info, path); + TableDesc desc = new TableDesc("table1", schema, info, path.toUri()); desc.setStats(stats); String json = desc.toJson(); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index c54a395..fa1cfd6 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -262,7 +262,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { if(client != null) client.release(); } TableMeta meta = new TableMeta(storeType, options); - TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path); + TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri()); if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) { tableDesc.setExternal(true); } @@ -442,12 +442,13 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { table.setTableType(TableType.EXTERNAL_TABLE.name()); table.putToParameters("EXTERNAL", "TRUE"); - FileSystem fs = tableDesc.getPath().getFileSystem(conf); - if (fs.isFile(tableDesc.getPath())) { + Path tablePath = new Path(tableDesc.getPath()); + FileSystem fs = tablePath.getFileSystem(conf); + if (fs.isFile(tablePath)) { LOG.warn("A table path is a file, but HCatalog does not allow a file path."); - sd.setLocation(tableDesc.getPath().getParent().toString()); + sd.setLocation(tablePath.getParent().toString()); } else { - sd.setLocation(tableDesc.getPath().toString()); + sd.setLocation(tablePath.toString()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java index 5f4ad45..725f665 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java @@ -97,7 +97,7 @@ public class TestHCatalogStore { schema.addColumn("c_comment", TajoDataTypes.Type.TEXT); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, - new Path(warehousePath, new Path(DB_NAME, CUSTOMER))); + new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, CUSTOMER)); @@ -126,7 +126,7 @@ public class TestHCatalogStore { schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, - new Path(warehousePath, new Path(DB_NAME, REGION))); + new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -155,7 +155,7 @@ public class TestHCatalogStore { schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, - new Path(warehousePath, new Path(DB_NAME, REGION))); + new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -188,7 +188,7 @@ public class TestHCatalogStore { schema.addColumn("s_comment", TajoDataTypes.Type.TEXT); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta, - new Path(warehousePath, new Path(DB_NAME, SUPPLIER))); + new Path(warehousePath, new Path(DB_NAME, SUPPLIER)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, SUPPLIER)); @@ -228,7 +228,7 @@ public class TestHCatalogStore { TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, NATION), schema, meta, - new Path(warehousePath, new Path(DB_NAME, NATION))); + new Path(warehousePath, new Path(DB_NAME, NATION)).toUri()); org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema(); expressionSchema.addColumn("n_nationkey", TajoDataTypes.Type.INT4); @@ -274,7 +274,7 @@ public class TestHCatalogStore { for(String tableName : tableNames){ TableDesc table = new TableDesc(CatalogUtil.buildFQName("default", tableName), schema, meta, - new Path(warehousePath, new Path(DB_NAME, tableName))); + new Path(warehousePath, new Path(DB_NAME, tableName)).toUri()); store.createTable(table.getProto()); } @@ -299,13 +299,13 @@ public class TestHCatalogStore { schema.addColumn("n_comment", TajoDataTypes.Type.TEXT); String tableName = "table1"; - TableDesc table = new TableDesc(DB_NAME + "." + tableName, schema, meta, warehousePath); + TableDesc table = new TableDesc(DB_NAME + "." + tableName, schema, meta, warehousePath.toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, tableName)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, tableName)); FileSystem fs = FileSystem.getLocal(new Configuration()); - assertTrue(fs.exists(table1.getPath())); + assertTrue(fs.exists(new Path(table1.getPath()))); store.dropTable(DB_NAME, tableName); assertFalse(store.existTable(DB_NAME, tableName)); @@ -324,7 +324,7 @@ public class TestHCatalogStore { schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, - new Path(warehousePath, new Path(DB_NAME, REGION))); + new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -353,7 +353,7 @@ public class TestHCatalogStore { schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, - new Path(warehousePath, new Path(DB_NAME, REGION))); + new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -385,7 +385,7 @@ public class TestHCatalogStore { schema.addColumn("c_comment", TajoDataTypes.Type.TEXT); TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, - new Path(warehousePath, new Path(DB_NAME, CUSTOMER))); + new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, CUSTOMER)); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 4a45abc..a044d64 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -246,7 +246,7 @@ public class TestCatalog { CatalogUtil.buildFQName(databaseName, tableName), schema1, new TableMeta(StoreType.CSV, new KeyValueSet()), - path, true); + path.toUri(), true); return table; } @@ -367,7 +367,7 @@ public class TestCatalog { schema1, StoreType.CSV, new KeyValueSet(), - path); + path.toUri()); assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); catalog.createTable(meta); @@ -407,7 +407,7 @@ public class TestCatalog { TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); return new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), schema, meta, - new Path(CommonTestingUtil.getTestDir(), "indexed")); + new Path(CommonTestingUtil.getTestDir(), "indexed").toUri()); } @Test @@ -587,7 +587,7 @@ public class TestCatalog { TableDesc desc = new TableDesc(tableName, schema, meta, - new Path(CommonTestingUtil.getTestDir(), "addedtable")); + new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri()); desc.setPartitionMethod(partitionDesc); assertFalse(catalog.existsTable(tableName)); @@ -625,7 +625,7 @@ public class TestCatalog { TableDesc desc = new TableDesc(tableName, schema, meta, - new Path(CommonTestingUtil.getTestDir(), "addedtable")); + new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri()); desc.setPartitionMethod(partitionDesc); assertFalse(catalog.existsTable(tableName)); @@ -663,7 +663,7 @@ public class TestCatalog { TableDesc desc = new TableDesc(tableName, schema, meta, - new Path(CommonTestingUtil.getTestDir(), "addedtable")); + new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri()); desc.setPartitionMethod(partitionDesc); assertFalse(catalog.existsTable(tableName)); catalog.createTable(desc); @@ -700,7 +700,7 @@ public class TestCatalog { TableDesc desc = new TableDesc(tableName, schema, meta, - new Path(CommonTestingUtil.getTestDir(), "addedtable")); + new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri()); desc.setPartitionMethod(partitionDesc); assertFalse(catalog.existsTable(tableName)); catalog.createTable(desc); @@ -738,7 +738,7 @@ public class TestCatalog { TableDesc desc = new TableDesc(tableName, schema, meta, - new Path(CommonTestingUtil.getTestDir(), "addedtable")); + new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri()); desc.setPartitionMethod(partitionDesc); assertFalse(catalog.existsTable(tableName)); catalog.createTable(desc); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java index d78b04f..e99c0ff 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java @@ -81,7 +81,7 @@ public class TajoResultSet extends TajoResultSetBase { private void initScanner() throws IOException { if(desc != null) { schema = desc.getSchema(); - fs = FileScanner.getFileSystem(conf, desc.getPath()); + fs = FileScanner.getFileSystem(conf, new Path(desc.getPath())); if (maxRowNum != null) { this.totalRow = maxRowNum; } else { @@ -92,7 +92,7 @@ public class TajoResultSet extends TajoResultSetBase { totalRow = INFINITE_ROW_NUM; } - List<FileFragment> frags = getFragments(desc.getPath()); + List<FileFragment> frags = getFragments(new Path(desc.getPath())); scanner = new MergeScanner(conf, desc.getSchema(), desc.getMeta(), frags); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index b33e4a0..c75b348 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -177,7 +177,7 @@ public class GlobalPlanner { "Channel schema (" + channel.getSrcId().getId() + " -> " + channel.getTargetId().getId() + ") is not initialized"); TableMeta meta = new TableMeta(channel.getStoreType(), new KeyValueSet()); - TableDesc desc = new TableDesc(channel.getSrcId().toString(), channel.getSchema(), meta, new Path("/")); + TableDesc desc = new TableDesc(channel.getSrcId().toString(), channel.getSchema(), meta, new Path("/").toUri()); ScanNode scanNode = plan.createNode(ScanNode.class); scanNode.init(desc); return scanNode; http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index da5e073..fe1f795 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -74,11 +74,12 @@ public class PhysicalPlanUtil { */ public static CatalogProtos.FragmentProto[] getNonZeroLengthDataFiles(TajoConf tajoConf,TableDesc tableDesc, int fileIndex, int numResultFiles) throws IOException { - FileSystem fs = tableDesc.getPath().getFileSystem(tajoConf); + Path path = new Path(tableDesc.getPath()); + FileSystem fs = path.getFileSystem(tajoConf); List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>(); - if (fs.exists(tableDesc.getPath())) { - getNonZeroLengthDataFiles(fs, tableDesc.getPath(), nonZeroLengthFiles, fileIndex, numResultFiles, + if (fs.exists(path)) { + getNonZeroLengthDataFiles(fs, path, nonZeroLengthFiles, fileIndex, numResultFiles, new AtomicInteger(0)); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 9831c00..47a8750 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -344,7 +344,7 @@ public class GlobalEngine extends AbstractService { Path finalOutputDir = null; if (insertNode.getTableName() != null) { tableDesc = this.catalog.getTableDesc(insertNode.getTableName()); - finalOutputDir = tableDesc.getPath(); + finalOutputDir = new Path(tableDesc.getPath()); } else { finalOutputDir = insertNode.getPath(); } @@ -655,7 +655,7 @@ public class GlobalEngine extends AbstractService { Path warehousePath = new Path(TajoConf.getWarehouseDir(context.getConf()), databaseName); TableDesc tableDesc = catalog.getTableDesc(databaseName, simpleTableName); - Path tablePath = tableDesc.getPath(); + Path tablePath = new Path(tableDesc.getPath()); if (tablePath.getParent() == null || !tablePath.getParent().toUri().getPath().equals(warehousePath.toUri().getPath())) { throw new IOException("Can't truncate external table:" + eachTableName + ", data dir=" + tablePath + @@ -665,7 +665,7 @@ public class GlobalEngine extends AbstractService { } for (TableDesc eachTable: tableDescList) { - Path path = eachTable.getPath(); + Path path = new Path(eachTable.getPath()); LOG.info("Truncate table: " + eachTable.getName() + ", delete all data files in " + path); FileSystem fs = path.getFileSystem(context.getConf()); @@ -768,7 +768,7 @@ public class GlobalEngine extends AbstractService { } TableDesc desc = new TableDesc(CatalogUtil.buildFQName(databaseName, simpleTableName), - schema, meta, path, isExternal); + schema, meta, path.toUri(), isExternal); desc.setStats(stats); if (partitionDesc != null) { desc.setPartitionMethod(partitionDesc); @@ -867,7 +867,7 @@ public class GlobalEngine extends AbstractService { } } - Path path = catalog.getTableDesc(qualifiedName).getPath(); + Path path = new Path(catalog.getTableDesc(qualifiedName).getPath()); catalog.dropTable(qualifiedName); if (purge) { http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index 7114d39..7db6d8b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -794,7 +794,7 @@ public class Query implements EventHandler<QueryEvent> { query.getId().toString(), lastStage.getSchema(), meta, - finalOutputDir); + finalOutputDir.toUri()); resultTableDesc.setExternal(true); stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir)); @@ -827,7 +827,7 @@ public class Query implements EventHandler<QueryEvent> { createTableNode.getTableName(), createTableNode.getTableSchema(), meta, - finalOutputDir); + finalOutputDir.toUri()); tableDescTobeCreated.setExternal(createTableNode.isExternal()); if (createTableNode.hasPartition()) { @@ -869,7 +869,7 @@ public class Query implements EventHandler<QueryEvent> { finalTable = catalog.getTableDesc(tableName); } else { String tableName = query.getId().toString(); - finalTable = new TableDesc(tableName, lastStage.getSchema(), meta, finalOutputDir); + finalTable = new TableDesc(tableName, lastStage.getSchema(), meta, finalOutputDir.toUri()); } long volume = getTableVolume(query.systemConf, finalOutputDir); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index 447c164..55b1895 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -107,7 +107,7 @@ public class Repartitioner { } fragments[i] = new FileFragment(scans[i].getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST}); } else { - tablePath = tableDesc.getPath(); + tablePath = new Path(tableDesc.getPath()); try { stats[i] = GlobalPlanner.computeDescendentVolume(scans[i]); } catch (PlanningException e) { @@ -411,7 +411,7 @@ public class Repartitioner { partitionScan.setInputPaths(partitionScanPaths); } else { Collection<FileFragment> scanFragments = subQuery.getStorageManager().getSplits(eachScan.getCanonicalName(), - tableDesc.getMeta(), tableDesc.getSchema(), tableDesc.getPath()); + tableDesc.getMeta(), tableDesc.getSchema(), new Path(tableDesc.getPath())); if (scanFragments != null) { rightFragments.addAll(scanFragments); } @@ -527,7 +527,7 @@ public class Repartitioner { scanFragments = getFragmentsFromPartitionedTable(subQuery.getStorageManager(), scan, desc); } else { scanFragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, desc.getSchema(), - desc.getPath()); + new Path(desc.getPath())); } if (scanFragments != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index 96534df..476da04 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -1059,7 +1059,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> { // After calling this method, partition paths are removed from the physical plan. fragments = Repartitioner.getFragmentsFromPartitionedTable(subQuery.getStorageManager(), scan, table); } else { - Path inputPath = table.getPath(); + Path inputPath = new Path(table.getPath()); fragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, table.getSchema(), inputPath); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java index c6cac32..f6a5fed 100644 --- a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java +++ b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java @@ -122,7 +122,7 @@ public class LocalTajoTestingUtility { stats.setNumBytes(TPCH.tableVolumes.get(names[i])); TableDesc tableDesc = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, names[i]), schemas[i], meta, - tablePath); + tablePath.toUri()); tableDesc.setStats(stats); util.getMaster().getCatalog().createTable(tableDesc); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 604a56b..4e4964e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -684,7 +684,7 @@ public class QueryTestCaseBase { return null; } - Path path = tableDesc.getPath(); + Path path = new Path(tableDesc.getPath()); return getTableFileContents(path); } @@ -694,7 +694,7 @@ public class QueryTestCaseBase { return null; } - Path path = tableDesc.getPath(); + Path path = new Path(tableDesc.getPath()); FileSystem fs = path.getFileSystem(conf); return listFiles(fs, path); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java index 83ce459..704980b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java @@ -58,7 +58,7 @@ public class TestDDLBuilder { @Test public void testBuildDDLForExternalTable() throws Exception { - TableDesc desc = new TableDesc("db1.table1", schema1, meta1, new Path("/table1")); + TableDesc desc = new TableDesc("db1.table1", schema1, meta1, new Path("/table1").toUri()); desc.setPartitionMethod(partitionMethod1); desc.setExternal(true); assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForExternalTable.result"), @@ -84,13 +84,13 @@ public class TestDDLBuilder { "key,key2", expressionSchema2); - TableDesc desc = new TableDesc("db1.TABLE2", schema2, meta1, new Path("/table1")); + TableDesc desc = new TableDesc("db1.TABLE2", schema2, meta1, new Path("/table1").toUri()); desc.setPartitionMethod(partitionMethod2); desc.setExternal(true); assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName1.result"), DDLBuilder.buildDDLForExternalTable(desc)); - desc = new TableDesc("db1.TABLE1", schema2, meta1, new Path("/table1")); + desc = new TableDesc("db1.TABLE1", schema2, meta1, new Path("/table1").toUri()); desc.setPartitionMethod(partitionMethod2); desc.setExternal(false); assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName2.result"), @@ -99,7 +99,7 @@ public class TestDDLBuilder { @Test public void testBuildDDLForBaseTable() throws Exception { - TableDesc desc = new TableDesc("db1.table2", schema1, meta1, new Path("/table1")); + TableDesc desc = new TableDesc("db1.table2", schema1, meta1, new Path("/table1").toUri()); assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForBaseTable.result"), DDLBuilder.buildDDLForBaseTable(desc)); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index 6e255ff..616799f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -324,7 +324,7 @@ public class TestTajoClient { client.updateQuery(sql); assertTrue(client.existTable(tableName)); - Path tablePath = client.getTableDesc(tableName).getPath(); + Path tablePath = new Path(client.getTableDesc(tableName).getPath()); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); @@ -345,7 +345,7 @@ public class TestTajoClient { client.updateQuery(sql); assertTrue(client.existTable(tableName)); - Path tablePath = client.getTableDesc(tableName).getPath(); + Path tablePath = new Path(client.getTableDesc(tableName).getPath()); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); @@ -422,7 +422,7 @@ public class TestTajoClient { client.updateQuery(sql); assertTrue(client.existTable(tableName)); - Path tablePath = client.getTableDesc(tableName).getPath(); + Path tablePath = new Path(client.getTableDesc(tableName).getPath()); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); @@ -446,7 +446,7 @@ public class TestTajoClient { client.updateQuery(sql); assertTrue(client.existTable(tableName)); - Path tablePath = client.getTableDesc(tableName).getPath(); + Path tablePath = new Path(client.getTableDesc(tableName).getPath()); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); @@ -470,7 +470,7 @@ public class TestTajoClient { client.updateQuery(sql); assertTrue(client.existTable(tableName)); - Path tablePath = client.getTableDesc(tableName).getPath(); + Path tablePath = new Path(client.getTableDesc(tableName).getPath()); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); @@ -495,7 +495,7 @@ public class TestTajoClient { client.updateQuery(sql); assertTrue(client.existTable(tableName)); - Path tablePath = client.getTableDesc(tableName).getPath(); + Path tablePath = new Path(client.getTableDesc(tableName).getPath()); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); @@ -521,7 +521,7 @@ public class TestTajoClient { client.updateQuery(sql); assertTrue(client.existTable(tableName)); - Path tablePath = client.getTableDesc(tableName).getPath(); + Path tablePath = new Path(client.getTableDesc(tableName).getPath()); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); @@ -574,7 +574,7 @@ public class TestTajoClient { client.updateQuery(sql); assertTrue(client.existTable(tableName)); - Path tablePath = client.getTableDesc(tableName).getPath(); + Path tablePath = new Path(client.getTableDesc(tableName).getPath()); FileSystem hdfs = tablePath.getFileSystem(conf); assertTrue(hdfs.exists(tablePath)); @@ -702,7 +702,7 @@ public class TestTajoClient { assertEquals(res.getTableDesc().getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T"); - Path path = res.getTableDesc().getPath(); + Path path = new Path(res.getTableDesc().getPath()); FileSystem fs = path.getFileSystem(tajoConf); FileStatus[] files = fs.listStatus(path); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 73d89ec..df9ef65 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -241,7 +241,7 @@ public class ExprTestBase { } } cat.createTable(new TableDesc(qualifiedTableName, inputSchema, - CatalogProtos.StoreType.CSV, new KeyValueSet(), CommonTestingUtil.getTestDir())); + CatalogProtos.StoreType.CSV, new KeyValueSet(), CommonTestingUtil.getTestDir().toUri())); } Target [] targets; http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index dcacff2..c8e3745 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -106,7 +106,7 @@ public class TestEvalTreeUtil { TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); TableDesc desc = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta, - CommonTestingUtil.getTestDir()); + CommonTestingUtil.getTestDir().toUri()); catalog.createTable(desc); FunctionDesc funcMeta = new FunctionDesc("test_sum", TestSum.class, http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java index 87bc757..c9b52fd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java @@ -82,19 +82,19 @@ public class TestLogicalOptimizer { TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); TableDesc people = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, - CommonTestingUtil.getTestDir()); + CommonTestingUtil.getTestDir().toUri()); catalog.createTable(people); TableDesc student = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, new KeyValueSet(), - CommonTestingUtil.getTestDir()); + CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, new KeyValueSet(), - CommonTestingUtil.getTestDir()); + CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.GENERAL, http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index e4e7916..180033d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -95,17 +95,17 @@ public class TestLogicalPlanner { TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); TableDesc people = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, - CommonTestingUtil.getTestDir()); + CommonTestingUtil.getTestDir().toUri()); catalog.createTable(people); TableDesc student = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, new KeyValueSet(), - CommonTestingUtil.getTestDir()); + CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, new KeyValueSet(), - CommonTestingUtil.getTestDir()); + CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.AGGREGATION, http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index 5cc89b8..3437e3a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -88,19 +88,19 @@ public class TestPlannerUtil { TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); TableDesc people = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, - CommonTestingUtil.getTestDir()); + CommonTestingUtil.getTestDir().toUri()); catalog.createTable(people); TableDesc student = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, - new KeyValueSet(), CommonTestingUtil.getTestDir()); + new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, - new KeyValueSet(), CommonTestingUtil.getTestDir()); + new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.AGGREGATION, @@ -338,7 +338,7 @@ public class TestPlannerUtil { TableDesc tableDesc = new TableDesc(); tableDesc.setName("Test"); - tableDesc.setPath(path); + tableDesc.setPath(path.toUri()); FileSystem fs = path.getFileSystem(util.getConfiguration()); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index df7f897..aef8064 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -150,9 +150,11 @@ public class TestBNLJoinExec { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN); - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), + FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), + FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), + new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLCrossJoin"); @@ -181,10 +183,10 @@ public class TestBNLJoinExec { LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context).getRootBlock().getRoot(); - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), - Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), - Integer.MAX_VALUE); + FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); + FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), + new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index e12d6eb..60ae849 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -146,7 +146,7 @@ public class TestBSTIndexExec { TableDesc desc = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, - sm.getTablePath("employee")); + sm.getTablePath("employee").toUri()); catalog.createTable(desc); analyzer = new SQLAnalyzer(); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index f09d104..bb40b28 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -103,7 +103,7 @@ public class TestExternalSortExec { System.out.println(appender.getStats().getNumRows() + " rows (" + (appender.getStats().getNumBytes() / 1048576) + " MB)"); - employee = new TableDesc("default.employee", schema, employeeMeta, employeePath); + employee = new TableDesc("default.employee", schema, employeeMeta, employeePath.toUri()); catalog.createTable(employee); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); @@ -121,8 +121,8 @@ public class TestExternalSortExec { @Test public final void testNext() throws IOException, PlanningException { - FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), - Integer.MAX_VALUE); + FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 104ca52..c4ce43b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -266,8 +266,10 @@ public class TestFullOuterHashJoinExec { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), + Integer.MAX_VALUE); + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec0"); @@ -303,8 +305,10 @@ public class TestFullOuterHashJoinExec { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), + Integer.MAX_VALUE); + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuter_HashJoinExec1"); @@ -339,8 +343,10 @@ public class TestFullOuterHashJoinExec { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + Integer.MAX_VALUE); + FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), + Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec2"); @@ -376,8 +382,9 @@ public class TestFullOuterHashJoinExec { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), + Integer.MAX_VALUE); + FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 23c0fb4..a82de92 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -30,16 +30,20 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.*; +import org.apache.tajo.engine.planner.PhysicalPlanner; +import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.plan.LogicalPlanner; -import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.*; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.TUtil; @@ -314,9 +318,9 @@ public class TestFullOuterMergeJoinExec { enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] dep3Frags = - StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin0"); @@ -351,9 +355,9 @@ public class TestFullOuterMergeJoinExec { enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] job3Frags = - StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin1"); @@ -388,9 +392,9 @@ public class TestFullOuterMergeJoinExec { enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] job3Frags = - StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin2"); @@ -426,9 +430,9 @@ public class TestFullOuterMergeJoinExec { enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] dep4Frags = - StorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), dep4.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin3"); @@ -466,9 +470,9 @@ public class TestFullOuterMergeJoinExec { enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] phone3Frags = - StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); @@ -505,9 +509,9 @@ public class TestFullOuterMergeJoinExec { enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); FileFragment[] emp3Frags = - StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] phone3Frags = - StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index e941767..20d4651 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -150,10 +150,10 @@ public class TestHashAntiJoinExec { @Test public final void testHashAntiJoin() throws IOException, PlanningException { - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), - Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), - Integer.MAX_VALUE); + FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); + FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), + new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index e25ca8f..d1fa28a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -152,8 +152,10 @@ public class TestHashJoinExec { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); + FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); + FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), + new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin"); @@ -193,10 +195,10 @@ public class TestHashJoinExec { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), - Integer.MAX_VALUE); - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), - Integer.MAX_VALUE); + FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), + new Path(people.getPath()), Integer.MAX_VALUE); + FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin"); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 4998012..7a43a55 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -29,15 +29,19 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.*; +import org.apache.tajo.engine.planner.PhysicalPlanner; +import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.plan.LogicalOptimizer; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.*; +import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.TUtil; @@ -154,10 +158,10 @@ public class TestHashSemiJoinExec { @Test public final void testHashSemiJoin() throws IOException, PlanningException { - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), - Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), - Integer.MAX_VALUE); + FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); + FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), + new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index f20984a..ec9daa7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -270,8 +270,10 @@ public class TestLeftOuterHashJoinExec { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); - FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE); + FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), + new Path(dep3.getPath()), Integer.MAX_VALUE); + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), + new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterHashJoinExec0"); @@ -298,10 +300,10 @@ public class TestLeftOuterHashJoinExec { @Test public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningException { - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), - Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), - Integer.MAX_VALUE); + FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), + new Path(job3.getPath()), Integer.MAX_VALUE); + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), + new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); @@ -339,10 +341,10 @@ public class TestLeftOuterHashJoinExec { @Test public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), - Integer.MAX_VALUE); - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), - Integer.MAX_VALUE); + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), + new Path(emp3.getPath()), Integer.MAX_VALUE); + FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), + new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); @@ -381,10 +383,10 @@ public class TestLeftOuterHashJoinExec { @Test public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), - Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), - Integer.MAX_VALUE); + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), + new Path(emp3.getPath()), Integer.MAX_VALUE); + FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), + new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); @@ -423,10 +425,10 @@ public class TestLeftOuterHashJoinExec { @Test public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, "default.emp3", emp3.getMeta(), emp3.getPath(), - Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, "default.phone3", phone3.getMeta(), phone3.getPath(), - Integer.MAX_VALUE); + FileFragment[] emp3Frags = StorageManager.splitNG(conf, "default.emp3", emp3.getMeta(), + new Path(emp3.getPath()), Integer.MAX_VALUE); + FileFragment[] phone3Frags = StorageManager.splitNG(conf, "default.phone3", phone3.getMeta(), + new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index 4b453fe..b3d1f33 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -254,9 +254,9 @@ public class TestLeftOuterNLJoinExec { @Test public final void testLeftOuterNLJoinExec0() throws IOException, PlanningException { - FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), + FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); @@ -295,9 +295,9 @@ public class TestLeftOuterNLJoinExec { @Test public final void testLeftOuterNLJoinExec1() throws IOException, PlanningException { - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), + FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); @@ -340,9 +340,9 @@ public class TestLeftOuterNLJoinExec { @Test public final void testLeftOuter_NLJoinExec2() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), + FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); @@ -385,9 +385,9 @@ public class TestLeftOuterNLJoinExec { @Test public final void testLeftOuter_NLJoinExec3() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); @@ -429,9 +429,9 @@ public class TestLeftOuterNLJoinExec { @Test public final void testLeftOuter_NLJoinExec4() throws IOException, PlanningException { - FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), + FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE); - FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(), + FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index 7a8426f..cae5de5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -165,8 +165,10 @@ public class TestMergeJoinExec { Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); - FileFragment[] empFrags = sm.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); - FileFragment[] peopleFrags = sm.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE); + FileFragment[] empFrags = sm.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()), + Integer.MAX_VALUE); + FileFragment[] peopleFrags = sm.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()), + Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testMergeInnerJoin"); http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index fc67a38..a5d1dc5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -144,10 +144,10 @@ public class TestNLJoinExec { @Test public final void testNLCrossJoin() throws IOException, PlanningException { - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), - Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), - Integer.MAX_VALUE); + FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); + FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), + new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); @@ -173,10 +173,10 @@ public class TestNLJoinExec { @Test public final void testNLInnerJoin() throws IOException, PlanningException { - FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), - Integer.MAX_VALUE); - FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), - Integer.MAX_VALUE); + FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), + new Path(employee.getPath()), Integer.MAX_VALUE); + FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), + new Path(people.getPath()), Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
