initial work for TAJO-1591.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/863e53ea Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/863e53ea Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/863e53ea Branch: refs/heads/index_support Commit: 863e53eae091bb6073efb31a1efd8acf4ff88ea0 Parents: 9b3824b Author: Hyunsik Choi <[email protected]> Authored: Thu May 7 21:59:21 2015 -0700 Committer: Hyunsik Choi <[email protected]> Committed: Thu May 7 21:59:21 2015 -0700 ---------------------------------------------------------------------- .../org/apache/tajo/catalog/CatalogUtil.java | 24 ++++---- .../org/apache/tajo/catalog/DDLBuilder.java | 6 +- .../java/org/apache/tajo/catalog/TableDesc.java | 5 +- .../java/org/apache/tajo/catalog/TableMeta.java | 8 +-- .../tajo/catalog/json/TableMetaAdapter.java | 7 +-- .../src/main/proto/CatalogProtos.proto | 2 +- .../org/apache/tajo/catalog/TestTableDesc.java | 5 +- .../org/apache/tajo/catalog/TestTableMeta.java | 12 ++-- .../tajo/catalog/store/HiveCatalogStore.java | 15 +++-- .../catalog/store/TestHiveCatalogStore.java | 22 +++---- .../dictionary/AbstractTableDescriptor.java | 2 +- .../tajo/catalog/store/AbstractDBStore.java | 8 +-- .../org/apache/tajo/catalog/store/MemStore.java | 2 +- .../org/apache/tajo/catalog/TestCatalog.java | 24 ++++---- .../org/apache/tajo/cli/tools/TajoDump.java | 2 +- .../cli/tsql/commands/DescTableCommand.java | 3 +- .../java/org/apache/tajo/benchmark/TPCH.java | 2 +- .../tajo/engine/planner/global/DataChannel.java | 6 +- .../engine/planner/global/GlobalPlanner.java | 10 ++-- .../planner/physical/ExternalSortExec.java | 4 +- .../planner/physical/PhysicalPlanUtil.java | 54 ++++++++--------- .../physical/RangeShuffleFileWriteExec.java | 2 +- .../org/apache/tajo/master/GlobalEngine.java | 2 +- .../apache/tajo/master/exec/DDLExecutor.java | 3 +- .../NonForwardQueryResultSystemScanner.java | 2 +- .../apache/tajo/master/exec/QueryExecutor.java | 6 +- .../java/org/apache/tajo/querymaster/Query.java | 4 +- .../tajo/querymaster/QueryMasterTask.java | 2 +- .../apache/tajo/querymaster/Repartitioner.java | 4 +- .../java/org/apache/tajo/querymaster/Stage.java | 4 +- .../src/main/proto/TajoWorkerProtocol.proto | 2 +- .../org/apache/tajo/BackendTestingUtil.java | 2 +- .../apache/tajo/LocalTajoTestingUtility.java | 2 +- .../org/apache/tajo/TajoTestingCluster.java | 2 +- .../apache/tajo/cli/tools/TestDDLBuilder.java | 2 +- .../apache/tajo/engine/eval/ExprTestBase.java | 4 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 2 +- .../engine/planner/TestLogicalOptimizer.java | 6 +- .../tajo/engine/planner/TestLogicalPlanner.java | 10 ++-- .../tajo/engine/planner/TestPlannerUtil.java | 9 ++- .../planner/global/TestBroadcastJoinPlan.java | 2 +- .../planner/physical/TestBNLJoinExec.java | 4 +- .../planner/physical/TestBSTIndexExec.java | 2 +- .../planner/physical/TestExternalSortExec.java | 2 +- .../physical/TestFullOuterHashJoinExec.java | 8 +-- .../physical/TestFullOuterMergeJoinExec.java | 11 ++-- .../planner/physical/TestHashAntiJoinExec.java | 4 +- .../planner/physical/TestHashJoinExec.java | 4 +- .../planner/physical/TestHashSemiJoinExec.java | 4 +- .../physical/TestLeftOuterHashJoinExec.java | 8 +-- .../physical/TestLeftOuterNLJoinExec.java | 8 +-- .../planner/physical/TestMergeJoinExec.java | 4 +- .../engine/planner/physical/TestNLJoinExec.java | 4 +- .../planner/physical/TestPhysicalPlanner.java | 14 ++--- .../physical/TestProgressExternalSortExec.java | 2 +- .../physical/TestRightOuterHashJoinExec.java | 6 +- .../physical/TestRightOuterMergeJoinExec.java | 10 ++-- .../engine/planner/physical/TestSortExec.java | 2 +- .../apache/tajo/engine/query/TestCTASQuery.java | 6 +- .../tajo/engine/query/TestHBaseTable.java | 10 ++-- .../org/apache/tajo/jdbc/TestResultSet.java | 3 +- .../tajo/master/TestExecutionBlockCursor.java | 2 +- .../apache/tajo/master/TestGlobalPlanner.java | 2 +- .../org/apache/tajo/storage/TestRowFile.java | 2 +- .../ws/rs/resources/TestTablesResource.java | 31 ++++------ .../org/apache/tajo/plan/LogicalPlanner.java | 8 +-- .../function/python/PythonScriptEngine.java | 2 +- .../tajo/plan/logical/PersistentStoreNode.java | 8 +-- .../tajo/plan/logical/ShuffleFileWriteNode.java | 2 +- .../tajo/plan/logical/StoreTableNode.java | 2 +- .../org/apache/tajo/plan/util/PlannerUtil.java | 16 ++--- .../plan/verifier/PreLogicalPlanVerifier.java | 3 +- tajo-plan/src/main/proto/Plan.proto | 2 +- .../org/apache/tajo/plan/TestLogicalNode.java | 6 +- .../org/apache/tajo/storage/StorageManager.java | 47 +++++---------- .../storage/hbase/AddSortForInsertRewriter.java | 3 +- .../tajo/storage/hbase/HBasePutAppender.java | 3 +- .../apache/tajo/storage/hbase/HBaseScanner.java | 3 +- .../tajo/storage/hbase/HBaseStorageManager.java | 3 +- .../tajo/storage/hbase/TestColumnMapping.java | 3 +- .../storage/hbase/TestHBaseStorageManager.java | 2 +- .../apache/tajo/storage/FileStorageManager.java | 7 +-- .../tajo/storage/TestCompressionStorages.java | 22 +++---- .../tajo/storage/TestDelimitedTextFile.java | 8 +-- .../tajo/storage/TestFileStorageManager.java | 10 ++-- .../apache/tajo/storage/TestFileSystems.java | 2 +- .../org/apache/tajo/storage/TestLineReader.java | 8 +-- .../apache/tajo/storage/TestMergeScanner.java | 37 ++++++------ .../org/apache/tajo/storage/TestStorages.java | 62 ++++++++++---------- .../apache/tajo/storage/avro/TestAvroUtil.java | 7 +-- .../apache/tajo/storage/index/TestBSTIndex.java | 10 ++-- .../index/TestSingleCSVFileBSTIndex.java | 4 +- .../apache/tajo/storage/json/TestJsonSerDe.java | 2 +- 93 files changed, 342 insertions(+), 398 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 12f36b1..5c1b9b3 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -295,13 +295,13 @@ public class CatalogUtil { } } - public static TableMeta newTableMeta(StoreType type) { - KeyValueSet defaultProperties = CatalogUtil.newPhysicalProperties(type); - return new TableMeta(type, defaultProperties); + public static TableMeta newTableMeta(String storeType) { + KeyValueSet defaultProperties = CatalogUtil.newPhysicalProperties(storeType); + return new TableMeta(storeType, defaultProperties); } - public static TableMeta newTableMeta(StoreType type, KeyValueSet options) { - return new TableMeta(type, options); + public static TableMeta newTableMeta(String storeType, KeyValueSet options) { + return new TableMeta(storeType, options); } public static TableDesc newTableDesc(String tableName, Schema schema, TableMeta meta, Path path) { @@ -851,21 +851,21 @@ public class CatalogUtil { /** * Create new table property with default configs. It is used to make TableMeta to be stored in Catalog. * - * @param type StoreType + * @param storeType StoreType * @return Table properties */ - public static KeyValueSet newPhysicalProperties(StoreType type) { + public static KeyValueSet newPhysicalProperties(String storeType) { KeyValueSet options = new KeyValueSet(); - if (StoreType.CSV == type || StoreType.TEXTFILE == type) { + if (storeType.equalsIgnoreCase("CSV") || storeType.equalsIgnoreCase("TEXT")) { options.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - } else if (StoreType.JSON == type) { + } else if (storeType.equalsIgnoreCase("JSON")) { options.set(StorageConstants.TEXT_SERDE_CLASS, "org.apache.tajo.storage.json.JsonLineSerDe"); - } else if (StoreType.RCFILE == type) { + } else if (storeType.equalsIgnoreCase("RCFILE")) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); - } else if (StoreType.SEQUENCEFILE == type) { + } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - } else if (type == StoreType.PARQUET) { + } else if (storeType.equalsIgnoreCase("PARQUET")) { options.set(BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE); options.set(PAGE_SIZE, StorageConstants.PARQUET_DEFAULT_PAGE_SIZE); options.set(COMPRESSION, StorageConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index 1a59e88..65640b9 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -31,7 +31,7 @@ public class DDLBuilder { sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") - .append(" Storage: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType())); + .append(" Storage: ").append(desc.getMeta().getStoreType()); sb.append("\n-- Path: ").append(desc.getPath()); sb.append("\n--\n"); sb.append("CREATE EXTERNAL TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); @@ -54,7 +54,7 @@ public class DDLBuilder { sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") - .append(" Storage: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType())); + .append(" Storage: ").append(desc.getMeta().getStoreType()); sb.append("\n--\n"); sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); @@ -88,7 +88,7 @@ public class DDLBuilder { } private static void buildUsingClause(StringBuilder sb, TableMeta meta) { - sb.append(" USING " + CatalogUtil.getStoreTypeString(meta.getStoreType())); + sb.append(" USING " + meta.getStoreType()); } private static void buildWithClause(StringBuilder sb, TableMeta meta) { http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java index ee096a2..17f9146 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java @@ -27,7 +27,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.partition.PartitionMethodDesc; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.ProtoObject; @@ -68,8 +67,8 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone this(tableName, schema, meta, path, true); } - public TableDesc(String tableName, Schema schema, StoreType type, KeyValueSet options, URI path) { - this(tableName, schema, new TableMeta(type, options), path); + public TableDesc(String tableName, Schema schema, String storeType, KeyValueSet options, URI path) { + this(tableName, schema, new TableMeta(storeType, options), path); } public TableDesc(TableDescProto proto) { http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java index 2d95e6b..59cba44 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java @@ -41,16 +41,16 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj private TableProto proto = TableProto.getDefaultInstance(); private boolean viaProto = false; - @Expose protected StoreType storeType; + @Expose protected String storeType; @Expose protected KeyValueSet options; private TableMeta() { builder = TableProto.newBuilder(); } - public TableMeta(StoreType type, KeyValueSet options) { + public TableMeta(String storeType, KeyValueSet options) { this(); - this.storeType = type; + this.storeType = storeType; this.options = new KeyValueSet(options); } @@ -59,7 +59,7 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj viaProto = true; } - public StoreType getStoreType() { + public String getStoreType() { TableProtoOrBuilder p = viaProto ? proto : builder; if (this.storeType != null) { return storeType; http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java index 3f4304a..2d164f6 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java @@ -36,18 +36,17 @@ public class TableMetaAdapter implements GsonSerDerAdapter<TableMeta> { Preconditions.checkNotNull(json); JsonObject jsonObject = json.getAsJsonObject(); - CatalogProtos.StoreType type = CatalogProtos.StoreType.valueOf( - CommonGsonHelper.getOrDie(jsonObject, "store").getAsString()); + String storeType = CommonGsonHelper.getOrDie(jsonObject, "store").getAsString(); KeyValueSet keyValueSet = context.deserialize(CommonGsonHelper.getOrDie(jsonObject, "options"), KeyValueSet.class); - return new TableMeta(type, keyValueSet); + return new TableMeta(storeType, keyValueSet); } @Override public JsonElement serialize(TableMeta src, Type typeOfSrc, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); - jsonObj.addProperty("store", src.getStoreType().name()); + jsonObj.addProperty("store", src.getStoreType()); jsonObj.add("options", context.serialize(src.getOptions(), KeyValueSet.class)); return jsonObj; } http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index c467b4e..b213916 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -80,7 +80,7 @@ message FragmentProto { } message TableProto { - required StoreType storeType = 1; + required string storeType = 1; optional KeyValueSetProto params = 2; } http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java index feeaadd..9d84de6 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java @@ -21,7 +21,6 @@ package org.apache.tajo.catalog; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.Type; @@ -45,7 +44,7 @@ public class TestTableDesc { schema = new Schema(); schema.addColumn("name", Type.BLOB); schema.addColumn("addr", Type.TEXT); - info = CatalogUtil.newTableMeta(StoreType.CSV); + info = CatalogUtil.newTableMeta("CSV"); path = new Path(CommonTestingUtil.getTestDir(), "table1"); desc = new TableDesc("table1", schema, info, path.toUri()); stats = new TableStats(); @@ -71,7 +70,7 @@ public class TestTableDesc { Schema schema = new Schema(); schema.addColumn("name", Type.BLOB); schema.addColumn("addr", Type.TEXT); - TableMeta info = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta info = CatalogUtil.newTableMeta("CSV"); testClone(info); Path path = new Path(CommonTestingUtil.getTestDir(), "tajo"); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java index 904b4bc..e1c0158 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java @@ -32,7 +32,7 @@ public class TestTableMeta { @Before public void setUp() { - meta = CatalogUtil.newTableMeta(StoreType.CSV); + meta = CatalogUtil.newTableMeta("CSV"); } @Test @@ -40,7 +40,7 @@ public class TestTableMeta { Schema schema1 = new Schema(); schema1.addColumn("name", Type.BLOB); schema1.addColumn("addr", Type.TEXT); - TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta1 = CatalogUtil.newTableMeta("CSV"); TableMeta meta2 = new TableMeta(meta1.getProto()); assertEquals(meta1, meta2); @@ -51,7 +51,7 @@ public class TestTableMeta { Schema schema1 = new Schema(); schema1.addColumn("name", Type.BLOB); schema1.addColumn("addr", Type.TEXT); - TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta1 = CatalogUtil.newTableMeta("CSV"); TableMeta meta2 = (TableMeta) meta1.clone(); assertEquals(meta1.getStoreType(), meta2.getStoreType()); @@ -63,7 +63,7 @@ public class TestTableMeta { Schema schema1 = new Schema(); schema1.addColumn("name", Type.BLOB); schema1.addColumn("addr", Type.TEXT); - TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta1 = CatalogUtil.newTableMeta("CSV"); TableMeta meta2 = (TableMeta) meta1.clone(); @@ -72,7 +72,7 @@ public class TestTableMeta { @Test public void testGetStorageType() { - assertEquals(StoreType.CSV, meta.getStoreType()); + assertEquals("CSV", meta.getStoreType()); } @Test @@ -80,7 +80,7 @@ public class TestTableMeta { Schema schema2 = new Schema(); schema2.addColumn("name", Type.BLOB); schema2.addColumn("addr", Type.TEXT); - TableMeta meta2 = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta2 = CatalogUtil.newTableMeta("CSV"); assertTrue(meta.equals(meta2)); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 5b1a996..835a158 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -110,7 +110,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { org.apache.hadoop.hive.ql.metadata.Table table = null; HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; Path path = null; - CatalogProtos.StoreType storeType = null; + String storeType = null; org.apache.tajo.catalog.Schema schema = null; KeyValueSet options = null; TableStats stats = null; @@ -185,12 +185,12 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { // set file output format String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); - storeType = CatalogUtil.getStoreType(HiveCatalogUtil.getStoreType(fileOutputformat)); + storeType = HiveCatalogUtil.getStoreType(fileOutputformat); - if (storeType.equals(CatalogProtos.StoreType.TEXTFILE)) { + if (storeType.equalsIgnoreCase("TEXT")) { options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat)); - } else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) { + } else if (storeType.equals("RCFILE")) { options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) { @@ -198,7 +198,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { } else if (ColumnarSerDe.class.getName().equals(serde)) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } - } else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE) ) { + } else if (storeType.equals("SEQUENCEFILE") ) { options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); @@ -543,13 +543,12 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL); } } else { - if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.PARQUET)) { + if (tableDesc.getMeta().getStoreType().equals("PARQUET")) { sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName()); sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName()); sd.getSerdeInfo().setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName()); } else { - throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType - ().name())); + throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType())); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index 05e4e0a..6e91b89 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -86,7 +86,7 @@ public class TestHiveCatalogStore { @Test public void testTableUsingTextFile() throws Exception { - TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet()); + TableMeta meta = new TableMeta("CSV", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); @@ -120,7 +120,7 @@ public class TestHiveCatalogStore { public void testTableUsingRCFileWithBinarySerde() throws Exception { KeyValueSet options = new KeyValueSet(); options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); - TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options); + TableMeta meta = new TableMeta("RCFILE", options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); @@ -149,7 +149,7 @@ public class TestHiveCatalogStore { public void testTableUsingRCFileWithTextSerde() throws Exception { KeyValueSet options = new KeyValueSet(); options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); - TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options); + TableMeta meta = new TableMeta("RCFILE", options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); @@ -178,7 +178,7 @@ public class TestHiveCatalogStore { KeyValueSet options = new KeyValueSet(); options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava("\u0002")); options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava("\u0003")); - TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options); + TableMeta meta = new TableMeta("CSV", options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4); @@ -221,7 +221,7 @@ public class TestHiveCatalogStore { @Test public void testAddTableByPartition() throws Exception { - TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet()); + TableMeta meta = new TableMeta("CSV", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("n_name", TajoDataTypes.Type.TEXT); @@ -329,7 +329,7 @@ public class TestHiveCatalogStore { @Test public void testGetAllTableNames() throws Exception{ - TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet()); + TableMeta meta = new TableMeta("CSV", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("n_name", TajoDataTypes.Type.TEXT); schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); @@ -357,7 +357,7 @@ public class TestHiveCatalogStore { @Test public void testDeleteTable() throws Exception { - TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet()); + TableMeta meta = new TableMeta("CSV", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("n_name", TajoDataTypes.Type.TEXT); schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); @@ -381,7 +381,7 @@ public class TestHiveCatalogStore { public void testTableUsingSequenceFileWithBinarySerde() throws Exception { KeyValueSet options = new KeyValueSet(); options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); - TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options); + TableMeta meta = new TableMeta("SEQUENCEFILE", options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); @@ -410,7 +410,7 @@ public class TestHiveCatalogStore { public void testTableUsingSequenceFileWithTextSerde() throws Exception { KeyValueSet options = new KeyValueSet(); options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); - TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options); + TableMeta meta = new TableMeta("SEQUENCEFILE", options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); @@ -437,7 +437,7 @@ public class TestHiveCatalogStore { @Test public void testTableUsingParquet() throws Exception { - TableMeta meta = new TableMeta(CatalogProtos.StoreType.PARQUET, new KeyValueSet()); + TableMeta meta = new TableMeta("PARQUET", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); @@ -469,7 +469,7 @@ public class TestHiveCatalogStore { public void testDataTypeCompatibility() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testDataTypeCompatibility"); - TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet()); + TableMeta meta = new TableMeta("CSV", new KeyValueSet()); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("col1", TajoDataTypes.Type.INT4); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java index a8b384c..f2dc3cf 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java @@ -60,7 +60,7 @@ abstract class AbstractTableDescriptor implements TableDescriptor { protected TableProto getTableProto() { TableProto.Builder metaBuilder = TableProto.newBuilder(); - metaBuilder.setStoreType(StoreType.SYSTEM); + metaBuilder.setStoreType("SYSTEM"); metaBuilder.setParams(KeyValueSetProto.newBuilder().build()); return metaBuilder.build(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index b10e5a5..7fe6ef3 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -764,7 +764,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo pstmt.setString(2, tableName); pstmt.setString(3, TableType.EXTERNAL_TABLE.name()); pstmt.setString(4, table.getPath()); - pstmt.setString(5, CatalogUtil.getStoreTypeString(table.getMeta().getStoreType())); + pstmt.setString(5, table.getMeta().getStoreType()); pstmt.executeUpdate(); pstmt.close(); } else { @@ -778,7 +778,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo pstmt.setInt(1, dbid); pstmt.setString(2, tableName); pstmt.setString(3, TableType.BASE_TABLE.name()); - pstmt.setString(4, CatalogUtil.getStoreTypeString(table.getMeta().getStoreType())); + pstmt.setString(4, table.getMeta().getStoreType()); pstmt.executeUpdate(); pstmt.close(); } @@ -1567,7 +1567,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo PreparedStatement pstmt = null; CatalogProtos.TableDescProto.Builder tableBuilder = null; - StoreType storeType; + String storeType; try { tableBuilder = CatalogProtos.TableDescProto.newBuilder(); @@ -1609,7 +1609,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo tableBuilder.setPath(res.getString(4).trim()); } - storeType = CatalogUtil.getStoreType(res.getString(5).trim()); + storeType = res.getString(5).trim(); res.close(); pstmt.close(); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 821b00c..55c4336 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -429,7 +429,7 @@ public class MemStore implements CatalogStore { builder.setName(tableName); builder.setPath(tableDesc.getPath()); builder.setTableType(tableDesc.getIsExternal()?"EXTERNAL":"BASE"); - builder.setStoreType(CatalogUtil.getStoreTypeString(tableDesc.getMeta().getStoreType())); + builder.setStoreType(tableDesc.getMeta().getStoreType()); tableList.add(builder.build()); tableId++; http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 306f581..cbcec83 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -264,7 +264,7 @@ public class TestCatalog { TableDesc table = new TableDesc( CatalogUtil.buildFQName(databaseName, tableName), schema1, - new TableMeta(StoreType.CSV, new KeyValueSet()), + new TableMeta("CSV", new KeyValueSet()), path.toUri(), true); return table; } @@ -317,7 +317,7 @@ public class TestCatalog { TableDesc table = new TableDesc( CatalogUtil.buildFQName(databaseName, tableName), schema, - new TableMeta(StoreType.CSV, new KeyValueSet()), + new TableMeta("CSV", new KeyValueSet()), path.toUri(), true); assertTrue(catalog.createTable(table)); @@ -331,7 +331,7 @@ public class TestCatalog { table = new TableDesc( CatalogUtil.buildFQName(databaseName, tableName), schema, - new TableMeta(StoreType.CSV, new KeyValueSet()), + new TableMeta("CSV", new KeyValueSet()), path.toUri(), true); assertTrue(catalog.createTable(table)); @@ -421,7 +421,7 @@ public class TestCatalog { TableDesc meta = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"), schema1, - StoreType.CSV, + "CSV", new KeyValueSet(), path.toUri()); @@ -441,7 +441,7 @@ public class TestCatalog { TableDesc tableDesc = new TableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName), schema, - StoreType.CSV, + "CSV", new KeyValueSet(), path.toUri()); @@ -561,7 +561,7 @@ public class TestCatalog { String tableName = "indexed"; - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta = CatalogUtil.newTableMeta("CSV"); return new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), schema, meta, new Path(CommonTestingUtil.getTestDir(), "indexed").toUri()); @@ -732,7 +732,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); + TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); Schema partSchema = new Schema(); @@ -772,7 +772,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); + TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); Schema partSchema = new Schema(); partSchema.addColumn("id", Type.INT4); @@ -810,7 +810,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); + TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); Schema partSchema = new Schema(); partSchema.addColumn("id", Type.INT4); @@ -847,7 +847,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); + TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); Schema partSchema = new Schema(); partSchema.addColumn("id", Type.INT4); @@ -884,7 +884,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); + TableMeta meta = CatalogUtil.newTableMeta("CSV", opts); Schema partSchema = new Schema(); partSchema.addColumn("id", Type.INT4); @@ -999,7 +999,7 @@ public class TestCatalog { TableDesc setPropertyDesc = catalog.getTableDesc("default","mynewcooltable"); KeyValueSet options = new KeyValueSet(); options.set("timezone", "GMT+9"); // Seoul, Korea - setPropertyDesc.setMeta(new TableMeta(StoreType.CSV, options)); + setPropertyDesc.setMeta(new TableMeta("CSV", options)); String prevTimeZone = setPropertyDesc.getMeta().getOption("timezone"); String newTimeZone = "GMT-7"; // Silicon Valley, California catalog.alterTable(createMockAlterTableSetProperty(newTimeZone)); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index 497ccb6..7fefc5a 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -177,7 +177,7 @@ public class TajoDump { try { TableDesc table = client.getTableDesc(CatalogUtil.buildFQName(databaseName, tableName)); - if (table.getMeta().getStoreType() == StoreType.SYSTEM) { + if (table.getMeta().getStoreType().equalsIgnoreCase("SYSTEM")) { continue; } http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java index a3960e6..f888c36 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java @@ -21,7 +21,6 @@ package org.apache.tajo.cli.tsql.commands; import org.apache.commons.lang.CharUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -80,7 +79,7 @@ public class DescTableCommand extends TajoShellCommand { StringBuilder sb = new StringBuilder(); sb.append("\ntable name: ").append(desc.getName()).append("\n"); sb.append("table path: ").append(desc.getPath()).append("\n"); - sb.append("store type: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType())).append("\n"); + sb.append("store type: ").append(desc.getMeta().getStoreType()).append("\n"); if (desc.getStats() != null) { long row = desc.getStats().getNumRows(); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java index 609e49c..9994634 100644 --- a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java +++ b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java @@ -207,7 +207,7 @@ public class TPCH extends BenchmarkSet { } public void loadTable(String tableName) throws ServiceException { - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta = CatalogUtil.newTableMeta("CSV"); meta.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); PartitionMethodDesc partitionMethodDesc = null; http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java index 11548d3..3738dd3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java @@ -40,7 +40,7 @@ public class DataChannel { private Schema schema; - private StoreType storeType = StoreType.RAW; + private String storeType = "RAW"; public DataChannel(ExecutionBlockId srcId, ExecutionBlockId targetId) { this.srcId = srcId; @@ -144,11 +144,11 @@ public class DataChannel { return this.storeType != null; } - public void setStoreType(StoreType storeType) { + public void setStoreType(String storeType) { this.storeType = storeType; } - public StoreType getStoreType() { + public String getStoreType() { return storeType; } http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 36bdf21..e18b2eb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -69,7 +69,7 @@ public class GlobalPlanner { private static Log LOG = LogFactory.getLog(GlobalPlanner.class); private final TajoConf conf; - private final CatalogProtos.StoreType storeType; + private final String storeType; private final CatalogService catalog; private final GlobalPlanRewriteEngine rewriteEngine; @@ -77,7 +77,7 @@ public class GlobalPlanner { public GlobalPlanner(final TajoConf conf, final CatalogService catalog) throws IOException { this.conf = conf; this.catalog = catalog; - this.storeType = CatalogProtos.StoreType.valueOf(conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase()); + this.storeType = conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase(); Preconditions.checkArgument(storeType != null); Class<? extends GlobalPlanRewriteRuleProvider> clazz = @@ -99,7 +99,7 @@ public class GlobalPlanner { return catalog; } - public CatalogProtos.StoreType getStoreType() { + public String getStoreType() { return storeType; } @@ -175,7 +175,7 @@ public class GlobalPlanner { private static void setFinalOutputChannel(DataChannel outputChannel, Schema outputSchema) { outputChannel.setShuffleType(NONE_SHUFFLE); outputChannel.setShuffleOutputNum(1); - outputChannel.setStoreType(CatalogProtos.StoreType.CSV); + outputChannel.setStoreType("CSV"); outputChannel.setSchema(outputSchema); } @@ -1184,7 +1184,7 @@ public class GlobalPlanner { for (DataChannel dataChannel : masterPlan.getIncomingChannels(execBlock.getId())) { // This data channel will be stored in staging directory, but RawFile, default file type, does not support // distributed file system. It needs to change the file format for distributed file system. - dataChannel.setStoreType(CatalogProtos.StoreType.CSV); + dataChannel.setStoreType("CSV"); ExecutionBlock subBlock = masterPlan.getExecBlock(dataChannel.getSrcId()); ProjectionNode copy = PlannerUtil.clone(plan, node); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java index 3da296c..13b5a3a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java @@ -110,7 +110,7 @@ public class ExternalSortExec extends SortExec { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); this.plan = plan; - this.meta = CatalogUtil.newTableMeta(StoreType.ROWFILE); + this.meta = CatalogUtil.newTableMeta("ROWFILE"); this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT); if (defaultFanout < 2) { @@ -163,7 +163,7 @@ public class ExternalSortExec extends SortExec { */ private Path sortAndStoreChunk(int chunkId, List<Tuple> tupleBlock) throws IOException { - TableMeta meta = CatalogUtil.newTableMeta(StoreType.RAW); + TableMeta meta = CatalogUtil.newTableMeta("RAW"); int rowNum = tupleBlock.size(); long sortStart = System.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 247b373..f2bd75f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -29,12 +29,15 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.PhysicalPlanningException; -import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.PersistentStoreNode; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.*; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.storage.BaseTupleComparator; +import org.apache.tajo.storage.FileStorageManager; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -205,20 +208,15 @@ public class PhysicalPlanUtil { * @param nullChar A character for NULL representation */ private static void setNullCharForTextSerializer(TableMeta meta, String nullChar) { - switch (meta.getStoreType()) { - case CSV: - meta.putOption(StorageConstants.TEXT_NULL, nullChar); - break; - case TEXTFILE: - meta.putOption(StorageConstants.TEXT_NULL, nullChar); - break; - case RCFILE: - meta.putOption(StorageConstants.RCFILE_NULL, nullChar); - break; - case SEQUENCEFILE: - meta.putOption(StorageConstants.SEQUENCEFILE_NULL, nullChar); - break; - default: // nothing to do + String storeType = meta.getStoreType(); + if (storeType.equalsIgnoreCase("CSV")) { + meta.putOption(StorageConstants.TEXT_NULL, nullChar); + } else if (storeType.equalsIgnoreCase("TEXT")) { + meta.putOption(StorageConstants.TEXT_NULL, nullChar); + } else if (storeType.equalsIgnoreCase("RCFILE")) { + meta.putOption(StorageConstants.RCFILE_NULL, nullChar); + } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) { + meta.putOption(StorageConstants.SEQUENCEFILE_NULL, nullChar); } } @@ -229,17 +227,17 @@ public class PhysicalPlanUtil { * @return True if TableMeta contains NULL char property according to file format */ public static boolean containsNullChar(TableMeta meta) { - switch (meta.getStoreType()) { - case CSV: - return meta.containsOption(StorageConstants.TEXT_NULL); - case TEXTFILE: - return meta.containsOption(StorageConstants.TEXT_NULL); - case RCFILE: - return meta.containsOption(StorageConstants.RCFILE_NULL); - case SEQUENCEFILE: - return meta.containsOption(StorageConstants.SEQUENCEFILE_NULL); - default: // nothing to do - return false; + String storeType = meta.getStoreType(); + if (storeType.equalsIgnoreCase("CSV")) { + return meta.containsOption(StorageConstants.TEXT_NULL); + } else if (storeType.equalsIgnoreCase("TEXT")) { + return meta.containsOption(StorageConstants.TEXT_NULL); + } else if (storeType.equalsIgnoreCase("RCFILE")) { + return meta.containsOption(StorageConstants.RCFILE_NULL); + } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) { + return meta.containsOption(StorageConstants.SEQUENCEFILE_NULL); + } else { + return false; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java index 8da1a03..cb1fa05 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java @@ -75,7 +75,7 @@ public class RangeShuffleFileWriteExec extends UnaryPhysicalExec { Path storeTablePath = new Path(context.getWorkDir(), "output"); LOG.info("Output data directory: " + storeTablePath); this.meta = CatalogUtil.newTableMeta(context.getDataChannel() != null ? - context.getDataChannel().getStoreType() : CatalogProtos.StoreType.RAW); + context.getDataChannel().getStoreType() : "RAW"); FileSystem fs = new RawLocalFileSystem(); fs.mkdirs(storeTablePath); this.appender = (FileAppender) ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf())) http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 074f34e..adbc8a4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -293,7 +293,7 @@ public class GlobalEngine extends AbstractService { } private void verifyInsertTableSchema(QueryContext queryContext, VerificationState state, LogicalPlan plan) { - StoreType storeType = PlannerUtil.getStoreType(plan); + String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); if (rootNode.getChild().getType() == NodeType.INSERT) { http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 9264c68..c84b0c7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -29,7 +29,6 @@ import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.query.QueryContext; @@ -210,7 +209,7 @@ public class DDLExecutor { createTable.getPartitionMethod(), ifNotExists); } - public TableDesc createTable(QueryContext queryContext, String tableName, CatalogProtos.StoreType storeType, + public TableDesc createTable(QueryContext queryContext, String tableName, String storeType, Schema schema, TableMeta meta, Path path, boolean isExternal, PartitionMethodDesc partitionDesc, boolean ifNotExists) throws IOException { String databaseName; http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 958c252..307a155 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -120,7 +120,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult .createPlan(taskContext, leafBlock.getPlan()); tableDesc = new TableDesc("table_"+System.currentTimeMillis(), physicalExec.getSchema(), - new TableMeta(StoreType.SYSTEM, new KeyValueSet()), null); + new TableMeta("SYSTEM", new KeyValueSet()), null); outSchema = physicalExec.getSchema(); encoder = RowStoreUtil.createEncoder(getLogicalSchema()); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index ad1a8e3..481bdbe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -415,7 +415,7 @@ public class QueryExecutor { List<CatalogProtos.ColumnProto> columns = new ArrayList<CatalogProtos.ColumnProto>(); CatalogProtos.TableDescProto tableDescProto = CatalogProtos.TableDescProto.newBuilder() .setTableName(nodeUniqName) - .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType(CatalogProtos.StoreType.CSV).build()) + .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType("CSV").build()) .setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build()) .setStats(stats.getProto()) .build(); @@ -436,7 +436,7 @@ public class QueryExecutor { SubmitQueryResponse.Builder responseBuilder) throws Exception { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); - CatalogProtos.StoreType storeType = PlannerUtil.getStoreType(plan); + String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { StorageManager sm = StorageManager.getStorageManager(context.getConf(), storeType); StorageProperty storageProperty = sm.getStorageProperty(); @@ -474,7 +474,7 @@ public class QueryExecutor { public static MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, GlobalPlanner planner) throws Exception { - CatalogProtos.StoreType storeType = PlannerUtil.getStoreType(plan); + String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { StorageManager sm = StorageManager.getStorageManager(planner.getConf(), storeType); StorageProperty storageProperty = sm.getStorageProperty(); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 1ce15fc..a2e434b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -420,7 +420,7 @@ public class Query implements EventHandler<QueryEvent> { if (finalState != QueryState.QUERY_SUCCEEDED) { Stage lastStage = query.getStage(stageEvent.getExecutionBlockId()); if (lastStage != null && lastStage.getTableMeta() != null) { - StoreType storeType = lastStage.getTableMeta().getStoreType(); + String storeType = lastStage.getTableMeta().getStoreType(); if (storeType != null) { LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); try { @@ -439,7 +439,7 @@ public class Query implements EventHandler<QueryEvent> { private QueryState finalizeQuery(Query query, QueryCompletedEvent event) { Stage lastStage = query.getStage(event.getExecutionBlockId()); - StoreType storeType = lastStage.getTableMeta().getStoreType(); + String storeType = lastStage.getTableMeta().getStoreType(); try { LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot(); CatalogService catalog = lastStage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 465fa84..8d20141 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -322,7 +322,7 @@ public class QueryMasterTask extends CompositeService { jsonExpr = null; // remove the possible OOM plan = planner.createPlan(queryContext, expr); - StoreType storeType = PlannerUtil.getStoreType(plan); + String storeType = PlannerUtil.getStoreType(plan); if (storeType != null) { sm = StorageManager.getStorageManager(systemConf, storeType); StorageProperty storageProperty = sm.getStorageProperty(); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 8e9e343..986d16a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -232,7 +232,7 @@ public class Repartitioner { int maxStatsScanIdx = -1; StringBuilder nonLeafScanNamesBuilder = new StringBuilder(); for (int i = 0; i < scans.length; i++) { - if (scans[i].getTableDesc().getMeta().getStoreType() == StoreType.RAW) { + if (scans[i].getTableDesc().getMeta().getStoreType().equalsIgnoreCase("RAW")) { // Intermediate data scan hasNonLeafNode = true; largeScanIndexList.add(i); @@ -670,7 +670,7 @@ public class Repartitioner { TupleRange mergedRange = TupleUtil.columnStatToRange(sortSpecs, sortSchema, totalStat.getColumnStats(), false); if (sortNode.getSortPurpose() == SortPurpose.STORAGE_SPECIFIED) { - StoreType storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); + String storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); CatalogService catalog = stage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalRootNode rootNode = masterPlan.getLogicalPlan().getRootBlock().getRoot(); TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, rootNode.getChild()); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index d82d078..f03ee2f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -721,10 +721,10 @@ public class Stage implements EventHandler<StageEvent> { DataChannel channel = masterPlan.getOutgoingChannels(getId()).get(0); // if store plan (i.e., CREATE or INSERT OVERWRITE) - StoreType storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); + String storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); if (storeType == null) { // get default or store type - storeType = StoreType.CSV; + storeType = "CSV"; } schema = channel.getSchema(); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/proto/TajoWorkerProtocol.proto ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto index fddef8f..e660a30 100644 --- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto +++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto @@ -189,7 +189,7 @@ message DataChannelProto { repeated ColumnProto shuffleKeys = 7; optional int32 numOutputs = 9 [default = 1]; - optional StoreType storeType = 10 [default = CSV]; + optional string storeType = 10; } message RunExecutionBlockRequestProto { http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java index 308ed28..0774eff 100644 --- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java @@ -42,7 +42,7 @@ public class BackendTestingUtil { mockupSchema = new Schema(); mockupSchema.addColumn("deptname", Type.TEXT); mockupSchema.addColumn("score", Type.INT4); - mockupMeta = CatalogUtil.newTableMeta(StoreType.CSV); + mockupMeta = CatalogUtil.newTableMeta("CSV"); } public static void writeTmpTable(TajoConf conf, Path tablePath) http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java index 5407ff5..6691df1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java +++ b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java @@ -116,7 +116,7 @@ public class LocalTajoTestingUtility { fs.mkdirs(tablePath); Path dfsPath = new Path(tablePath, localPath.getName()); fs.copyFromLocalFile(localPath, dfsPath); - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.TEXTFILE, option); + TableMeta meta = CatalogUtil.newTableMeta("TEXT", option); // Add fake table statistic data to tables. // It gives more various situations to unit tests. http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 54f9e92..59f6da5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -669,7 +669,7 @@ public class TajoTestingCluster { out.close(); } } - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV, tableOption); + TableMeta meta = CatalogUtil.newTableMeta("CSV", tableOption); client.createExternalTable(tableName, schema, tablePath.toUri(), meta); } finally { client.close(); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java index 704980b..aac5f90 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java @@ -41,7 +41,7 @@ public class TestDDLBuilder { schema1.addColumn("name", TajoDataTypes.Type.BLOB); schema1.addColumn("addr", TajoDataTypes.Type.TEXT); - meta1 = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV); + meta1 = CatalogUtil.newTableMeta("CSV"); meta1.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); meta1.putOption(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName()); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 36ffd0c..af86abc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -264,8 +264,8 @@ public class ExprTestBase { vtuple.put(i, lazyTuple.get(i)); } } - cat.createTable(new TableDesc(qualifiedTableName, inputSchema, - CatalogProtos.StoreType.CSV, new KeyValueSet(), CommonTestingUtil.getTestDir().toUri())); + cat.createTable(new TableDesc(qualifiedTableName, inputSchema,"CSV", + new KeyValueSet(), CommonTestingUtil.getTestDir().toUri())); } Target [] targets; http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index 0466a24..dd5a75a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -103,7 +103,7 @@ public class TestEvalTreeUtil { schema.addColumn("score", TajoDataTypes.Type.INT4); schema.addColumn("age", TajoDataTypes.Type.INT4); - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta = CatalogUtil.newTableMeta("CSV"); TableDesc desc = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta, CommonTestingUtil.getTestDir().toUri()); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java index c9b52fd..a408fd6 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java @@ -79,7 +79,7 @@ public class TestLogicalOptimizer { schema3.addColumn("score", Type.INT4); schema3.addColumn("phone", Type.INT4); - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta = CatalogUtil.newTableMeta("CSV"); TableDesc people = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); @@ -87,13 +87,13 @@ public class TestLogicalOptimizer { TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, new KeyValueSet(), + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "CSV", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, new KeyValueSet(), + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "CSV", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index af0aa6a..d5d5779 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -92,19 +92,19 @@ public class TestLogicalPlanner { schema3.addColumn("deptname", Type.TEXT); schema3.addColumn("score", Type.INT4); - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta = CatalogUtil.newTableMeta("CSV"); TableDesc people = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); catalog.createTable(people); TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, new KeyValueSet(), + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "CSV", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, new KeyValueSet(), + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "CSV", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); @@ -121,7 +121,7 @@ public class TestLogicalPlanner { tpch.loadSchemas(); tpch.loadOutSchema(); for (String table : tpchTables) { - TableMeta m = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta m = CatalogUtil.newTableMeta("CSV"); TableDesc d = CatalogUtil.newTableDesc( CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir()); @@ -979,7 +979,7 @@ public class TestLogicalPlanner { assertEquals(Type.INT8, def.getColumn(2).getDataType().getType()); assertEquals("score", def.getColumn(3).getSimpleName()); assertEquals(Type.FLOAT4, def.getColumn(3).getDataType().getType()); - assertEquals(StoreType.CSV, createTable.getStorageType()); + assertTrue("CSV".equalsIgnoreCase(createTable.getStorageType())); assertEquals("/tmp/data", createTable.getPath().toString()); assertTrue(createTable.hasOptions()); assertEquals("|", createTable.getOptions().get("csv.delimiter")); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index 9ce7b5b..0082800 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -29,7 +29,6 @@ import org.apache.tajo.algebra.Expr; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.function.builtin.SumInt; @@ -40,10 +39,10 @@ import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; @@ -85,7 +84,7 @@ public class TestPlannerUtil { schema3.addColumn("deptname", Type.TEXT); schema3.addColumn("score", CatalogUtil.newSimpleDataType(Type.INT4)); - TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta meta = CatalogUtil.newTableMeta("CSV"); TableDesc people = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); @@ -93,13 +92,13 @@ public class TestPlannerUtil { TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "CSV", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, + CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "CSV", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index eb92bcf..9ff360f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -128,7 +128,7 @@ public class TestBroadcastJoinPlan { } private TableDesc makeTestData(String tableName, Schema schema, int dataSize) throws Exception { - TableMeta tableMeta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV); + TableMeta tableMeta = CatalogUtil.newTableMeta("CSV"); Path dataPath = new Path(testDir, tableName + ".csv"); String contentsData = ""; http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index 842a120..a50d813 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -85,7 +85,7 @@ public class TestBNLJoinExec { schema.addColumn("memid", Type.INT4); schema.addColumn("deptname", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); @@ -107,7 +107,7 @@ public class TestBNLJoinExec { peopleSchema.addColumn("fk_memid", Type.INT4); peopleSchema.addColumn("name", Type.TEXT); peopleSchema.addColumn("age", Type.INT4); - TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV"); Path peoplePath = new Path(testDir, "people.csv"); appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) .getAppender(peopleMeta, peopleSchema, peoplePath); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index b22a87e..ac8860f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -112,7 +112,7 @@ public class TestBSTIndexExec { writer.open(); long offset; - meta = CatalogUtil.newTableMeta(StoreType.CSV); + meta = CatalogUtil.newTableMeta("CSV"); tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv"); fs = tablePath.getFileSystem(conf); fs.mkdirs(tablePath.getParent()); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 0ef8294..0b93c76 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -81,7 +81,7 @@ public class TestExternalSortExec { schema.addColumn("empid", Type.INT4); schema.addColumn("deptname", Type.TEXT); - TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV"); Path employeePath = new Path(testDir, "employee.csv"); Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) .getAppender(employeeMeta, schema, employeePath); http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 780e698..e5ba9b3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -103,7 +103,7 @@ public class TestFullOuterHashJoinExec { dep3Schema.addColumn("loc_id", Type.INT4); - TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) .getAppender(dep3Meta, dep3Schema, dep3Path); @@ -133,7 +133,7 @@ public class TestFullOuterHashJoinExec { job3Schema.addColumn("job_title", Type.TEXT); - TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta job3Meta = CatalogUtil.newTableMeta("CSV"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) .getAppender(job3Meta, job3Schema, job3Path); @@ -173,7 +173,7 @@ public class TestFullOuterHashJoinExec { emp3Schema.addColumn("job_id", Type.INT4); - TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) .getAppender(emp3Meta, emp3Schema, emp3Path); @@ -226,7 +226,7 @@ public class TestFullOuterHashJoinExec { phone3Schema.addColumn("phone_number", Type.TEXT); - TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); + TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV"); Path phone3Path = new Path(testDir, "phone3.csv"); Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf)) .getAppender(phone3Meta, phone3Schema, phone3Path);
