TAJO-1663: Change the variable name storeType to dataFormat. Closes #790
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/b418c499 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/b418c499 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/b418c499 Branch: refs/heads/branch-0.11.0 Commit: b418c499170afe51bf38163876ce71a7aa639bdc Parents: 9b4c8ab Author: Hyunsik Choi <[email protected]> Authored: Mon Sep 28 14:38:45 2015 -0700 Committer: Hyunsik Choi <[email protected]> Committed: Mon Sep 28 14:41:59 2015 -0700 ---------------------------------------------------------------------- CHANGES | 2 + .../java/org/apache/tajo/algebra/Insert.java | 2 +- .../org/apache/tajo/catalog/CatalogUtil.java | 77 ++++++------ .../org/apache/tajo/catalog/DDLBuilder.java | 6 +- .../java/org/apache/tajo/catalog/TableDesc.java | 4 +- .../java/org/apache/tajo/catalog/TableMeta.java | 22 ++-- .../tajo/catalog/json/TableMetaAdapter.java | 7 +- .../src/main/proto/CatalogProtos.proto | 8 +- .../org/apache/tajo/catalog/TestTableMeta.java | 6 +- .../tajo/catalog/store/HiveCatalogStore.java | 22 ++-- .../tajo/catalog/store/HiveCatalogUtil.java | 10 +- .../dictionary/AbstractTableDescriptor.java | 2 +- .../InfoSchemaMetadataDictionary.java | 4 +- .../dictionary/TablesTableDescriptor.java | 2 +- .../tajo/catalog/store/AbstractDBStore.java | 23 ++-- .../src/main/resources/schemas/derby/derby.xml | 5 +- .../main/resources/schemas/mariadb/mariadb.xml | 5 +- .../src/main/resources/schemas/mysql/mysql.xml | 5 +- .../main/resources/schemas/oracle/oracle.xml | 5 +- .../resources/schemas/postgresql/postgresql.xml | 5 +- .../TestCatalogAgainstCaseSensitivity.java | 2 +- .../schemas/derbytest/loadtest/derby.xml | 2 +- .../org/apache/tajo/cli/tools/TajoDump.java | 3 +- .../cli/tsql/commands/DescTableCommand.java | 2 +- .../engine/planner/TestQueryValidation.java | 2 +- .../planner/physical/TestPhysicalPlanner.java | 4 +- .../apache/tajo/engine/query/TestCTASQuery.java | 10 +- .../tajo/engine/query/TestCreateTable.java | 4 +- .../TestCTASQuery/CtasWithDataFormat.sql | 12 ++ .../queries/TestCTASQuery/CtasWithStoreType.sql | 12 -- .../TestCTASQuery/testCtasWithDataFormat.sql | 1 + .../TestCTASQuery/testCtasWithStoreType.sql | 1 - .../engine/planner/PhysicalPlannerImpl.java | 2 +- .../tajo/engine/planner/global/DataChannel.java | 22 ++-- .../engine/planner/global/GlobalPlanner.java | 36 +++--- .../global/builder/DistinctGroupbyBuilder.java | 6 +- .../rewriter/rules/GlobalPlanRewriteUtil.java | 2 +- .../planner/physical/BSTIndexScanExec.java | 2 +- .../planner/physical/PhysicalPlanUtil.java | 16 +-- .../engine/planner/physical/StoreTableExec.java | 3 +- .../org/apache/tajo/master/GlobalEngine.java | 4 +- .../NonForwardQueryResultSystemScanner.java | 4 +- .../apache/tajo/master/exec/QueryExecutor.java | 2 +- .../apache/tajo/querymaster/Repartitioner.java | 6 +- .../java/org/apache/tajo/querymaster/Stage.java | 8 +- .../java/org/apache/tajo/querymaster/Task.java | 2 +- .../resources/webapps/admin/catalogview.jsp | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 13 +-- .../org/apache/tajo/plan/TablePropertyUtil.java | 4 +- .../tajo/plan/logical/CreateTableNode.java | 2 +- .../apache/tajo/plan/logical/InsertNode.java | 2 +- .../tajo/plan/logical/PersistentStoreNode.java | 2 +- .../plan/serder/LogicalNodeDeserializer.java | 4 +- .../org/apache/tajo/plan/util/PlannerUtil.java | 4 +- .../plan/verifier/PreLogicalPlanVerifier.java | 6 +- tajo-plan/src/main/proto/Plan.proto | 2 +- .../org/apache/tajo/plan/TestLogicalNode.java | 2 - .../apache/tajo/storage/OldStorageManager.java | 20 ++-- .../org/apache/tajo/storage/Tablespace.java | 16 +-- .../storage/fragment/FragmentConvertor.java | 14 +-- .../tajo/storage/hbase/HBaseFragment.java | 2 +- .../tajo/storage/fragment/FileFragment.java | 2 +- .../apache/tajo/storage/text/TextLineSerDe.java | 2 +- .../tajo/storage/TestCompressionStorages.java | 26 ++--- .../apache/tajo/storage/TestMergeScanner.java | 22 ++-- .../org/apache/tajo/storage/TestStorages.java | 94 +++++++-------- .../apache/tajo/storage/index/TestBSTIndex.java | 116 +++++++++---------- .../index/TestSingleCSVFileBSTIndex.java | 8 +- .../apache/tajo/storage/jdbc/JdbcFragment.java | 2 +- 69 files changed, 376 insertions(+), 383 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index fe0a525..3c52241 100644 --- a/CHANGES +++ b/CHANGES @@ -40,6 +40,8 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1663: Change the variable name storeType to dataFormat. (hyunsik) + TAJO-1885: Simple query with projection should be supported. (hyunsik) TAJO-1890: Clean up debug and test modes and unhandled exceptions. http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java index 8b9df97..4f76886 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java @@ -98,7 +98,7 @@ public class Insert extends Expr { this.storageType = storageType; } - public String getStorageType() { + public String getDataFormat() { return storageType; } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 665f9ec..12fb23d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -30,13 +30,12 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.catalog.proto.CatalogProtos.DataFormat; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableIdentifierProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; -import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UndefinedOperatorException; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; @@ -282,53 +281,53 @@ public class CatalogUtil { } - public static String getBackwardCompitablityStoreType(String storeType) { - return getStoreTypeString(getStoreType(storeType)); + public static String getBackwardCompitableDataFormat(String dataFormat) { + return getDataFormatAsString(asDataFormat(dataFormat)); } - public static String getStoreTypeString(final StoreType type) { - if (type == StoreType.TEXTFILE) { + public static String getDataFormatAsString(final DataFormat type) { + if (type == DataFormat.TEXTFILE) { return BuiltinStorages.TEXT; } else { return type.name(); } } - public static StoreType getStoreType(final String typeStr) { + public static DataFormat asDataFormat(final String typeStr) { if (typeStr.equalsIgnoreCase("CSV")) { - return StoreType.TEXTFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.RAW.name())) { - return StoreType.RAW; - } else if (typeStr.equalsIgnoreCase(StoreType.ROWFILE.name())) { - return StoreType.ROWFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.RCFILE.name())) { - return StoreType.RCFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.ORC.name())) { - return StoreType.ORC; - } else if (typeStr.equalsIgnoreCase(StoreType.PARQUET.name())) { - return StoreType.PARQUET; - } else if (typeStr.equalsIgnoreCase(StoreType.SEQUENCEFILE.name())) { - return StoreType.SEQUENCEFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.AVRO.name())) { - return StoreType.AVRO; + return DataFormat.TEXTFILE; + } else if (typeStr.equalsIgnoreCase(DataFormat.RAW.name())) { + return CatalogProtos.DataFormat.RAW; + } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.ROWFILE.name())) { + return DataFormat.ROWFILE; + } else if (typeStr.equalsIgnoreCase(DataFormat.RCFILE.name())) { + return DataFormat.RCFILE; + } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.ORC.name())) { + return CatalogProtos.DataFormat.ORC; + } else if (typeStr.equalsIgnoreCase(DataFormat.PARQUET.name())) { + return DataFormat.PARQUET; + } else if (typeStr.equalsIgnoreCase(DataFormat.SEQUENCEFILE.name())) { + return DataFormat.SEQUENCEFILE; + } else if (typeStr.equalsIgnoreCase(DataFormat.AVRO.name())) { + return CatalogProtos.DataFormat.AVRO; } else if (typeStr.equalsIgnoreCase(BuiltinStorages.TEXT)) { - return StoreType.TEXTFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.JSON.name())) { - return StoreType.JSON; - } else if (typeStr.equalsIgnoreCase(StoreType.HBASE.name())) { - return StoreType.HBASE; + return CatalogProtos.DataFormat.TEXTFILE; + } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.JSON.name())) { + return CatalogProtos.DataFormat.JSON; + } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.HBASE.name())) { + return CatalogProtos.DataFormat.HBASE; } else { return null; } } - public static TableMeta newTableMeta(String storeType) { - KeyValueSet defaultProperties = CatalogUtil.newDefaultProperty(storeType); - return new TableMeta(storeType, defaultProperties); + public static TableMeta newTableMeta(String dataFormat) { + KeyValueSet defaultProperties = CatalogUtil.newDefaultProperty(dataFormat); + return new TableMeta(dataFormat, defaultProperties); } - public static TableMeta newTableMeta(String storeType, KeyValueSet options) { - return new TableMeta(storeType, options); + public static TableMeta newTableMeta(String dataFormat, KeyValueSet options) { + return new TableMeta(dataFormat, options); } public static TableDesc newTableDesc(String tableName, Schema schema, TableMeta meta, Path path) { @@ -980,21 +979,21 @@ public class CatalogUtil { /** * Create new table property with default configs. It is used to make TableMeta to be stored in Catalog. * - * @param storeType StoreType + * @param dataFormat DataFormat * @return Table properties */ - public static KeyValueSet newDefaultProperty(String storeType) { + public static KeyValueSet newDefaultProperty(String dataFormat) { KeyValueSet options = new KeyValueSet(); - if (storeType.equalsIgnoreCase(BuiltinStorages.TEXT)) { + if (dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT)) { options.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - } else if (storeType.equalsIgnoreCase("JSON")) { + } else if (dataFormat.equalsIgnoreCase("JSON")) { options.set(StorageConstants.TEXT_SERDE_CLASS, "org.apache.tajo.storage.json.JsonLineSerDe"); - } else if (storeType.equalsIgnoreCase("RCFILE")) { + } else if (dataFormat.equalsIgnoreCase("RCFILE")) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); - } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) { + } else if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - } else if (storeType.equalsIgnoreCase("PARQUET")) { + } else if (dataFormat.equalsIgnoreCase("PARQUET")) { options.set(BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE); options.set(PAGE_SIZE, StorageConstants.PARQUET_DEFAULT_PAGE_SIZE); options.set(COMPRESSION, StorageConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index 7aecc01..7159aae 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -33,7 +33,7 @@ public class DDLBuilder { sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") - .append(" Storage: ").append(desc.getMeta().getStoreType()); + .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n-- Path: ").append(desc.getUri()); sb.append("\n--\n"); sb.append("CREATE EXTERNAL TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); @@ -56,7 +56,7 @@ public class DDLBuilder { sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") - .append(" Storage: ").append(desc.getMeta().getStoreType()); + .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n--\n"); sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); @@ -112,7 +112,7 @@ public class DDLBuilder { } private static void buildUsingClause(StringBuilder sb, TableMeta meta) { - sb.append(" USING " + CatalogUtil.getBackwardCompitablityStoreType(meta.getStoreType())); + sb.append(" USING " + CatalogUtil.getBackwardCompitableDataFormat(meta.getDataFormat())); } private static void buildWithClause(StringBuilder sb, TableMeta meta) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java index ccf084e..3cdc00b 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java @@ -64,9 +64,9 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone this(tableName, schema, meta, path, true); } - public TableDesc(String tableName, @Nullable Schema schema, String storeType, KeyValueSet options, + public TableDesc(String tableName, @Nullable Schema schema, String dataFormat, KeyValueSet options, @Nullable URI path) { - this(tableName, schema, new TableMeta(storeType, options), path); + this(tableName, schema, new TableMeta(dataFormat, options), path); } public TableDesc(TableDescProto proto) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java index 6838fe0..cfbbf75 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java @@ -35,21 +35,21 @@ import java.util.Map; * It contains all information for scanning a fragmented table */ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObject, Cloneable { - @Expose protected String storeType; + @Expose protected String dataFormat; @Expose protected KeyValueSet options; - public TableMeta(String storeType, KeyValueSet options) { - this.storeType = storeType; + public TableMeta(String dataFormat, KeyValueSet options) { + this.dataFormat = dataFormat; this.options = new KeyValueSet(options); } public TableMeta(TableProto proto) { - this.storeType = proto.getStoreType(); + this.dataFormat = proto.getDataFormat(); this.options = new KeyValueSet(proto.getParams()); } - public String getStoreType() { - return this.storeType; + public String getDataFormat() { + return this.dataFormat; } public void setOptions(KeyValueSet options) { @@ -84,7 +84,7 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj if(object instanceof TableMeta) { TableMeta other = (TableMeta) object; - boolean eq = this.getStoreType().equals(other.getStoreType()); + boolean eq = this.getDataFormat().equals(other.getDataFormat()); eq = eq && this.getOptions().equals(other.getOptions()); return eq; } @@ -93,13 +93,13 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj } public int hashCode() { - return Objects.hashCode(getStoreType(), getOptions()); + return Objects.hashCode(getDataFormat(), getOptions()); } @Override public Object clone() throws CloneNotSupportedException { TableMeta meta = (TableMeta) super.clone(); - meta.storeType = getStoreType(); + meta.dataFormat = getDataFormat(); meta.options = (KeyValueSet) (toMap() != null ? options.clone() : null); return meta; } @@ -115,7 +115,7 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj //////////////////////////////////////////////////////////////////////// public TableProto getProto() { TableProto.Builder builder = TableProto.newBuilder(); - builder.setStoreType(storeType); + builder.setDataFormat(dataFormat); builder.setParams(options.getProto()); return builder.build(); } @@ -127,7 +127,7 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj } public void mergeProtoToLocal() { - getStoreType(); + getDataFormat(); toMap(); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java index 2d164f6..766a8a9 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java @@ -23,7 +23,6 @@ import com.google.gson.*; import org.apache.tajo.json.CommonGsonHelper; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.json.GsonSerDerAdapter; import java.lang.reflect.Type; @@ -36,17 +35,17 @@ public class TableMetaAdapter implements GsonSerDerAdapter<TableMeta> { Preconditions.checkNotNull(json); JsonObject jsonObject = json.getAsJsonObject(); - String storeType = CommonGsonHelper.getOrDie(jsonObject, "store").getAsString(); + String dataFormat = CommonGsonHelper.getOrDie(jsonObject, "store").getAsString(); KeyValueSet keyValueSet = context.deserialize(CommonGsonHelper.getOrDie(jsonObject, "options"), KeyValueSet.class); - return new TableMeta(storeType, keyValueSet); + return new TableMeta(dataFormat, keyValueSet); } @Override public JsonElement serialize(TableMeta src, Type typeOfSrc, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); - jsonObj.addProperty("store", src.getStoreType()); + jsonObj.addProperty("store", src.getDataFormat()); jsonObj.add("options", context.serialize(src.getOptions(), KeyValueSet.class)); return jsonObj; } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 08818b1..9a7b66a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -25,7 +25,7 @@ option java_generate_equals_and_hash = true; import "DataTypes.proto"; import "PrimitiveProtos.proto"; -enum StoreType { +enum DataFormat { MEM = 0; TEXTFILE = 1; RAW = 2; @@ -75,12 +75,12 @@ message SchemaProto { message FragmentProto { required string id = 1; - required string storeType = 2; + required string dataFormat = 2; required bytes contents = 3; } message TableProto { - required string storeType = 1; + required string dataFormat = 1; optional KeyValueSetProto params = 2; } @@ -293,7 +293,7 @@ message TableDescriptorProto { required string name = 3; optional string tableType = 4; required string path = 5; - required string storeType = 6; + required string dataFormat = 6; } message AlterTablespaceProto { http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java index 9087262..3f1d5c5 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java @@ -55,7 +55,7 @@ public class TestTableMeta { TableMeta meta1 = CatalogUtil.newTableMeta("TEXT"); TableMeta meta2 = (TableMeta) meta1.clone(); - assertEquals(meta1.getStoreType(), meta2.getStoreType()); + assertEquals(meta1.getDataFormat(), meta2.getDataFormat()); assertEquals(meta1, meta2); } @@ -73,7 +73,7 @@ public class TestTableMeta { @Test public void testGetStorageType() { - assertEquals("TEXT", meta.getStoreType()); + assertEquals("TEXT", meta.getDataFormat()); } @Test @@ -108,7 +108,7 @@ public class TestTableMeta { optionBuilder.addKeyval(keyValueBuilder); } TableProto.Builder builder = TableProto.newBuilder(); - builder.setStoreType(BuiltinStorages.TEXT); + builder.setDataFormat(BuiltinStorages.TEXT); builder.setParams(optionBuilder); TableMeta meta2 = new TableMeta(builder.build()); assertTrue(meta1.equals(meta2)); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 87cc93f..e377502 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -111,7 +111,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { org.apache.hadoop.hive.ql.metadata.Table table = null; HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; Path path = null; - String storeType = null; + String dataFormat = null; org.apache.tajo.catalog.Schema schema = null; KeyValueSet options = null; TableStats stats = null; @@ -186,12 +186,12 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { // set file output format String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); - storeType = HiveCatalogUtil.getStoreType(fileOutputformat); + dataFormat = HiveCatalogUtil.getDataFormat(fileOutputformat); - if (storeType.equalsIgnoreCase("TEXT")) { + if (dataFormat.equalsIgnoreCase("TEXT")) { options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat)); - } else if (storeType.equals("RCFILE")) { + } else if (dataFormat.equals("RCFILE")) { options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) { @@ -199,7 +199,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { } else if (ColumnarSerDe.class.getName().equals(serde)) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } - } else if (storeType.equals("SEQUENCEFILE")) { + } else if (dataFormat.equals("SEQUENCEFILE")) { options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); @@ -258,7 +258,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { } finally { if(client != null) client.release(); } - TableMeta meta = new TableMeta(storeType, options); + TableMeta meta = new TableMeta(dataFormat, options); TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri()); if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) { tableDesc.setExternal(true); @@ -479,7 +479,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { table.setPartitionKeys(partitionKeys); } - if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.RCFILE)) { + if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.RCFILE)) { String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE); sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName()); @@ -494,7 +494,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); } - } else if (tableDesc.getMeta().getStoreType().equals(BuiltinStorages.TEXT)) { + } else if (tableDesc.getMeta().getDataFormat().equals(BuiltinStorages.TEXT)) { sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName()); @@ -518,7 +518,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL))); table.getParameters().remove(StorageConstants.TEXT_NULL); } - } else if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { + } else if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE); sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName()); @@ -549,12 +549,12 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore { table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL); } } else { - if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.PARQUET)) { + if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.PARQUET)) { sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName()); sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName()); sd.getSerdeInfo().setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName()); } else { - throw new UnsupportedException(tableDesc.getMeta().getStoreType() + " in HivecatalogStore"); + throw new UnsupportedException(tableDesc.getMeta().getDataFormat() + " in HivecatalogStore"); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java index 67a1488..9e1da2b 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java @@ -32,8 +32,6 @@ import org.apache.tajo.exception.*; import org.apache.thrift.TException; import parquet.hadoop.mapred.DeprecatedParquetOutputFormat; -import static org.apache.tajo.exception.ExceptionUtil.makeNotSupported; - public class HiveCatalogUtil { public static void validateSchema(Table tblSchema) { for (FieldSchema fieldSchema : tblSchema.getCols()) { @@ -101,7 +99,7 @@ public class HiveCatalogUtil { } } - public static String getStoreType(String fileFormat) { + public static String getDataFormat(String fileFormat) { Preconditions.checkNotNull(fileFormat); String[] fileFormatArrary = fileFormat.split("\\."); @@ -113,11 +111,11 @@ public class HiveCatalogUtil { if(outputFormatClass.equals(HiveIgnoreKeyTextOutputFormat.class.getSimpleName())) { return BuiltinStorages.TEXT; } else if(outputFormatClass.equals(HiveSequenceFileOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.SEQUENCEFILE.name(); + return CatalogProtos.DataFormat.SEQUENCEFILE.name(); } else if(outputFormatClass.equals(RCFileOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.RCFILE.name(); + return CatalogProtos.DataFormat.RCFILE.name(); } else if(outputFormatClass.equals(DeprecatedParquetOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.PARQUET.name(); + return CatalogProtos.DataFormat.PARQUET.name(); } else { throw new TajoRuntimeException(new UnknownDataFormatException(fileFormat)); } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java index f9b7c62..cb071aa 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java @@ -55,7 +55,7 @@ abstract class AbstractTableDescriptor implements TableDescriptor { protected TableProto getTableProto() { TableProto.Builder metaBuilder = TableProto.newBuilder(); - metaBuilder.setStoreType("SYSTEM"); + metaBuilder.setDataFormat("SYSTEM"); metaBuilder.setParams(KeyValueSetProto.newBuilder().build()); return metaBuilder.build(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java index 17f84d8..6a6ad78 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java @@ -24,7 +24,7 @@ import java.util.List; import org.apache.tajo.exception.UndefinedTableException; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.catalog.proto.CatalogProtos.DataFormat; import org.apache.tajo.util.TUtil; public class InfoSchemaMetadataDictionary { @@ -131,6 +131,6 @@ public class InfoSchemaMetadataDictionary { } protected String getTablePath() { - return StoreType.SYSTEM.name().toUpperCase(); + return DataFormat.SYSTEM.name().toUpperCase(); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/TablesTableDescriptor.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/TablesTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/TablesTableDescriptor.java index 7485248..5522e00 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/TablesTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/TablesTableDescriptor.java @@ -29,7 +29,7 @@ class TablesTableDescriptor extends AbstractTableDescriptor { new ColumnDescriptor("table_name", Type.TEXT, 0), new ColumnDescriptor("table_type", Type.TEXT, 0), new ColumnDescriptor("path", Type.TEXT, 0), - new ColumnDescriptor("store_type", Type.TEXT, 0) + new ColumnDescriptor("data_format", Type.TEXT, 0) }; public TablesTableDescriptor(InfoSchemaMetadataDictionary metadataDictionary) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 19bb678..f9b04ef 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -38,7 +38,6 @@ import org.apache.tajo.exception.*; import org.apache.tajo.util.JavaResourceUtil; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; -import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; @@ -839,7 +838,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo conn.setAutoCommit(false); String sql = "INSERT INTO TABLES (DB_ID, " + COL_TABLES_NAME + - ", TABLE_TYPE, PATH, STORE_TYPE, HAS_SELF_DESCRIBE_SCHEMA) VALUES(?, ?, ?, ?, ?, ?) "; + ", TABLE_TYPE, PATH, DATA_FORMAT, HAS_SELF_DESCRIBE_SCHEMA) VALUES(?, ?, ?, ?, ?, ?) "; if (LOG.isDebugEnabled()) { LOG.debug(sql); @@ -854,7 +853,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo pstmt.setString(3, TableType.MANAGED.name()); } pstmt.setString(4, table.getPath()); - pstmt.setString(5, table.getMeta().getStoreType()); + pstmt.setString(5, table.getMeta().getDataFormat()); pstmt.setBoolean(6, table.getSchema() == null); pstmt.executeUpdate(); pstmt.close(); @@ -1667,7 +1666,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo ResultSet res = null; PreparedStatement pstmt = null; CatalogProtos.TableDescProto.Builder tableBuilder = null; - String storeType; + String dataFormat; Pair<Integer, String> databaseIdAndUri = getDatabaseIdAndUri(databaseName); @@ -1678,7 +1677,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo // Geting Table Description ////////////////////////////////////////// String sql = - "SELECT TID, " + COL_TABLES_NAME + ", TABLE_TYPE, PATH, STORE_TYPE, HAS_SELF_DESCRIBE_SCHEMA FROM TABLES " + + "SELECT TID, " + COL_TABLES_NAME + ", TABLE_TYPE, PATH, DATA_FORMAT, HAS_SELF_DESCRIBE_SCHEMA FROM TABLES " + "WHERE DB_ID = ? AND " + COL_TABLES_NAME + "=?"; if (LOG.isDebugEnabled()) { @@ -1703,7 +1702,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } tableBuilder.setPath(res.getString(4).trim()); - storeType = res.getString(5).trim(); + dataFormat = res.getString(5).trim(); boolean hasSelfDescSchema = res.getBoolean(6); res.close(); @@ -1741,7 +1740,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo ////////////////////////////////////////// CatalogProtos.TableProto.Builder metaBuilder = CatalogProtos.TableProto.newBuilder(); - metaBuilder.setStoreType(storeType); + metaBuilder.setDataFormat(dataFormat); sql = "SELECT key_, value_ FROM " + TB_OPTIONS + " WHERE " + COL_TABLES_PK + " = ?"; if (LOG.isDebugEnabled()) { @@ -1855,7 +1854,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo List<TableDescriptorProto> tables = new ArrayList<TableDescriptorProto>(); try { - String sql = "SELECT t.TID, t.DB_ID, t." + COL_TABLES_NAME + ", t.TABLE_TYPE, t.PATH, t.STORE_TYPE, " + + String sql = "SELECT t.TID, t.DB_ID, t." + COL_TABLES_NAME + ", t.TABLE_TYPE, t.PATH, t.DATA_FORMAT, " + " s.SPACE_URI FROM " + TB_TABLES + " t, " + TB_DATABASES + " d, " + TB_SPACES + " s WHERE t.DB_ID = d.DB_ID AND d.SPACE_ID = s.SPACE_ID"; @@ -1878,10 +1877,10 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } else { builder.setPath(resultSet.getString("PATH")); } - String storeType = resultSet.getString("STORE_TYPE"); - if (storeType != null) { - storeType = storeType.trim(); - builder.setStoreType(storeType); + String dataFormat = resultSet.getString("DATA_FORMAT"); + if (dataFormat != null) { + dataFormat = dataFormat.trim(); + builder.setDataFormat(dataFormat); } tables.add(builder.build()); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index 921d98e..ff0b20b 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -19,6 +19,7 @@ <tns:store xmlns:tns="http://tajo.apache.org/catalogstore"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd "> <!-- Catalog base version history + * 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663) * 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493) * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832) * 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813) @@ -31,7 +32,7 @@ * 2 - 2014-06-09: First versioning * 1- Before 2013-03-20 --> - <tns:base version="11"> + <tns:base version="12"> <tns:objects> <tns:Object order="0" type="table" name="META"> <tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql> @@ -68,7 +69,7 @@ TABLE_NAME VARCHAR(128) NOT NULL, TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096), - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), HAS_SELF_DESCRIBE_SCHEMA BOOLEAN, CONSTRAINT TABLES_PK PRIMARY KEY (TID), CONSTRAINT C_TABLE_ID_UNIQ UNIQUE (DB_ID, TABLE_NAME) http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml index a23d393..dbf82b6 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml @@ -19,6 +19,7 @@ <tns:store xmlns:tns="http://tajo.apache.org/catalogstore"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd "> <!-- Catalog base version history + * 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663) * 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493) * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832) * 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813) @@ -31,7 +32,7 @@ * 2 - 2014-06-09: First versioning * 1- Before 2013-03-20 --> - <tns:base version="11"> + <tns:base version="12"> <tns:objects> <tns:Object order="0" type="table" name="META"> <tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql> @@ -66,7 +67,7 @@ TABLE_NAME VARCHAR(128) BINARY NOT NULL, TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096) BINARY, - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), HAS_SELF_DESCRIBE_SCHEMA BOOLEAN, FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID), INDEX IDX_DB_ID (DB_ID), http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml index 126a5e4..59ceae4 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml @@ -19,6 +19,7 @@ <tns:store xmlns:tns="http://tajo.apache.org/catalogstore"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd "> <!-- Catalog base version history + * 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663) * 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493) * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832) * 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813) @@ -31,7 +32,7 @@ * 2 - 2014-06-09: First versioning * 1- Before 2013-03-20 --> - <tns:base version="11"> + <tns:base version="12"> <tns:objects> <tns:Object order="0" type="table" name="META"> <tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql> @@ -66,7 +67,7 @@ TABLE_NAME VARCHAR(128) BINARY NOT NULL, TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096) BINARY, - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), HAS_SELF_DESCRIBE_SCHEMA BOOLEAN, FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID), INDEX IDX_DB_ID (DB_ID), http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml index b5396ef..b61778a 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml @@ -19,6 +19,7 @@ <tns:store xmlns:tns="http://tajo.apache.org/catalogstore"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd "> <!-- Catalog base version history + * 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663) * 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493) * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832) * 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813) @@ -31,7 +32,7 @@ * 2 - 2014-06-09: First versioning * 1- Before 2013-03-20 --> - <tns:base version="11"> + <tns:base version="12"> <tns:objects> <tns:Object order="0" type="table" name="meta"> <tns:sql><![CDATA[ @@ -98,7 +99,7 @@ TABLE_TYPE VARCHAR2(128) NOT NULL, PATH VARCHAR2(4000), HAS_SELF_DESCRIBE_SCHEMA CHAR NOT NULL, - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID), CONSTRAINT C_TABLE_ID_UNIQ UNIQUE (DB_ID, TABLE_NAME) )]]> http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml index 033c233..90449c1 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml @@ -21,6 +21,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd "> <!-- Catalog base version history + * 12 - 2015-09-28: Change the variable name storeType to dataFormat (TAJO-1663) * 11 - 2015-09-23: Add contents length and file count for partition directory (TAJO-1493) * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832) * 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813) @@ -34,7 +35,7 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition. * 2 - 2014-06-09: First versioning * 1- Before 2013-03-20 --> - <tns:base version="11"> + <tns:base version="12"> <tns:objects> <tns:Object name="META" type="table" order="0"> <tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql> @@ -75,7 +76,7 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition. TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096), HAS_SELF_DESCRIBE_SCHEMA BOOLEAN, - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID), UNIQUE (DB_ID, TABLE_NAME) )]]> http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index 0ca6baa..794ae4a 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -115,7 +115,7 @@ public class TestCatalogAgainstCaseSensitivity { assertTrue(tableDescs.containsKey(qualifiedTableName)); TableDesc desc = tableDescs.get(qualifiedTableName); assertEquals(desc.getUri().toString(), eachTableDescriptor.getPath()); - assertEquals(desc.getMeta().getStoreType(), eachTableDescriptor.getStoreType()); + assertEquals(desc.getMeta().getDataFormat(), eachTableDescriptor.getDataFormat()); } ////////////////////////////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-catalog/tajo-catalog-server/src/test/resources/schemas/derbytest/loadtest/derby.xml ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/resources/schemas/derbytest/loadtest/derby.xml b/tajo-catalog/tajo-catalog-server/src/test/resources/schemas/derbytest/loadtest/derby.xml index 86fd5e8..f7a2e14 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/resources/schemas/derbytest/loadtest/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/test/resources/schemas/derbytest/loadtest/derby.xml @@ -54,7 +54,7 @@ TABLE_NAME VARCHAR(128) NOT NULL, TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096), - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), CONSTRAINT TABLES_PK PRIMARY KEY (TID), CONSTRAINT C_TABLE_ID_UNIQ UNIQUE (DB_ID, TABLE_NAME) )]]> http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index 99288e8..e28e7f1 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -22,7 +22,6 @@ import com.google.protobuf.ServiceException; import org.apache.commons.cli.*; import org.apache.tajo.auth.UserRoleInfo; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.client.TajoClient; @@ -178,7 +177,7 @@ public class TajoDump { String fqName = CatalogUtil.buildFQName(databaseName, tableName); TableDesc table = client.getTableDesc(fqName); - if (table.getMeta().getStoreType().equalsIgnoreCase("SYSTEM")) { + if (table.getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")) { continue; } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java index b17a336..55553aa 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java @@ -99,7 +99,7 @@ public class DescTableCommand extends TajoShellCommand { StringBuilder sb = new StringBuilder(); sb.append("\ntable name: ").append(desc.getName()).append("\n"); sb.append("table uri: ").append(desc.getUri()).append("\n"); - sb.append("store type: ").append(desc.getMeta().getStoreType()).append("\n"); + sb.append("store type: ").append(desc.getMeta().getDataFormat()).append("\n"); if (desc.getStats() != null) { long row = desc.getStats().getNumRows(); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java index cb299b4..0ef1150 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java @@ -63,7 +63,7 @@ public class TestQueryValidation extends QueryTestCaseBase { } @Test - public void testUnsupportedStoreType() throws IOException { + public void testUnsupportedDataFormat() throws IOException { // See TAJO-1249 assertInvalidSQLFromFile("invalid_store_format.sql"); } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 0b94c52..ae28912 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -645,7 +645,7 @@ public class TestPhysicalPlanner { ctx.setDataChannel(dataChannel); LogicalNode rootNode = optimizer.optimize(plan); - TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType()); + TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getDataFormat()); FileSystem fs = sm.getFileSystem(); QueryId queryId = id.getTaskId().getExecutionBlockId().getQueryId(); @@ -779,7 +779,7 @@ public class TestPhysicalPlanner { ctx.setDataChannel(dataChannel); optimizer.optimize(plan); - TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType()); + TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getDataFormat()); FileSystem fs = sm.getFileSystem(); QueryId queryId = id.getTaskId().getExecutionBlockId().getQueryId(); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java index f327c85..667d193 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java @@ -210,8 +210,8 @@ public class TestCTASQuery extends QueryTestCaseBase { } @Test - public final void testCtasWithStoreType() throws Exception { - ResultSet res = executeFile("CtasWithStoreType.sql"); + public final void testCtasWithDataFormat() throws Exception { + ResultSet res = executeFile("CtasWithDataFormat.sql"); res.close(); ResultSet res2 = executeQuery(); @@ -220,7 +220,7 @@ public class TestCTASQuery extends QueryTestCaseBase { TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); - assertTrue("RCFILE".equalsIgnoreCase(desc.getMeta().getStoreType())); + assertTrue("RCFILE".equalsIgnoreCase(desc.getMeta().getDataFormat())); } @Test @@ -234,7 +234,7 @@ public class TestCTASQuery extends QueryTestCaseBase { TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); - assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getStoreType())); + assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getDataFormat())); } @Test @@ -248,7 +248,7 @@ public class TestCTASQuery extends QueryTestCaseBase { TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); - assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getStoreType())); + assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getDataFormat())); KeyValueSet options = desc.getMeta().getOptions(); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index f2f54ba..3450b03 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -20,7 +20,6 @@ package org.apache.tajo.engine.query; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -28,7 +27,6 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.KeyValueSet; import org.junit.Test; -import org.junit.experimental.categories.Category; import java.sql.ResultSet; import java.util.List; @@ -434,7 +432,7 @@ public class TestCreateTable extends QueryTestCaseBase { if(origTableDesc.getMeta() != null) { TableMeta origMeta = origTableDesc.getMeta(); TableMeta newMeta = newTableDesc.getMeta(); - if(origMeta.getStoreType().equals(newMeta.getStoreType()) == false) { + if(origMeta.getDataFormat().equals(newMeta.getDataFormat()) == false) { fail("Store type of input tables not equal"); return false; } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithDataFormat.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithDataFormat.sql b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithDataFormat.sql new file mode 100644 index 0000000..c09305e --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithDataFormat.sql @@ -0,0 +1,12 @@ +create table testCtasWithDataFormat (col1 float, col2 float) using rcfile as +select + sum(l_orderkey) as total1, + avg(l_partkey) as total2 +from + lineitem +group by + l_quantity +order by + l_quantity +limit + 3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql deleted file mode 100644 index bee1a89..0000000 --- a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql +++ /dev/null @@ -1,12 +0,0 @@ -create table testCtasWithStoreType (col1 float, col2 float) using rcfile as -select - sum(l_orderkey) as total1, - avg(l_partkey) as total2 -from - lineitem -group by - l_quantity -order by - l_quantity -limit - 3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithDataFormat.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithDataFormat.sql b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithDataFormat.sql new file mode 100644 index 0000000..5c484f3 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithDataFormat.sql @@ -0,0 +1 @@ +select * from testCtasWithDataFormat; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql deleted file mode 100644 index 9e7e337..0000000 --- a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql +++ /dev/null @@ -1 +0,0 @@ -select * from testCtasWithStoreType; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 461e405..f471e45 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -106,7 +106,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { PhysicalExec execPlan) throws IOException { DataChannel channel = context.getDataChannel(); ShuffleFileWriteNode shuffleFileWriteNode = LogicalPlan.createNodeWithoutPID(ShuffleFileWriteNode.class); - shuffleFileWriteNode.setStorageType(context.getDataChannel().getStoreType()); + shuffleFileWriteNode.setDataFormat(context.getDataChannel().getDataFormat()); shuffleFileWriteNode.setInSchema(plan.getOutSchema()); shuffleFileWriteNode.setOutSchema(plan.getOutSchema()); shuffleFileWriteNode.setShuffle(channel.getShuffleType(), channel.getShuffleKeys(), channel.getShuffleOutputNum()); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java index 2d32785..c779d2f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java @@ -40,7 +40,7 @@ public class DataChannel { private Schema schema; - private String storeType = BuiltinStorages.RAW; + private String dataFormat = BuiltinStorages.RAW; public DataChannel(ExecutionBlockId srcId, ExecutionBlockId targetId) { this.srcId = srcId; @@ -82,8 +82,8 @@ public class DataChannel { this.numOutputs = proto.getNumOutputs(); } - if (proto.hasStoreType()) { - this.storeType = proto.getStoreType(); + if (proto.hasDataFormat()) { + this.dataFormat = proto.getDataFormat(); } } @@ -144,16 +144,16 @@ public class DataChannel { return numOutputs; } - public boolean hasStoreType() { - return this.storeType != null; + public boolean hasDataFormat() { + return this.dataFormat != null; } - public void setStoreType(String storeType) { - this.storeType = storeType; + public void setDataFormat(String dataFormat) { + this.dataFormat = dataFormat; } - public String getStoreType() { - return storeType; + public String getDataFormat() { + return dataFormat; } public DataChannelProto getProto() { @@ -176,8 +176,8 @@ public class DataChannel { builder.setNumOutputs(numOutputs); } - if(storeType != null){ - builder.setStoreType(storeType); + if(dataFormat != null){ + builder.setDataFormat(dataFormat); } return builder.build(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 5049995..15354fc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -68,8 +68,8 @@ public class GlobalPlanner { private static Log LOG = LogFactory.getLog(GlobalPlanner.class); private final TajoConf conf; - private final String storeType; - private final String finalOutputStoreType; + private final String dataFormat; + private final String finalOutputDataFormat; private final CatalogService catalog; private final GlobalPlanRewriteEngine rewriteEngine; @@ -77,8 +77,8 @@ public class GlobalPlanner { public GlobalPlanner(final TajoConf conf, final CatalogService catalog) throws IOException { this.conf = conf; this.catalog = catalog; - this.storeType = conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase(); - this.finalOutputStoreType = conf.getVar(ConfVars.QUERY_OUTPUT_DEFAULT_FILE_FORMAT).toUpperCase(); + this.dataFormat = conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase(); + this.finalOutputDataFormat = conf.getVar(ConfVars.QUERY_OUTPUT_DEFAULT_FILE_FORMAT).toUpperCase(); Class<? extends GlobalPlanRewriteRuleProvider> clazz = (Class<? extends GlobalPlanRewriteRuleProvider>) conf.getClassVar(GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS); @@ -99,8 +99,8 @@ public class GlobalPlanner { return catalog; } - public String getStoreType() { - return storeType; + public String getDataFormat() { + return dataFormat; } public static class GlobalPlanContext { @@ -163,7 +163,7 @@ public class GlobalPlanner { private void setFinalOutputChannel(DataChannel outputChannel, Schema outputSchema) { outputChannel.setShuffleType(NONE_SHUFFLE); outputChannel.setShuffleOutputNum(1); - outputChannel.setStoreType(finalOutputStoreType); + outputChannel.setDataFormat(finalOutputDataFormat); outputChannel.setSchema(outputSchema); } @@ -171,7 +171,7 @@ public class GlobalPlanner { Preconditions.checkArgument(channel.getSchema() != null, "Channel schema (" + channel.getSrcId().getId() + " -> " + channel.getTargetId().getId() + ") is not initialized"); - TableMeta meta = new TableMeta(channel.getStoreType(), new KeyValueSet()); + TableMeta meta = new TableMeta(channel.getDataFormat(), new KeyValueSet()); TableDesc desc = new TableDesc( channel.getSrcId().toString(), channel.getSchema(), meta, StorageConstants.LOCAL_FS_URI); ScanNode scanNode = plan.createNode(ScanNode.class); @@ -184,7 +184,7 @@ public class GlobalPlanner { ExecutionBlock childBlock = leftTable ? leftBlock : rightBlock; DataChannel channel = new DataChannel(childBlock, parent, HASH_SHUFFLE, 32); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); if (join.getJoinType() != JoinType.CROSS) { // ShuffleKeys need to not have thea-join condition because Tajo supports only equi-join. Column [][] joinColumns = PlannerUtil.joinJoinKeyForEachTable(join.getJoinQual(), @@ -331,7 +331,7 @@ public class GlobalPlanner { // create other side channel if (otherSideBlock != null) { DataChannel otherSideChannel = new DataChannel(otherSideBlock, targetBlock, HASH_SHUFFLE, 32); - otherSideChannel.setStoreType(storeType); + otherSideChannel.setDataFormat(dataFormat); if (otherSideShuffleKeys != null) { otherSideChannel.setShuffleKeys(otherSideShuffleKeys); } @@ -587,7 +587,7 @@ public class GlobalPlanner { channel = new DataChannel(firstStage, secondStage, HASH_SHUFFLE, 32); channel.setShuffleKeys(secondPhaseGroupby.getGroupingColumns().clone()); channel.setSchema(firstStage.getPlan().getOutSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); // Setting for the second phase's logical plan ScanNode scanNode = buildInputExecutor(context.plan.getLogicalPlan(), channel); @@ -735,7 +735,7 @@ public class GlobalPlanner { channel.setShuffleKeys(firstPhaseGroupby.getGroupingColumns()); } channel.setSchema(firstPhaseGroupby.getOutSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); ScanNode scanNode = buildInputExecutor(masterPlan.getLogicalPlan(), channel); secondPhaseGroupby.setChild(scanNode); @@ -888,7 +888,7 @@ public class GlobalPlanner { ExecutionBlock childBlock = masterPlan.getExecBlock(channel.getSrcId()); setShuffleKeysFromPartitionedTableStore(currentNode, channel); channel.setSchema(childBlock.getPlan().getOutSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); lastChannel = channel; } @@ -911,7 +911,7 @@ public class GlobalPlanner { DataChannel channel = new DataChannel(lastBlock, nextBlock, HASH_SHUFFLE, 32); setShuffleKeysFromPartitionedTableStore(currentNode, channel); channel.setSchema(lastBlock.getPlan().getOutSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); ScanNode scanNode = buildInputExecutor(masterPlan.getLogicalPlan(), channel); currentNode.setChild(scanNode); @@ -990,7 +990,7 @@ public class GlobalPlanner { MasterPlan masterPlan = context.plan; for (DataChannel dataChannel : masterPlan.getIncomingChannels(execBlock.getId())) { - dataChannel.setStoreType(finalOutputStoreType); + dataChannel.setDataFormat(finalOutputDataFormat); ExecutionBlock subBlock = masterPlan.getExecBlock(dataChannel.getSrcId()); ProjectionNode copy = PlannerUtil.clone(plan, node); @@ -1035,7 +1035,7 @@ public class GlobalPlanner { DataChannel newChannel = new DataChannel(execBlock, newExecBlock, HASH_SHUFFLE, 1); newChannel.setShuffleKeys(new Column[]{}); newChannel.setSchema(node.getOutSchema()); - newChannel.setStoreType(storeType); + newChannel.setDataFormat(dataFormat); ScanNode scanNode = buildInputExecutor(plan, newChannel); LimitNode parentLimit = PlannerUtil.clone(context.plan.getLogicalPlan(), node); @@ -1103,7 +1103,7 @@ public class GlobalPlanner { channel.setShuffleKeys(null); } channel.setSchema(windowAgg.getInSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); LogicalNode childNode = windowAgg.getChild(); ScanNode scanNode = buildInputExecutor(masterPlan.getLogicalPlan(), channel); @@ -1236,7 +1236,7 @@ public class GlobalPlanner { for (ExecutionBlock childBlocks : queryBlockBlocks) { DataChannel channel = new DataChannel(childBlocks, execBlock, NONE_SHUFFLE, 1); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); masterPlan.addConnect(channel); } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 6fbecc1..210ac2c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -142,7 +142,7 @@ public class DistinctGroupbyBuilder { firstChannel.setShuffleKeys(firstStageDistinctNode.getFirstStageShuffleKeyColumns()); firstChannel.setSchema(firstStageDistinctNode.getOutSchema()); - firstChannel.setStoreType(globalPlanner.getStoreType()); + firstChannel.setDataFormat(globalPlanner.getDataFormat()); ScanNode scanNode = GlobalPlanner.buildInputExecutor(context.getPlan().getLogicalPlan(), firstChannel); secondStageDistinctNode.setChild(scanNode); @@ -161,7 +161,7 @@ public class DistinctGroupbyBuilder { secondChannel.setShuffleKeys(firstStageDistinctNode.getGroupingColumns()); } secondChannel.setSchema(secondStageDistinctNode.getOutSchema()); - secondChannel.setStoreType(globalPlanner.getStoreType()); + secondChannel.setDataFormat(globalPlanner.getDataFormat()); scanNode = GlobalPlanner.buildInputExecutor(context.getPlan().getLogicalPlan(), secondChannel); thirdStageDistinctNode.setChild(scanNode); @@ -333,7 +333,7 @@ public class DistinctGroupbyBuilder { channel.setShuffleKeys(firstStageDistinctNode.getGroupingColumns()); } channel.setSchema(firstStageDistinctNode.getOutSchema()); - channel.setStoreType(globalPlanner.getStoreType()); + channel.setDataFormat(globalPlanner.getDataFormat()); ScanNode scanNode = GlobalPlanner.buildInputExecutor(context.getPlan().getLogicalPlan(), channel); secondStageDistinctNode.setChild(scanNode); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java index df1f33b..b13cb0f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java @@ -48,7 +48,7 @@ public class GlobalPlanRewriteUtil { originalChannel.getShuffleOutputNum()); newChannel.setSchema(originalChannel.getSchema()); newChannel.setShuffleKeys(originalChannel.getShuffleKeys()); - newChannel.setStoreType(originalChannel.getStoreType()); + newChannel.setDataFormat(originalChannel.getDataFormat()); newChannel.setTransmitType(originalChannel.getTransmitType()); plan.addConnect(newChannel); plan.disconnect(eachGrandChild, child); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index e171338..5352649 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -181,7 +181,7 @@ public class BSTIndexScanExec extends ScanExec { Schema fileScanOutSchema = mergeSubSchemas(projected, keySchema, plan.getTargets(), qual); this.fileScanner = OldStorageManager.getStorageManager(context.getConf(), - plan.getTableDesc().getMeta().getStoreType()) + plan.getTableDesc().getMeta().getDataFormat()) .getSeekableScanner(plan.getTableDesc().getMeta(), plan.getPhysicalSchema(), fragment, fileScanOutSchema); this.fileScanner.init(); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index e9f4fb9..1bd38d1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -209,12 +209,12 @@ public class PhysicalPlanUtil { * @param nullChar A character for NULL representation */ private static void setNullCharForTextSerializer(TableMeta meta, String nullChar) { - String storeType = meta.getStoreType(); - if (storeType.equalsIgnoreCase(BuiltinStorages.TEXT)) { + String dataFormat = meta.getDataFormat(); + if (dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT)) { meta.putOption(StorageConstants.TEXT_NULL, nullChar); - } else if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) { + } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) { meta.putOption(StorageConstants.RCFILE_NULL, nullChar); - } else if (storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { + } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { meta.putOption(StorageConstants.SEQUENCEFILE_NULL, nullChar); } } @@ -226,12 +226,12 @@ public class PhysicalPlanUtil { * @return True if TableMeta contains NULL char property according to file format */ public static boolean containsNullChar(TableMeta meta) { - String storeType = meta.getStoreType(); - if (storeType.equalsIgnoreCase(BuiltinStorages.TEXT)) { + String dataFormat = meta.getDataFormat(); + if (dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT)) { return meta.containsOption(StorageConstants.TEXT_NULL); - } else if (storeType.equalsIgnoreCase(BuiltinStorages.RCFILE)) { + } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) { return meta.containsOption(StorageConstants.RCFILE_NULL); - } else if (storeType.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { + } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { return meta.containsOption(StorageConstants.SEQUENCEFILE_NULL); } else { return false; http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index 8db1f5d..2ebad1e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -18,7 +18,6 @@ package org.apache.tajo.engine.planner.physical; -import com.google.common.base.Optional; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -83,7 +82,7 @@ public class StoreTableExec extends UnaryPhysicalExec { public void openNewFile(int suffixId) throws IOException { Schema appenderSchema = (plan instanceof InsertNode) ? ((InsertNode) plan).getTableSchema() : outSchema; - if (PlannerUtil.isFileStorageType(meta.getStoreType())) { + if (PlannerUtil.isFileStorageType(meta.getDataFormat())) { String prevFile = null; lastFileName = context.getOutputPath(); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 4f6873b..1966244 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -303,8 +303,8 @@ public class GlobalEngine extends AbstractService { } private void verifyInsertTableSchema(VerificationState state, LogicalPlan plan) { - String storeType = PlannerUtil.getStoreType(plan); - if (storeType != null) { + String dataFormat = PlannerUtil.getDataFormat(plan); + if (dataFormat != null) { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); if (rootNode.getChild().getType() == NodeType.INSERT) { try { http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index bac124f..7f8c61c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -245,8 +245,8 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult } } else if ("path".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(table.getPath())); - } else if ("store_type".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createText(table.getStoreType())); + } else if ("data_format".equalsIgnoreCase(column.getSimpleName())) { + aTuple.put(fieldId, DatumFactory.createText(table.getDataFormat())); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index 50f4a85..2716800 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -528,7 +528,7 @@ public class QueryExecutor { List<CatalogProtos.ColumnProto> columns = new ArrayList<CatalogProtos.ColumnProto>(); CatalogProtos.TableDescProto tableDescProto = CatalogProtos.TableDescProto.newBuilder() .setTableName(nodeUniqName) - .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType(BuiltinStorages.TEXT).build()) + .setMeta(CatalogProtos.TableProto.newBuilder().setDataFormat(BuiltinStorages.TEXT).build()) .setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build()) .setStats(stats.getProto()) .build(); http://git-wip-us.apache.org/repos/asf/tajo/blob/b418c499/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index bddf198..3de63fb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -195,7 +195,7 @@ public class Repartitioner { int maxStatsScanIdx = -1; StringBuilder nonLeafScanNamesBuilder = new StringBuilder(); for (int i = 0; i < scans.length; i++) { - if (scans[i].getTableDesc().getMeta().getStoreType().equalsIgnoreCase("RAW")) { + if (scans[i].getTableDesc().getMeta().getDataFormat().equalsIgnoreCase("RAW")) { // Intermediate data scan hasNonLeafNode = true; largeScanIndexList.add(i); @@ -636,7 +636,7 @@ public class Repartitioner { TupleRange mergedRange = TupleUtil.columnStatToRange(sortSpecs, sortSchema, totalStat.getColumnStats(), false); if (sortNode.getSortPurpose() == SortPurpose.STORAGE_SPECIFIED) { - String storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); + String dataFormat = PlannerUtil.getDataFormat(masterPlan.getLogicalPlan()); CatalogService catalog = stage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalRootNode rootNode = masterPlan.getLogicalPlan().getRootBlock().getRoot(); TableDesc tableDesc = null; @@ -647,7 +647,7 @@ public class Repartitioner { PlannerUtil.getStoreTableName(masterPlan.getLogicalPlan())); } - Tablespace space = TablespaceManager.getAnyByScheme(storeType).get(); + Tablespace space = TablespaceManager.getAnyByScheme(dataFormat).get(); ranges = space.getInsertSortRanges( stage.getContext().getQueryContext(), tableDesc,
