Repository: tajo Updated Branches: refs/heads/master af4083bfd -> b7c8cf723
TAJO-1247: Store type 'TEXTFILE' should be TEXT while keeping enum 'TEXTFILE' in protobuf. (DaeMyung Kang via hyunsik) Closes #302 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/b7c8cf72 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/b7c8cf72 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/b7c8cf72 Branch: refs/heads/master Commit: b7c8cf7239483c6b44925175658f21dbbf252665 Parents: af4083b Author: Hyunsik Choi <[email protected]> Authored: Fri Dec 19 10:07:53 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Fri Dec 19 10:10:52 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 +++ .../org/apache/tajo/catalog/CatalogUtil.java | 24 ++++++++++++++++---- .../org/apache/tajo/catalog/DDLBuilder.java | 6 ++--- .../apache/tajo/catalog/store/HCatalogUtil.java | 3 ++- .../TestSelectQuery/datetime_table_ddl.sql | 2 +- .../datetime_table_timezoned_ddl.sql | 2 +- .../src/main/sphinx/table_management/csv.rst | 4 ++-- .../sphinx/table_management/table_overview.rst | 4 ++-- tajo-docs/src/main/sphinx/time_zone.rst | 6 ++--- 9 files changed, 37 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 29b66d1..54a4ef4 100644 --- a/CHANGES +++ b/CHANGES @@ -24,6 +24,9 @@ Release 0.9.1 - unreleased IMPROVEMENT + TAJO-1247: Store type 'TEXTFILE' should be TEXT while keeping enum 'TEXTFILE' in protobuf. + (DaeMyung Kang via hyunsik) + TAJO-1221: HA TajoClient should not connect TajoMaster at the first. (jaehwa) http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index cbcac63..bb15ec1 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -18,7 +18,6 @@ package org.apache.tajo.catalog; -import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; import org.apache.tajo.DataTypeUtil; @@ -32,11 +31,9 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; -import org.mortbay.util.ajax.JSON; import java.sql.Connection; import java.sql.ResultSet; @@ -52,6 +49,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type; public class CatalogUtil { + public static final String TEXTFILE_NAME = "TEXT"; /** * Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name. * @@ -264,6 +262,24 @@ public class CatalogUtil { return sb.toString(); } + public static String getStoreTypeString(final StoreType type) { + if (type == StoreType.TEXTFILE) { + return TEXTFILE_NAME; + } else if (type == StoreType.CSV || + type == StoreType.RAW || + type == StoreType.ROWFILE || + type == StoreType.RCFILE || + type == StoreType.PARQUET || + type == StoreType.SEQUENCEFILE || + type == StoreType.AVRO || + type == StoreType.JSON || + type == StoreType.HBASE) { + return type.name(); + } else { + return null; + } + } + public static StoreType getStoreType(final String typeStr) { if (typeStr.equalsIgnoreCase(StoreType.CSV.name())) { return StoreType.CSV; @@ -279,7 +295,7 @@ public class CatalogUtil { return StoreType.SEQUENCEFILE; } else if (typeStr.equalsIgnoreCase(StoreType.AVRO.name())) { return StoreType.AVRO; - } else if (typeStr.equalsIgnoreCase(StoreType.TEXTFILE.name())) { + } else if (typeStr.equalsIgnoreCase(TEXTFILE_NAME)) { return StoreType.TEXTFILE; } else if (typeStr.equalsIgnoreCase(StoreType.JSON.name())) { return StoreType.JSON; http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index bd7364a..978092a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -31,7 +31,7 @@ public class DDLBuilder { sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") - .append(" Storage: ").append(desc.getMeta().getStoreType().name()); + .append(" Storage: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType())); sb.append("\n-- Path: ").append(desc.getPath()); sb.append("\n--\n"); sb.append("CREATE EXTERNAL TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); @@ -54,7 +54,7 @@ public class DDLBuilder { sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") - .append(" Storage: ").append(desc.getMeta().getStoreType().name()); + .append(" Storage: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType())); sb.append("\n--\n"); sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); @@ -91,7 +91,7 @@ public class DDLBuilder { } private static void buildUsingClause(StringBuilder sb, TableMeta meta) { - sb.append(" USING " + meta.getStoreType().name()); + sb.append(" USING " + CatalogUtil.getStoreTypeString(meta.getStoreType())); } private static void buildWithClause(StringBuilder sb, TableMeta meta) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java index c7f2590..ff2a2c4 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java @@ -31,6 +31,7 @@ import org.apache.hcatalog.data.schema.HCatFieldSchema; import org.apache.hcatalog.data.schema.HCatSchema; import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.common.TajoDataTypes; import org.apache.thrift.TException; import parquet.hadoop.mapred.DeprecatedParquetOutputFormat; @@ -128,7 +129,7 @@ public class HCatalogUtil { String outputFormatClass = fileFormatArrary[fileFormatArrary.length-1]; if(outputFormatClass.equals(HiveIgnoreKeyTextOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.TEXTFILE.name(); + return CatalogUtil.TEXTFILE_NAME; } else if(outputFormatClass.equals(HiveSequenceFileOutputFormat.class.getSimpleName())) { return CatalogProtos.StoreType.SEQUENCEFILE.name(); } else if(outputFormatClass.equals(RCFileOutputFormat.class.getSimpleName())) { http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql index beb19b7..d2c97c5 100644 --- a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql @@ -1,4 +1,4 @@ CREATE EXTERNAL TABLE ${0} ( t_timestamp TIMESTAMP, t_date DATE -) USING TEXTFILE LOCATION ${table.path} \ No newline at end of file +) USING TEXT LOCATION ${table.path} http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql index 11014d8..131e619 100644 --- a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql @@ -1,4 +1,4 @@ CREATE EXTERNAL TABLE ${0} ( t_timestamp TIMESTAMP, t_date DATE -) USING TEXTFILE WITH ('timezone' = 'GMT+9') LOCATION ${table.path} \ No newline at end of file +) USING TEXT WITH ('timezone' = 'GMT+9') LOCATION ${table.path} http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-docs/src/main/sphinx/table_management/csv.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/table_management/csv.rst b/tajo-docs/src/main/sphinx/table_management/csv.rst index 37dc2ba..53c6e1d 100644 --- a/tajo-docs/src/main/sphinx/table_management/csv.rst +++ b/tajo-docs/src/main/sphinx/table_management/csv.rst @@ -110,6 +110,6 @@ clause in a Hive's ``CREATE TABLE`` statement as follows: CREATE TABLE table1 (id int, name string, score float, type string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' - STORED AS TEXTFILE + STORED AS TEXT -To the best of our knowledge, there is not way to specify a custom NULL character in Hive. \ No newline at end of file +To the best of our knowledge, there is not way to specify a custom NULL character in Hive. http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-docs/src/main/sphinx/table_management/table_overview.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/table_management/table_overview.rst b/tajo-docs/src/main/sphinx/table_management/table_overview.rst index 704d6aa..3d933c2 100644 --- a/tajo-docs/src/main/sphinx/table_management/table_overview.rst +++ b/tajo-docs/src/main/sphinx/table_management/table_overview.rst @@ -57,7 +57,7 @@ You can specify a table time zone as follows: CREATE EXTERNAL TABLE table1 ( t_timestamp TIMESTAMP, t_date DATE - ) USING TEXTFILE WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/' + ) USING TEXT WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/' -In order to learn time zone, please refer to :doc:`/time_zone`. \ No newline at end of file +In order to learn time zone, please refer to :doc:`/time_zone`. http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-docs/src/main/sphinx/time_zone.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/time_zone.rst b/tajo-docs/src/main/sphinx/time_zone.rst index 8225dee..101012f 100644 --- a/tajo-docs/src/main/sphinx/time_zone.rst +++ b/tajo-docs/src/main/sphinx/time_zone.rst @@ -36,7 +36,7 @@ You can specify a table time zone as follows: CREATE EXTERNAL TABLE table1 ( t_timestamp TIMESTAMP, t_date DATE - ) USING TEXTFILE WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/' + ) USING TEXT WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/' In order to learn table properties, please refer to :doc:`/table_management/table_overview`. @@ -102,7 +102,7 @@ In order to register the table, we should put a table property ``'timezone'='Asi CREATE EXTERNAL TABLE table1 ( t_timestamp TIMESTAMP, t_date DATE - ) USING TEXTFILE WITH('text.delimiter'='|', 'timezone'='ASIA/Seoul') LOCATION '/path-to-table/' + ) USING TEXT WITH('text.delimiter'='|', 'timezone'='ASIA/Seoul') LOCATION '/path-to-table/' By default, ``tsql`` and ``TajoClient`` API use UTC time zone. So, timestamp values in the result are adjusted by the time zone offset. But, date is not adjusted because date type does not consider time zone. @@ -126,4 +126,4 @@ In addition, users can set client-side time zone by setting a session variable ' ---------------------------------- 1980-04-01 01:50:30.01, 1980-04-01 1980-04-01 01:50:30 , 1980-04-01 - 1980-04-01 01:50:30 , 1980-04-01 \ No newline at end of file + 1980-04-01 01:50:30 , 1980-04-01
