Repository: tajo Updated Branches: refs/heads/branch-0.8.0 b97faf5b3 -> 67c2a2134
TAJO-705: CTAS always stores tables with CSV storage type into catalog. (jinho) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/67c2a213 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/67c2a213 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/67c2a213 Branch: refs/heads/branch-0.8.0 Commit: 67c2a2134afab5a281122ae99b0cdcfa72c8a1e5 Parents: b97faf5 Author: jinossy <[email protected]> Authored: Tue Mar 25 23:46:57 2014 +0900 Committer: jinossy <[email protected]> Committed: Tue Mar 25 23:46:57 2014 +0900 ---------------------------------------------------------------------- CHANGES.txt | 3 ++ .../org/apache/tajo/catalog/CatalogUtil.java | 14 +++++-- .../tajo/engine/planner/LogicalPlanner.java | 6 ++- .../apache/tajo/master/querymaster/Query.java | 2 +- .../apache/tajo/engine/query/TestCTASQuery.java | 41 +++++++++++++++++--- .../queries/TestCTASQuery/CtasWithOptions.sql | 13 +++++++ .../queries/TestCTASQuery/CtasWithStoreType.sql | 12 ++++++ .../TestCTASQuery/testCtasWithOptions.sql | 1 + .../TestCTASQuery/testCtasWithStoreType.sql | 1 + .../TestCTASQuery/testCtasWithOptions.result | 5 +++ .../TestCTASQuery/testCtasWithStoreType.result | 5 +++ 11 files changed, 92 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index ea49240..515fd64 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -283,6 +283,9 @@ Release 0.8.0 - unreleased BUG FIXES + TAJO-705: CTAS always stores tables with CSV storage type into catalog. + (jinho) + TAJO-707: Jenkins build failure in TestNetTypes. (jihoon) TAJO-693: StatusUpdateTransition in QueryUnitAttempt handles TA_UPDATE http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 2ddbb17..5ad2eca 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -183,11 +183,9 @@ public class CatalogUtil { return StoreType.CSV; } else if (typeStr.equalsIgnoreCase(StoreType.RAW.name())) { return StoreType.RAW; - } else if (typeStr.equalsIgnoreCase(StoreType.CSV.name())) { - return StoreType.CSV; } else if (typeStr.equalsIgnoreCase(StoreType.ROWFILE.name())) { return StoreType.ROWFILE; - }else if (typeStr.equalsIgnoreCase(StoreType.RCFILE.name())) { + } else if (typeStr.equalsIgnoreCase(StoreType.RCFILE.name())) { return StoreType.RCFILE; } else if (typeStr.equalsIgnoreCase(StoreType.TREVNI.name())) { return StoreType.TREVNI; @@ -195,6 +193,16 @@ public class CatalogUtil { return null; } } + public static Options newOptionsWithDefault(StoreType type) { + Options options = new Options(); + if(StoreType.CSV == type){ + options.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.CSVFILE_DELIMITER_DEFAULT); + } else if(StoreType.RCFILE == type){ + options.put(CatalogConstants.RCFILE_SERDE, CatalogConstants.RCFILE_BINARY_SERDE); + } + + return options; + } public static TableMeta newTableMeta(StoreType type) { return new TableMeta(type, new Options()); http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 48e3c93..c479f1c 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -1313,12 +1313,14 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex createTableNode.setStorageType(CatalogProtos.StoreType.CSV); } + // Set default options to be created. + Options options = CatalogUtil.newOptionsWithDefault(createTableNode.getStorageType()); if (expr.hasParams()) { - Options options = new Options(); options.putAll(expr.getParams()); - createTableNode.setOptions(options); } + createTableNode.setOptions(options); + if (expr.hasPartition()) { if (expr.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) { createTableNode.setPartitionMethod(getPartitionMethod(context, expr.getTableName(), expr.getPartitionMethod())); http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java index 8467b4b..3a4df7b 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -482,10 +482,10 @@ public class Query implements EventHandler<QueryEvent> { Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { CatalogService catalog = context.getWorkerContext().getCatalog(); SubQuery lastStage = query.getSubQuery(finalExecBlockId); - TableMeta meta = lastStage.getTableMeta(); TableStats stats = lastStage.getResultStats(); CreateTableNode createTableNode = (CreateTableNode) lastStage.getBlock().getPlan(); + TableMeta meta = new TableMeta(createTableNode.getStorageType(), createTableNode.getOptions()); TableDesc tableDescTobeCreated = new TableDesc( http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java index e61f509..82c4be9 100644 --- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java +++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java @@ -19,15 +19,14 @@ package org.apache.tajo.engine.query; import com.google.common.collect.Maps; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.junit.Test; @@ -38,8 +37,7 @@ import java.util.Map; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.catalog.CatalogUtil.buildFQName; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** @@ -178,4 +176,37 @@ public class TestCTASQuery extends QueryTestCaseBase { resultSetToString(res2); res2.close(); } + + @Test + public final void testCtasWithStoreType() throws Exception { + ResultSet res = executeFile("CtasWithStoreType.sql"); + res.close(); + + ResultSet res2 = executeQuery(); + resultSetToString(res2); + res2.close(); + + TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); + assertNotNull(desc); + assertEquals(CatalogProtos.StoreType.RCFILE, desc.getMeta().getStoreType()); + } + + @Test + public final void testCtasWithOptions() throws Exception { + ResultSet res = executeFile("CtasWithOptions.sql"); + res.close(); + + ResultSet res2 = executeQuery(); + resultSetToString(res2); + res2.close(); + + TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); + assertNotNull(desc); + assertEquals(CatalogProtos.StoreType.CSV, desc.getMeta().getStoreType()); + + + Options options = desc.getMeta().getOptions(); + assertNotNull(options); + assertEquals(StringEscapeUtils.escapeJava("\u0001"), options.get(CatalogConstants.CSVFILE_DELIMITER)); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql new file mode 100644 index 0000000..a393b9d --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithOptions.sql @@ -0,0 +1,13 @@ +create table testCtasWithOptions (col1 float, col2 float) +using csv with ('csvfile.delimiter'='\u0001') as +select + sum(l_orderkey) as total1, + avg(l_partkey) as total2 +from + lineitem +group by + l_quantity +order by + l_quantity +limit + 3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql new file mode 100644 index 0000000..bee1a89 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql @@ -0,0 +1,12 @@ +create table testCtasWithStoreType (col1 float, col2 float) using rcfile as +select + sum(l_orderkey) as total1, + avg(l_partkey) as total2 +from + lineitem +group by + l_quantity +order by + l_quantity +limit + 3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql new file mode 100644 index 0000000..ac25e10 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithOptions.sql @@ -0,0 +1 @@ +select * from testCtasWithOptions; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql new file mode 100644 index 0000000..9e7e337 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql @@ -0,0 +1 @@ +select * from testCtasWithStoreType; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result new file mode 100644 index 0000000..ea78854 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithOptions.result @@ -0,0 +1,5 @@ +col1,col2,key +------------------------------- +1.0,1.0,17.0 +1.0,1.0,36.0 +2.0,2.0,38.0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/67c2a213/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result new file mode 100644 index 0000000..ea78854 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestCTASQuery/testCtasWithStoreType.result @@ -0,0 +1,5 @@ +col1,col2,key +------------------------------- +1.0,1.0,17.0 +1.0,1.0,36.0 +2.0,2.0,38.0 \ No newline at end of file
