Repository: tajo Updated Branches: refs/heads/branch-0.11.0 cae8e56f0 -> 7b27c8ce9
TAJO-1674: Validation of CTAS schema mismatch. Closes #726 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/7b27c8ce Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/7b27c8ce Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/7b27c8ce Branch: refs/heads/branch-0.11.0 Commit: 7b27c8ce9768aa56ed584757caa33f44d56a14ab Parents: cae8e56 Author: Hyunsik Choi <[email protected]> Authored: Wed Sep 9 10:51:25 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Wed Sep 9 10:59:51 2015 +0900 ---------------------------------------------------------------------- CHANGES | 6 +++- .../org/apache/tajo/catalog/CatalogUtil.java | 15 +++++++-- .../tajo/engine/query/TestTablePartitions.java | 32 +++----------------- .../TestCreateTable/negative/type_mismatch.sql | 2 ++ .../tajo/plan/verifier/LogicalPlanVerifier.java | 16 +++++++++- .../tajo/plan/verifier/SyntaxErrorUtil.java | 5 +-- 6 files changed, 43 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/7b27c8ce/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 2ea1a95..0d4f71d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,7 @@ Tajo Change Log +Release 0.12.0 - unreleased + Release 0.11.0 - unreleased NEW FEATURES @@ -250,7 +252,9 @@ Release 0.11.0 - unreleased BUG FIXES - TAJO-1819: TAJO-1819: Cannot find existing tables when pgsql catalog starts up. + TAJO-1674: Validation of CTAS schema mismatch. (hyunsik) + + TAJO-1819: Cannot find existing tables when pgsql catalog starts up. (jihoon) TAJO-1821: Temporary data is not cleared after TestCatalog. (jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/7b27c8ce/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index ac76387..a6487b4 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -873,7 +873,11 @@ public class CatalogUtil { return pair; } - /* It is the relationship graph of type conversions. */ + /* + * It is the relationship graph of type conversions. + * It contains tuples, each of which (LHS type, RHS type, Result type). + */ + public static final Map<Type, Map<Type, Type>> OPERATION_CASTING_MAP = Maps.newHashMap(); static { @@ -929,9 +933,16 @@ public class CatalogUtil { TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.FLOAT8, Type.TEXT, Type.TEXT); TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TIMESTAMP, Type.TIMESTAMP); + TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TEXT, Type.TEXT); + TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.VARCHAR, Type.TEXT); + + TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.TIMESTAMP, Type.TIMESTAMP); + TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.TEXT, Type.TEXT); + TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.VARCHAR, Type.VARCHAR, Type.TEXT); + TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.TIMESTAMP, Type.TIMESTAMP); TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.TEXT, Type.TEXT); - TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TEXT, Type.TEXT, Type.TEXT); + TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.VARCHAR, Type.TEXT); TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIME, Type.TIME, Type.TIME); TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.DATE, Type.DATE, Type.DATE); http://git-wip-us.apache.org/repos/asf/tajo/blob/7b27c8ce/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index 52e7b54..94e5e71 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -37,14 +37,11 @@ import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.ipc.ClientProtos; -import org.apache.tajo.jdbc.FetchResultSet; -import org.apache.tajo.jdbc.TajoMemoryResultSet; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.querymaster.QueryMasterTask; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.worker.TajoWorker; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -211,7 +208,7 @@ public class TestTablePartitions extends QueryTestCaseBase { TableDesc tableDesc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"}, - tableDesc.getStats().getNumRows()); + tableDesc.getStats().getNumRows()); executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -254,7 +251,7 @@ public class TestTablePartitions extends QueryTestCaseBase { } verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, - new String[]{"key"}, desc.getStats().getNumRows()); + new String[]{"key"}, desc.getStats().getNumRows()); executeString("DROP TABLE " + tableName + " PURGE").close(); res.close(); @@ -674,7 +671,7 @@ public class TestTablePartitions extends QueryTestCaseBase { } verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1"}, - desc.getStats().getNumRows()); + desc.getStats().getNumRows()); executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -733,7 +730,7 @@ public class TestTablePartitions extends QueryTestCaseBase { } verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1", "col2"}, - desc.getStats().getNumRows()); + desc.getStats().getNumRows()); executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -1045,7 +1042,7 @@ public class TestTablePartitions extends QueryTestCaseBase { TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col2"}, - desc.getStats().getNumRows()); + desc.getStats().getNumRows()); executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -1081,25 +1078,6 @@ public class TestTablePartitions extends QueryTestCaseBase { executeString("DROP TABLE " + tableName + " PURGE").close(); } - private MasterPlan getQueryPlan(ResultSet res) { - QueryId queryId; - if (res instanceof TajoMemoryResultSet) { - queryId = ((TajoMemoryResultSet) res).getQueryId(); - } else { - queryId = ((FetchResultSet) res).getQueryId(); - } - - for (TajoWorker eachWorker : testingCluster.getTajoWorkers()) { - QueryMasterTask queryMasterTask = eachWorker.getWorkerContext().getQueryMaster().getQueryMasterTask(queryId, true); - if (queryMasterTask != null) { - return queryMasterTask.getQuery().getPlan(); - } - } - - fail("Can't find query from workers" + queryId); - return null; - } - @Test public void testScatteredHashShuffle() throws Exception { testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME.varname, "2"); http://git-wip-us.apache.org/repos/asf/tajo/blob/7b27c8ce/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/type_mismatch.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/type_mismatch.sql b/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/type_mismatch.sql new file mode 100644 index 0000000..1bff25c --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCreateTable/negative/type_mismatch.sql @@ -0,0 +1,2 @@ +CREATE TABLE MISMATCH1 (n_name text, n_comment text, n_nationkey int8, n_regionkey int8) AS SELECT * FROM default.nation; +CREATE TABLE MISMATCH2 (n_name text, n_comment text) PARTITION BY COLUMN (n_nationkey int8, n_regionkey int8) AS SELECT * FROM default.nation; http://git-wip-us.apache.org/repos/asf/tajo/blob/7b27c8ce/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java index 0b0968e..c87ceb3 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java @@ -19,6 +19,7 @@ package org.apache.tajo.plan.verifier; import com.google.common.base.Preconditions; +import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes.Type; @@ -31,6 +32,7 @@ import org.apache.tajo.plan.Target; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.util.TUtil; import java.util.Stack; @@ -241,7 +243,15 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri private static void ensureDomains(VerificationState state, Schema targetTableScheme, Schema schema) throws TajoException { for (int i = 0; i < schema.size(); i++) { - if (!schema.getColumn(i).getDataType().equals(targetTableScheme.getColumn(i).getDataType())) { + + // null can be used anywhere + if (schema.getColumn(i).getDataType().getType() == Type.NULL_TYPE) { + continue; + } + + // checking castable between two data types + if (!TUtil.containsInNestedMap(CatalogUtil.OPERATION_CASTING_MAP, + schema.getColumn(i).getDataType().getType(), targetTableScheme.getColumn(i).getDataType().getType())) { Column targetColumn = targetTableScheme.getColumn(i); Column insertColumn = schema.getColumn(i); state.addVerification(makeDataTypeMisMatch(insertColumn, targetColumn)); @@ -254,6 +264,10 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri CreateTableNode node, Stack<LogicalNode> stack) throws TajoException { super.visitCreateTable(context, plan, block, node, stack); // here, we don't need check table existence because this check is performed in PreLogicalPlanVerifier. + + if (node.hasSubQuery()) { + ensureDomains(context.state, node.getLogicalSchema(), node.getChild(0).getOutSchema()); + } return node; } http://git-wip-us.apache.org/repos/asf/tajo/blob/7b27c8ce/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java index 3c11b8c..bf4a0d6 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/SyntaxErrorUtil.java @@ -21,6 +21,7 @@ package org.apache.tajo.plan.verifier; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.error.Errors.ResultCode; +import org.apache.tajo.exception.DataTypeMismatchException; import org.apache.tajo.plan.logical.NodeType; public class SyntaxErrorUtil { @@ -29,8 +30,8 @@ public class SyntaxErrorUtil { return new SyntaxErrorException(ResultCode.SYNTAX_ERROR, message); } - public static SyntaxErrorException makeDataTypeMisMatch(Column src, Column target) { - return new SyntaxErrorException(ResultCode.DATATYPE_MISMATCH, + public static DataTypeMismatchException makeDataTypeMisMatch(Column src, Column target) { + return new DataTypeMismatchException( src.getSimpleName(), src.getDataType().getType().name(), target.getSimpleName(), target.getDataType().getType().name()); }
