Repository: tajo Updated Branches: refs/heads/master 31e247cd6 -> a191b1dc7
TAJO-699: Create a table using LIKE. (Prafulla T via hyunsik) Closes #21 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a191b1dc Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a191b1dc Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a191b1dc Branch: refs/heads/master Commit: a191b1dc7ad3189c825069084fc6fc6167a4eb87 Parents: 31e247c Author: Hyunsik Choi <[email protected]> Authored: Wed May 28 03:01:17 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Wed May 28 03:01:17 2014 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/algebra/CreateTable.java | 11 ++ .../org/apache/tajo/engine/parser/SQLParser.g4 | 3 +- .../tajo/engine/parser/HiveQLAnalyzer.java | 7 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 6 +- .../tajo/engine/planner/LogicalPlanner.java | 31 +++- .../tajo/engine/parser/TestHiveQLAnalyzer.java | 13 ++ .../tajo/engine/parser/TestSQLAnalyzer.java | 9 + .../tajo/engine/query/TestCreateTable.java | 176 +++++++++++++++++++ .../queries/default/create_table_like_1.sql | 1 + 10 files changed, 255 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 8107f81..66b5641 100644 --- a/CHANGES +++ b/CHANGES @@ -15,6 +15,8 @@ Release 0.9.0 - unreleased IMPROVEMENT + TAJO-699: Create a table using LIKE. (Prafulla T via hyunsik) + TAJO-825: Datetime type refactoring. (Hyoungjun Kim via jihoon) TAJO-811: add simple fifo scheduler support. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java index f60b571..c74677d 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java @@ -47,6 +47,8 @@ public class CreateTable extends Expr { private PartitionMethodDescExpr partition; @Expose @SerializedName("IfNotExists") private boolean ifNotExists; + @Expose @SerializedName("LikeParentTable") + private String likeParentTable; public CreateTable(final String tableName, boolean ifNotExists) { super(OpType.CreateTable); @@ -147,6 +149,15 @@ public class CreateTable extends Expr { return ifNotExists; } + public void setLikeParentTable(String parentTable) { + this.likeParentTable = parentTable; + } + + public String getLikeParentTableName() { + return likeParentTable; + } + + @Override public int hashCode() { return Objects.hashCode( http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 0076794..9570457 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -94,6 +94,7 @@ create_table_statement (param_clause)? (table_partitioning_clauses)? (AS query_expression)? | CREATE TABLE (if_not_exists)? table_name (USING file_type=identifier)? (param_clause)? (table_partitioning_clauses)? AS query_expression + | CREATE TABLE (if_not_exists)? table_name LIKE like_table_name=table_name ; table_elements @@ -804,7 +805,7 @@ boolean_primary ; boolean_predicand - : parenthesized_boolean_value_expression + : parenthesized_boolean_value_expression | nonparenthesized_value_expression_primary ; http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java index de4b159..2cec22b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java @@ -1425,6 +1425,11 @@ public class HiveQLAnalyzer extends HiveQLParserBaseVisitor<Expr> { createTable.setExternal(); } + if(ctx.KW_LIKE() != null) { + createTable.setLikeParentTable(ctx.likeName.getText()); + return createTable; + } + if (ctx.tableFileFormat() != null) { if (ctx.tableFileFormat().KW_RCFILE() != null) { createTable.setStorageType("rcfile"); @@ -1548,4 +1553,4 @@ public class HiveQLAnalyzer extends HiveQLParserBaseVisitor<Expr> { return Character.toUpperCase((char) returnChar); } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index e7de8f6..fe65d47 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1006,8 +1006,12 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { @Override public Expr visitCreate_table_statement(SQLParser.Create_table_statementContext ctx) { - String tableName = ctx.table_name().getText(); + String tableName = ctx.table_name(0).getText(); CreateTable createTable = new CreateTable(tableName, checkIfExist(ctx.if_not_exists())); + if(checkIfExist(ctx.LIKE())) { + createTable.setLikeParentTable(ctx.like_table_name.getText()); + return createTable; + } if (checkIfExist(ctx.EXTERNAL())) { createTable.setExternal(); http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 0780d4f..76dae24 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -1315,6 +1315,33 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return dropDatabaseNode; } + public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, CreateTableNode createTableNode) + throws PlanningException { + String parentTableName = expr.getLikeParentTableName(); + + if (CatalogUtil.isFQTableName(parentTableName) == false) { + parentTableName = + CatalogUtil.buildFQName(context.session.getCurrentDatabase(), + parentTableName); + } + TableDesc parentTableDesc = catalog.getTableDesc(parentTableName); + if(parentTableDesc == null) + throw new PlanningException("Table '"+parentTableName+"' does not exist"); + PartitionMethodDesc partitionDesc = parentTableDesc.getPartitionMethod(); + createTableNode.setTableSchema(parentTableDesc.getSchema()); + createTableNode.setPartitionMethod(partitionDesc); + + createTableNode.setStorageType(parentTableDesc.getMeta().getStoreType()); + createTableNode.setOptions(parentTableDesc.getMeta().getOptions()); + + createTableNode.setExternal(parentTableDesc.isExternal()); + if(parentTableDesc.isExternal()) { + createTableNode.setPath(parentTableDesc.getPath()); + } + return createTableNode; + } + + @Override public LogicalNode visitCreateTable(PlanContext context, Stack<Expr> stack, CreateTable expr) throws PlanningException { @@ -1329,7 +1356,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex createTableNode.setTableName( CatalogUtil.buildFQName(context.session.getCurrentDatabase(), expr.getTableName())); } - + // This is CREATE TABLE <tablename> LIKE <parentTable> + if(expr.getLikeParentTableName() != null) + return handleCreateTableLike(context, expr, createTableNode); if (expr.hasStorageType()) { // If storage type (using clause) is specified createTableNode.setStorageType(CatalogUtil.getStoreType(expr.getStorageType())); http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java index ef21dc3..45b5f44 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java @@ -19,12 +19,15 @@ package org.apache.tajo.engine.parser; import com.google.common.base.Preconditions; + import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CommonTokenStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.engine.parser.SQLParser.SqlContext; +import org.apache.tajo.algebra.CreateTable; import org.apache.tajo.algebra.Expr; +import org.apache.tajo.algebra.OpType; import org.apache.tajo.util.FileUtil; import org.junit.Test; @@ -268,4 +271,14 @@ public class TestHiveQLAnalyzer { public void testDrop() throws IOException { compareJsonResult("drop_table.sql"); } + + @Test + public void testCreateTableLike1() throws IOException { + String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_like_1.sql")); + Expr expr = parseQuery(sql); + assertEquals(OpType.CreateTable, expr.getType()); + CreateTable createTable = (CreateTable) expr; + assertEquals("orig_name", createTable.getLikeParentTableName()); + } + } http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index 2010502..5423813 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -270,6 +270,15 @@ public class TestSQLAnalyzer { } @Test + public void testCreateTableLike1() throws IOException { + String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_like_1.sql")); + Expr expr = parseQuery(sql); + assertEquals(OpType.CreateTable, expr.getType()); + CreateTable createTable = (CreateTable) expr; + assertEquals("orig_name", createTable.getLikeParentTableName()); + } + + @Test public void testCreateTablePartitionByHash1() throws IOException { String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_partition_by_hash_1.sql")); Expr expr = parseQuery(sql); http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index 2d289ba..0655700 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -23,9 +23,14 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.util.KeyValueSet; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -34,6 +39,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; @Category(IntegrationTest.class) public class TestCreateTable extends QueryTestCaseBase { @@ -357,4 +363,174 @@ public class TestCreateTable extends QueryTestCaseBase { createdNames = executeDDL("table1_ddl.sql", "table1", "varchar"); assertTableExists(createdNames.get(0)); } + + private boolean isClonedSchema(Schema origSchema, Schema newSchema) { + // Check schema of tables + boolean schemaEqual = + (origSchema.size() == newSchema.size()); + if(schemaEqual == false) { + fail("Number of columns in schema not equal"); + return false; + } + + for(int col = 0; col < origSchema.size(); col++) { + Column colA = origSchema.getColumn(col); + Column colB = newSchema.getColumn(col); + if(colA.getSimpleName().equals(colB.getSimpleName()) == false) { + fail("Column names at index " + col + " do not match"); + return false; + } + if(colA.getDataType().equals(colB.getDataType()) == false) { + fail("Column datatypes at index " + col + " do not match"); + return false; + } + } + return true; + } + + private boolean isClonedTable(String orignalTable, String newTable) throws Exception { + assertTableExists(newTable); + TableDesc origTableDesc = client.getTableDesc(orignalTable); + TableDesc newTableDesc = client.getTableDesc(newTable); + + if(isClonedSchema(origTableDesc.getSchema(), newTableDesc.getSchema()) == false) { + fail("Schema of input tables do not match"); + return false; + } + + // Check partition information + PartitionMethodDesc origPartMethod = origTableDesc.getPartitionMethod(); + PartitionMethodDesc newPartMethod = newTableDesc.getPartitionMethod(); + if(origPartMethod != null) { + if(newPartMethod == null) { + fail("New table does not have partition info"); + return false; + } + if(isClonedSchema(origPartMethod.getExpressionSchema(), + newPartMethod.getExpressionSchema()) == false) { + fail("Partition columns of input tables do not match"); + return false; + } + + if(origPartMethod.getPartitionType().equals(newPartMethod.getPartitionType()) == false) { + fail("Partition type of input tables do not match"); + return false; + } + } + + // Check external flag + if(origTableDesc.isExternal() != newTableDesc.isExternal()) { + fail("External table flag on input tables not equal"); + return false; + } + + if(origTableDesc.getMeta() != null) { + TableMeta origMeta = origTableDesc.getMeta(); + TableMeta newMeta = newTableDesc.getMeta(); + if(origMeta.getStoreType().equals(newMeta.getStoreType()) == false) { + fail("Store type of input tables not equal"); + return false; + } + + KeyValueSet origOptions = origMeta.getOptions(); + KeyValueSet newOptions = newMeta.getOptions(); + if(origOptions.equals(newOptions) == false) { + fail("Meta options of input tables not equal"); + return false; + } + } + return true; + } + + @Test + public final void testCreateTableLike1() throws Exception { + // Basic create table with default database + executeString("CREATE TABLE table1 (c1 int, c2 varchar);").close(); + executeString("CREATE TABLE table2 LIKE table1"); + String testMsg = "testCreateTableLike1: Basic create table with default db"; + assertTrue(testMsg,isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + // Basic create table with database + executeString("CREATE DATABASE d1").close(); + executeString("CREATE TABLE d1.table1 (c1 int, c2 varchar);").close(); + executeString("CREATE TABLE d1.table2 LIKE d1.table1"); + testMsg = "testCreateTableLike1: Basic create table with db test failed"; + assertTrue(testMsg, isClonedTable("d1.table1","d1.table2")); + executeString("DROP TABLE d1.table1"); + executeString("DROP TABLE d1.table2"); + + // Table with non-default store type + executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING rcfile;").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table with non-default store type test failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + // Table with non-default meta options + executeString("CREATE TABLE table1 (c1 int, c2 varchar) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec');").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table with non-default meta options test failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + + // Table with partitions (default partition type) + executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int, c4 float, c5 text);").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table with partitions test failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + + + // Table with external flag + // Use existing file as input for creating external table + String className = getClass().getSimpleName(); + Path currentDatasetPath = new Path(datasetBasePath, className); + Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1"); + executeString("CREATE EXTERNAL TABLE table3 (c1 int, c2 varchar) USING rcfile LOCATION '" + filePath.toUri() + "'").close(); + executeString("CREATE TABLE table2 LIKE table3"); + testMsg = "testCreateTableLike1: Table with external table flag test failed"; + assertTrue(testMsg, isClonedTable("table3","table2")); + executeString("DROP TABLE table3"); + executeString("DROP TABLE table2"); + + + // Table created using CTAS + executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close(); + executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2,c3 from table3;").close(); + executeString("CREATE TABLE table2 LIKE table4"); + testMsg = "testCreateTableLike1: Table using CTAS test failed"; + assertTrue(testMsg, isClonedTable("table4","table2")); + executeString("DROP TABLE table3"); + executeString("DROP TABLE table4"); + executeString("DROP TABLE table2"); + + + /* Enable when view is supported + // View + executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close(); + executeString("CREATE VIEW table4(c1,c2,c3) AS SELECT c1*c1, c2, c2,c3 from table3;").close(); + executeString("CREATE TABLE table2 LIKE table4"); + testMsg = "testCreateTableLike1: Table using VIEW test failed"; + assertTrue(testMsg, isClonedTable("table4","table2")); + executeString("DROP TABLE table3"); + executeString("DROP TABLE table4"); + executeString("DROP TABLE table2"); + */ + + /* Enable when partition type other than column is supported + // Table with partitions (range partition) + executeString("CREATE TABLE table1 (c1 int, c2 varchar) PARTITION BY RANGE (c1) ( PARTITION c1 VALUES LESS THAN (2), PARTITION c1 VALUES LESS THAN (5), PARTITION c1 VALUES LESS THAN (MAXVALUE) );").close(); + executeString("CREATE TABLE table2 LIKE table1"); + testMsg = "testCreateTableLike1: Table using non-default partition type failed"; + assertTrue(testMsg, isClonedTable("table1","table2")); + executeString("DROP TABLE table1"); + executeString("DROP TABLE table2"); + */ + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/a191b1dc/tajo-core/src/test/resources/queries/default/create_table_like_1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/default/create_table_like_1.sql b/tajo-core/src/test/resources/queries/default/create_table_like_1.sql new file mode 100644 index 0000000..73f2f46 --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/create_table_like_1.sql @@ -0,0 +1 @@ +create table new_table like orig_name;
