TAJO-1421: Add 'ALTER TABLE SET PROPERTY' statement. Closes #450
Signed-off-by: JaeHwa Jung <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/dd433734 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/dd433734 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/dd433734 Branch: refs/heads/index_support Commit: dd43373418b1857a1f694a08c06da77a86d3cfeb Parents: b0abff8 Author: Yongjin Choi <[email protected]> Authored: Sun Apr 5 22:13:25 2015 +0900 Committer: JaeHwa Jung <[email protected]> Committed: Sun Apr 5 22:13:25 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../org/apache/tajo/algebra/AlterTable.java | 24 ++++++- .../apache/tajo/algebra/AlterTableOpType.java | 2 +- .../org/apache/tajo/catalog/AlterTableDesc.java | 30 ++++++++ .../org/apache/tajo/catalog/AlterTableType.java | 2 +- .../org/apache/tajo/catalog/CatalogUtil.java | 8 +++ .../src/main/proto/CatalogProtos.proto | 2 + .../tajo/catalog/store/HCatalogStore.java | 3 + .../tajo/catalog/store/AbstractDBStore.java | 72 ++++++++++++++++++++ .../org/apache/tajo/catalog/store/MemStore.java | 16 +++++ .../org/apache/tajo/catalog/TestCatalog.java | 19 ++++++ .../org/apache/tajo/engine/parser/SQLLexer.g4 | 1 + .../org/apache/tajo/engine/parser/SQLParser.g4 | 9 +++ .../apache/tajo/engine/parser/SQLAnalyzer.java | 23 +++++++ .../apache/tajo/master/exec/DDLExecutor.java | 3 + .../java/org/apache/tajo/QueryTestCaseBase.java | 5 ++ .../tajo/engine/parser/TestSQLAnalyzer.java | 39 +++++++++++ .../tajo/engine/query/TestAlterTable.java | 16 ++++- .../resources/dataset/TestAlterTable/table2.tbl | 1 + .../alter_table_set_property_delimiter.sql | 1 + .../alter_table_set_property_timezone.sql | 1 + .../queries/TestAlterTable/table2_ddl.sql | 1 + .../testAlterTableSetProperty.sql | 1 + .../testAlterTableSetProperty2.sql | 1 + .../default/alter_table_set_property_1.sql | 1 + .../default/alter_table_set_property_2.sql | 1 + .../default/alter_table_set_property_3.sql | 1 + .../after_set_property_delimiter.result | 3 + .../before_set_property_delimiter.result | 3 + .../org/apache/tajo/plan/LogicalPlanner.java | 1 + .../tajo/plan/logical/AlterTableNode.java | 16 +++++ .../plan/serder/LogicalNodeDeserializer.java | 3 + .../tajo/plan/serder/LogicalNodeSerializer.java | 4 ++ tajo-plan/src/main/proto/Plan.proto | 2 + 34 files changed, 313 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 3def16c..e8a2c1f 100644 --- a/CHANGES +++ b/CHANGES @@ -11,6 +11,9 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1421: Add 'ALTER TABLE SET PROPERTY' statement. + (Contributed by Yongjin Choi. Committed by jaehwa) + TAJO-1501: Too many log message of HashShuffleAppenderManager. (Contributed by Jongyoung Park. Committed by jaehwa) http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java index dc68fc1..3ca9a7b 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java @@ -24,6 +24,9 @@ import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; import org.apache.tajo.util.TUtil; +import java.util.HashMap; +import java.util.Map; + public class AlterTable extends Expr { @Expose @SerializedName("OldTableName") @@ -38,6 +41,8 @@ public class AlterTable extends Expr { private ColumnDefinition addNewColumn; @Expose @SerializedName("AlterTableType") private AlterTableOpType alterTableOpType; + @Expose @SerializedName("TableProperties") + private Map<String, String> params; @Expose @SerializedName("Columns") ColumnReferenceExpr [] columns; @@ -112,6 +117,18 @@ public class AlterTable extends Expr { public void setLocation(String location) { this.location = location; } + public boolean hasParams() { + return params != null; + } + + public Map<String, String> getParams() { + return params; + } + + public void setParams(Map<String, String> params) { + this.params = params; + } + @Override public int hashCode() { return Objects.hashCode(tableName, @@ -122,7 +139,8 @@ public class AlterTable extends Expr { null != alterTableOpType ? Objects.hashCode(alterTableOpType) : alterTableOpType, null != columns ? Objects.hashCode(columns) : columns, null != values ? Objects.hashCode(values) : values, - null != location ? Objects.hashCode(location) : location + null != location ? Objects.hashCode(location) : location, + null != params ? Objects.hashCode(params) : params ); } @@ -138,7 +156,8 @@ public class AlterTable extends Expr { TUtil.checkEquals(alterTableOpType, another.alterTableOpType) && TUtil.checkEquals(columns, another.columns) && TUtil.checkEquals(values, another.values) && - TUtil.checkEquals(location, another.location) + TUtil.checkEquals(location, another.location) && + TUtil.checkEquals(params, another.params) ; } @@ -154,6 +173,7 @@ public class AlterTable extends Expr { alter.columns = columns; alter.values = values; alter.location = location; + alter.params = new HashMap<String, String>(params); return alter; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java index 5d4f381..679ab4b 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java @@ -18,5 +18,5 @@ package org.apache.tajo.algebra; public enum AlterTableOpType { - RENAME_TABLE, RENAME_COLUMN, ADD_COLUMN, ADD_PARTITION, DROP_PARTITION + RENAME_TABLE, RENAME_COLUMN, ADD_COLUMN, ADD_PARTITION, DROP_PARTITION, SET_PROPERTY } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java index f1265fb..9133b6f 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java @@ -25,6 +25,7 @@ import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.util.KeyValueSet; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTableDescProto; @@ -44,10 +45,16 @@ public class AlterTableDesc implements ProtoObject<AlterTableDescProto>, GsonObj protected Column addColumn = null; //optional @Expose protected PartitionDesc partitionDesc; //optional + @Expose + protected KeyValueSet properties; public AlterTableDesc() { + init(); } + private void init() { + this.properties = new KeyValueSet(); + } public String getTableName() { return tableName; @@ -101,6 +108,22 @@ public class AlterTableDesc implements ProtoObject<AlterTableDescProto>, GsonObj public void setPartitionDesc(PartitionDesc partitionDesc) { this.partitionDesc = partitionDesc; } + public void setProperties(KeyValueSet properties) { + this.properties = properties; + } + + public KeyValueSet getProperties() { + return properties; + } + + public void setProperty(String key, String value) { + this.properties.set(key, value); + } + + public String getProperty(String key) { + return this.properties.get(key); + } + @Override public String toString() { Gson gson = new GsonBuilder().setPrettyPrinting(). @@ -117,6 +140,7 @@ public class AlterTableDesc implements ProtoObject<AlterTableDescProto>, GsonObj newAlter.columnName = newColumnName; newAlter.addColumn = addColumn; newAlter.partitionDesc = partitionDesc; + newAlter.properties = (KeyValueSet)properties.clone(); return newAlter; } @@ -144,6 +168,9 @@ public class AlterTableDesc implements ProtoObject<AlterTableDescProto>, GsonObj if (null != this.addColumn) { builder.setAddColumn(addColumn.getProto()); } + if (null != this.properties) { + builder.setParams(properties.getProto()); + } switch (alterTableType) { case RENAME_TABLE: @@ -155,6 +182,9 @@ public class AlterTableDesc implements ProtoObject<AlterTableDescProto>, GsonObj case ADD_COLUMN: builder.setAlterTableType(CatalogProtos.AlterTableType.ADD_COLUMN); break; + case SET_PROPERTY: + builder.setAlterTableType(CatalogProtos.AlterTableType.SET_PROPERTY); + break; case ADD_PARTITION: builder.setAlterTableType(CatalogProtos.AlterTableType.ADD_PARTITION); break; http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableType.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableType.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableType.java index 7e3be91..a562d9f 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableType.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableType.java @@ -18,5 +18,5 @@ package org.apache.tajo.catalog; public enum AlterTableType { - RENAME_TABLE, RENAME_COLUMN, ADD_COLUMN, ADD_PARTITION, DROP_PARTITION + RENAME_TABLE, RENAME_COLUMN, ADD_COLUMN, ADD_PARTITION, DROP_PARTITION, SET_PROPERTY } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index afcff2d..628d710 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -764,6 +764,14 @@ public class CatalogUtil { return alterTableDesc; } + public static AlterTableDesc setProperty(String tableName, KeyValueSet params, AlterTableType alterTableType) { + final AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(tableName); + alterTableDesc.setProperties(params); + alterTableDesc.setAlterTableType(alterTableType); + return alterTableDesc; + } + /* It is the relationship graph of type conversions. */ public static final Map<Type, Map<Type, Type>> OPERATION_CASTING_MAP = Maps.newHashMap(); http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 3abd840..1402bbc 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -60,6 +60,7 @@ enum AlterTableType { ADD_COLUMN = 2; ADD_PARTITION = 3; DROP_PARTITION = 4; + SET_PROPERTY = 5; } message ColumnProto { @@ -357,6 +358,7 @@ message AlterTableDescProto { optional AlterColumnProto alterColumnName = 4; required AlterTableType alterTableType = 5; optional PartitionDescProto partitionDesc = 6; + optional KeyValueSetProto params = 7; } message AlterColumnProto { http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index 2761517..e829b94 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -646,6 +646,9 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { } dropPartition(databaseName, tableName, partitionDesc); break; + case SET_PROPERTY: + // TODO - not implemented yet + break; default: //TODO } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 518b499..1773033 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1015,6 +1015,9 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } dropPartition(tableId, alterTableDescProto.getPartitionDesc().getPartitionName()); break; + case SET_PROPERTY: + setProperties(tableId, alterTableDescProto.getParams()); + break; default: } } catch (SQLException sqlException) { @@ -1023,6 +1026,75 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo } + private Map<String, String> getTableOptions(final int tableId) throws CatalogException { + Connection conn = null; + PreparedStatement pstmt = null; + ResultSet res = null; + Map<String, String> options = TUtil.newHashMap(); + + try { + String tidSql = "SELECT key_, value_ FROM " + TB_OPTIONS + " WHERE TID=?"; + + conn = getConnection(); + pstmt = conn.prepareStatement(tidSql); + pstmt.setInt(1, tableId); + res = pstmt.executeQuery(); + + while (res.next()) { + options.put(res.getString("KEY_"), res.getString("VALUE_")); + } + } catch (SQLException se) { + throw new CatalogException(se); + } finally { + CatalogUtil.closeQuietly(pstmt, res); + } + + return options; + } + + private void setProperties(final int tableId, final KeyValueSetProto properties) { + final String updateSql = "UPDATE " + TB_OPTIONS + " SET VALUE_=? WHERE TID=? AND KEY_=?"; + final String insertSql = "INSERT INTO " + TB_OPTIONS + " (TID, KEY_, VALUE_) VALUES(?, ?, ?)"; + + Connection conn; + PreparedStatement pstmt = null; + + Map<String, String> oldProperties = getTableOptions(tableId); + + try { + conn = getConnection(); + conn.setAutoCommit(false); + + for (KeyValueProto entry : properties.getKeyvalList()) { + if (oldProperties.containsKey(entry.getKey())) { + // replace old property with new one + pstmt = conn.prepareStatement(updateSql); + + pstmt.setString(1, entry.getValue()); + pstmt.setInt(2, tableId); + pstmt.setString(3, entry.getKey()); + pstmt.executeUpdate(); + pstmt.close(); + } else { + // insert new property + pstmt = conn.prepareStatement(insertSql); + + pstmt.setInt(1, tableId); + pstmt.setString(2, entry.getKey()); + pstmt.setString(3, entry.getValue()); + pstmt.executeUpdate(); + pstmt.close(); + } + } + + conn.commit(); + } catch (SQLException sqlException) { + throw new CatalogException(sqlException); + } finally { + CatalogUtil.closeQuietly(pstmt); + } + } + private void renameTable(final int tableId, final String tableName) throws CatalogException { final String updtaeRenameTableSql = "UPDATE " + TB_TABLES + " SET " + COL_TABLES_NAME + " = ? " + " WHERE TID = ?"; http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 470f09d..b058504 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -41,6 +42,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto; +import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; import java.io.IOException; @@ -350,6 +352,20 @@ public class MemStore implements CatalogStore { partitions.remove(partitionName); } break; + case SET_PROPERTY: + KeyValueSet properties = new KeyValueSet(tableDescProto.getMeta().getParams()); + KeyValueSet newProperties = new KeyValueSet(alterTableDescProto.getParams()); + + for (String key : newProperties.getAllKeyValus().keySet()) { + if (properties.containsKey(key)) + properties.remove(key); + properties.set(key, newProperties.get(key)); + } + + TableMeta newMeta = new TableMeta(tableDescProto.getMeta().getStoreType(), properties); + newTableDescProto = tableDescProto.toBuilder().setMeta(newMeta.getProto()).build(); + database.put(tableName, newTableDescProto); + break; default: } } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 48a11ee..cef4027 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -995,6 +995,17 @@ public class TestCatalog { TableDesc addColumnDesc = catalog.getTableDesc("default","mynewcooltable"); assertTrue(addColumnDesc.getSchema().containsByName("mynewcol")); + //SET_PROPERTY + TableDesc setPropertyDesc = catalog.getTableDesc("default","mynewcooltable"); + KeyValueSet options = new KeyValueSet(); + options.set("timezone", "GMT+9"); // Seoul, Korea + setPropertyDesc.setMeta(new TableMeta(StoreType.CSV, options)); + String prevTimeZone = setPropertyDesc.getMeta().getOption("timezone"); + String newTimeZone = "GMT-7"; // Silicon Valley, California + catalog.alterTable(createMockAlterTableSetProperty(newTimeZone)); + setPropertyDesc = catalog.getTableDesc("default","mynewcooltable"); + assertNotEquals(prevTimeZone, setPropertyDesc.getMeta().getOption("timezone")); + assertEquals(newTimeZone, setPropertyDesc.getMeta().getOption("timezone")); } private AlterTableDesc createMockAlterTableName(){ @@ -1022,6 +1033,14 @@ public class TestCatalog { return alterTableDesc; } + private AlterTableDesc createMockAlterTableSetProperty(String newTimeZone) { + AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName("default.mynewcooltable"); + alterTableDesc.setProperty("timezone", newTimeZone); + alterTableDesc.setAlterTableType(AlterTableType.SET_PROPERTY); + return alterTableDesc; + } + public static class TestIntFunc extends Function { public TestIntFunc() { super( http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 index 6fccaad..d1daeb6 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 @@ -280,6 +280,7 @@ PERCENT_RANK : P E R C E N T UNDERLINE R A N K; PRECEDING : P R E C E D I N G; PRECISION : P R E C I S I O N; PURGE : P U R G E; +PROPERTY : P R O P E R T Y; QUARTER : Q U A R T E R; http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 9ac3f8c..1a63470 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -1604,6 +1604,7 @@ alter_table_statement | ALTER TABLE table_name ADD COLUMN field_element | ALTER TABLE table_name (if_not_exists)? ADD PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (LOCATION path=Character_String_Literal)? | ALTER TABLE table_name (if_exists)? DROP PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN + | ALTER TABLE table_name SET PROPERTY property_list ; partition_column_value_list @@ -1613,3 +1614,11 @@ partition_column_value_list partition_column_value : identifier EQUAL row_value_predicand ; + +property_list + : property (COMMA property)* + ; + +property + : key=Character_String_Literal EQUAL value=Character_String_Literal + ; http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 23c2eec..673b22f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1831,11 +1831,24 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { } } + if (checkIfExist(ctx.property_list())) { + alterTable.setParams(getProperties(ctx.property_list())); + } + alterTable.setAlterTableOpType(determineAlterTableType(ctx)); return alterTable; } + private Map<String, String> getProperties(SQLParser.Property_listContext ctx) { + Map<String, String> params = new HashMap<String, String>(); + for (int i = 0; i < ctx.property().size(); i++) { + params.put(stripQuote(ctx.property(i).key.getText()), stripQuote(ctx.property(i).value.getText())); + } + + return params; + } + private AlterTableOpType determineAlterTableType(SQLParser.Alter_table_statementContext ctx) { final int RENAME_MASK = 00000001; @@ -1844,6 +1857,8 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { final int ADD_MASK = 00001000; final int DROP_MASK = 00001001; final int PARTITION_MASK = 00000020; + final int SET_MASK = 00000002; + final int PROPERTY_MASK = 00010000; int val = 00000000; @@ -1869,6 +1884,12 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { case PARTITION: val = val | PARTITION_MASK; break; + case SET: + val = val | SET_MASK; + break; + case PROPERTY: + val = val | PROPERTY_MASK; + break; default: break; } @@ -1890,6 +1911,8 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { return AlterTableOpType.ADD_PARTITION; case 529: return AlterTableOpType.DROP_PARTITION; + case 4098: + return AlterTableOpType.SET_PROPERTY; default: return null; } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 2030b55..9264c68 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -421,6 +421,9 @@ public class DDLExecutor { } catalog.alterTable(CatalogUtil.addNewColumn(qualifiedName, alterTable.getAddNewColumn(), AlterTableType.ADD_COLUMN)); break; + case SET_PROPERTY: + catalog.alterTable(CatalogUtil.setProperty(qualifiedName, alterTable.getProperties(), AlterTableType.SET_PROPERTY)); + break; default: //TODO } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 4e104dd..727192a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -546,6 +546,11 @@ public class QueryTestCaseBase { return client.getTableDesc(tableName); } + public void assertTablePropertyEquals(String tableName, String key, String expectedValue) throws ServiceException { + TableDesc tableDesc = fetchTableMetaData(tableName); + assertEquals(expectedValue, tableDesc.getMeta().getOption(key)); + } + /** * It transforms a ResultSet instance to rows represented as strings. * http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index d223554..f698ed5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -485,6 +485,45 @@ public class TestSQLAnalyzer { } @Test + public void testAlterTableSetProperty1() throws IOException { + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_set_property_1.sql"); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.SET_PROPERTY); + assertTrue(alterTable.hasParams()); + assertTrue(alterTable.getParams().containsKey("timezone")); + assertEquals("GMT-7", alterTable.getParams().get("timezone")); + } + + @Test + public void testAlterTableSetProperty2() throws IOException { + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_set_property_2.sql"); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.SET_PROPERTY); + assertTrue(alterTable.hasParams()); + assertTrue(alterTable.getParams().containsKey("text.delimiter")); + assertEquals("&", alterTable.getParams().get("text.delimiter")); + } + + @Test + public void testAlterTableSetProperty3() throws IOException { + // update multiple table properties with a single 'SET PROPERTY' sql + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_set_property_3.sql"); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.SET_PROPERTY); + assertTrue(alterTable.hasParams()); + assertTrue(alterTable.getParams().containsKey("compression.type")); + assertEquals("RECORD", alterTable.getParams().get("compression.type")); + assertTrue(alterTable.getParams().containsKey("compression.codec")); + assertEquals("org.apache.hadoop.io.compress.SnappyCodec", alterTable.getParams().get("compression.codec")); + } + + @Test public void testTableSubQuery1() throws IOException { String sql = FileUtil.readTextFileFromResource("queries/default/table_subquery1.sql"); parseQuery(sql); http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 57d8b32..44fa1f3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -23,6 +23,7 @@ import org.apache.tajo.QueryTestCaseBase; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.sql.ResultSet; import java.util.List; @Category(IntegrationTest.class) @@ -39,7 +40,7 @@ public class TestAlterTable extends QueryTestCaseBase { public final void testAlterTableColumnName() throws Exception { List<String> createdNames = executeDDL("table1_ddl.sql", "table1.tbl", "XYZ"); executeDDL("alter_table_rename_column_ddl.sql", null); - assertColumnExists(createdNames.get(0),"renum"); + assertColumnExists(createdNames.get(0), "renum"); } @Test @@ -49,4 +50,17 @@ public class TestAlterTable extends QueryTestCaseBase { assertColumnExists(createdNames.get(0),"cool"); } + @Test + public final void testAlterTableSetProperty() throws Exception { + executeDDL("table2_ddl.sql", "table2.tbl", "ALTX"); + ResultSet before_res = executeQuery(); + assertResultSet(before_res, "before_set_property_delimiter.result"); + cleanupQuery(before_res); + + executeDDL("alter_table_set_property_delimiter.sql", null); + + ResultSet after_res = executeQuery(); + assertResultSet(after_res, "after_set_property_delimiter.result"); + cleanupQuery(after_res); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/dataset/TestAlterTable/table2.tbl ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/dataset/TestAlterTable/table2.tbl b/tajo-core/src/test/resources/dataset/TestAlterTable/table2.tbl new file mode 100644 index 0000000..59f652e --- /dev/null +++ b/tajo-core/src/test/resources/dataset/TestAlterTable/table2.tbl @@ -0,0 +1 @@ +aaa+AAA|bbb+BBB|ccc http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_set_property_delimiter.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_set_property_delimiter.sql b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_set_property_delimiter.sql new file mode 100644 index 0000000..031e700 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_set_property_delimiter.sql @@ -0,0 +1 @@ +ALTER TABLE ALTX SET PROPERTY 'text.delimiter'='|'; http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_set_property_timezone.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_set_property_timezone.sql b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_set_property_timezone.sql new file mode 100644 index 0000000..24b237e --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_set_property_timezone.sql @@ -0,0 +1 @@ +ALTER TABLE ALTY SET PROPERTY 'timezone'='GMT-7'; http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/queries/TestAlterTable/table2_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/table2_ddl.sql b/tajo-core/src/test/resources/queries/TestAlterTable/table2_ddl.sql new file mode 100644 index 0000000..ae563e7 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/table2_ddl.sql @@ -0,0 +1 @@ +CREATE EXTERNAL TABLE ${0} (xx text, yy text, zz text) USING CSV WITH('text.delimiter'='+') LOCATION ${table.path}; http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/queries/TestAlterTable/testAlterTableSetProperty.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/testAlterTableSetProperty.sql b/tajo-core/src/test/resources/queries/TestAlterTable/testAlterTableSetProperty.sql new file mode 100644 index 0000000..dd0bd83 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/testAlterTableSetProperty.sql @@ -0,0 +1 @@ +select * from ALTX; http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/queries/TestAlterTable/testAlterTableSetProperty2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/testAlterTableSetProperty2.sql b/tajo-core/src/test/resources/queries/TestAlterTable/testAlterTableSetProperty2.sql new file mode 100644 index 0000000..2e854aa --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/testAlterTableSetProperty2.sql @@ -0,0 +1 @@ +select * from ALTY; http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/queries/default/alter_table_set_property_1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/default/alter_table_set_property_1.sql b/tajo-core/src/test/resources/queries/default/alter_table_set_property_1.sql new file mode 100644 index 0000000..ac7aa4d --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_set_property_1.sql @@ -0,0 +1 @@ +ALTER TABLE table1 SET PROPERTY 'timezone' = 'GMT-7' http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/queries/default/alter_table_set_property_2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/default/alter_table_set_property_2.sql b/tajo-core/src/test/resources/queries/default/alter_table_set_property_2.sql new file mode 100644 index 0000000..77df213 --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_set_property_2.sql @@ -0,0 +1 @@ +ALTER TABLE table1 SET PROPERTY 'text.delimiter' = '&' http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/queries/default/alter_table_set_property_3.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/default/alter_table_set_property_3.sql b/tajo-core/src/test/resources/queries/default/alter_table_set_property_3.sql new file mode 100644 index 0000000..6230a34 --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_set_property_3.sql @@ -0,0 +1 @@ +ALTER TABLE table1 SET PROPERTY 'compression.type'='RECORD','compression.codec'='org.apache.hadoop.io.compress.SnappyCodec' http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/results/TestAlterTable/after_set_property_delimiter.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestAlterTable/after_set_property_delimiter.result b/tajo-core/src/test/resources/results/TestAlterTable/after_set_property_delimiter.result new file mode 100644 index 0000000..6729298 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestAlterTable/after_set_property_delimiter.result @@ -0,0 +1,3 @@ +xx,yy,zz +------------------------------- +aaa+AAA,bbb+BBB,ccc http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-core/src/test/resources/results/TestAlterTable/before_set_property_delimiter.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestAlterTable/before_set_property_delimiter.result b/tajo-core/src/test/resources/results/TestAlterTable/before_set_property_delimiter.result new file mode 100644 index 0000000..af74e40 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestAlterTable/before_set_property_delimiter.result @@ -0,0 +1,3 @@ +xx,yy,zz +------------------------------- +aaa,AAA|bbb,BBB|ccc http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 7edf74a..d6a04a6 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1928,6 +1928,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex alterTableNode.setNewTableName(alterTable.getNewTableName()); alterTableNode.setColumnName(alterTable.getColumnName()); alterTableNode.setNewColumnName(alterTable.getNewColumnName()); + alterTableNode.setProperties(new KeyValueSet(alterTable.getParams())); if (null != alterTable.getAddNewColumn()) { alterTableNode.setAddNewColumn(convertColumn(alterTable.getAddNewColumn())); http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java index 69cff54..9cbde0c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java @@ -24,6 +24,7 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.algebra.AlterTableOpType; import org.apache.tajo.catalog.Column; import org.apache.tajo.plan.PlanString; +import org.apache.tajo.util.KeyValueSet; public class AlterTableNode extends LogicalNode { @@ -38,6 +39,8 @@ public class AlterTableNode extends LogicalNode { @Expose private Column addNewColumn; @Expose + private KeyValueSet properties = new KeyValueSet(); + @Expose private AlterTableOpType alterTableOpType; public AlterTableNode(int pid) { @@ -102,6 +105,18 @@ public class AlterTableNode extends LogicalNode { this.alterTableOpType = alterTableOpType; } + public boolean hasProperties() { + return this.properties != null; + } + + public KeyValueSet getProperties() { + return this.properties; + } + + public void setProperties(KeyValueSet properties) { + this.properties = properties; + } + @Override public PlanString getPlanString() { return new PlanString(this); @@ -117,6 +132,7 @@ public class AlterTableNode extends LogicalNode { result = prime * result + ((newColumnName == null) ? 0 : newColumnName.hashCode()); result = prime * result + ((newTableName == null) ? 0 : newTableName.hashCode()); result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); + result = prime * result + ((properties == null) ? 0 : properties.hashCode()); return result; } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index 3387157..f96626b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -583,6 +583,9 @@ public class LogicalNodeDeserializer { alterTable.setColumnName(alterTableProto.getRenameColumn().getOldName()); alterTable.setNewColumnName(alterTableProto.getRenameColumn().getNewName()); break; + case SET_PROPERTY: + alterTable.setProperties(new KeyValueSet(alterTableProto.getProperties())); + break; default: throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + alterTableProto.getSetType().name()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index 1bde955..60171de 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -567,6 +567,10 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe .setOldName(node.getColumnName()) .setNewName(node.getNewColumnName())); break; + case SET_PROPERTY: + alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.SET_PROPERTY); + alterTableBuilder.setProperties(node.getProperties().getProto()); + break; default: throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + node.getAlterTableOpType().name()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/dd433734/tajo-plan/src/main/proto/Plan.proto ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 02f52ff..c555a1a 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -280,6 +280,7 @@ message AlterTableNode { RENAME_TABLE = 0; RENAME_COLUMN = 1; ADD_COLUMN = 2; + SET_PROPERTY = 3; } message RenameTable { @@ -300,6 +301,7 @@ message AlterTableNode { optional RenameTable renameTable = 3; optional RenameColumn renameColumn = 4; optional AddColumn addColumn = 5; + optional KeyValueSetProto properties = 6; } enum EvalType {
