TAJO-1345: Implement logical plan part and DDL executor for alter partition. (jaehwa)
Closes #618 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3dba8c7e Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3dba8c7e Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3dba8c7e Branch: refs/heads/index_support Commit: 3dba8c7e6bc539b07b136b959aa91dc4402be380 Parents: 92e3bc1 Author: JaeHwa Jung <[email protected]> Authored: Fri Jul 17 14:48:44 2015 +0900 Committer: JaeHwa Jung <[email protected]> Committed: Fri Jul 17 14:48:44 2015 +0900 ---------------------------------------------------------------------- CHANGES | 5 +- .../org/apache/tajo/algebra/AlterTable.java | 27 ++-- .../org/apache/tajo/catalog/CatalogUtil.java | 71 ++++++++- ...xistsAssumedPartitionDirectoryException.java | 28 ++++ .../AlreadyExistsPartitionException.java | 6 +- .../exception/NoPartitionedTableException.java | 2 +- .../exception/NoSuchPartitionException.java | 12 +- .../exception/NoSuchPartitionKeyException.java | 29 ++++ .../tajo/catalog/partition/PartitionDesc.java | 50 ++----- .../tajo/catalog/partition/PartitionKey.java | 147 ------------------- .../catalog/store/TestHiveCatalogStore.java | 10 +- .../org/apache/tajo/catalog/store/MemStore.java | 6 +- .../org/apache/tajo/catalog/TestCatalog.java | 12 +- .../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 1 + .../apache/tajo/master/exec/DDLExecutor.java | 111 +++++++++++++- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 30 ++-- .../org/apache/tajo/client/TestTajoClient.java | 62 ++++---- .../tajo/engine/parser/TestSQLAnalyzer.java | 4 + .../tajo/engine/planner/TestLogicalPlanner.java | 102 +++++++++++++ .../tajo/engine/query/TestAlterTable.java | 47 ++++++ .../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 13 +- .../alter_table_add_partition1.sql | 1 + .../alter_table_drop_partition1.sql | 1 + .../TestAlterTable/create_partitioned_table.sql | 1 + .../default/alter_table_drop_partition_3.sql | 2 +- .../testAlterTableAddDropPartition.result | 9 ++ .../testAlterTableAddPartition.result | 2 - .../testAlterTableDropPartition.result | 2 - tajo-docs/src/main/sphinx/sql_language.rst | 3 +- .../main/sphinx/sql_language/alter_table.rst | 100 +++++++++++++ .../org/apache/tajo/plan/LogicalPlanner.java | 50 +++++++ .../tajo/plan/logical/AlterTableNode.java | 53 +++++-- .../plan/serder/LogicalNodeDeserializer.java | 19 +++ .../tajo/plan/serder/LogicalNodeSerializer.java | 29 ++++ .../plan/verifier/PreLogicalPlanVerifier.java | 6 - tajo-plan/src/main/proto/Plan.proto | 10 ++ 37 files changed, 769 insertions(+), 296 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 1dc63b7..39f2617 100644 --- a/CHANGES +++ b/CHANGES @@ -324,7 +324,10 @@ Release 0.11.0 - unreleased TASKS - TAJO-1450: Encapsulate Datum in Tuple. (Contributed by navis, + TAJO-1345: Implement logical plan part and DDL executor + for alter partition. (jaehwa) + + TAJO-1450: Encapsulate Datum in Tuple. (Contributed by navis, Committed by jihoon) TAJO-1583: Remove ServerCallable in RPC client. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java index 9440257..260025f 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java @@ -51,6 +51,9 @@ public class AlterTable extends Expr { @Expose @SerializedName("location") private String location; + @Expose @SerializedName("IsPurge") + private boolean purge; + public AlterTable(final String tableName) { super(OpType.AlterTable); this.tableName = tableName; @@ -129,18 +132,18 @@ public class AlterTable extends Expr { this.params = params; } + public boolean isPurge() { + return purge; + } + + public void setPurge(boolean purge) { + this.purge = purge; + } + @Override public int hashCode() { - return Objects.hashCode(tableName, - null != newTableName ? Objects.hashCode(newTableName) : newTableName, - null != columnName ? Objects.hashCode(columnName) : columnName, - null != newColumnName ? Objects.hashCode(newColumnName) : newColumnName, - null != addNewColumn ? Objects.hashCode(addNewColumn) : addNewColumn, - null != alterTableOpType ? Objects.hashCode(alterTableOpType) : alterTableOpType, - null != columns ? Objects.hashCode(columns) : columns, - null != values ? Objects.hashCode(values) : values, - null != location ? Objects.hashCode(location) : location, - null != params ? Objects.hashCode(params) : params + return Objects.hashCode(tableName, newTableName, columnName, newColumnName, addNewColumn, alterTableOpType, + columns, values, location, params, purge ); } @@ -157,7 +160,8 @@ public class AlterTable extends Expr { TUtil.checkEquals(columns, another.columns) && TUtil.checkEquals(values, another.values) && TUtil.checkEquals(location, another.location) && - TUtil.checkEquals(params, another.params) + TUtil.checkEquals(params, another.params) && + TUtil.checkEquals(purge, another.purge) ; } @@ -178,6 +182,7 @@ public class AlterTable extends Expr { if (params != null) { alter.params = new HashMap<String, String>(params); } + alter.purge = purge; return alter; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 638ebca..ecf8890 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -23,10 +23,12 @@ import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; import org.apache.tajo.DataTypeUtil; import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; @@ -40,10 +42,7 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import static org.apache.tajo.common.TajoDataTypes.Type; @@ -792,6 +791,70 @@ public class CatalogUtil { return alterTableDesc; } + /** + * Converts passed parameters to a AlterTableDesc. This method would be called when adding a partition or dropping + * a table. This creates AlterTableDesc that is a wrapper class for protocol buffer. + * + * @param tableName table name + * @param columns partition column names + * @param values partition values + * @param location partition location + * @param alterTableType ADD_PARTITION or DROP_PARTITION + * @return AlterTableDesc + */ + public static AlterTableDesc addOrDropPartition(String tableName, String[] columns, + String[] values, String location, AlterTableType alterTableType) { + final AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(tableName); + + PartitionDesc partitionDesc = new PartitionDesc(); + Pair<List<PartitionKeyProto>, String> pair = getPartitionKeyNamePair(columns, values); + + partitionDesc.setPartitionKeys(pair.getFirst()); + partitionDesc.setPartitionName(pair.getSecond()); + + if (alterTableType.equals(AlterTableType.ADD_PARTITION) && location != null) { + partitionDesc.setPath(location); + } + + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(alterTableType); + return alterTableDesc; + } + + /** + * Get partition key/value list and partition name + * + * ex) partition key/value list : + * - col1, 2015-07-01 + * - col2, tajo + * partition name : col1=2015-07-01/col2=tajo + * + * @param columns partition column names + * @param values partition values + * @return partition key/value list and partition name + */ + public static Pair<List<PartitionKeyProto>, String> getPartitionKeyNamePair(String[] columns, String[] values) { + Pair<List<PartitionKeyProto>, String> pair = null; + List<PartitionKeyProto> partitionKeyList = TUtil.newList(); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < columns.length; i++) { + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + builder.setColumnName(columns[i]); + builder.setPartitionValue(values[i]); + + if (i > 0) { + sb.append("/"); + } + sb.append(columns[i]).append("=").append(values[i]); + partitionKeyList.add(builder.build()); + } + + pair = new Pair<List<PartitionKeyProto>, String>(partitionKeyList, sb.toString()); + return pair; + } + /* It is the relationship graph of type conversions. */ public static final Map<Type, Map<Type, Type>> OPERATION_CASTING_MAP = Maps.newHashMap(); http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java new file mode 100644 index 0000000..df13f82 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog.exception; + +public class AlreadyExistsAssumedPartitionDirectoryException extends RuntimeException { + + private static final long serialVersionUID = 277182608283894931L; + + public AlreadyExistsAssumedPartitionDirectoryException(String message) { + super(String.format("ERROR: There is a directory which is assumed to be a partitioned directory : %s", message)); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java index ab6144f..a7f0d87 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java @@ -26,8 +26,12 @@ public class AlreadyExistsPartitionException extends RuntimeException { super(message); } + public AlreadyExistsPartitionException(String tableName, String partitionName) { + super(String.format("ERROR: partition \"%s already exist in \"%s\"", partitionName, tableName)); + } + public AlreadyExistsPartitionException(String databaseName, String tableName, String partitionName) { - super(String.format("ERROR: \"%s already exist in \"%s.%s\"", partitionName, databaseName, tableName)); + super(String.format("ERROR: partition \"%s already exist in \"%s.%s\"", partitionName, databaseName, tableName)); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java index faa8bc4..eded789 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java @@ -23,6 +23,6 @@ public class NoPartitionedTableException extends Exception { public NoPartitionedTableException() {} public NoPartitionedTableException(String databaseName, String relName) { - super(String.format("ERROR: \"%s.%s\" is not a partitioned table", databaseName, relName)); + super(String.format("ERROR: table \"%s.%s\" is not a partitioned table", databaseName, relName)); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java index 45c9299..70e0d26 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java @@ -18,12 +18,6 @@ package org.apache.tajo.catalog.exception; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.function.FunctionUtil; -import org.codehaus.jackson.schema.JsonSerializableSchema; - -import java.util.Collection; - public class NoSuchPartitionException extends RuntimeException { private static final long serialVersionUID = 277182608283894938L; @@ -32,8 +26,12 @@ public class NoSuchPartitionException extends RuntimeException { super(message); } + public NoSuchPartitionException(String tableName, String partitionName) { + super(String.format("ERROR: \"%s\" is not the partition of \"%s\".", partitionName, tableName)); + } + public NoSuchPartitionException(String databaseName, String tableName, String partitionName) { - super(String.format("ERROR: \"%s\" does not exist in \"%s.%s\".", partitionName, databaseName, tableName)); + super(String.format("ERROR: \"%s\" is not the partition of \"%s.%s\".", partitionName, databaseName, tableName)); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java new file mode 100644 index 0000000..94574dc --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog.exception; + +public class NoSuchPartitionKeyException extends RuntimeException { + + private static final long serialVersionUID = 277182608283894939L; + + public NoSuchPartitionKeyException(String tableName, String partitionKey) { + super(String.format("ERROR: \"%s\" column is not the partition key of \"%s\".", + partitionKey, tableName)); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java index b6d883d..7287fce 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java @@ -22,13 +22,12 @@ import com.google.common.base.Objects; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; -import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; -import java.util.ArrayList; import java.util.List; /** @@ -55,36 +54,11 @@ import java.util.List; */ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescProto>, Cloneable, GsonObject { @Expose protected String partitionName; - @Expose protected List<PartitionKey> partitionKeys; + @Expose protected List<PartitionKeyProto> partitionKeys; @Expose protected String path; //optional private CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); - public PartitionDesc() { - } - - public PartitionDesc(PartitionDesc partition) { - this.partitionName = partition.partitionName; - this.partitionKeys = partition.partitionKeys; - this.path = partition.path; - } - - public PartitionDesc(CatalogProtos.PartitionDescProto proto) { - if(proto.hasPartitionName()) { - this.partitionName = proto.getPartitionName(); - } - - this.partitionKeys = new ArrayList<PartitionKey>(); - for(CatalogProtos.PartitionKeyProto keyProto : proto.getPartitionKeysList()) { - PartitionKey partitionKey = new PartitionKey(keyProto); - this.partitionKeys.add(partitionKey); - } - - if(proto.hasPath()) { - this.path = proto.getPath(); - } - } - public String getPartitionName() { return partitionName; } @@ -93,14 +67,6 @@ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescPro this.partitionName = partitionName; } - public List<PartitionKey> getPartitionKeys() { - return partitionKeys; - } - - public void setPartitionKeys(List<PartitionKey> partitionKeys) { - this.partitionKeys = partitionKeys; - } - public void setPath(String path) { this.path = path; } @@ -109,6 +75,14 @@ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescPro return path; } + public List<PartitionKeyProto> getPartitionKeys() { + return partitionKeys; + } + + public void setPartitionKeys(List<PartitionKeyProto> partitionKeys) { + this.partitionKeys = partitionKeys; + } + public int hashCode() { return Objects.hashCode(partitionName, partitionKeys, path); } @@ -142,8 +116,8 @@ public class PartitionDesc implements ProtoObject<CatalogProtos.PartitionDescPro builder.clearPartitionKeys(); if (this.partitionKeys != null) { - for(PartitionKey partitionKey : this.partitionKeys) { - builder.addPartitionKeys(partitionKey.getProto()); + for(PartitionKeyProto partitionKey : this.partitionKeys) { + builder.addPartitionKeys(partitionKey); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java deleted file mode 100644 index 085598b..0000000 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.catalog.partition; - -import com.google.common.base.Objects; -import com.google.gson.annotations.Expose; -import org.apache.tajo.catalog.json.CatalogGsonHelper; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.common.ProtoObject; -import org.apache.tajo.json.GsonObject; -import org.apache.tajo.util.TUtil; - - -/** - * This presents column name and partition value pairs of column partitioned table. - * - * For example, consider you have a partitioned table as follows: - * - * create external table table1 (id text, name text) PARTITION BY COLUMN (dt text, phone text, - * gender text) USING RCFILE LOCATION '/tajo/data/table1'; - * - * Then, its data will be stored on HDFS as follows: - * - /tajo/data/table1/dt=20150301/phone=1300/gender=m - * - /tajo/data/table1/dt=20150301/phone=1300/gender=f - * - /tajo/data/table1/dt=20150302/phone=1500/gender=m - * - /tajo/data/table1/dt=20150302/phone=1500/gender=f - * - * In such as above, first directory can be presented with this class as follows: - * The first pair: column name = dt, partition value = 20150301 - * The second pair: column name = phone, partition value = 1300 - * The thris pair: column name = gender, partition value = m - * - */ -public class PartitionKey implements ProtoObject<CatalogProtos.PartitionKeyProto>, Cloneable, GsonObject { - @Expose protected String columnName; // required - @Expose protected String partitionValue; // required - - private CatalogProtos.PartitionKeyProto.Builder builder = CatalogProtos.PartitionKeyProto.newBuilder(); - - public PartitionKey() { - } - - public PartitionKey(String columnName, String partitionValue) { - this.columnName = columnName; - this.partitionValue = partitionValue; - } - - public PartitionKey(PartitionKey partition) { - this.columnName = partition.columnName; - this.partitionValue = partition.partitionValue; - } - - public PartitionKey(CatalogProtos.PartitionKeyProto proto) { - if (proto.hasColumnName()) { - this.columnName = proto.getColumnName(); - } - if (proto.hasPartitionValue()) { - this.partitionValue = proto.getPartitionValue(); - } - } - - public String getPartitionValue() { - return partitionValue; - } - - public void setPartitionValue(String partitionValue) { - this.partitionValue = partitionValue; - } - - public String getColumnName() { - return columnName; - } - - public void setColumnName(String columnName) { - this.columnName = columnName; - } - - public int hashCode() { - return Objects.hashCode(partitionValue, columnName); - } - - public boolean equals(Object o) { - if (o instanceof PartitionKey) { - PartitionKey another = (PartitionKey) o; - return TUtil.checkEquals(columnName, another.columnName) && - TUtil.checkEquals(partitionValue, another.partitionValue); - } - return false; - } - - @Override - public CatalogProtos.PartitionKeyProto getProto() { - if (builder == null) { - builder = CatalogProtos.PartitionKeyProto.newBuilder(); - } - - if (this.columnName != null) { - builder.setColumnName(this.columnName); - } - - if (this.partitionValue != null) { - builder.setPartitionValue(this.partitionValue); - } - - return builder.build(); - } - - public String toString() { - StringBuilder sb = new StringBuilder("name: " + partitionValue); - return sb.toString(); - } - - @Override - public String toJson() { - return CatalogGsonHelper.toJson(this, PartitionKey.class); - } - - public static PartitionKey fromJson(String strVal) { - return strVal != null ? CatalogGsonHelper.fromJson(strVal, PartitionKey.class) : null; - } - - public Object clone() throws CloneNotSupportedException { - PartitionKey desc = (PartitionKey) super.clone(); - desc.builder = CatalogProtos.PartitionKeyProto.newBuilder(); - desc.partitionValue = partitionValue; - desc.columnName = columnName; - - return desc; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index 946d271..1a8282a 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -26,9 +26,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionDesc; -import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageConstants; @@ -286,11 +286,15 @@ public class TestHiveCatalogStore { PartitionDesc partitionDesc = new PartitionDesc(); partitionDesc.setPartitionName(partitionName); - List<PartitionKey> partitionKeyList = new ArrayList<PartitionKey>(); + List<PartitionKeyProto> partitionKeyList = new ArrayList<PartitionKeyProto>(); String[] partitionNames = partitionName.split("/"); for(int i = 0; i < partitionNames.length; i++) { String[] eachPartitionName = partitionNames[i].split("="); - partitionKeyList.add(new PartitionKey(eachPartitionName[0], eachPartitionName[1])); + + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + builder.setColumnName(eachPartitionName[0]); + builder.setPartitionValue(eachPartitionName[1]); + partitionKeyList.add(builder.build()); } partitionDesc.setPartitionKeys(partitionKeyList); partitionDesc.setPath(path.toString()); http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 55c4336..8f1ac95 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -322,13 +322,11 @@ public class MemStore implements CatalogStore { builder.setPath(partitionDesc.getPath()); if (partitionDesc.getPartitionKeysCount() > 0) { - int i = 0; for (CatalogProtos.PartitionKeyProto eachKey : partitionDesc.getPartitionKeysList()) { CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(eachKey.getColumnName()); keyBuilder.setPartitionValue(eachKey.getPartitionValue()); - builder.setPartitionKeys(i, keyBuilder.build()); - i++; + builder.addPartitionKeys(keyBuilder.build()); } } @@ -348,7 +346,7 @@ public class MemStore implements CatalogStore { if(!partitions.containsKey(tableName)) { throw new NoSuchPartitionException(databaseName, tableName, partitionName); } else { - partitions.remove(partitionName); + partitions.get(tableName).remove(partitionName); } break; case SET_PROPERTY: http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index cbcec83..2d52e9c 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -26,13 +26,12 @@ import org.apache.tajo.catalog.dictionary.InfoSchemaMetadataDictionary; import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.partition.PartitionDesc; -import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.store.PostgreSQLStore; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.catalog.store.DerbyStore; import org.apache.tajo.catalog.store.MySQLStore; import org.apache.tajo.catalog.store.MariaDBStore; @@ -936,10 +935,15 @@ public class TestCatalog { String[] partitionNames = partitionName.split("/"); - List<PartitionKey> partitionKeyList = new ArrayList<PartitionKey>(); + List<PartitionKeyProto> partitionKeyList = new ArrayList<PartitionKeyProto>(); for(int i = 0; i < partitionNames.length; i++) { String columnName = partitionNames[i].split("=")[0]; - partitionKeyList.add(new PartitionKey(partitionNames[i], columnName)); + + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + builder.setColumnName(partitionNames[i]); + builder.setPartitionValue(columnName); + + partitionKeyList.add(builder.build()); } partitionDesc.setPartitionKeys(partitionKeyList); http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 469b2a2..1137b30 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -1603,7 +1603,7 @@ alter_table_statement | ALTER TABLE table_name RENAME COLUMN column_name TO column_name | ALTER TABLE table_name ADD COLUMN field_element | ALTER TABLE table_name (if_not_exists)? ADD PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (LOCATION path=Character_String_Literal)? - | ALTER TABLE table_name (if_exists)? DROP PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN + | ALTER TABLE table_name (if_exists)? DROP PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (PURGE)? | ALTER TABLE table_name SET PROPERTY property_list ; http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 62bb0f9..9f850f9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1847,6 +1847,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { String path = stripQuote(ctx.path.getText()); alterTable.setLocation(path); } + alterTable.setPurge(checkIfExist(ctx.PURGE())); } if (checkIfExist(ctx.property_list())) { http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 7104412..af8d796 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -24,12 +24,15 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.algebra.AlterTableOpType; import org.apache.tajo.algebra.AlterTablespaceSetType; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.TajoMaster; @@ -39,6 +42,7 @@ import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.util.Pair; import java.io.IOException; import java.net.URI; @@ -375,12 +379,17 @@ public class DDLExecutor { } } + /** - * ALTER TABLE SET ... + * Execute alter table statement using catalog api. + * + * @param context + * @param queryContext + * @param alterTable + * @throws IOException */ public void alterTable(TajoMaster.MasterContext context, final QueryContext queryContext, final AlterTableNode alterTable) throws IOException { - final CatalogService catalog = context.getCatalog(); final String tableName = alterTable.getTableName(); @@ -400,6 +409,25 @@ public class DDLExecutor { throw new NoSuchTableException(qualifiedName); } + Path partitionPath = null; + TableDesc desc = null; + Pair<List<PartitionKeyProto>, String> pair = null; + CatalogProtos.PartitionDescProto partitionDescProto = null; + + if (alterTable.getAlterTableOpType() == AlterTableOpType.RENAME_TABLE + || alterTable.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION + || alterTable.getAlterTableOpType() == AlterTableOpType.DROP_PARTITION) { + desc = catalog.getTableDesc(databaseName, simpleTableName); + } + + // When adding a partition or dropping a partition, check existing partition column information. + if (alterTable.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION + || alterTable.getAlterTableOpType() == AlterTableOpType.DROP_PARTITION) { + pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), alterTable.getPartitionValues()); + partitionDescProto = catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); + existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); + } + switch (alterTable.getAlterTableOpType()) { case RENAME_TABLE: if (!catalog.existsTable(databaseName, simpleTableName)) { @@ -409,8 +437,6 @@ public class DDLExecutor { throw new AlreadyExistsTableException(alterTable.getNewTableName()); } - TableDesc desc = catalog.getTableDesc(databaseName, simpleTableName); - if (!desc.isExternal()) { // if the table is the managed table Path oldPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR), databaseName, simpleTableName); @@ -446,11 +472,88 @@ public class DDLExecutor { case SET_PROPERTY: catalog.alterTable(CatalogUtil.setProperty(qualifiedName, alterTable.getProperties(), AlterTableType.SET_PROPERTY)); break; + case ADD_PARTITION: + if (partitionDescProto != null) { + throw new AlreadyExistsPartitionException(tableName, pair.getSecond()); + } + + if (alterTable.getLocation() != null) { + partitionPath = new Path(alterTable.getLocation()); + } else { + // If location is not specified, the partition's location will be set using the table location. + partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); + alterTable.setLocation(partitionPath.toString()); + } + + FileSystem fs = partitionPath.getFileSystem(context.getConf()); + + // If there is a directory which was assumed to be a partitioned directory and users don't input another + // location, this will throw exception. + Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond()); + boolean result1 = fs.exists(assumedDirectory); + boolean result2 = fs.exists(partitionPath); + if (fs.exists(assumedDirectory) && !assumedDirectory.equals(partitionPath)) { + throw new AlreadyExistsAssumedPartitionDirectoryException(assumedDirectory.toString()); + } + + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), + alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION)); + + // If the partition's path doesn't exist, this would make the directory by force. + if (!fs.exists(partitionPath)) { + fs.mkdirs(partitionPath); + } + break; + case DROP_PARTITION: + if (partitionDescProto == null) { + throw new NoSuchPartitionException(tableName, pair.getSecond()); + } + + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), + alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.DROP_PARTITION)); + + // When dropping partition on an managed table, the data will be delete from file system. + if (!desc.isExternal()) { + deletePartitionPath(partitionDescProto); + } else { + // When dropping partition on an external table, the data in the table will NOT be deleted from the file + // system. But if PURGE is specified, the partition data will be deleted. + if (alterTable.isPurge()) { + deletePartitionPath(partitionDescProto); + } + } + break; default: //TODO } } + private void deletePartitionPath(CatalogProtos.PartitionDescProto partitionDescProto) throws IOException { + Path partitionPath = new Path(partitionDescProto.getPath()); + FileSystem fs = partitionPath.getFileSystem(context.getConf()); + if (fs.exists(partitionPath)) { + fs.delete(partitionPath, true); + } + } + + private boolean existPartitionColumnNames(String tableName, String[] columnNames) { + for(String columnName : columnNames) { + if (!existPartitionColumnName(tableName, columnName)) { + throw new NoSuchPartitionKeyException(tableName, columnName); + } + } + return true; + } + + private boolean existPartitionColumnName(String tableName, String columnName) { + final TableDesc tableDesc = catalog.getTableDesc(tableName); + if (tableDesc.getPartitionMethod().getExpressionSchema().contains(columnName)) { + return true; + } else { + return false; + } + } + private boolean existColumnName(String tableName, String columnName) { final TableDesc tableDesc = catalog.getTableDesc(tableName); return tableDesc.getSchema().containsByName(columnName) ? true : false; http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index 26e25a4..5005670 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -30,6 +30,7 @@ import org.apache.tajo.TpchTestBase; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; +import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TablespaceManager; @@ -85,7 +86,7 @@ public class TestTajoCli { } private static void setVar(TajoCli cli, ConfigKey key, String val) throws Exception { - cli.executeMetaCommand("\\set " + key.keyname() +" " + val); + cli.executeMetaCommand("\\set " + key.keyname() + " " + val); } private static void assertSessionVar(TajoCli cli, String key, String expectedVal) { @@ -431,22 +432,31 @@ public class TestTajoCli { } @Test - public void testAlterTableAddPartition() throws Exception { + public void testAlterTableAddDropPartition() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition"); tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); + tajoCli.executeScript("alter table " + tableName + " add partition (key2 = 0.1)"); tajoCli.executeScript("alter table " + tableName + " add partition (key = 0.1)"); + tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)"); + tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)"); - String consoleResult = new String(out.toByteArray()); - assertOutputResult(consoleResult); - } + tajoCli.executeScript("drop table " + tableName); + tajoCli.executeScript("create table " + tableName + + " (col1 int4, col2 int4) partition by column(col3 float8, col4 int4)"); - @Test - public void testAlterTableDropPartition() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testAlterTableDropPartition"); + TajoClient client = testBase.getTestingCluster().newTajoClient(); + TableDesc tableDesc = client.getTableDesc(tableName); - tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); - tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)"); + String partitionLocation = tableDesc.getUri().toString() + "/col5=0.1/col6=10"; + tajoCli.executeScript("alter table " + tableName + " add partition (col3 = 0.1, col4 = 10)" + + " location '" + partitionLocation + "'"); + + Path partitionPath = new Path(partitionLocation); + FileSystem fs = testBase.getTestingCluster().getDefaultFileSystem(); + assertTrue(fs.exists(partitionPath)); + + tajoCli.executeScript("alter table " + tableName + " drop partition (col3 = 0.1, col4 = 10)"); String consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index 73b97fa..6ae52bc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -227,8 +227,8 @@ public class TestTajoClient { assertFalse(client.existTable(tableName)); String sql = - "create external table " + tableName + " (deptname text, score integer) " - + "using csv location '" + tablePath + "'"; + "create external table " + tableName + " (deptname text, score integer) " + + "using csv location '" + tablePath + "'"; client.updateQuery(sql); assertTrue(client.existTable(tableName)); client.dropTable(tableName); @@ -237,7 +237,7 @@ public class TestTajoClient { @Test public final void testCreateAndDropExternalTable() - throws IOException, ServiceException, SQLException { + throws IOException, ServiceException, SQLException { final String tableName = "testCreateAndDropExternalTable"; Path tablePath = writeTmpTable(tableName); LOG.error("Full path:" + tablePath.toUri().getRawPath()); @@ -281,7 +281,7 @@ public class TestTajoClient { assertFalse(client.existTable(tableName)); String sql = "create external table " + tableName + " (deptname text, score int4) " + "using csv location '" - + tablePath + "'"; + + tablePath + "'"; client.executeQueryAndGetResult(sql); assertTrue(client.existTable(tableName)); @@ -301,7 +301,7 @@ public class TestTajoClient { assertFalse(client.existTable(tableName)); String sql = "create external table " + tableName + " (deptname text, score int4) " + "using csv location '" - + tablePath + "'"; + + tablePath + "'"; client.executeQueryAndGetResult(sql); assertTrue(client.existTable(tableName)); @@ -361,8 +361,8 @@ public class TestTajoClient { assertFalse(client.existTable(tableName)); String sql = - "create external table " + tableName + " (deptname text, score int4) " - + "using csv location '" + tablePath + "'"; + "create external table " + tableName + " (deptname text, score int4) " + + "using csv location '" + tablePath + "'"; client.executeQueryAndGetResult(sql); assertTrue(client.existTable(tableName)); } @@ -409,7 +409,7 @@ public class TestTajoClient { //@Test public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndDropTablePartitionedHash1ByExecuteQuery"; @@ -433,7 +433,7 @@ public class TestTajoClient { //@Test public final void testCreateAndPurgeTablePartitionedHash1ByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndPurgeTablePartitionedHash1ByExecuteQuery"; @@ -457,7 +457,7 @@ public class TestTajoClient { //@Test public final void testCreateAndDropTablePartitionedHash2ByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndDropTablePartitionedHash2ByExecuteQuery"; @@ -481,7 +481,7 @@ public class TestTajoClient { //@Test public final void testCreateAndDropTablePartitionedListByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndDropTablePartitionedListByExecuteQuery"; @@ -506,7 +506,7 @@ public class TestTajoClient { //@Test public final void testCreateAndDropTablePartitionedRangeByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndDropTablePartitionedRangeByExecuteQuery"; @@ -532,7 +532,7 @@ public class TestTajoClient { @Test public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testFailCreateTablePartitionedOtherExceptColumn"; @@ -545,7 +545,7 @@ public class TestTajoClient { rangeSql += "PARTITION sub_part2 VALUES LESS THAN (MAXVALUE) )"; assertFalse(client.updateQuery(rangeSql)); - + String listSql = "create table " + tableName + " (deptname text, score int4)"; listSql += "PARTITION BY LIST (deptname)"; listSql += "( PARTITION sub_part1 VALUES('r&d', 'design'),"; @@ -562,7 +562,7 @@ public class TestTajoClient { @Test public final void testCreateAndDropTablePartitionedColumnByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = CatalogUtil.normalizeIdentifier("testCreateAndDropTablePartitionedColumnByExecuteQuery"); @@ -585,7 +585,7 @@ public class TestTajoClient { @Test public final void testGetFunctions() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { Collection<FunctionDesc> catalogFunctions = cluster.getMaster().getCatalog().getFunctions(); String functionName = "sum"; int numFunctions = 0; @@ -607,7 +607,7 @@ public class TestTajoClient { @Test public final void testGetFinishedQueryList() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { final String tableName = CatalogUtil.normalizeIdentifier("testGetFinishedQueryList"); String sql = "create table " + tableName + " (deptname text, score int4)"; @@ -678,15 +678,15 @@ public class TestTajoClient { @Test public void testNullCharSessionInCTAS() throws Exception { String sql = - "create table nullcharsession as select\n" + - " c_custkey,\n" + - " orders.o_orderkey,\n" + - " orders.o_orderstatus \n" + - "from\n" + - " orders full outer join customer on c_custkey = o_orderkey\n" + - "order by\n" + - " c_custkey,\n" + - " orders.o_orderkey;\n"; + "create table nullcharsession as select\n" + + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; Map<String, String> variables = new HashMap<String, String>(); variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T"); @@ -720,10 +720,10 @@ public class TestTajoClient { // text type field's value is replaced with \T String expected = "1|1|O\n" + - "2|2|O\n" + - "3|3|F\n" + - "4||\\T\n" + - "5||\\T\n"; + "2|2|O\n" + + "3|3|F\n" + + "4||\\T\n" + + "5||\\T\n"; String resultDatas = new String(buf, 0, readBytes); @@ -757,7 +757,7 @@ public class TestTajoClient { assertEquals(2, queryHistory.getStageHistoriesCount()); List<ClientProtos.StageHistoryProto> taskHistories = - new ArrayList<StageHistoryProto>(queryHistory.getStageHistoriesList()); + new ArrayList<StageHistoryProto>(queryHistory.getStageHistoriesList()); Collections.sort(taskHistories, new Comparator<StageHistoryProto>() { @Override public int compare(ClientProtos.StageHistoryProto o1, StageHistoryProto o2) { http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index f698ed5..0e37b47 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.Iterator; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -448,6 +449,7 @@ public class TestSQLAnalyzer { assertEquals("1", value1.getValue()); LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; assertEquals("2", value2.getValue()); + assertFalse(alterTable.isPurge()); } @Test @@ -468,6 +470,7 @@ public class TestSQLAnalyzer { assertEquals("01", value2.getValue()); LiteralValue value3 = (LiteralValue)alterTable.getValues()[2]; assertEquals("11", value3.getValue()); + assertFalse(alterTable.isPurge()); } @Test @@ -482,6 +485,7 @@ public class TestSQLAnalyzer { assertEquals("col1", alterTable.getColumns()[0].getName()); LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; assertEquals("TAJO", value1.getValue()); + assertTrue(alterTable.isPurge()); } @Test http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 0cbecba..fb0579c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -25,10 +25,13 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.QueryVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.algebra.AlterTableOpType; import org.apache.tajo.algebra.Expr; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.benchmark.TPCH; import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.TextDatum; @@ -1230,4 +1233,103 @@ public class TestLogicalPlanner { assertEquals(NodeType.INSERT, root.getChild().getType()); return root.getChild(); } + + String [] ALTER_PARTITIONS = { + "ALTER TABLE partitioned_table ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx" + + ".com/warehouse/partitioned_table/col1=1/col2=2'", //0 + "ALTER TABLE partitioned_table DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' )", //1 + }; + + @Test + public final void testAddPartitionAndDropPartition() throws PlanningException { + String tableName = CatalogUtil.normalizeIdentifier("partitioned_table"); + String qualifiedTableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName); + + Schema schema = new Schema(); + schema.addColumn("id", Type.INT4) + .addColumn("name", Type.TEXT) + .addColumn("age", Type.INT4) + .addColumn("score", Type.FLOAT8); + + KeyValueSet opts = new KeyValueSet(); + opts.set("file.delimiter", ","); + + Schema partSchema = new Schema(); + partSchema.addColumn("id", Type.INT4); + partSchema.addColumn("name", Type.TEXT); + + PartitionMethodDesc partitionMethodDesc = + new PartitionMethodDesc(DEFAULT_DATABASE_NAME, tableName, + CatalogProtos.PartitionType.COLUMN, "id,name", partSchema); + + TableDesc desc = null; + + try { + desc = new TableDesc(qualifiedTableName, schema, "CSV", new KeyValueSet(), + CommonTestingUtil.getTestDir().toUri()); + } catch (Exception e) { + throw new PlanningException(e.getMessage()); + } + + desc.setPartitionMethod(partitionMethodDesc); + assertFalse(catalog.existsTable(qualifiedTableName)); + catalog.createTable(desc); + assertTrue(catalog.existsTable(qualifiedTableName)); + + TableDesc retrieved = catalog.getTableDesc(qualifiedTableName); + assertEquals(retrieved.getName(), qualifiedTableName); + assertEquals(retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN); + assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id"); + + QueryContext qc = new QueryContext(util.getConfiguration(), session); + + // Testing alter table add partition + Expr expr = sqlAnalyzer.parse(ALTER_PARTITIONS[0]); + LogicalPlan rootNode = planner.createPlan(qc, expr); + LogicalNode plan = rootNode.getRootBlock().getRoot(); + testJsonSerDerObject(plan); + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + assertEquals(NodeType.ALTER_TABLE, root.getChild().getType()); + + AlterTableNode alterTableNode = root.getChild(); + + assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); + + assertEquals(alterTableNode.getPartitionColumns().length, 2); + assertEquals(alterTableNode.getPartitionValues().length, 2); + + assertEquals(alterTableNode.getPartitionColumns()[0], "col1"); + assertEquals(alterTableNode.getPartitionColumns()[1], "col2"); + + assertEquals(alterTableNode.getPartitionValues()[0], "1"); + assertEquals(alterTableNode.getPartitionValues()[1], "2"); + + assertEquals(alterTableNode.getLocation(), "hdfs://xxx.com/warehouse/partitioned_table/col1=1/col2=2"); + + // Testing alter table drop partition + expr = sqlAnalyzer.parse(ALTER_PARTITIONS[1]); + rootNode = planner.createPlan(qc, expr); + plan = rootNode.getRootBlock().getRoot(); + testJsonSerDerObject(plan); + assertEquals(NodeType.ROOT, plan.getType()); + root = (LogicalRootNode) plan; + assertEquals(NodeType.ALTER_TABLE, root.getChild().getType()); + + alterTableNode = root.getChild(); + + assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.DROP_PARTITION); + + assertEquals(alterTableNode.getPartitionColumns().length, 3); + assertEquals(alterTableNode.getPartitionValues().length, 3); + + assertEquals(alterTableNode.getPartitionColumns()[0], "col1"); + assertEquals(alterTableNode.getPartitionColumns()[1], "col2"); + assertEquals(alterTableNode.getPartitionColumns()[2], "col3"); + + assertEquals(alterTableNode.getPartitionValues()[0], "2015"); + assertEquals(alterTableNode.getPartitionValues()[1], "01"); + assertEquals(alterTableNode.getPartitionValues()[2], "11"); + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 44fa1f3..8cdaf80 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -18,14 +18,22 @@ package org.apache.tajo.engine.query; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.junit.Test; import org.junit.experimental.categories.Category; import java.sql.ResultSet; import java.util.List; +import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; +import static org.junit.Assert.*; + @Category(IntegrationTest.class) public class TestAlterTable extends QueryTestCaseBase { @Test @@ -63,4 +71,43 @@ public class TestAlterTable extends QueryTestCaseBase { assertResultSet(after_res, "after_set_property_delimiter.result"); cleanupQuery(after_res); } + + @Test + public final void testAlterTableAddPartition() throws Exception { + executeDDL("create_partitioned_table.sql", null); + + String tableName = CatalogUtil.buildFQName("TestAlterTable", "partitioned_table"); + assertTrue(catalog.existsTable(tableName)); + + TableDesc retrieved = catalog.getTableDesc(tableName); + assertEquals(retrieved.getName(), tableName); + assertEquals(retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN); + assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getAllColumns().size(), 2); + assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "col3"); + assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(1).getSimpleName(), "col4"); + + executeDDL("alter_table_add_partition1.sql", null); + + List<CatalogProtos.PartitionDescProto> partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + assertNotNull(partitions); + assertEquals(partitions.size(), 1); + assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2"); + assertEquals(partitions.get(0).getPartitionKeysList().get(0).getColumnName(), "col3"); + assertEquals(partitions.get(0).getPartitionKeysList().get(0).getPartitionValue(), "1"); + assertEquals(partitions.get(0).getPartitionKeysList().get(1).getColumnName(), "col4"); + assertEquals(partitions.get(0).getPartitionKeysList().get(1).getPartitionValue(), "2"); + + assertNotNull(partitions.get(0).getPath()); + Path partitionPath = new Path(partitions.get(0).getPath()); + FileSystem fs = partitionPath.getFileSystem(conf); + assertTrue(fs.exists(partitionPath)); + assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); + + executeDDL("alter_table_drop_partition1.sql", null); + + partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + assertNotNull(partitions); + assertEquals(partitions.size(), 0); + assertFalse(fs.exists(partitionPath)); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index ad74046..0f49bc1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -604,8 +604,9 @@ public class TestTajoJdbc extends QueryTestCaseBase { // It is because HiveCatalogStore does not support Time data type. try { if (!testingCluster.isHiveCatalogStoreRunning()) { - String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), - DEFAULT_DATABASE_NAME); + String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), + tajoMasterAddress.getPort(), "TestTajoJdbc"); + conn = DriverManager.getConnection(connUri); assertTrue(conn.isValid(100)); @@ -615,12 +616,10 @@ public class TestTajoJdbc extends QueryTestCaseBase { resultSet.close(); stmt = conn.createStatement(); - resultSet = stmt.executeQuery("alter table " + tableName + " add partition (key = 0.1)"); - } - } catch (SQLException e) { - errorMessage = e.getMessage(); + result = stmt.executeUpdate("alter table " + tableName + " add partition (key = 0.1)"); + assertEquals(result, 1); + } } finally { - assertEquals(errorMessage, "ADD_PARTITION is not supported yet\n"); cleanupQuery(resultSet); if (stmt != null) { stmt.close(); http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition1.sql b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition1.sql new file mode 100644 index 0000000..1154683 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition1.sql @@ -0,0 +1 @@ +ALTER TABLE partitioned_table ADD PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition1.sql b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition1.sql new file mode 100644 index 0000000..b5d672f --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition1.sql @@ -0,0 +1 @@ +ALTER TABLE partitioned_table DROP PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/resources/queries/TestAlterTable/create_partitioned_table.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/create_partitioned_table.sql b/tajo-core/src/test/resources/queries/TestAlterTable/create_partitioned_table.sql new file mode 100644 index 0000000..b349349 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/create_partitioned_table.sql @@ -0,0 +1 @@ +create table partitioned_table (col1 int4, col2 int4) partition by column(col3 int4, col4 int4) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql index 1942e16..8a1a6a5 100644 --- a/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql +++ b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql @@ -1 +1 @@ -ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) \ No newline at end of file +ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) PURGE \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result new file mode 100644 index 0000000..c16c311 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result @@ -0,0 +1,9 @@ +OK +ERROR: "key2" column is not the partition key of "default.testaltertableaddpartition". +OK +OK +ERROR: "key=0.1" is not the partition of "testaltertableaddpartition". +OK +OK +OK +OK \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result deleted file mode 100644 index 31f46bc..0000000 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result +++ /dev/null @@ -1,2 +0,0 @@ -OK -ERROR: ADD_PARTITION is not supported yet \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result deleted file mode 100644 index 1fadcea..0000000 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result +++ /dev/null @@ -1,2 +0,0 @@ -OK -ERROR: DROP_PARTITION is not supported yet \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-docs/src/main/sphinx/sql_language.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/sql_language.rst b/tajo-docs/src/main/sphinx/sql_language.rst index 1b405e6..b565765 100644 --- a/tajo-docs/src/main/sphinx/sql_language.rst +++ b/tajo-docs/src/main/sphinx/sql_language.rst @@ -10,4 +10,5 @@ SQL Language sql_language/insert sql_language/queries sql_language/sql_expression - sql_language/predicates \ No newline at end of file + sql_language/predicates + sql_language/alter_table http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-docs/src/main/sphinx/sql_language/alter_table.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst new file mode 100644 index 0000000..947255c --- /dev/null +++ b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst @@ -0,0 +1,100 @@ +************************ +ALTER TABLE Statement +************************ + +======================== +RENAME TABLE +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE <table_name> RENAME TO <new_table_name> + + For example: + ALTER TABLE table1 RENAME TO table2; + +This statement lets you change the name of a table to a different name. + +======================== +RENAME COLUMN +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE <table_name> RENAME COLUMN <column_name> TO <new_column_name> + + For example: + ALTER TABLE table1 RENAME COLUMN id TO id2; + +This statement will allow users to change a column's name. + +======================== +ADD COLUMN +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE <table_name> ADD COLUMN <column_name> <data_type> + + For example: + ALTER TABLE table1 ADD COLUMN id text; + +This statement lets you add new columns to the end of the existing column. + +======================== +SET PROPERTY +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE <table_name> SET PROPERTY (<key> = <value>, ...) + + For example: + ALTER TABLE table1 SET PROPERTY 'timezone' = 'GMT-7' + ALTER TABLE table1 SET PROPERTY 'text.delimiter' = '&' + ALTER TABLE table1 SET PROPERTY 'compression.type'='RECORD','compression.codec'='org.apache.hadoop.io.compress.SnappyCodec' + + +This statement will allow users to change a table property. + +======================== +ADD PARTITION +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE <table_name> [IF NOT EXISTS] ADD PARTITION (<partition column> = <partition value>, ...) [LOCATION = <partition's path>] + + For example: + ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) + ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx.com/warehouse/table1/col1=1/col2=2' + +You can use ``ALTER TABLE ADD PARTITION`` to add partitions to a table. The location must be a directory inside of which data files reside. If the location doesn't exist on the file system, Tajo will make the location by force. ``ADD PARTITION`` changes the table metadata, but does not load data. If the data does not exist in the partition's location, queries will not return any results. + +======================== + DROP PARTITION +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE <table_name> [IF NOT EXISTS] DROP PARTITION (<partition column> = <partition value>, ...) [PURGE] + + For example: + ALTER TABLE table1 DROP PARTITION (col1 = 1 , col2 = 2) + ALTER TABLE table1 DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) + ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) PURGE + +You can use ``ALTER TABLE DROP PARTITION`` to drop a partition for a table. This removes the data for a managed table + and this doesn't remove the data for an external table. But if ``PURGE`` is specified for an external table, the partition data will be removed. The metadata is completely lost in all cases. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 9d2161e..581fde7 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -2015,6 +2015,42 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex return schema; } + /** + * It transforms ColumnDefinition array to String array. + * + * @param columnReferenceExprs + * @return + */ + private static String[] convertColumnsToStrings(ColumnReferenceExpr[] columnReferenceExprs) { + int columnCount = columnReferenceExprs.length; + String[] columns = new String[columnCount]; + + for(int i = 0; i < columnCount; i++) { + ColumnReferenceExpr columnReferenceExpr = columnReferenceExprs[i]; + columns[i] = columnReferenceExpr.getName(); + } + + return columns; + } + + /** + * It transforms Expr array to String array. + * + * @param exprs + * @return + */ + private static String[] convertExprsToStrings(Expr[] exprs) { + int exprCount = exprs.length; + String[] values = new String[exprCount]; + + for(int i = 0; i < exprCount; i++) { + LiteralValue expr = (LiteralValue)exprs[i]; + values[i] = expr.getValue(); + } + + return values; + } + private static Column convertColumn(ColumnDefinition columnDefinition) { return new Column(columnDefinition.getColumnName(), convertDataType(columnDefinition)); } @@ -2078,6 +2114,20 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex if (null != alterTable.getAddNewColumn()) { alterTableNode.setAddNewColumn(convertColumn(alterTable.getAddNewColumn())); } + + if (alterTable.getColumns() != null) { + alterTableNode.setPartitionColumns(convertColumnsToStrings(alterTable.getColumns())); + } + + if (alterTable.getValues() != null) { + alterTableNode.setPartitionValues(convertExprsToStrings(alterTable.getValues())); + } + + if (alterTable.getLocation() != null) { + alterTableNode.setLocation(alterTable.getLocation()); + } + + alterTableNode.setPurge(alterTable.isPurge()); alterTableNode.setAlterTableOpType(alterTable.getAlterTableOpType()); return alterTableNode; } http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java index 9cbde0c..ecb173a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java @@ -19,6 +19,7 @@ package org.apache.tajo.plan.logical; +import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.algebra.AlterTableOpType; @@ -42,6 +43,14 @@ public class AlterTableNode extends LogicalNode { private KeyValueSet properties = new KeyValueSet(); @Expose private AlterTableOpType alterTableOpType; + @Expose + private String[] partitionColumns; + @Expose + private String[] partitionValues; + @Expose + private String location; + @Expose + private boolean isPurge; public AlterTableNode(int pid) { super(pid, NodeType.ALTER_TABLE); @@ -117,6 +126,38 @@ public class AlterTableNode extends LogicalNode { this.properties = properties; } + public String[] getPartitionColumns() { + return partitionColumns; + } + + public void setPartitionColumns(String[] partitionColumns) { + this.partitionColumns = partitionColumns; + } + + public String[] getPartitionValues() { + return partitionValues; + } + + public void setPartitionValues(String[] partitionValues) { + this.partitionValues = partitionValues; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public boolean isPurge() { + return isPurge; + } + + public void setPurge(boolean isPurge) { + this.isPurge = isPurge; + } + @Override public PlanString getPlanString() { return new PlanString(this); @@ -124,16 +165,8 @@ public class AlterTableNode extends LogicalNode { @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((addNewColumn == null) ? 0 : addNewColumn.hashCode()); - result = prime * result + ((alterTableOpType == null) ? 0 : alterTableOpType.hashCode()); - result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()); - result = prime * result + ((newColumnName == null) ? 0 : newColumnName.hashCode()); - result = prime * result + ((newTableName == null) ? 0 : newTableName.hashCode()); - result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); - result = prime * result + ((properties == null) ? 0 : properties.hashCode()); - return result; + return Objects.hashCode(tableName, addNewColumn, alterTableOpType, columnName, newColumnName, newTableName, + tableName, properties, partitionColumns, partitionValues, location, isPurge); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index c1d9f9a..fe900c0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -579,6 +579,8 @@ public class LogicalNodeDeserializer { PlanProto.AlterTableNode alterTableProto = protoNode.getAlterTable(); alterTable.setTableName(alterTableProto.getTableName()); + PlanProto.AlterTableNode.AlterPartition alterPartition = null; + switch (alterTableProto.getSetType()) { case RENAME_TABLE: alterTable.setNewTableName(alterTableProto.getRenameTable().getNewName()); @@ -593,6 +595,23 @@ public class LogicalNodeDeserializer { case SET_PROPERTY: alterTable.setProperties(new KeyValueSet(alterTableProto.getProperties())); break; + case ADD_PARTITION: + alterPartition = alterTableProto.getAlterPartition(); + alterTable.setPartitionColumns(alterPartition.getColumnNamesList().toArray(new String[alterPartition + .getColumnNamesCount()])); + alterTable.setPartitionValues(alterPartition.getPartitionValuesList().toArray(new String[alterPartition + .getPartitionValuesCount()])); + if (alterPartition.getLocation() != null) { + alterTable.setLocation(alterPartition.getLocation()); + } + break; + case DROP_PARTITION: + alterPartition = alterTableProto.getAlterPartition(); + alterTable.setPartitionColumns(alterPartition.getColumnNamesList().toArray(new String[alterPartition + .getColumnNamesCount()])); + alterTable.setPartitionValues(alterPartition.getPartitionValuesList().toArray(new String[alterPartition + .getPartitionValuesCount()])); + break; default: throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + alterTableProto.getSetType().name()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index 6737756..76370a9 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -549,6 +549,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe AlterTableNode node, Stack<LogicalNode> stack) { PlanProto.AlterTableNode.Builder alterTableBuilder = PlanProto.AlterTableNode.newBuilder(); alterTableBuilder.setTableName(node.getTableName()); + PlanProto.AlterTableNode.AlterPartition.Builder partitionBuilder = null; switch (node.getAlterTableOpType()) { case RENAME_TABLE: @@ -569,6 +570,34 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.SET_PROPERTY); alterTableBuilder.setProperties(node.getProperties().getProto()); break; + case ADD_PARTITION: + alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.ADD_PARTITION); + partitionBuilder = PlanProto.AlterTableNode.AlterPartition.newBuilder(); + for (String columnName : node.getPartitionColumns()) { + partitionBuilder.addColumnNames(columnName); + } + + for (String partitionValue : node.getPartitionValues()) { + partitionBuilder.addPartitionValues(partitionValue); + } + if (node.getLocation() != null) { + partitionBuilder.setLocation(node.getLocation()); + } + alterTableBuilder.setAlterPartition(partitionBuilder); + break; + case DROP_PARTITION: + alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.DROP_PARTITION); + partitionBuilder = PlanProto.AlterTableNode.AlterPartition.newBuilder(); + for (String columnName : node.getPartitionColumns()) { + partitionBuilder.addColumnNames(columnName); + } + + for (String partitionValue : node.getPartitionValues()) { + partitionBuilder.addPartitionValues(partitionValue); + } + partitionBuilder.setPurge(node.isPurge()); + alterTableBuilder.setAlterPartition(partitionBuilder); + break; default: throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + node.getAlterTableOpType().name()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/3dba8c7e/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 09160ab..18ca627 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -320,12 +320,6 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer @Override public Expr visitAlterTable(Context context, Stack<Expr> stack, AlterTable expr) throws PlanningException { super.visitAlterTable(context, stack, expr); - - if (expr.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION - || expr.getAlterTableOpType() == AlterTableOpType.DROP_PARTITION) { - context.state.addVerification(expr.getAlterTableOpType().name() + " is not supported yet"); - } - return expr; } }
