Repository: hive Updated Branches: refs/heads/master 674138a65 -> db24ce722
HIVE-16372: Enable DDL statement for non-native tables (add/remove table properties) (Pengcheng Xiong, reviewed by Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db24ce72 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db24ce72 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db24ce72 Branch: refs/heads/master Commit: db24ce7224758a3c6bde4be2e796675ce4265e1f Parents: 674138a Author: Pengcheng Xiong <pxi...@hortonworks.com> Authored: Wed Apr 5 16:39:08 2017 -0700 Committer: Pengcheng Xiong <pxi...@hortonworks.com> Committed: Wed Apr 5 16:39:08 2017 -0700 ---------------------------------------------------------------------- .../src/test/queries/negative/hbase_ddl.q | 9 + .../src/test/queries/positive/hbase_ddl.q | 20 ++ .../src/test/results/negative/hbase_ddl.q.out | 29 +++ .../src/test/results/positive/hbase_ddl.q.out | 186 +++++++++++++++++++ .../org/apache/hadoop/hive/ql/ErrorMsg.java | 4 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/plan/AlterTableDesc.java | 6 + .../clientnegative/alter_non_native.q.out | 2 +- 8 files changed, 255 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/hbase-handler/src/test/queries/negative/hbase_ddl.q ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/queries/negative/hbase_ddl.q b/hbase-handler/src/test/queries/negative/hbase_ddl.q new file mode 100644 index 0000000..2913bcd --- /dev/null +++ b/hbase-handler/src/test/queries/negative/hbase_ddl.q @@ -0,0 +1,9 @@ +DROP TABLE hbase_table_1; +CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value') +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string") +TBLPROPERTIES ("hbase.table.name" = "hbase_table_0"); + +DESCRIBE EXTENDED hbase_table_1; + +alter table hbase_table_1 change column key newkey string; http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/hbase-handler/src/test/queries/positive/hbase_ddl.q ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/queries/positive/hbase_ddl.q b/hbase-handler/src/test/queries/positive/hbase_ddl.q new file mode 100644 index 0000000..a8bae75 --- /dev/null +++ b/hbase-handler/src/test/queries/positive/hbase_ddl.q @@ -0,0 +1,20 @@ +DROP TABLE hbase_table_1; +CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value') +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string") +TBLPROPERTIES ("hbase.table.name" = "hbase_table_0"); + +DESCRIBE EXTENDED hbase_table_1; + +select * from hbase_table_1; + +EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0; +FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0; + +ALTER TABLE hbase_table_1 SET TBLPROPERTIES('hbase.mapred.output.outputtable'='kkk'); + +desc formatted hbase_table_1; + +ALTER TABLE hbase_table_1 unset TBLPROPERTIES('hbase.mapred.output.outputtable'); + +desc formatted hbase_table_1; http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/hbase-handler/src/test/results/negative/hbase_ddl.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/negative/hbase_ddl.q.out b/hbase-handler/src/test/results/negative/hbase_ddl.q.out new file mode 100644 index 0000000..b5aad70 --- /dev/null +++ b/hbase-handler/src/test/results/negative/hbase_ddl.q.out @@ -0,0 +1,29 @@ +PREHOOK: query: DROP TABLE hbase_table_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE hbase_table_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value') +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string") +TBLPROPERTIES ("hbase.table.name" = "hbase_table_0") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hbase_table_1 +POSTHOOK: query: CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value') +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string") +TBLPROPERTIES ("hbase.table.name" = "hbase_table_0") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_table_1 +PREHOOK: query: DESCRIBE EXTENDED hbase_table_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@hbase_table_1 +POSTHOOK: query: DESCRIBE EXTENDED hbase_table_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@hbase_table_1 +key int It is a column key +value string It is the column string value + +#### A masked pattern was here #### +FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS] to a non-native table hbase_table_1 http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/hbase-handler/src/test/results/positive/hbase_ddl.q.out ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/results/positive/hbase_ddl.q.out b/hbase-handler/src/test/results/positive/hbase_ddl.q.out new file mode 100644 index 0000000..8cb88ed --- /dev/null +++ b/hbase-handler/src/test/results/positive/hbase_ddl.q.out @@ -0,0 +1,186 @@ +PREHOOK: query: DROP TABLE hbase_table_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE hbase_table_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value') +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string") +TBLPROPERTIES ("hbase.table.name" = "hbase_table_0") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hbase_table_1 +POSTHOOK: query: CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value') +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string") +TBLPROPERTIES ("hbase.table.name" = "hbase_table_0") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_table_1 +PREHOOK: query: DESCRIBE EXTENDED hbase_table_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@hbase_table_1 +POSTHOOK: query: DESCRIBE EXTENDED hbase_table_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@hbase_table_1 +key int It is a column key +value string It is the column string value + +#### A masked pattern was here #### +PREHOOK: query: select * from hbase_table_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from hbase_table_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table_1 +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 + Stage-1 is a root stage + Stage-3 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Alter Table Operator: + Alter Table + type: drop props + old name: default.hbase_table_1 + properties: + COLUMN_STATS_ACCURATE + + Stage: Stage-2 + Insert operator: + Insert + + Stage: Stage-1 + Pre Insert operator: + Pre-Insert task + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) % 2.0) = 0.0) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat + output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat + serde: org.apache.hadoop.hive.hbase.HBaseSerDe + name: default.hbase_table_1 + +PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table_1 +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table_1 +PREHOOK: query: ALTER TABLE hbase_table_1 SET TBLPROPERTIES('hbase.mapred.output.outputtable'='kkk') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@hbase_table_1 +PREHOOK: Output: default@hbase_table_1 +POSTHOOK: query: ALTER TABLE hbase_table_1 SET TBLPROPERTIES('hbase.mapred.output.outputtable'='kkk') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@hbase_table_1 +POSTHOOK: Output: default@hbase_table_1 +PREHOOK: query: desc formatted hbase_table_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@hbase_table_1 +POSTHOOK: query: desc formatted hbase_table_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@hbase_table_1 +# col_name data_type comment + +key int It is a column key +value string It is the column string value + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + hbase.mapred.output.outputtable kkk + hbase.table.name hbase_table_0 +#### A masked pattern was here #### + numFiles 0 + numRows 0 + rawDataSize 0 + storage_handler org.apache.hadoop.hive.hbase.HBaseStorageHandler + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe +InputFormat: null +OutputFormat: null +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + hbase.columns.mapping cf:string + serialization.format 1 +PREHOOK: query: ALTER TABLE hbase_table_1 unset TBLPROPERTIES('hbase.mapred.output.outputtable') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@hbase_table_1 +PREHOOK: Output: default@hbase_table_1 +POSTHOOK: query: ALTER TABLE hbase_table_1 unset TBLPROPERTIES('hbase.mapred.output.outputtable') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@hbase_table_1 +POSTHOOK: Output: default@hbase_table_1 +PREHOOK: query: desc formatted hbase_table_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@hbase_table_1 +POSTHOOK: query: desc formatted hbase_table_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@hbase_table_1 +# col_name data_type comment + +key int It is a column key +value string It is the column string value + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + hbase.table.name hbase_table_0 +#### A masked pattern was here #### + numFiles 0 + numRows 0 + rawDataSize 0 + storage_handler org.apache.hadoop.hive.hbase.HBaseStorageHandler + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe +InputFormat: null +OutputFormat: null +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + hbase.columns.mapping cf:string + serialization.format 1 http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 6a43385..746c199 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql; import java.text.MessageFormat; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -27,6 +28,7 @@ import java.util.regex.Pattern; import org.antlr.runtime.tree.Tree; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.ASTNodeOrigin; +import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; /** * List of all error messages. @@ -217,7 +219,7 @@ public enum ErrorMsg { ALTER_COMMAND_FOR_VIEWS(10131, "To alter a view you need to use the ALTER VIEW command."), ALTER_COMMAND_FOR_TABLES(10132, "To alter a base table you need to use the ALTER TABLE command."), ALTER_VIEW_DISALLOWED_OP(10133, "Cannot use this form of ALTER on a view"), - ALTER_TABLE_NON_NATIVE(10134, "ALTER TABLE cannot be used for a non-native table"), + ALTER_TABLE_NON_NATIVE(10134, "ALTER TABLE can only be used for " + AlterTableTypes.nonNativeTableAllowedTypes + " to a non-native table "), SORTMERGE_MAPJOIN_FAILED(10135, "Sort merge bucketed join could not be performed. " + "If you really want to perform the operation, either set " + http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 6e72d07..0cf9205 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1350,7 +1350,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { throw new SemanticException(ErrorMsg.ALTER_COMMAND_FOR_TABLES.getMsg()); } } - if (tbl.isNonNative()) { + if (tbl.isNonNative() && !AlterTableTypes.nonNativeTableAllowedTypes.contains(op)) { throw new SemanticException(ErrorMsg.ALTER_TABLE_NON_NATIVE.getMsg(tbl.getTableName())); } } http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java index b83c16d..d5a6679 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java @@ -29,8 +29,11 @@ import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.Explain.Level; +import com.google.common.collect.ImmutableList; + import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -64,6 +67,9 @@ public class AlterTableDesc extends DDLDesc implements Serializable { private final String name; private AlterTableTypes(String name) { this.name = name; } public String getName() { return name; } + + public static final List<AlterTableTypes> nonNativeTableAllowedTypes = + ImmutableList.of(ADDPROPS, DROPPROPS); } public static enum ProtectModeType { http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/ql/src/test/results/clientnegative/alter_non_native.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/alter_non_native.q.out b/ql/src/test/results/clientnegative/alter_non_native.q.out index 64c4cee..0a32067 100644 --- a/ql/src/test/results/clientnegative/alter_non_native.q.out +++ b/ql/src/test/results/clientnegative/alter_non_native.q.out @@ -8,4 +8,4 @@ STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler' POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@non_native1 -FAILED: SemanticException [Error 10134]: ALTER TABLE cannot be used for a non-native table non_native1 +FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS] to a non-native table non_native1
