HIVE-11745 : Alter table Exchange partition with multiple partition_spec is not working (Yongzhi Chen via Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c6895897 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c6895897 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c6895897 Branch: refs/heads/spark Commit: c6895897bc8178546b10780c5f463353c4ee1b9a Parents: 5548a9c Author: Szehon Ho <[email protected]> Authored: Tue Sep 15 16:39:50 2015 -0700 Committer: Szehon Ho <[email protected]> Committed: Tue Sep 15 16:40:26 2015 -0700 ---------------------------------------------------------------------- .../hive/ql/security/FolderPermissionBase.java | 17 +- .../test/resources/testconfiguration.properties | 1 + .../hadoop/hive/metastore/HiveMetaStore.java | 6 + .../queries/clientpositive/exchgpartition2lel.q | 32 ++++ .../clientpositive/exchgpartition2lel.q.out | 182 +++++++++++++++++++ 5 files changed, 237 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c6895897/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java index f28edc6..d98082f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java @@ -370,7 +370,7 @@ public abstract class FolderPermissionBase { } @Test - public void testAlterPartition() throws Exception { + public void testPartition() throws Exception { String tableName = "alterpart"; CommandProcessorResponse ret = driver.run("CREATE TABLE " + tableName + " (key string, value string) partitioned by (part1 int, part2 int, part3 int)"); Assert.assertEquals(0,ret.getResponseCode()); @@ -396,6 +396,21 @@ public abstract class FolderPermissionBase { for (String child : listStatus(warehouseDir + "/" + tableName + "/part1=2/part2=2/part3=2")) { verifyPermission(child, 1); } + + String tableName2 = "alterpart2"; + ret = driver.run("CREATE TABLE " + tableName2 + " (key string, value string) partitioned by (part1 int, part2 int, part3 int)"); + Assert.assertEquals(0,ret.getResponseCode()); + + assertExistence(warehouseDir + "/" + tableName2); + setPermission(warehouseDir + "/" + tableName2); + ret = driver.run("alter table " + tableName2 + " exchange partition (part1='2',part2='2',part3='2') with table " + tableName); + Assert.assertEquals(0,ret.getResponseCode()); + + //alter exchange can not change base table's permission + //alter exchange can only control final partition folder's permission + verifyPermission(warehouseDir + "/" + tableName2 + "/part1=2", 0); + verifyPermission(warehouseDir + "/" + tableName2 + "/part1=2/part2=2", 0); + verifyPermission(warehouseDir + "/" + tableName2 + "/part1=2/part2=2/part3=2", 1); } @Test http://git-wip-us.apache.org/repos/asf/hive/blob/c6895897/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index bed621d..2851720 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -12,6 +12,7 @@ minimr.query.files=auto_sortmerge_join_16.q,\ constprog_partitioner.q,\ disable_merge_for_bucketing.q,\ empty_dir_in_table.q,\ + exchgpartition2lel.q,\ external_table_with_space_in_location_path.q,\ file_with_header_footer.q,\ groupby2.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/c6895897/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 1840e76..a80f686 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -2596,6 +2596,12 @@ public class HiveMetaStore extends ThriftHiveMetastore { ms.dropPartition(partition.getDbName(), sourceTable.getTableName(), partition.getValues()); } + Path destParentPath = destPath.getParent(); + if (!wh.isDir(destParentPath)) { + if (!wh.mkdirs(destParentPath, true)) { + throw new MetaException("Unable to create path " + destParentPath); + } + } /** * TODO: Use the hard link feature of hdfs * once https://issues.apache.org/jira/browse/HDFS-3370 is done http://git-wip-us.apache.org/repos/asf/hive/blob/c6895897/ql/src/test/queries/clientpositive/exchgpartition2lel.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/exchgpartition2lel.q b/ql/src/test/queries/clientpositive/exchgpartition2lel.q new file mode 100644 index 0000000..2b15894 --- /dev/null +++ b/ql/src/test/queries/clientpositive/exchgpartition2lel.q @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS t4; + +CREATE TABLE t1 (a int) PARTITIONED BY (d1 int); +CREATE TABLE t2 (a int) PARTITIONED BY (d1 int); +CREATE TABLE t3 (a int) PARTITIONED BY (d1 int, d2 int); +CREATE TABLE t4 (a int) PARTITIONED BY (d1 int, d2 int); +CREATE TABLE t5 (a int) PARTITIONED BY (d1 int, d2 int, d3 int); +CREATE TABLE t6 (a int) PARTITIONED BY (d1 int, d2 int, d3 int); + +INSERT OVERWRITE TABLE t1 PARTITION (d1 = 1) SELECT key FROM src where key = 100 limit 1; +INSERT OVERWRITE TABLE t3 PARTITION (d1 = 1, d2 = 1) SELECT key FROM src where key = 100 limit 1; +INSERT OVERWRITE TABLE t5 PARTITION (d1 = 1, d2 = 1, d3=1) SELECT key FROM src where key = 100 limit 1; + +SELECT * FROM t1; + +SELECT * FROM t3; + +ALTER TABLE t2 EXCHANGE PARTITION (d1 = 1) WITH TABLE t1; +SELECT * FROM t1; +SELECT * FROM t2; + +ALTER TABLE t4 EXCHANGE PARTITION (d1 = 1, d2 = 1) WITH TABLE t3; +SELECT * FROM t3; +SELECT * FROM t4; + +ALTER TABLE t6 EXCHANGE PARTITION (d1 = 1, d2 = 1, d3 = 1) WITH TABLE t5; +SELECT * FROM t5; +SELECT * FROM t6; + http://git-wip-us.apache.org/repos/asf/hive/blob/c6895897/ql/src/test/results/clientpositive/exchgpartition2lel.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/exchgpartition2lel.q.out b/ql/src/test/results/clientpositive/exchgpartition2lel.q.out new file mode 100644 index 0000000..5997d6b --- /dev/null +++ b/ql/src/test/results/clientpositive/exchgpartition2lel.q.out @@ -0,0 +1,182 @@ +PREHOOK: query: DROP TABLE IF EXISTS t1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS t1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS t2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS t2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS t3 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS t3 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS t4 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS t4 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE t1 (a int) PARTITIONED BY (d1 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: CREATE TABLE t1 (a int) PARTITIONED BY (d1 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: CREATE TABLE t2 (a int) PARTITIONED BY (d1 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: CREATE TABLE t2 (a int) PARTITIONED BY (d1 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: CREATE TABLE t3 (a int) PARTITIONED BY (d1 int, d2 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t3 +POSTHOOK: query: CREATE TABLE t3 (a int) PARTITIONED BY (d1 int, d2 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t3 +PREHOOK: query: CREATE TABLE t4 (a int) PARTITIONED BY (d1 int, d2 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t4 +POSTHOOK: query: CREATE TABLE t4 (a int) PARTITIONED BY (d1 int, d2 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t4 +PREHOOK: query: CREATE TABLE t5 (a int) PARTITIONED BY (d1 int, d2 int, d3 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t5 +POSTHOOK: query: CREATE TABLE t5 (a int) PARTITIONED BY (d1 int, d2 int, d3 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t5 +PREHOOK: query: CREATE TABLE t6 (a int) PARTITIONED BY (d1 int, d2 int, d3 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t6 +POSTHOOK: query: CREATE TABLE t6 (a int) PARTITIONED BY (d1 int, d2 int, d3 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t6 +PREHOOK: query: INSERT OVERWRITE TABLE t1 PARTITION (d1 = 1) SELECT key FROM src where key = 100 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1@d1=1 +POSTHOOK: query: INSERT OVERWRITE TABLE t1 PARTITION (d1 = 1) SELECT key FROM src where key = 100 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1@d1=1 +POSTHOOK: Lineage: t1 PARTITION(d1=1).a EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE t3 PARTITION (d1 = 1, d2 = 1) SELECT key FROM src where key = 100 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t3@d1=1/d2=1 +POSTHOOK: query: INSERT OVERWRITE TABLE t3 PARTITION (d1 = 1, d2 = 1) SELECT key FROM src where key = 100 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t3@d1=1/d2=1 +POSTHOOK: Lineage: t3 PARTITION(d1=1,d2=1).a EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE t5 PARTITION (d1 = 1, d2 = 1, d3=1) SELECT key FROM src where key = 100 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t5@d1=1/d2=1/d3=1 +POSTHOOK: query: INSERT OVERWRITE TABLE t5 PARTITION (d1 = 1, d2 = 1, d3=1) SELECT key FROM src where key = 100 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t5@d1=1/d2=1/d3=1 +POSTHOOK: Lineage: t5 PARTITION(d1=1,d2=1,d3=1).a EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: SELECT * FROM t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t1@d1=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t1@d1=1 +#### A masked pattern was here #### +100 1 +PREHOOK: query: SELECT * FROM t3 +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +PREHOOK: Input: default@t3@d1=1/d2=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +POSTHOOK: Input: default@t3@d1=1/d2=1 +#### A masked pattern was here #### +100 1 1 +PREHOOK: query: ALTER TABLE t2 EXCHANGE PARTITION (d1 = 1) WITH TABLE t1 +PREHOOK: type: ALTERTABLE_EXCHANGEPARTITION +POSTHOOK: query: ALTER TABLE t2 EXCHANGE PARTITION (d1 = 1) WITH TABLE t1 +POSTHOOK: type: ALTERTABLE_EXCHANGEPARTITION +PREHOOK: query: SELECT * FROM t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +PREHOOK: query: SELECT * FROM t2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t2 +PREHOOK: Input: default@t2@d1=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@t2@d1=1 +#### A masked pattern was here #### +100 1 +PREHOOK: query: ALTER TABLE t4 EXCHANGE PARTITION (d1 = 1, d2 = 1) WITH TABLE t3 +PREHOOK: type: ALTERTABLE_EXCHANGEPARTITION +POSTHOOK: query: ALTER TABLE t4 EXCHANGE PARTITION (d1 = 1, d2 = 1) WITH TABLE t3 +POSTHOOK: type: ALTERTABLE_EXCHANGEPARTITION +PREHOOK: query: SELECT * FROM t3 +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +PREHOOK: query: SELECT * FROM t4 +PREHOOK: type: QUERY +PREHOOK: Input: default@t4 +PREHOOK: Input: default@t4@d1=1/d2=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t4 +POSTHOOK: Input: default@t4@d1=1/d2=1 +#### A masked pattern was here #### +100 1 1 +PREHOOK: query: ALTER TABLE t6 EXCHANGE PARTITION (d1 = 1, d2 = 1, d3 = 1) WITH TABLE t5 +PREHOOK: type: ALTERTABLE_EXCHANGEPARTITION +POSTHOOK: query: ALTER TABLE t6 EXCHANGE PARTITION (d1 = 1, d2 = 1, d3 = 1) WITH TABLE t5 +POSTHOOK: type: ALTERTABLE_EXCHANGEPARTITION +PREHOOK: query: SELECT * FROM t5 +PREHOOK: type: QUERY +PREHOOK: Input: default@t5 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t5 +#### A masked pattern was here #### +PREHOOK: query: SELECT * FROM t6 +PREHOOK: type: QUERY +PREHOOK: Input: default@t6 +PREHOOK: Input: default@t6@d1=1/d2=1/d3=1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t6 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t6 +POSTHOOK: Input: default@t6@d1=1/d2=1/d3=1 +#### A masked pattern was here #### +100 1 1 1
