Repository: hive Updated Branches: refs/heads/master a81f53ac1 -> 4137c212c
HIVE-20603: "Wrong FS" error when inserting to partition after changing table location filesystem (Jason Dere, reviewed by Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4137c212 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4137c212 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4137c212 Branch: refs/heads/master Commit: 4137c212ccbb1955ffba9c4f6e1b9876b07c122b Parents: a81f53a Author: Jason Dere <jd...@hortonworks.com> Authored: Tue Sep 25 13:21:42 2018 -0700 Committer: Jason Dere <jd...@hortonworks.com> Committed: Tue Sep 25 13:21:42 2018 -0700 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 2 + .../apache/hadoop/hive/ql/metadata/Hive.java | 2 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 21 ++++- .../clientpositive/alter_table_location2.q | 21 +++++ .../clientpositive/alter_table_location3.q | 16 ++++ .../llap/alter_table_location2.q.out | 97 ++++++++++++++++++++ .../llap/alter_table_location3.q.out | 95 +++++++++++++++++++ .../hive/metastore/utils/MetaStoreUtils.java | 2 +- 8 files changed, 250 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4137c212/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index df89748..def3561 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -374,6 +374,8 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ vectorized_timestamp_ints_casts.q minillap.query.files=acid_bucket_pruning.q,\ + alter_table_location2.q,\ + alter_table_location3.q,\ bucket5.q,\ bucket6.q,\ dynamic_semijoin_user_level.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/4137c212/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 76541de..3c32de9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1942,7 +1942,7 @@ public class Hive { replaceFiles(tbl.getPath(), loadPath, destPath, oldPartPath, getConf(), isSrcLocal, isAutoPurge, newFiles, FileUtils.HIDDEN_FILES_PATH_FILTER, needRecycle, isManaged); } else { - FileSystem fs = tbl.getDataLocation().getFileSystem(conf); + FileSystem fs = destPath.getFileSystem(conf); copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation, (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles, tbl.getNumBuckets() > 0, isFullAcidTable, isManaged); http://git-wip-us.apache.org/repos/asf/hive/blob/4137c212/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 3873282..344e9fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7344,10 +7344,23 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { checkImmutableTable(qb, destinationTable, partPath, true); - // if the table is in a different dfs than the partition, - // replace the partition's dfs with the table's dfs. - destinationPath = new Path(tabPath.toUri().getScheme(), tabPath.toUri() - .getAuthority(), partPath.toUri().getPath()); + // Previous behavior (HIVE-1707) used to replace the partition's dfs with the table's dfs. + // The changes in HIVE-19891 appears to no longer support that behavior. + destinationPath = partPath; + + if (MetaStoreUtils.isArchived(destinationPartition.getTPartition())) { + try { + String conflictingArchive = ArchiveUtils.conflictingArchiveNameOrNull( + db, destinationTable, destinationPartition.getSpec()); + String message = String.format("Insert conflict with existing archive: %s", + conflictingArchive); + throw new SemanticException(message); + } catch (SemanticException err) { + throw err; + } catch (HiveException err) { + throw new SemanticException(err); + } + } isMmTable = AcidUtils.isInsertOnlyTable(destinationTable.getParameters()); queryTmpdir = isMmTable ? destinationPath : ctx.getTempDirForFinalJobPath(destinationPath); http://git-wip-us.apache.org/repos/asf/hive/blob/4137c212/ql/src/test/queries/clientpositive/alter_table_location2.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/alter_table_location2.q b/ql/src/test/queries/clientpositive/alter_table_location2.q new file mode 100644 index 0000000..80089f9 --- /dev/null +++ b/ql/src/test/queries/clientpositive/alter_table_location2.q @@ -0,0 +1,21 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +CREATE TABLE alter_table_location2 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS ORC + LOCATION 'pfile://${system:test.tmp.dir}/alter_table_location2' + TBLPROPERTIES ("transactional"="true"); +; + +INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (8,'Henry','CSE'); +ALTER TABLE alter_table_location2 ADD PARTITION (year=2017); + +ALTER TABLE alter_table_location2 SET LOCATION 'hdfs:///tmp/alter_table_location2'; + +INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (9,'Horace','CSE'); +INSERT INTO alter_table_location2 PARTITION (year=2017) VALUES (10,'Harris','CSE'); +INSERT INTO alter_table_location2 PARTITION (year=2018) VALUES (11,'Humphrey','CSE'); + +SELECT * from alter_table_location2 order by id; + http://git-wip-us.apache.org/repos/asf/hive/blob/4137c212/ql/src/test/queries/clientpositive/alter_table_location3.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/alter_table_location3.q b/ql/src/test/queries/clientpositive/alter_table_location3.q new file mode 100644 index 0000000..bb15d51 --- /dev/null +++ b/ql/src/test/queries/clientpositive/alter_table_location3.q @@ -0,0 +1,16 @@ + +CREATE TABLE alter_table_location3 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS TEXTFILE + LOCATION 'pfile://${system:test.tmp.dir}/alter_table_location3' +; + +INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (8,'Henry','CSE'); +ALTER TABLE alter_table_location3 ADD PARTITION (year=2017); + +ALTER TABLE alter_table_location3 SET LOCATION 'hdfs:///tmp/alter_table_location3'; +INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (9,'Horace','CSE'); +INSERT INTO alter_table_location3 PARTITION (year=2017) VALUES (10,'Harris','CSE'); +INSERT INTO alter_table_location3 PARTITION (year=2018) VALUES (11,'Humphrey','CSE'); + +SELECT * from alter_table_location3 order by id; http://git-wip-us.apache.org/repos/asf/hive/blob/4137c212/ql/src/test/results/clientpositive/llap/alter_table_location2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/alter_table_location2.q.out b/ql/src/test/results/clientpositive/llap/alter_table_location2.q.out new file mode 100644 index 0000000..e4eee18 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/alter_table_location2.q.out @@ -0,0 +1,97 @@ +PREHOOK: query: CREATE TABLE alter_table_location2 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS ORC +#### A masked pattern was here #### + TBLPROPERTIES ("transactional"="true") +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@alter_table_location2 +POSTHOOK: query: CREATE TABLE alter_table_location2 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS ORC +#### A masked pattern was here #### + TBLPROPERTIES ("transactional"="true") +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter_table_location2 +PREHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (8,'Henry','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location2@year=2016 +POSTHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (8,'Henry','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location2@year=2016 +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).name SCRIPT [] +PREHOOK: query: ALTER TABLE alter_table_location2 ADD PARTITION (year=2017) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@alter_table_location2 +POSTHOOK: query: ALTER TABLE alter_table_location2 ADD PARTITION (year=2017) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@alter_table_location2 +POSTHOOK: Output: default@alter_table_location2@year=2017 +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_LOCATION +PREHOOK: Input: default@alter_table_location2 +PREHOOK: Output: default@alter_table_location2 +PREHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_LOCATION +POSTHOOK: Input: default@alter_table_location2 +POSTHOOK: Output: default@alter_table_location2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (9,'Horace','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location2@year=2016 +POSTHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2016) VALUES (9,'Horace','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location2@year=2016 +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2016).name SCRIPT [] +PREHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2017) VALUES (10,'Harris','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location2@year=2017 +POSTHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2017) VALUES (10,'Harris','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location2@year=2017 +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2017).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2017).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2017).name SCRIPT [] +PREHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2018) VALUES (11,'Humphrey','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location2@year=2018 +POSTHOOK: query: INSERT INTO alter_table_location2 PARTITION (year=2018) VALUES (11,'Humphrey','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location2@year=2018 +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2018).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2018).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location2 PARTITION(year=2018).name SCRIPT [] +PREHOOK: query: SELECT * from alter_table_location2 order by id +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_table_location2 +PREHOOK: Input: default@alter_table_location2@year=2016 +PREHOOK: Input: default@alter_table_location2@year=2017 +PREHOOK: Input: default@alter_table_location2@year=2018 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * from alter_table_location2 order by id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_table_location2 +POSTHOOK: Input: default@alter_table_location2@year=2016 +POSTHOOK: Input: default@alter_table_location2@year=2017 +POSTHOOK: Input: default@alter_table_location2@year=2018 +POSTHOOK: Output: hdfs://### HDFS PATH ### +8 Henry CSE 2016 +9 Horace CSE 2016 +10 Harris CSE 2017 +11 Humphrey CSE 2018 http://git-wip-us.apache.org/repos/asf/hive/blob/4137c212/ql/src/test/results/clientpositive/llap/alter_table_location3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/alter_table_location3.q.out b/ql/src/test/results/clientpositive/llap/alter_table_location3.q.out new file mode 100644 index 0000000..d9db631 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/alter_table_location3.q.out @@ -0,0 +1,95 @@ +PREHOOK: query: CREATE TABLE alter_table_location3 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS TEXTFILE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@alter_table_location3 +POSTHOOK: query: CREATE TABLE alter_table_location3 (id int, name string, dept string) + PARTITIONED BY (year int) + STORED AS TEXTFILE +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter_table_location3 +PREHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (8,'Henry','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location3@year=2016 +POSTHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (8,'Henry','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location3@year=2016 +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).name SCRIPT [] +PREHOOK: query: ALTER TABLE alter_table_location3 ADD PARTITION (year=2017) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@alter_table_location3 +POSTHOOK: query: ALTER TABLE alter_table_location3 ADD PARTITION (year=2017) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@alter_table_location3 +POSTHOOK: Output: default@alter_table_location3@year=2017 +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_LOCATION +PREHOOK: Input: default@alter_table_location3 +PREHOOK: Output: default@alter_table_location3 +PREHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_LOCATION +POSTHOOK: Input: default@alter_table_location3 +POSTHOOK: Output: default@alter_table_location3 +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (9,'Horace','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location3@year=2016 +POSTHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2016) VALUES (9,'Horace','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location3@year=2016 +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2016).name SCRIPT [] +PREHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2017) VALUES (10,'Harris','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location3@year=2017 +POSTHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2017) VALUES (10,'Harris','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location3@year=2017 +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2017).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2017).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2017).name SCRIPT [] +PREHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2018) VALUES (11,'Humphrey','CSE') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@alter_table_location3@year=2018 +POSTHOOK: query: INSERT INTO alter_table_location3 PARTITION (year=2018) VALUES (11,'Humphrey','CSE') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@alter_table_location3@year=2018 +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2018).dept SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2018).id SCRIPT [] +POSTHOOK: Lineage: alter_table_location3 PARTITION(year=2018).name SCRIPT [] +PREHOOK: query: SELECT * from alter_table_location3 order by id +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_table_location3 +PREHOOK: Input: default@alter_table_location3@year=2016 +PREHOOK: Input: default@alter_table_location3@year=2017 +PREHOOK: Input: default@alter_table_location3@year=2018 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * from alter_table_location3 order by id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_table_location3 +POSTHOOK: Input: default@alter_table_location3@year=2016 +POSTHOOK: Input: default@alter_table_location3@year=2017 +POSTHOOK: Input: default@alter_table_location3@year=2018 +POSTHOOK: Output: hdfs://### HDFS PATH ### +8 Henry CSE 2016 +9 Horace CSE 2016 +10 Harris CSE 2017 +11 Humphrey CSE 2018 http://git-wip-us.apache.org/repos/asf/hive/blob/4137c212/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index c681a87..a92f34b 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -325,7 +325,7 @@ public class MetaStoreUtils { public static boolean isArchived(Partition part) { Map<String, String> params = part.getParameters(); - return "TRUE".equalsIgnoreCase(params.get(hive_metastoreConstants.IS_ARCHIVED)); + return (params != null && "TRUE".equalsIgnoreCase(params.get(hive_metastoreConstants.IS_ARCHIVED))); } public static Path getOriginalLocation(Partition part) {
