Repository: hive Updated Branches: refs/heads/master 637224fcc -> 1faadb074
HIVE-17063 : insert overwrite partition onto a external table fail when drop partition first (Wang Haihua, Deepak Jaiswal, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1faadb07 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1faadb07 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1faadb07 Branch: refs/heads/master Commit: 1faadb0744975538a0ec4b2de63f05caf37b6f40 Parents: 637224f Author: djaiswal <djais...@apache.org> Authored: Thu Feb 8 00:10:57 2018 -0800 Committer: djaiswal <djais...@apache.org> Committed: Thu Feb 8 00:10:57 2018 -0800 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../apache/hadoop/hive/ql/metadata/Hive.java | 22 ++++++++++-- .../insert_after_drop_partition.q | 7 ++++ .../llap/insert_after_drop_partition.q.out | 36 ++++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1faadb07/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 2a22db9..974bfac 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -555,6 +555,7 @@ minillaplocal.query.files=\ is_distinct_from.q,\ infer_bucket_sort_bucketed_table.q,\ input16_cc.q,\ + insert_after_drop_partition.q,\ insert_dir_distcp.q,\ insert_into_with_schema.q,\ insert_values_orig_table.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/1faadb07/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 109f4c7..c8d1589 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -61,8 +61,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.common.JavaUtils; @@ -3670,7 +3672,15 @@ private void constructOneLBLocationMap(FileStatus fSta, "Unable to move source " + srcStatus.getPath() + " to destination " + destFile; if (null == pool) { - if(!destFs.rename(srcStatus.getPath(), destFile)) { + boolean success = false; + if (destFs instanceof DistributedFileSystem) { + ((DistributedFileSystem)destFs).rename(srcStatus.getPath(), destFile, Options.Rename.OVERWRITE); + success = true; + } else { + destFs.delete(destFile, false); + success = destFs.rename(srcStatus.getPath(), destFile); + } + if(!success) { throw new IOException("rename for src path: " + srcStatus.getPath() + " to dest:" + destf + " returned false"); } @@ -3681,7 +3691,15 @@ private void constructOneLBLocationMap(FileStatus fSta, SessionState.setCurrentSessionState(parentSession); final String group = srcStatus.getGroup(); try { - if (!destFs.rename(srcStatus.getPath(), destFile)) { + boolean success = false; + if (destFs instanceof DistributedFileSystem) { + ((DistributedFileSystem)destFs).rename(srcStatus.getPath(), destFile, Options.Rename.OVERWRITE); + success = true; + } else { + destFs.delete(destFile, false); + success = destFs.rename(srcStatus.getPath(), destFile); + } + if (!success) { throw new IOException( "rename for src path: " + srcStatus.getPath() + " to dest path:" + destFile + " returned false"); http://git-wip-us.apache.org/repos/asf/hive/blob/1faadb07/ql/src/test/queries/clientpositive/insert_after_drop_partition.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/insert_after_drop_partition.q b/ql/src/test/queries/clientpositive/insert_after_drop_partition.q new file mode 100644 index 0000000..0087560 --- /dev/null +++ b/ql/src/test/queries/clientpositive/insert_after_drop_partition.q @@ -0,0 +1,7 @@ +create external table insert_after_drop_partition(key string, val string) partitioned by (insertdate string); + +insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10; + +alter table insert_after_drop_partition drop partition (insertdate='2008-01-01'); + +insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/1faadb07/ql/src/test/results/clientpositive/llap/insert_after_drop_partition.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/insert_after_drop_partition.q.out b/ql/src/test/results/clientpositive/llap/insert_after_drop_partition.q.out new file mode 100644 index 0000000..abe0f8d --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/insert_after_drop_partition.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: create external table insert_after_drop_partition(key string, val string) partitioned by (insertdate string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_after_drop_partition +POSTHOOK: query: create external table insert_after_drop_partition(key string, val string) partitioned by (insertdate string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_after_drop_partition +PREHOOK: query: insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01 +POSTHOOK: query: insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01 +POSTHOOK: Lineage: insert_after_drop_partition PARTITION(insertdate=2008-01-01).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_after_drop_partition PARTITION(insertdate=2008-01-01).val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: alter table insert_after_drop_partition drop partition (insertdate='2008-01-01') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@insert_after_drop_partition +PREHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01 +POSTHOOK: query: alter table insert_after_drop_partition drop partition (insertdate='2008-01-01') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@insert_after_drop_partition +POSTHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01 +PREHOOK: query: insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01 +POSTHOOK: query: insert overwrite table insert_after_drop_partition partition (insertdate='2008-01-01') select * from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_after_drop_partition@insertdate=2008-01-01 +POSTHOOK: Lineage: insert_after_drop_partition PARTITION(insertdate=2008-01-01).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_after_drop_partition PARTITION(insertdate=2008-01-01).val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
