This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 157d257a0de HIVE-25600: Compaction job creates redundant base/delta
folder within base/delta folder (Nikhil Gupta, reviewed by Sankar Hariappan)
(#4340)
157d257a0de is described below
commit 157d257a0de697da24be3597d60d8a92d4025fb8
Author: Diksha628 <[email protected]>
AuthorDate: Tue May 23 19:57:07 2023 +0530
HIVE-25600: Compaction job creates redundant base/delta folder within
base/delta folder (Nikhil Gupta, reviewed by Sankar Hariappan) (#4340)
Signed-off-by: Sankar Hariappan <[email protected]>
Closes (#2705)
Co-authored-by: guptanikhil007 <[email protected]>
---
.../org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 95870ad46f9..474f6c53426 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -1184,6 +1184,16 @@ public class CompactorMR {
Path tmpPath = fileStatus.getPath();
//newPath is the base/delta dir
Path newPath = new Path(finalLocation, tmpPath.getName());
+ /* rename(A, B) has "interesting" behavior if A and B are directories.
If B doesn't exist,
+ * it does the expected operation and everything that was in A is now
in B. If B exists,
+ * it will make A a child of B.
+ * This issue can happen if the previous MR job succeeded but HMS was
unable to persist compaction result.
+ * We will delete the directory B if it exists to avoid the above issue
+ */
+ if (fs.exists(newPath)) {
+ LOG.info(String.format("Final path %s already exists. Deleting the
path to avoid redundant base creation", newPath.toString()));
+ fs.delete(newPath, true);
+ }
/* Create the markers in the tmp location and rename everything in the
end to prevent race condition between
* marker creation and split read. */
AcidUtils.OrcAcidVersion.writeVersionFile(tmpPath, fs);
@@ -1192,6 +1202,7 @@ public class CompactorMR {
}
fs.delete(tmpLocation, true);
}
+
private void createCompactorMarker(JobConf conf, Path finalLocation,
FileSystem fs)
throws IOException {
if(conf.getBoolean(IS_MAJOR, false)) {
