This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new ee71318d720 HADOOP-18636 LocalDirAllocator cannot recover from
directory tree deletion (#5412)
ee71318d720 is described below
commit ee71318d720ea2a19f209116134fffb9785146d7
Author: Steve Loughran <ste...@cloudera.com>
AuthorDate: Wed Feb 22 11:48:12 2023 +0000
HADOOP-18636 LocalDirAllocator cannot recover from directory tree deletion
(#5412)
Even though DiskChecker.mkdirsWithExistsCheck() will create the directory
tree,
it is only called *after* the enumeration of directories with available
space has completed.
Directories which don't exist are reported as having 0 space, therefore
the mkdirs code is never reached.
Adding a simple mkdirs() -without bothering to check the outcome-
ensures that if a dir has been deleted then it will be reconstructed
if possible. If it can't it will still have 0 bytes of space
reported and so be excluded from the allocation.
Contributed by Steve Loughran
---
.../java/org/apache/hadoop/fs/LocalDirAllocator.java | 9 ++++++++-
.../org/apache/hadoop/fs/TestLocalDirAllocator.java | 19 +++++++++++++++++++
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
index 774e015b373..d8ab16f41d3 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
@@ -414,7 +414,14 @@ public class LocalDirAllocator {
//build the "roulette wheel"
for(int i =0; i < ctx.dirDF.length; ++i) {
- availableOnDisk[i] = ctx.dirDF[i].getAvailable();
+ final DF target = ctx.dirDF[i];
+ // attempt to recreate the dir so that getAvailable() is valid
+ // if it fails, getAvailable() will return 0, so the dir will
+ // be declared unavailable.
+ // return value is logged at debug to keep spotbugs quiet.
+ final boolean b = new File(target.getDirPath()).mkdirs();
+ LOG.debug("mkdirs of {}={}", target, b);
+ availableOnDisk[i] = target.getAvailable();
totalAvailable += availableOnDisk[i];
}
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
index 939881f39df..3693b4f0acd 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
@@ -548,5 +548,24 @@ public class TestLocalDirAllocator {
"p1/x", Long.MAX_VALUE - 1), "Expect a DiskErrorException.",
() -> dirAllocator.getLocalPathForWrite("p1/x", Long.MAX_VALUE - 1,
conf));
}
+
+ /**
+ * Test for HADOOP-18636 LocalDirAllocator cannot recover from directory
tree deletion.
+ */
+ @Test(timeout = 30000)
+ public void testDirectoryRecovery() throws Throwable {
+ String dir0 = buildBufferDir(ROOT, 0);
+ String subdir = dir0 + "/subdir1/subdir2";
+
+ conf.set(CONTEXT, subdir);
+ // get local path and an ancestor
+ final Path pathForWrite = dirAllocator.getLocalPathForWrite("file", -1,
conf);
+ final Path ancestor = pathForWrite.getParent().getParent();
+
+ // delete that ancestor
+ localFs.delete(ancestor, true);
+ // and expect to get a new file back
+ dirAllocator.getLocalPathForWrite("file2", -1, conf);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
