This is an automated email from the ASF dual-hosted git repository.
bteke pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ff8eac517a3 YARN-11463. Node Labels root directory creation doesn't
have a retry logic - 2nd addendum (#5670)
ff8eac517a3 is described below
commit ff8eac517a3b86aa8167a6bc1b981d8884181ec2
Author: Peter Szucs <116345192+p-sz...@users.noreply.github.com>
AuthorDate: Thu May 18 14:48:43 2023 +0200
YARN-11463. Node Labels root directory creation doesn't have a retry logic
- 2nd addendum (#5670)
---
.../yarn/nodelabels/store/AbstractFSNodeStore.java | 14 ++++++------
.../nodelabels/TestFileSystemNodeLabelsStore.java | 26 ++++++----------------
2 files changed, 14 insertions(+), 26 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
index 6e47b2afb08..972a0d0d5f5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
@@ -65,6 +65,12 @@ public abstract class AbstractFSNodeStore<M> {
this.fsWorkingPath = fsStorePath;
this.manager = mgr;
initFileSystem(conf);
+ initNodeStoreRootDirectory(conf);
+ this.replication = conf.getInt(YarnConfiguration.FS_STORE_FILE_REPLICATION,
+ YarnConfiguration.DEFAULT_FS_STORE_FILE_REPLICATION);
+ }
+
+ private void initNodeStoreRootDirectory(Configuration conf) throws
IOException {
// mkdir of root dir path with retry logic
int maxRetries =
conf.getInt(YarnConfiguration.NODE_STORE_ROOT_DIR_NUM_RETRIES,
YarnConfiguration.NODE_STORE_ROOT_DIR_NUM_DEFAULT_RETRIES);
@@ -73,11 +79,7 @@ public abstract class AbstractFSNodeStore<M> {
while (!success && retryCount <= maxRetries) {
try {
- if (!fs.exists(fsWorkingPath)) {
- success = fs.mkdirs(fsWorkingPath);
- } else {
- success = true;
- }
+ success = fs.mkdirs(fsWorkingPath);
} catch (IOException e) {
retryCount++;
if (retryCount > maxRetries) {
@@ -91,8 +93,6 @@ public abstract class AbstractFSNodeStore<M> {
}
}
}
- this.replication = conf.getInt(YarnConfiguration.FS_STORE_FILE_REPLICATION,
- YarnConfiguration.DEFAULT_FS_STORE_FILE_REPLICATION);
LOG.info("Created store directory :" + fsWorkingPath);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
index 517958367d8..4f964d3478b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
+import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Timeout;
@@ -348,27 +349,13 @@ public class TestFileSystemNodeLabelsStore extends
NodeLabelTestBase {
@MethodSource("getParameters")
@ParameterizedTest
- void testRootMkdirOnInitStoreWhenRootDirectoryAlreadyExists(String
className) throws Exception {
- initTestFileSystemNodeLabelsStore(className);
- final FileSystem mockFs = Mockito.mock(FileSystem.class);
- final FileSystemNodeLabelsStore mockStore =
createMockNodeLabelsStore(mockFs);
- final int expectedMkdirsCount = 0;
-
- Mockito.when(mockStore.getFs().exists(Mockito.any(Path.class)))
- .thenReturn(true);
- verifyMkdirsCount(mockStore, expectedMkdirsCount);
- }
-
- @MethodSource("getParameters")
- @ParameterizedTest
- void testRootMkdirOnInitStoreWhenRootDirectoryNotExists(String className)
throws Exception {
+ void testRootMkdirOnInitStore(String className) throws Exception {
initTestFileSystemNodeLabelsStore(className);
final FileSystem mockFs = Mockito.mock(FileSystem.class);
final FileSystemNodeLabelsStore mockStore =
createMockNodeLabelsStore(mockFs);
final int expectedMkdirsCount = 1;
- Mockito.when(mockStore.getFs().exists(Mockito.any(Path.class)))
- .thenReturn(false).thenReturn(true);
+
Mockito.when(mockStore.getFs().mkdirs(Mockito.any(Path.class))).thenReturn(true);
verifyMkdirsCount(mockStore, expectedMkdirsCount);
}
@@ -378,10 +365,11 @@ public class TestFileSystemNodeLabelsStore extends
NodeLabelTestBase {
initTestFileSystemNodeLabelsStore(className);
final FileSystem mockFs = Mockito.mock(FileSystem.class);
final FileSystemNodeLabelsStore mockStore =
createMockNodeLabelsStore(mockFs);
- final int expectedMkdirsCount = 2;
+ final int expectedMkdirsCount = 3;
- Mockito.when(mockStore.getFs().exists(Mockito.any(Path.class)))
- .thenReturn(false).thenReturn(false).thenReturn(true);
+ Mockito.when(mockStore.getFs().mkdirs(Mockito.any(Path.class)))
+ .thenThrow(SafeModeException.class).thenThrow(SafeModeException.class)
+ .thenReturn(true);
verifyMkdirsCount(mockStore, expectedMkdirsCount);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
