This is an automated email from the ASF dual-hosted git repository.
zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 917456c5e [#1675] fix(test): Fix tests which may be flaky on different
machines (#1676)
917456c5e is described below
commit 917456c5eb0d7b70f7faa5da489aaa29d54cde40
Author: RickyMa <[email protected]>
AuthorDate: Wed May 8 13:04:04 2024 +0800
[#1675] fix(test): Fix tests which may be flaky on different machines
(#1676)
### What changes were proposed in this pull request?
Fix tests which may be flaky on different machines.
### Why are the changes needed?
Fix: https://github.com/apache/incubator-uniffle/issues/1675.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unnecessary.
---
.../server/storage/LocalStorageManagerTest.java | 42 +++++++++++++++++-----
1 file changed, 33 insertions(+), 9 deletions(-)
diff --git
a/server/src/test/java/org/apache/uniffle/server/storage/LocalStorageManagerTest.java
b/server/src/test/java/org/apache/uniffle/server/storage/LocalStorageManagerTest.java
index 552265863..5584e2609 100644
---
a/server/src/test/java/org/apache/uniffle/server/storage/LocalStorageManagerTest.java
+++
b/server/src/test/java/org/apache/uniffle/server/storage/LocalStorageManagerTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.SystemUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@@ -190,12 +191,18 @@ public class LocalStorageManagerTest {
}
@Test
- public void testInitializeLocalStorage() {
+ public void testInitializeLocalStorage() throws IOException {
+ final File storageBaseDir1 =
Files.createTempDirectory("rss-data-1").toFile();
+ final File storageBaseDir2 =
Files.createTempDirectory("rss-data-2").toFile();
+ final File rootRestrictedDir1 = new
File("/proc/rss-data-mock-restricted-dir-1");
+ final File rootRestrictedDir2 = new
File("/proc/rss-data-mock-restricted-dir-2");
// case1: when no candidates, it should throw exception.
ShuffleServerConf conf = new ShuffleServerConf();
conf.setLong(ShuffleServerConf.FLUSH_COLD_STORAGE_THRESHOLD_SIZE, 2000L);
- conf.set(ShuffleServerConf.RSS_STORAGE_BASE_PATH,
Arrays.asList("/a/rss-data", "/b/rss-data"));
+ conf.set(
+ ShuffleServerConf.RSS_STORAGE_BASE_PATH,
+ Arrays.asList(rootRestrictedDir1.getAbsolutePath(),
rootRestrictedDir2.getAbsolutePath()));
conf.setLong(ShuffleServerConf.DISK_CAPACITY, 1024L);
conf.setLong(ShuffleServerConf.LOCAL_STORAGE_INITIALIZE_MAX_FAIL_NUMBER,
1);
conf.setString(
@@ -209,21 +216,33 @@ public class LocalStorageManagerTest {
// case2: when candidates exist, it should initialize successfully.
conf.set(
- ShuffleServerConf.RSS_STORAGE_BASE_PATH, Arrays.asList("/a/rss-data",
"/tmp/rss-data-1"));
+ ShuffleServerConf.RSS_STORAGE_BASE_PATH,
+ Arrays.asList(storageBaseDir1.getAbsolutePath(),
rootRestrictedDir1.getAbsolutePath()));
LocalStorageManager localStorageManager = new LocalStorageManager(conf);
assertEquals(1, localStorageManager.getStorages().size());
// case3: all ok
conf.set(
ShuffleServerConf.RSS_STORAGE_BASE_PATH,
- Arrays.asList("/tmp/rss-data-1", "/tmp/rss-data-2"));
+ Arrays.asList(storageBaseDir1.getAbsolutePath(),
storageBaseDir2.getAbsolutePath()));
localStorageManager = new LocalStorageManager(conf);
assertEquals(2, localStorageManager.getStorages().size());
- // case4: only have 1 candidate, but exceed the number of
+ // case4: after https://github.com/apache/incubator-uniffle/pull/616
+ // dirs will be created automatically if they do not exist
+ FileUtils.deleteQuietly(storageBaseDir1);
+ FileUtils.deleteQuietly(storageBaseDir2);
+ conf.set(
+ ShuffleServerConf.RSS_STORAGE_BASE_PATH,
+ Arrays.asList(storageBaseDir1.getAbsolutePath(),
storageBaseDir2.getAbsolutePath()));
+ localStorageManager = new LocalStorageManager(conf);
+ assertEquals(2, localStorageManager.getStorages().size());
+
+ // case5: only have 1 candidate, but exceed the number of
// rss.server.localstorage.initialize.max.fail.number
conf.set(
- ShuffleServerConf.RSS_STORAGE_BASE_PATH, Arrays.asList("/a/rss-data",
"/tmp/rss-data-1"));
+ ShuffleServerConf.RSS_STORAGE_BASE_PATH,
+ Arrays.asList(storageBaseDir1.getAbsolutePath(),
rootRestrictedDir1.getAbsolutePath()));
conf.setLong(ShuffleServerConf.LOCAL_STORAGE_INITIALIZE_MAX_FAIL_NUMBER,
0L);
try {
localStorageManager = new LocalStorageManager(conf);
@@ -232,10 +251,11 @@ public class LocalStorageManagerTest {
// ignore
}
- // case5: if failed=2, but lower than
rss.server.localstorage.initialize.max.fail.number, should
- // exit
+ // case6: if failed=2, but lower than
rss.server.localstorage.initialize.max.fail.number,
+ // it should fail too
conf.set(
- ShuffleServerConf.RSS_STORAGE_BASE_PATH, Arrays.asList("/a/rss-data",
"/b/rss-data-1"));
+ ShuffleServerConf.RSS_STORAGE_BASE_PATH,
+ Arrays.asList(rootRestrictedDir1.getAbsolutePath(),
rootRestrictedDir2.getAbsolutePath()));
conf.setLong(ShuffleServerConf.LOCAL_STORAGE_INITIALIZE_MAX_FAIL_NUMBER,
10L);
try {
localStorageManager = new LocalStorageManager(conf);
@@ -243,6 +263,10 @@ public class LocalStorageManagerTest {
} catch (Exception e) {
// ignore
}
+
+ // clear temp dirs
+ FileUtils.deleteQuietly(storageBaseDir1);
+ FileUtils.deleteQuietly(storageBaseDir2);
}
@Test
