This is an automated email from the ASF dual-hosted git repository.
ckj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 76559ce3 [#722] test: cleanup residue files in tmp directory after
tests (#1134)
76559ce3 is described below
commit 76559ce3b16c1595818947eb8a5d208d02659a39
Author: Neo Chien <[email protected]>
AuthorDate: Mon Aug 21 19:34:06 2023 +0800
[#722] test: cleanup residue files in tmp directory after tests (#1134)
### What changes were proposed in this pull request?
Cleanup residue files in tmp directory after tests
### Why are the changes needed?
Fix: #722
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
current UT
---
.../java/org/apache/uniffle/common/util/ChecksumUtilsTest.java | 5 +++--
.../test/java/org/apache/uniffle/test/IntegrationTestBase.java | 7 ++++---
.../uniffle/test/ShuffleServerWithKerberizedHadoopTest.java | 8 ++++----
.../src/test/java/org/apache/uniffle/test/RepartitionTest.java | 5 +++--
.../src/test/java/org/apache/uniffle/test/SparkSQLTest.java | 5 +++--
.../uniffle/test/ContinuousSelectPartitionStrategyTest.java | 8 ++++----
.../apache/uniffle/test/GetShuffleReportForMultiPartTest.java | 8 ++++----
.../java/org/apache/uniffle/server/ShuffleTaskManagerTest.java | 10 ++++++----
8 files changed, 31 insertions(+), 25 deletions(-)
diff --git
a/common/src/test/java/org/apache/uniffle/common/util/ChecksumUtilsTest.java
b/common/src/test/java/org/apache/uniffle/common/util/ChecksumUtilsTest.java
index 37c31052..100be0f7 100644
--- a/common/src/test/java/org/apache/uniffle/common/util/ChecksumUtilsTest.java
+++ b/common/src/test/java/org/apache/uniffle/common/util/ChecksumUtilsTest.java
@@ -21,18 +21,20 @@ import java.io.File;
import java.io.FileOutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Random;
import java.util.zip.CRC32;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class ChecksumUtilsTest {
+ @TempDir File tempDir;
+
@Test
public void crc32TestWithByte() {
byte[] data = new byte[32 * 1024 * 1024];
@@ -56,7 +58,6 @@ public class ChecksumUtilsTest {
byte[] data = new byte[length];
new Random().nextBytes(data);
- String tempDir = Files.createTempDirectory("rss").toString();
File file = new File(tempDir, "crc_test.txt");
file.createNewFile();
file.deleteOnExit();
diff --git
a/integration-test/common/src/test/java/org/apache/uniffle/test/IntegrationTestBase.java
b/integration-test/common/src/test/java/org/apache/uniffle/test/IntegrationTestBase.java
index e73ccfa7..269a7d12 100644
---
a/integration-test/common/src/test/java/org/apache/uniffle/test/IntegrationTestBase.java
+++
b/integration-test/common/src/test/java/org/apache/uniffle/test/IntegrationTestBase.java
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Lists;
import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.io.TempDir;
import org.apache.uniffle.common.util.RssUtils;
import org.apache.uniffle.coordinator.CoordinatorConf;
@@ -64,6 +65,8 @@ public abstract class IntegrationTestBase extends
HadoopTestBase {
protected static final int NETTY_PORT = 21000;
protected static AtomicInteger nettyPortCounter = new AtomicInteger();
+ static @TempDir File tempDir;
+
public static void startServers() throws Exception {
for (CoordinatorServer coordinator : coordinators) {
coordinator.start();
@@ -106,12 +109,10 @@ public abstract class IntegrationTestBase extends
HadoopTestBase {
}
protected static ShuffleServerConf getShuffleServerConf() throws Exception {
- File dataFolder = Files.createTempDirectory("rssdata").toFile();
ShuffleServerConf serverConf = new ShuffleServerConf();
- dataFolder.deleteOnExit();
serverConf.setInteger("rss.rpc.server.port", SHUFFLE_SERVER_PORT);
serverConf.setString("rss.storage.type",
StorageType.MEMORY_LOCALFILE_HDFS.name());
- serverConf.setString("rss.storage.basePath", dataFolder.getAbsolutePath());
+ serverConf.setString("rss.storage.basePath", tempDir.getAbsolutePath());
serverConf.setString("rss.server.buffer.capacity", "671088640");
serverConf.setString("rss.server.memory.shuffle.highWaterMark", "50.0");
serverConf.setString("rss.server.memory.shuffle.lowWaterMark", "0.0");
diff --git
a/integration-test/common/src/test/java/org/apache/uniffle/test/ShuffleServerWithKerberizedHadoopTest.java
b/integration-test/common/src/test/java/org/apache/uniffle/test/ShuffleServerWithKerberizedHadoopTest.java
index 8bced501..60ad0535 100644
---
a/integration-test/common/src/test/java/org/apache/uniffle/test/ShuffleServerWithKerberizedHadoopTest.java
+++
b/integration-test/common/src/test/java/org/apache/uniffle/test/ShuffleServerWithKerberizedHadoopTest.java
@@ -18,7 +18,6 @@
package org.apache.uniffle.test;
import java.io.File;
-import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -31,6 +30,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.roaringbitmap.longlong.Roaring64NavigableMap;
import org.apache.uniffle.client.TestUtils;
@@ -80,13 +80,13 @@ public class ShuffleServerWithKerberizedHadoopTest extends
KerberizedHadoopBase
private static CoordinatorServer coordinatorServer;
private static ShuffleServer shuffleServer;
+ static @TempDir File tempDir;
+
private static ShuffleServerConf getShuffleServerConf() throws Exception {
- File dataFolder = Files.createTempDirectory("rssdata").toFile();
ShuffleServerConf serverConf = new ShuffleServerConf();
- dataFolder.deleteOnExit();
serverConf.setInteger("rss.rpc.server.port", SHUFFLE_SERVER_PORT);
serverConf.setString("rss.storage.type",
StorageType.MEMORY_LOCALFILE_HDFS.name());
- serverConf.setString("rss.storage.basePath", dataFolder.getAbsolutePath());
+ serverConf.setString("rss.storage.basePath", tempDir.getAbsolutePath());
serverConf.setString("rss.server.buffer.capacity", "671088640");
serverConf.setString("rss.server.memory.shuffle.highWaterMark", "50.0");
serverConf.setString("rss.server.memory.shuffle.lowWaterMark", "0.0");
diff --git
a/integration-test/spark-common/src/test/java/org/apache/uniffle/test/RepartitionTest.java
b/integration-test/spark-common/src/test/java/org/apache/uniffle/test/RepartitionTest.java
index 277649fc..a5d72a42 100644
---
a/integration-test/spark-common/src/test/java/org/apache/uniffle/test/RepartitionTest.java
+++
b/integration-test/spark-common/src/test/java/org/apache/uniffle/test/RepartitionTest.java
@@ -20,7 +20,6 @@ package org.apache.uniffle.test;
import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
-import java.nio.file.Files;
import java.util.Arrays;
import java.util.Map;
import java.util.Random;
@@ -33,6 +32,7 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,6 +40,8 @@ public abstract class RepartitionTest extends
SparkIntegrationTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(RepartitionTest.class);
+ static @TempDir File tempDir;
+
@Test
public void resultCompareTest() throws Exception {
run();
@@ -63,7 +65,6 @@ public abstract class RepartitionTest extends
SparkIntegrationTestBase {
public abstract void updateRssStorage(SparkConf sparkConf);
protected String generateTextFile(int wordsPerRow, int rows) throws
Exception {
- String tempDir = Files.createTempDirectory("rss").toString();
File file = new File(tempDir, "wordcount.txt");
file.createNewFile();
LOG.info("Create file:" + file.getAbsolutePath());
diff --git
a/integration-test/spark-common/src/test/java/org/apache/uniffle/test/SparkSQLTest.java
b/integration-test/spark-common/src/test/java/org/apache/uniffle/test/SparkSQLTest.java
index effce773..1231a6cd 100644
---
a/integration-test/spark-common/src/test/java/org/apache/uniffle/test/SparkSQLTest.java
+++
b/integration-test/spark-common/src/test/java/org/apache/uniffle/test/SparkSQLTest.java
@@ -20,7 +20,6 @@ package org.apache.uniffle.test;
import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
-import java.nio.file.Files;
import java.util.Map;
import java.util.Random;
@@ -31,6 +30,7 @@ import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +38,8 @@ public abstract class SparkSQLTest extends
SparkIntegrationTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(SparkSQLTest.class);
+ static @TempDir File tempDir;
+
@Test
public void resultCompareTest() throws Exception {
run();
@@ -76,7 +78,6 @@ public abstract class SparkSQLTest extends
SparkIntegrationTestBase {
protected String generateCsvFile() throws Exception {
int rows = 1000;
- String tempDir = Files.createTempDirectory("rss").toString();
File file = new File(tempDir, "test.csv");
file.createNewFile();
LOG.info("Create file:" + file.getAbsolutePath());
diff --git
a/integration-test/spark3/src/test/java/org/apache/uniffle/test/ContinuousSelectPartitionStrategyTest.java
b/integration-test/spark3/src/test/java/org/apache/uniffle/test/ContinuousSelectPartitionStrategyTest.java
index f15d26bf..e1dd2ac3 100644
---
a/integration-test/spark3/src/test/java/org/apache/uniffle/test/ContinuousSelectPartitionStrategyTest.java
+++
b/integration-test/spark3/src/test/java/org/apache/uniffle/test/ContinuousSelectPartitionStrategyTest.java
@@ -18,7 +18,6 @@
package org.apache.uniffle.test;
import java.io.File;
-import java.nio.file.Files;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -37,6 +36,7 @@ import org.apache.spark.sql.functions;
import org.apache.spark.sql.internal.SQLConf;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.apache.uniffle.coordinator.CoordinatorConf;
import
org.apache.uniffle.coordinator.strategy.assignment.AbstractAssignmentStrategy;
@@ -54,6 +54,8 @@ public class ContinuousSelectPartitionStrategyTest extends
SparkIntegrationTestB
private static final int replicateWrite = 3;
private static final int replicateRead = 2;
+ static @TempDir File tempDir;
+
@BeforeAll
public static void setupServers() throws Exception {
CoordinatorConf coordinatorConf = getCoordinatorConf();
@@ -75,13 +77,11 @@ public class ContinuousSelectPartitionStrategyTest extends
SparkIntegrationTestB
private static void createShuffleServers() throws Exception {
for (int i = 0; i < 3; i++) {
// Copy from IntegrationTestBase#getShuffleServerConf
- File dataFolder = Files.createTempDirectory("rssdata" + i).toFile();
ShuffleServerConf serverConf = new ShuffleServerConf();
- dataFolder.deleteOnExit();
serverConf.setInteger("rss.rpc.server.port", SHUFFLE_SERVER_PORT + i);
serverConf.setInteger("rss.server.netty.port", NETTY_PORT + i);
serverConf.setString("rss.storage.type",
StorageType.MEMORY_LOCALFILE_HDFS.name());
- serverConf.setString("rss.storage.basePath",
dataFolder.getAbsolutePath());
+ serverConf.setString("rss.storage.basePath", tempDir.getAbsolutePath());
serverConf.setString("rss.server.buffer.capacity",
String.valueOf(671088640 - i));
serverConf.setString("rss.server.memory.shuffle.highWaterMark", "50.0");
serverConf.setString("rss.server.memory.shuffle.lowWaterMark", "0.0");
diff --git
a/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetShuffleReportForMultiPartTest.java
b/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetShuffleReportForMultiPartTest.java
index f7944ceb..d9791702 100644
---
a/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetShuffleReportForMultiPartTest.java
+++
b/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetShuffleReportForMultiPartTest.java
@@ -18,7 +18,6 @@
package org.apache.uniffle.test;
import java.io.File;
-import java.nio.file.Files;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -43,6 +42,7 @@ import org.apache.spark.sql.functions;
import org.apache.spark.sql.internal.SQLConf;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.roaringbitmap.longlong.Roaring64NavigableMap;
import org.apache.uniffle.common.ShuffleServerInfo;
@@ -62,6 +62,8 @@ public class GetShuffleReportForMultiPartTest extends
SparkIntegrationTestBase {
private static final int replicateWrite = 3;
private static final int replicateRead = 2;
+ static @TempDir File tempDir;
+
@BeforeAll
public static void setupServers() throws Exception {
CoordinatorConf coordinatorConf = getCoordinatorConf();
@@ -79,13 +81,11 @@ public class GetShuffleReportForMultiPartTest extends
SparkIntegrationTestBase {
private static void createShuffleServers() throws Exception {
for (int i = 0; i < 4; i++) {
// Copy from IntegrationTestBase#getShuffleServerConf
- File dataFolder = Files.createTempDirectory("rssdata" + i).toFile();
ShuffleServerConf serverConf = new ShuffleServerConf();
- dataFolder.deleteOnExit();
serverConf.setInteger("rss.rpc.server.port", SHUFFLE_SERVER_PORT + i);
serverConf.setInteger("rss.server.netty.port", NETTY_PORT + i);
serverConf.setString("rss.storage.type",
StorageType.MEMORY_LOCALFILE_HDFS.name());
- serverConf.setString("rss.storage.basePath",
dataFolder.getAbsolutePath());
+ serverConf.setString("rss.storage.basePath", tempDir.getAbsolutePath());
serverConf.setString("rss.server.buffer.capacity", "671088640");
serverConf.setString("rss.server.memory.shuffle.highWaterMark", "50.0");
serverConf.setString("rss.server.memory.shuffle.lowWaterMark", "0.0");
diff --git
a/server/src/test/java/org/apache/uniffle/server/ShuffleTaskManagerTest.java
b/server/src/test/java/org/apache/uniffle/server/ShuffleTaskManagerTest.java
index 3458655d..ea32911e 100644
--- a/server/src/test/java/org/apache/uniffle/server/ShuffleTaskManagerTest.java
+++ b/server/src/test/java/org/apache/uniffle/server/ShuffleTaskManagerTest.java
@@ -18,7 +18,6 @@
package org.apache.uniffle.server;
import java.io.File;
-import java.nio.file.Files;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
@@ -80,10 +79,15 @@ public class ShuffleTaskManagerTest extends HadoopTestBase {
private ShuffleServer shuffleServer;
+ @TempDir File tempDir1;
+ @TempDir File tempDir2;
+
@BeforeEach
public void beforeEach() {
ShuffleServerMetrics.clear();
ShuffleServerMetrics.register();
+ assertTrue(this.tempDir1.isDirectory());
+ assertTrue(this.tempDir2.isDirectory());
}
@AfterEach
@@ -463,11 +467,9 @@ public class ShuffleTaskManagerTest extends HadoopTestBase
{
conf.setString(ShuffleServerConf.RSS_STORAGE_TYPE.key(), "LOCALFILE");
conf.set(ShuffleServerConf.RSS_TEST_MODE_ENABLE, true);
- java.nio.file.Path path1 =
Files.createTempDirectory("removeShuffleDataWithLocalfileTest");
- java.nio.file.Path path2 =
Files.createTempDirectory("removeShuffleDataWithLocalfileTest");
conf.setString(
ShuffleServerConf.RSS_STORAGE_BASE_PATH.key(),
- path1.toAbsolutePath().toString() + "," +
path2.toAbsolutePath().toString());
+ tempDir1.getAbsolutePath() + "," + tempDir2.getAbsolutePath());
shuffleServer = new ShuffleServer(conf);
ShuffleTaskManager shuffleTaskManager =
shuffleServer.getShuffleTaskManager();
