squito commented on a change in pull request #24499: [SPARK-25888][Core] Serve
local disk persisted blocks by the external service after releasing executor by
dynamic allocation
URL: https://github.com/apache/spark/pull/24499#discussion_r281801229
##########
File path:
common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/CleanupNonShuffleServiceServedFilesSuite.java
##########
@@ -156,66 +156,50 @@ private static void
assertStillThere(TestShuffleDataContext dataContext) {
}
}
- private static FilenameFilter filter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- // Don't delete shuffle data or shuffle index files.
- return !name.endsWith(".index") && !name.endsWith(".data");
- }
+ private static FilenameFilter filter = (dir, name) -> {
+ // Don't delete shuffle data or shuffle index files.
+ return !name.endsWith(".index") && !name.endsWith(".data") &&
!name.startsWith("rdd_");
};
- private static boolean assertOnlyShuffleDataInDir(File[] dirs) {
+ private static boolean assertOnlyFilesToKeepInDir(File[] dirs) {
for (File dir : dirs) {
assertTrue(dir.getName() + " wasn't cleaned up", !dir.exists() ||
- dir.listFiles(filter).length == 0 ||
assertOnlyShuffleDataInDir(dir.listFiles()));
+ dir.listFiles(filter).length == 0 ||
assertOnlyFilesToKeepInDir(dir.listFiles()));
}
return true;
}
private static void assertCleanedUp(TestShuffleDataContext dataContext) {
for (String localDir : dataContext.localDirs) {
File[] dirs = new File[] {new File(localDir)};
- assertOnlyShuffleDataInDir(dirs);
+ assertOnlyFilesToKeepInDir(dirs);
}
}
- private static TestShuffleDataContext initDataContext(boolean
withShuffleFiles)
- throws IOException {
- if (withShuffleFiles) {
- return initDataContextWithShuffleFiles();
- } else {
- return initDataContextWithoutShuffleFiles();
- }
- }
-
- private static TestShuffleDataContext initDataContextWithShuffleFiles()
throws IOException {
- TestShuffleDataContext dataContext = createDataContext();
- createShuffleFiles(dataContext);
- createNonShuffleFiles(dataContext);
- return dataContext;
- }
-
- private static TestShuffleDataContext initDataContextWithoutShuffleFiles()
throws IOException {
- TestShuffleDataContext dataContext = createDataContext();
- createNonShuffleFiles(dataContext);
- return dataContext;
- }
-
- private static TestShuffleDataContext createDataContext() {
+ private static TestShuffleDataContext initDataContext(boolean
withFilesToKeep)
+ throws IOException {
Review comment:
nit: indent more
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
