This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git
The following commit(s) were added to refs/heads/master by this push:
new 499a79375 RATIS-1752. Clean md5 file created by old ratis version.
(#790)
499a79375 is described below
commit 499a79375a58bb37718abe2176f8df279b66d521
Author: qian0817 <[email protected]>
AuthorDate: Tue Nov 29 02:33:50 2022 +0800
RATIS-1752. Clean md5 file created by old ratis version. (#790)
---
.../main/java/org/apache/ratis/util/FileUtils.java | 22 +++++++++--
.../impl/SimpleStateMachineStorage.java | 43 ++++++++++++++++------
.../ratis/server/storage/TestRaftStorage.java | 13 ++++++-
3 files changed, 62 insertions(+), 16 deletions(-)
diff --git a/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java
b/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java
index f978a79c3..15ec6ea94 100644
--- a/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java
+++ b/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java
@@ -143,13 +143,29 @@ public interface FileUtils {
deleteFully(source);
}
- /** The same as passing f.toPath() to {@link #delete(Path)}. */
+ /**
+ * The same as passing f.toPath() to {@link #deletePathQuietly(Path)}.
+ *
+ * @param f file to delete
+ * @return true if the file is successfully deleted false otherwise
+ */
static boolean deleteFileQuietly(File f) {
+ return deletePathQuietly(f.toPath());
+ }
+
+ /**
+ * Delete the given path quietly.
+ * Only print a debug message in case that there is an exception,
+ *
+ * @param p path to delete
+ * @return true if the path is successfully deleted false otherwise
+ */
+ static boolean deletePathQuietly(Path p) {
try {
- delete(f.toPath());
+ delete(p);
return true;
} catch (Exception ex) {
- LOG.debug("File delete was not successful {}", f.getAbsoluteFile(), ex);
+ LOG.debug("Failed to delete " + p.toAbsolutePath(), ex);
return false;
}
}
diff --git
a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
index 0a6ce2936..37e94a8a0 100644
---
a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
+++
b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
@@ -17,6 +17,8 @@
*/
package org.apache.ratis.statemachine.impl;
+import static org.apache.ratis.util.MD5FileUtil.MD5_SUFFIX;
+
import org.apache.ratis.io.MD5Hash;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.storage.FileInfo;
@@ -44,7 +46,6 @@ import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
/**
* A StateMachineStorage that stores the snapshot in a single file.
@@ -58,6 +59,14 @@ public class SimpleStateMachineStorage implements
StateMachineStorage {
/** snapshot.term_index */
public static final Pattern SNAPSHOT_REGEX =
Pattern.compile(SNAPSHOT_FILE_PREFIX + "\\.(\\d+)_(\\d+)");
+ public static final Pattern SNAPSHOT_MD5_REGEX =
+ Pattern.compile(SNAPSHOT_FILE_PREFIX + "\\.(\\d+)_(\\d+)" + MD5_SUFFIX);
+ private static final DirectoryStream.Filter<Path> SNAPSHOT_MD5_FILTER
+ = entry -> Optional.ofNullable(entry.getFileName())
+ .map(Path::toString)
+ .map(SNAPSHOT_MD5_REGEX::matcher)
+ .filter(Matcher::matches)
+ .isPresent();
private volatile File stateMachineDir = null;
private final AtomicReference<SingleFileSnapshotInfo> latestSnapshot = new
AtomicReference<>();
@@ -109,16 +118,28 @@ public class SimpleStateMachineStorage implements
StateMachineStorage {
if (allSnapshotFiles.size() >
snapshotRetentionPolicy.getNumSnapshotsRetained()) {
allSnapshotFiles.sort(Comparator.comparing(SingleFileSnapshotInfo::getIndex).reversed());
- List<File> snapshotFilesToBeCleaned = allSnapshotFiles.subList(
- snapshotRetentionPolicy.getNumSnapshotsRetained(),
allSnapshotFiles.size()).stream()
- .map(singleFileSnapshotInfo ->
singleFileSnapshotInfo.getFile().getPath().toFile())
- .collect(Collectors.toList());
- for (File snapshotFile : snapshotFilesToBeCleaned) {
- LOG.info("Deleting old snapshot at {}",
snapshotFile.getAbsolutePath());
- final boolean deleted = FileUtils.deleteFileQuietly(snapshotFile);
- if (deleted) {
- final File md5file = MD5FileUtil.getDigestFileForFile(snapshotFile);
- FileUtils.deleteFileQuietly(md5file);
+ allSnapshotFiles.subList(numSnapshotsRetained, allSnapshotFiles.size())
+ .stream()
+ .map(SingleFileSnapshotInfo::getFile)
+ .map(FileInfo::getPath)
+ .forEach(snapshotPath -> {
+ LOG.info("Deleting old snapshot at {}",
snapshotPath.toAbsolutePath());
+ FileUtils.deletePathQuietly(snapshotPath);
+ });
+ // clean up the md5 files if the corresponding snapshot file does not
exist
+ try (DirectoryStream<Path> stream =
Files.newDirectoryStream(stateMachineDir.toPath(),
+ SNAPSHOT_MD5_FILTER)) {
+ for (Path md5path : stream) {
+ Path md5FileNamePath = md5path.getFileName();
+ if (md5FileNamePath == null) {
+ continue;
+ }
+ final String md5FileName = md5FileNamePath.toString();
+ final File snapshotFile = new File(stateMachineDir,
+ md5FileName.substring(0, md5FileName.length() -
MD5_SUFFIX.length()));
+ if (!snapshotFile.exists()) {
+ FileUtils.deletePathQuietly(md5path);
+ }
}
}
}
diff --git
a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
index 7027bd8ea..4465cfdf3 100644
---
a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
+++
b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
@@ -18,6 +18,7 @@
package org.apache.ratis.server.storage;
import static
org.apache.ratis.statemachine.impl.SimpleStateMachineStorage.SNAPSHOT_REGEX;
+import static org.apache.ratis.util.MD5FileUtil.MD5_SUFFIX;
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
@@ -237,14 +238,22 @@ public class TestRaftStorage extends BaseTest {
//Create 5 snapshot files in storage dir.
for (int i = 0; i < 5; i++) {
final long term = ThreadLocalRandom.current().nextLong(10L);
- final long index = ThreadLocalRandom.current().nextLong(1000L);
+ final long index = ThreadLocalRandom.current().nextLong(100, 1000L);
indices.add(index);
File file = simpleStateMachineStorage.getSnapshotFile(term, index);
file.createNewFile();
}
+ // create MD5 files that will not be deleted in older version
+ for (int i = 0; i < 2; i++) {
+ final long index = ThreadLocalRandom.current().nextLong(0, 100L);
+ indices.add(index);
+ File file = simpleStateMachineStorage.getSnapshotFile(1, index);
+ File snapshotFile = new File(file.getParent(), file.getName() +
MD5_SUFFIX);
+ snapshotFile.createNewFile();
+ }
File stateMachineDir = storage.getStorageDir().getStateMachineDir();
- Assert.assertTrue(stateMachineDir.listFiles().length == 5);
+ Assert.assertTrue(stateMachineDir.listFiles().length == 7);
simpleStateMachineStorage.cleanupOldSnapshots(snapshotRetentionPolicy);
File[] remainingFiles = stateMachineDir.listFiles();
Assert.assertTrue(remainingFiles.length == 3);
