This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git
The following commit(s) were added to refs/heads/master by this push:
new bf15765a5 RATIS-2178. Add a log message whenever deleting a RaftLog
segment. (#1170)
bf15765a5 is described below
commit bf15765a58ac1476a13239d812df0c0b172acb22
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Sun Oct 27 20:56:18 2024 -0700
RATIS-2178. Add a log message whenever deleting a RaftLog segment. (#1170)
---
.../main/java/org/apache/ratis/util/FileUtils.java | 6 ++-
.../ratis/server/raftlog/segmented/LogSegment.java | 7 +--
.../raftlog/segmented/LogSegmentStartEnd.java | 39 ++++++++++-----
.../server/raftlog/segmented/SegmentedRaftLog.java | 3 +-
.../raftlog/segmented/SegmentedRaftLogCache.java | 25 ++++++----
.../segmented/SegmentedRaftLogInputStream.java | 57 ++++++----------------
.../raftlog/segmented/SegmentedRaftLogWorker.java | 23 +++++----
.../segmented/SegmentedRaftLogTestUtils.java | 14 ++++--
.../apache/ratis/server/ServerRestartTests.java | 14 ++----
.../server/raftlog/segmented/TestLogSegment.java | 12 +++--
.../raftlog/segmented/TestRaftLogReadWrite.java | 12 +++--
11 files changed, 108 insertions(+), 104 deletions(-)
diff --git a/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java
b/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java
index d5141e917..63bf3a4e5 100644
--- a/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java
+++ b/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java
@@ -201,8 +201,10 @@ public interface FileUtils {
}
/** The same as passing f.toPath() to {@link #delete(Path)}. */
- static void deleteFile(File f) throws IOException {
- delete(f.toPath());
+ static Path deleteFile(File f) throws IOException {
+ final Path path = f.toPath();
+ delete(path);
+ return path;
}
/**
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegment.java
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegment.java
index a88ade587..2eea0f90f 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegment.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegment.java
@@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Path;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
@@ -172,8 +173,7 @@ public final class LogSegment {
Consumer<ReferenceCountedObject<LogEntryProto>> entryConsumer)
throws IOException {
int count = 0;
- try (SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(
- file, startEnd.getStartIndex(), startEnd.getEndIndex(),
startEnd.isOpen(), maxOpSize, raftLogMetrics)) {
+ try(SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(file,
startEnd, maxOpSize, raftLogMetrics)) {
for(LogEntryProto prev = null, next; (next = in.nextEntry()) != null;
prev = next) {
if (prev != null) {
Preconditions.assertTrue(next.getIndex() == prev.getIndex() + 1,
@@ -224,7 +224,8 @@ public final class LogSegment {
if (entryCount == 0) {
// The segment does not have any entries, delete the file.
- FileUtils.deleteFile(file);
+ final Path deleted = FileUtils.deleteFile(file);
+ LOG.info("Deleted RaftLog segment since entry count is zero:
startEnd={}, path={}", startEnd, deleted);
return null;
} else if (file.length() > segment.getTotalFileSize()) {
// The segment has extra padding, truncate it.
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentStartEnd.java
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentStartEnd.java
index 6fbaeab62..4f0734ca9 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentStartEnd.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentStartEnd.java
@@ -22,14 +22,12 @@ import org.apache.ratis.server.storage.RaftStorage;
import org.apache.ratis.util.Preconditions;
import java.io.File;
-import java.util.Comparator;
import java.util.Objects;
-import java.util.Optional;
import java.util.regex.Pattern;
/**
* The start index and an end index of a log segment.
- *
+ * <p>
* This is a value-based class.
*/
public final class LogSegmentStartEnd implements
Comparable<LogSegmentStartEnd> {
@@ -76,21 +74,24 @@ public final class LogSegmentStartEnd implements
Comparable<LogSegmentStartEnd>
private final Long endIndex;
private LogSegmentStartEnd(long startIndex, Long endIndex) {
- Preconditions.assertTrue(startIndex >= RaftLog.LEAST_VALID_LOG_INDEX);
- Preconditions.assertTrue(endIndex == null || endIndex >= startIndex);
this.startIndex = startIndex;
this.endIndex = endIndex;
+
+ Preconditions.assertTrue(startIndex >= RaftLog.LEAST_VALID_LOG_INDEX,
this);
+ if (endIndex != null) {
+ Preconditions.assertTrue(endIndex >= startIndex, this);
+ }
}
- public long getStartIndex() {
+ long getStartIndex() {
return startIndex;
}
- public long getEndIndex() {
- return Optional.ofNullable(endIndex).orElse(RaftLog.INVALID_LOG_INDEX);
+ long getEndIndex() {
+ return Objects.requireNonNull(endIndex, "endIndex");
}
- public boolean isOpen() {
+ boolean isOpen() {
return endIndex == null;
}
@@ -108,9 +109,21 @@ public final class LogSegmentStartEnd implements
Comparable<LogSegmentStartEnd>
@Override
public int compareTo(LogSegmentStartEnd that) {
- return Comparator.comparingLong(LogSegmentStartEnd::getStartIndex)
- .thenComparingLong(LogSegmentStartEnd::getEndIndex)
- .compare(this, that);
+ if (this == that) {
+ return 0;
+ }
+ // startIndex always non-null
+ final int diff = Long.compare(this.getStartIndex(), that.getStartIndex());
+ if (diff != 0) {
+ return diff;
+ }
+
+ // same startIndex, compare endIndex
+ if (this.isOpen()) {
+ return that.isOpen()? 0 : -1; //open first
+ } else {
+ return that.isOpen() ? 1 : Long.compare(this.endIndex, that.endIndex);
+ }
}
@Override
@@ -131,6 +144,6 @@ public final class LogSegmentStartEnd implements
Comparable<LogSegmentStartEnd>
@Override
public String toString() {
- return startIndex + "-" +
Optional.ofNullable(endIndex).map(Object::toString).orElse("");
+ return startIndex + "-" + (endIndex != null? endIndex : "");
}
}
\ No newline at end of file
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java
index 485eb53d1..01143c753 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java
@@ -405,12 +405,13 @@ public final class SegmentedRaftLog extends RaftLogBase {
try (AutoCloseableLock writeLock = writeLock()) {
SegmentedRaftLogCache.TruncationSegments ts = cache.purge(index);
updateSnapshotIndexFromStateMachine();
- LOG.debug("purging segments:{}", ts);
if (ts != null) {
+ LOG.info("{}: {}", getName(), ts);
Task task = fileLogWorker.purge(ts);
return task.getFuture();
}
}
+ LOG.debug("{}: purge({}) found nothing to purge.", getName(), index);
return CompletableFuture.completedFuture(index);
}
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogCache.java
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogCache.java
index 8b194bbc9..b574da545 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogCache.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogCache.java
@@ -124,9 +124,14 @@ public class SegmentedRaftLogCache {
}
static class TruncationSegments {
+ private final String reason;
private final SegmentFileInfo toTruncate; // name of the file to be
truncated
private final SegmentFileInfo[] toDelete; // names of the files to be
deleted
+ String getReason() {
+ return reason;
+ }
+
public SegmentFileInfo getToTruncate() {
return toTruncate;
}
@@ -135,10 +140,10 @@ public class SegmentedRaftLogCache {
return toDelete;
}
- TruncationSegments(SegmentFileInfo toTruncate,
- List<SegmentFileInfo> toDelete) {
- this.toDelete = SegmentFileInfo.toSortedArray(toDelete);
+ TruncationSegments(String reason, SegmentFileInfo toTruncate,
List<SegmentFileInfo> toDelete) {
+ this.reason = reason;
this.toTruncate = toTruncate;
+ this.toDelete = SegmentFileInfo.toSortedArray(toDelete);
}
long maxEndIndex() {
@@ -154,8 +159,9 @@ public class SegmentedRaftLogCache {
@Override
public String toString() {
- return "toTruncate: " + toTruncate
- + "\n toDelete: " + Arrays.toString(toDelete);
+ return reason
+ + "\n toTruncate: " + toTruncate
+ + "\n toDelete: " + Arrays.toString(toDelete);
}
}
@@ -299,6 +305,7 @@ public class SegmentedRaftLogCache {
}
TruncationSegments truncate(long index, LogSegment openSegment, Runnable
clearOpenSegment) {
+ final String reason = "truncate(" + index + ")";
try(AutoCloseableLock writeLock = writeLock()) {
final int segmentIndex = binarySearch(index);
if (segmentIndex == -segments.size() - 1) {
@@ -307,7 +314,7 @@ public class SegmentedRaftLogCache {
if (index == openSegment.getStartIndex()) {
// the open segment should be deleted
final SegmentFileInfo deleted = deleteOpenSegment(openSegment,
clearOpenSegment);
- return new TruncationSegments(null,
Collections.singletonList(deleted));
+ return new TruncationSegments(reason, null,
Collections.singletonList(deleted));
} else {
openSegment.truncate(index);
Preconditions.assertTrue(!openSegment.isOpen(),
@@ -317,7 +324,7 @@ public class SegmentedRaftLogCache {
segments.add(openSegment);
sizeInBytes += openSegment.getTotalFileSize();
clearOpenSegment.run();
- return new TruncationSegments(info, Collections.emptyList());
+ return new TruncationSegments(reason, info,
Collections.emptyList());
}
}
} else if (segmentIndex >= 0) {
@@ -342,7 +349,7 @@ public class SegmentedRaftLogCache {
}
SegmentFileInfo t = ts.numOfEntries() == 0? null:
new SegmentFileInfo(ts.getStartIndex(), oldEnd, false,
ts.getTotalFileSize(), ts.getEndIndex());
- return new TruncationSegments(t, list);
+ return new TruncationSegments(reason, t, list);
}
return null;
}
@@ -376,7 +383,7 @@ public class SegmentedRaftLogCache {
list.forEach(LogSegment::evictCache);
List<SegmentFileInfo> toDelete =
list.stream().map(SegmentFileInfo::newClosedSegmentFileInfo)
.collect(Collectors.toList());
- return list.isEmpty() ? null : new TruncationSegments(null, toDelete);
+ return list.isEmpty() ? null : new TruncationSegments("purge(" + index
+ ")", null, toDelete);
}
}
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogInputStream.java
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogInputStream.java
index 481f837f5..050c472dd 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogInputStream.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogInputStream.java
@@ -33,6 +33,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.ratis.server.raftlog.RaftLog.INVALID_LOG_INDEX;
+import static org.apache.ratis.server.raftlog.RaftLog.LEAST_VALID_LOG_INDEX;
public class SegmentedRaftLogInputStream implements Closeable {
static final Logger LOG =
LoggerFactory.getLogger(SegmentedRaftLogInputStream.class);
@@ -62,27 +63,17 @@ public class SegmentedRaftLogInputStream implements
Closeable {
}
private final File logFile;
- private final long startIndex;
- private final long endIndex;
- private final boolean isOpen;
+ private final LogSegmentStartEnd startEnd;
private final OpenCloseState state;
private SegmentedRaftLogReader reader;
private final SizeInBytes maxOpSize;
private final SegmentedRaftLogMetrics raftLogMetrics;
- SegmentedRaftLogInputStream(File log, long startIndex, long endIndex,
boolean isOpen,
+ SegmentedRaftLogInputStream(File log, LogSegmentStartEnd startEnd,
SizeInBytes maxOpSize, SegmentedRaftLogMetrics raftLogMetrics) {
this.maxOpSize = maxOpSize;
- if (isOpen) {
- Preconditions.assertTrue(endIndex == INVALID_LOG_INDEX);
- } else {
- Preconditions.assertTrue(endIndex >= startIndex);
- }
-
this.logFile = log;
- this.startIndex = startIndex;
- this.endIndex = endIndex;
- this.isOpen = isOpen;
+ this.startEnd = startEnd;
this.state = new OpenCloseState(getName());
this.raftLogMetrics = raftLogMetrics;
}
@@ -104,14 +95,6 @@ public class SegmentedRaftLogInputStream implements
Closeable {
}
}
- long getStartIndex() {
- return startIndex;
- }
-
- long getEndIndex() {
- return endIndex;
- }
-
String getName() {
return logFile.getName();
}
@@ -131,7 +114,7 @@ public class SegmentedRaftLogInputStream implements
Closeable {
final LogEntryProto entry = reader.readEntry();
if (entry != null) {
long index = entry.getIndex();
- if (!isOpen() && index >= endIndex) {
+ if (!startEnd.isOpen() && index >= startEnd.getEndIndex()) {
/*
* The end index may be derived from the segment recovery
* process. It is possible that we still have some uncleaned
garbage
@@ -139,8 +122,8 @@ public class SegmentedRaftLogInputStream implements
Closeable {
*/
long skipAmt = logFile.length() - reader.getPos();
if (skipAmt > 0) {
- LOG.debug("skipping {} bytes at the end of log '{}': reached" +
- " entry {} out of {}", skipAmt, getName(), index, endIndex);
+ LOG.info("Skipping {} bytes at the end of log '{}': reached
entry {} out of [{}]",
+ skipAmt, getName(), index, startEnd);
reader.skipFully(skipAmt);
}
}
@@ -172,10 +155,6 @@ public class SegmentedRaftLogInputStream implements
Closeable {
}
}
- boolean isOpen() {
- return isOpen;
- }
-
@Override
public String toString() {
return getName();
@@ -188,24 +167,18 @@ public class SegmentedRaftLogInputStream implements
Closeable {
* ID. The file portion beyond this ID is
* potentially being updated.
* @return Result of the validation
- * @throws IOException
*/
static LogValidation scanEditLog(File file, long maxTxIdToScan, SizeInBytes
maxOpSize)
throws IOException {
- SegmentedRaftLogInputStream in;
- try {
- in = new SegmentedRaftLogInputStream(file, INVALID_LOG_INDEX,
INVALID_LOG_INDEX, false, maxOpSize, null);
- // read the header, initialize the inputstream
- in.init();
- } catch (EOFException e) {
- LOG.warn("Log file " + file + " has no valid header", e);
- return new LogValidation(0, INVALID_LOG_INDEX, true);
- }
-
- try {
+ final LogSegmentStartEnd startEnd =
LogSegmentStartEnd.valueOf(LEAST_VALID_LOG_INDEX);
+ try(SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(file,
startEnd, maxOpSize, null)) {
+ try {
+ in.init();
+ } catch (EOFException e) {
+ LOG.warn("Invalid header for RaftLog segment {}", file, e);
+ return new LogValidation(0, INVALID_LOG_INDEX, true);
+ }
return scanEditLog(in, maxTxIdToScan);
- } finally {
- IOUtils.cleanup(LOG, in);
}
}
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java
index a3d13de9f..a44be6ee5 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java
@@ -46,6 +46,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.file.Path;
import java.util.LinkedList;
import java.util.Objects;
import java.util.Optional;
@@ -482,7 +483,8 @@ class SegmentedRaftLogWorker {
if (segments.getToDelete() != null) {
try(UncheckedAutoCloseable ignored = raftLogMetrics.startPurgeTimer())
{
for (SegmentFileInfo fileInfo : segments.getToDelete()) {
- FileUtils.deleteFile(fileInfo.getFile(storage));
+ final Path deleted =
FileUtils.deleteFile(fileInfo.getFile(storage));
+ LOG.info("{}: Purged RaftLog segment: info={}, path={}", name,
fileInfo, deleted);
}
}
}
@@ -590,8 +592,8 @@ class SegmentedRaftLogWorker {
}
}
- File getFile(long startIndex, Long endIndex) {
- return LogSegmentStartEnd.valueOf(startIndex, endIndex).getFile(storage);
+ private File getFile(LogSegmentStartEnd startEnd) {
+ return startEnd.getFile(storage);
}
private class FinalizeLogSegment extends Task {
@@ -608,19 +610,20 @@ class SegmentedRaftLogWorker {
public void execute() throws IOException {
freeSegmentedRaftLogOutputStream();
- final File openFile = getFile(startIndex, null);
+ final LogSegmentStartEnd openStartEnd =
LogSegmentStartEnd.valueOf(startIndex);
+ final File openFile = getFile(openStartEnd);
Preconditions.assertTrue(openFile.exists(),
() -> name + ": File " + openFile + " to be rolled does not exist");
if (endIndex - startIndex + 1 > 0) {
// finalize the current open segment
- final File dstFile = getFile(startIndex, endIndex);
+ final File dstFile = getFile(LogSegmentStartEnd.valueOf(startIndex,
endIndex));
Preconditions.assertTrue(!dstFile.exists());
FileUtils.move(openFile, dstFile);
LOG.info("{}: Rolled log segment from {} to {}", name, openFile,
dstFile);
} else { // delete the file of the empty segment
- FileUtils.deleteFile(openFile);
- LOG.info("{}: Deleted empty log segment {}", name, openFile);
+ final Path deleted = FileUtils.deleteFile(openFile);
+ LOG.info("{}: Deleted empty RaftLog segment: startEnd={}, path={}",
name, openStartEnd, deleted);
}
updateFlushedIndexIncreasingly();
safeCacheEvictIndex.updateToMax(endIndex, traceIndexChange);
@@ -653,7 +656,7 @@ class SegmentedRaftLogWorker {
@Override
void execute() throws IOException {
- final File openFile = getFile(newStartIndex, null);
+ final File openFile = getFile(LogSegmentStartEnd.valueOf(newStartIndex));
Preconditions.assertTrue(!openFile.exists(), "open file %s exists for
%s",
openFile, name);
Preconditions.assertTrue(pendingFlushNum == 0);
@@ -695,8 +698,8 @@ class SegmentedRaftLogWorker {
final File delFile = del.getFile(storage);
Preconditions.assertTrue(delFile.exists(),
"File %s to be deleted does not exist", delFile);
- FileUtils.deleteFile(delFile);
- LOG.info("{}: Deleted log file {}", name, delFile);
+ final Path deleted = FileUtils.deleteFile(delFile);
+ LOG.info("{}: Deleted RaftLog segment for {}: path={}", name,
segments.getReason(), deleted);
minStart = Math.min(minStart, del.getStartIndex());
}
if (segments.getToTruncate() == null) {
diff --git
a/ratis-server/src/test/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogTestUtils.java
b/ratis-server/src/test/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogTestUtils.java
index e242eddf5..473aa0a46 100644
---
a/ratis-server/src/test/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogTestUtils.java
+++
b/ratis-server/src/test/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogTestUtils.java
@@ -17,25 +17,33 @@
*/
package org.apache.ratis.server.raftlog.segmented;
+import org.apache.ratis.server.RaftServer;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.Slf4jUtils;
import org.slf4j.event.Level;
import java.io.File;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
public interface SegmentedRaftLogTestUtils {
SizeInBytes MAX_OP_SIZE = SizeInBytes.valueOf("32MB");
static SegmentedRaftLogInputStream newSegmentedRaftLogInputStream(File log,
long startIndex, long endIndex, boolean isOpen) {
- return new SegmentedRaftLogInputStream(log, startIndex, endIndex, isOpen,
MAX_OP_SIZE, null);
+ final LogSegmentStartEnd startEnd = LogSegmentStartEnd.valueOf(startIndex,
endIndex, isOpen);
+ return new SegmentedRaftLogInputStream(log, startEnd, MAX_OP_SIZE, null);
}
static void setRaftLogWorkerLogLevel(Level level) {
Slf4jUtils.setLogLevel(SegmentedRaftLogWorker.LOG, level);
}
- static String getLogFlushTimeMetric(String memberId) {
- return SegmentedRaftLogWorker.class.getName() + "." + memberId +
".flush-time";
+ static List<Path> getOpenLogFiles(RaftServer.Division server) throws
Exception {
+ return LogSegmentPath.getLogSegmentPaths(server.getRaftStorage()).stream()
+ .filter(p -> p.getStartEnd().isOpen())
+ .map(LogSegmentPath::getPath)
+ .collect(Collectors.toList());
}
}
diff --git
a/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java
b/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java
index 005c5cf41..71be6884a 100644
--- a/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java
+++ b/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java
@@ -35,8 +35,8 @@ import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.raftlog.RaftLogIOException;
import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogFormat;
import org.apache.ratis.server.RaftServerConfigKeys.Log;
+import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogTestUtils;
import org.apache.ratis.server.raftlog.segmented.TestSegmentedRaftLog;
-import org.apache.ratis.server.raftlog.segmented.LogSegmentPath;
import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing;
import org.apache.ratis.statemachine.StateMachine;
import org.apache.ratis.util.FileUtils;
@@ -63,7 +63,6 @@ import java.util.concurrent.CompletionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
-import java.util.stream.Collectors;
import static
org.apache.ratis.server.storage.RaftStorageTestUtils.getLogUnsafe;
@@ -178,15 +177,8 @@ public abstract class ServerRestartTests<CLUSTER extends
MiniRaftCluster>
server.getRaftServer().close();
}
- static List<Path> getOpenLogFiles(RaftServer.Division server) throws
Exception {
- return LogSegmentPath.getLogSegmentPaths(server.getRaftStorage()).stream()
- .filter(p -> p.getStartEnd().isOpen())
- .map(LogSegmentPath::getPath)
- .collect(Collectors.toList());
- }
-
static File getOpenLogFile(RaftServer.Division server) throws Exception {
- final List<Path> openLogs = getOpenLogFiles(server);
+ final List<Path> openLogs =
SegmentedRaftLogTestUtils.getOpenLogFiles(server);
Assertions.assertEquals(1, openLogs.size());
return openLogs.get(0).toFile();
}
@@ -213,7 +205,7 @@ public abstract class ServerRestartTests<CLUSTER extends
MiniRaftCluster>
10, HUNDRED_MILLIS, impl.getId() + "-getOpenLogFile", LOG);
for(int i = 0; i < SegmentedRaftLogFormat.getHeaderLength(); i++) {
assertCorruptedLogHeader(impl.getId(), openLogFile, i, cluster, LOG);
- Assertions.assertTrue(getOpenLogFiles(impl).isEmpty());
+
Assertions.assertTrue(SegmentedRaftLogTestUtils.getOpenLogFiles(impl).isEmpty());
}
}
}
diff --git
a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestLogSegment.java
b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestLogSegment.java
index 608ff9f9c..dd4f075db 100644
---
a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestLogSegment.java
+++
b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestLogSegment.java
@@ -60,6 +60,8 @@ import static
org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogTestUtil
* Test basic functionality of {@link LogSegment}
*/
public class TestLogSegment extends BaseTest {
+ public static final LogSegmentStartEnd ZERO_START_NULL_END =
LogSegmentStartEnd.valueOf(0);
+
private File storageDir;
private long segmentMaxSize;
private long preallocatedSize;
@@ -175,7 +177,7 @@ public class TestLogSegment extends BaseTest {
final File openSegmentFile = prepareLog(true, 0, 100, 0,
isLastEntryPartiallyWritten);
RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
final LogSegment openSegment = LogSegment.loadSegment(storage,
openSegmentFile,
- LogSegmentStartEnd.valueOf(0), MAX_OP_SIZE, loadInitial, null, null);
+ ZERO_START_NULL_END, MAX_OP_SIZE, loadInitial, null, null);
final int delta = isLastEntryPartiallyWritten? 1: 0;
checkLogSegment(openSegment, 0, 99 - delta, true,
openSegmentFile.length(), 0);
storage.close();
@@ -220,7 +222,7 @@ public class TestLogSegment extends BaseTest {
final File openSegmentFile = prepareLog(true, 0, 100, 0, true);
RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
final LogSegment openSegment = LogSegment.loadSegment(storage,
openSegmentFile,
- LogSegmentStartEnd.valueOf(0), MAX_OP_SIZE, true, null,
raftLogMetrics);
+ ZERO_START_NULL_END, MAX_OP_SIZE, true, null, raftLogMetrics);
checkLogSegment(openSegment, 0, 98, true, openSegmentFile.length(), 0);
storage.close();
@@ -289,7 +291,7 @@ public class TestLogSegment extends BaseTest {
@Test
public void testPreallocateSegment() throws Exception {
RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
- final File file = LogSegmentStartEnd.valueOf(0).getFile(storage);
+ final File file = ZERO_START_NULL_END.getFile(storage);
final int[] maxSizes = new int[]{1024, 1025, 1024 * 1024 - 1, 1024 * 1024,
1024 * 1024 + 1, 2 * 1024 * 1024 - 1, 2 * 1024 * 1024,
2 * 1024 * 1024 + 1, 8 * 1024 * 1024};
@@ -340,7 +342,7 @@ public class TestLogSegment extends BaseTest {
public void testPreallocationAndAppend() throws Exception {
final SizeInBytes max = SizeInBytes.valueOf(2,
TraditionalBinaryPrefix.MEGA);
RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
- final File file = LogSegmentStartEnd.valueOf(0).getFile(storage);
+ final File file = ZERO_START_NULL_END.getFile(storage);
final byte[] content = new byte[1024];
Arrays.fill(content, (byte) 1);
@@ -370,7 +372,7 @@ public class TestLogSegment extends BaseTest {
@Test
public void testZeroSizeInProgressFile() throws Exception {
final RaftStorage storage =
RaftStorageTestUtils.newRaftStorage(storageDir);
- final File file = LogSegmentStartEnd.valueOf(0).getFile(storage);
+ final File file = ZERO_START_NULL_END.getFile(storage);
storage.close();
// create zero size in-progress file
diff --git
a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestRaftLogReadWrite.java
b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestRaftLogReadWrite.java
index 4f151c733..560091407 100644
---
a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestRaftLogReadWrite.java
+++
b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestRaftLogReadWrite.java
@@ -44,6 +44,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import static
org.apache.ratis.server.raftlog.segmented.TestLogSegment.ZERO_START_NULL_END;
+
/**
* Test basic functionality of LogReader, SegmentedRaftLogInputStream, and
SegmentedRaftLogOutputStream.
*/
@@ -105,7 +107,7 @@ public class TestRaftLogReadWrite extends BaseTest {
@Test
public void testReadWriteLog() throws IOException {
final RaftStorage storage =
RaftStorageTestUtils.newRaftStorage(storageDir);
- final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage);
+ final File openSegment = ZERO_START_NULL_END.getFile(storage);
long size = SegmentedRaftLogFormat.getHeaderLength();
final LogEntryProto[] entries = new LogEntryProto[100];
@@ -125,7 +127,7 @@ public class TestRaftLogReadWrite extends BaseTest {
@Test
public void testAppendLog() throws IOException {
final RaftStorage storage =
RaftStorageTestUtils.newRaftStorage(storageDir);
- final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage);
+ final File openSegment = ZERO_START_NULL_END.getFile(storage);
LogEntryProto[] entries = new LogEntryProto[200];
try (SegmentedRaftLogOutputStream out = new
SegmentedRaftLogOutputStream(openSegment, false,
segmentMaxSize, preallocatedSize,
ByteBuffer.allocateDirect(bufferSize))) {
@@ -158,7 +160,7 @@ public class TestRaftLogReadWrite extends BaseTest {
@Test
public void testReadWithPadding() throws IOException {
final RaftStorage storage =
RaftStorageTestUtils.newRaftStorage(storageDir);
- final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage);
+ final File openSegment = ZERO_START_NULL_END.getFile(storage);
long size = SegmentedRaftLogFormat.getHeaderLength();
LogEntryProto[] entries = new LogEntryProto[100];
@@ -187,7 +189,7 @@ public class TestRaftLogReadWrite extends BaseTest {
@Test
public void testReadWithCorruptPadding() throws IOException {
final RaftStorage storage =
RaftStorageTestUtils.newRaftStorage(storageDir);
- final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage);
+ final File openSegment = ZERO_START_NULL_END.getFile(storage);
LogEntryProto[] entries = new LogEntryProto[10];
final SegmentedRaftLogOutputStream out = new
SegmentedRaftLogOutputStream(openSegment, false,
@@ -236,7 +238,7 @@ public class TestRaftLogReadWrite extends BaseTest {
@Test
public void testReadWithEntryCorruption() throws IOException {
RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir);
- final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage);
+ final File openSegment = ZERO_START_NULL_END.getFile(storage);
try (SegmentedRaftLogOutputStream out = new
SegmentedRaftLogOutputStream(openSegment, false,
segmentMaxSize, preallocatedSize,
ByteBuffer.allocateDirect(bufferSize))) {
for (int i = 0; i < 100; i++) {
