This is an automated email from the ASF dual-hosted git repository.
apolovtsev pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 7ec4438c65d IGNITE-26762 Use separate folders for index and segment
files (#6803)
7ec4438c65d is described below
commit 7ec4438c65d76b50de6b1cd49fb05ce7cfd3a0e5
Author: Alexander Polovtcev <[email protected]>
AuthorDate: Fri Oct 17 14:40:55 2025 +0300
IGNITE-26762 Use separate folders for index and segment files (#6803)
---
.../raft/storage/segstore/IndexFileManager.java | 16 +++++++++++-----
.../raft/storage/segstore/SegmentFileManager.java | 22 +++++++++++++++++-----
.../storage/segstore/IndexFileManagerTest.java | 8 ++++----
.../storage/segstore/SegmentFileManagerTest.java | 11 ++++-------
.../storage/segstore/SegstoreLogStorageTest.java | 6 +++---
5 files changed, 39 insertions(+), 24 deletions(-)
diff --git
a/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/IndexFileManager.java
b/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/IndexFileManager.java
index 1e1ed1e4f16..95ae8d2228b 100644
---
a/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/IndexFileManager.java
+++
b/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/IndexFileManager.java
@@ -100,7 +100,7 @@ class IndexFileManager {
static final ByteOrder BYTE_ORDER = ByteOrder.LITTLE_ENDIAN;
- private final Path baseDir;
+ private final Path indexFilesDir;
/**
* Current index file ordinal (used to generate index file names).
@@ -114,8 +114,14 @@ class IndexFileManager {
*/
private final Map<Long, GroupIndexMeta> groupIndexMetas = new
ConcurrentHashMap<>();
- IndexFileManager(Path baseDir) {
- this.baseDir = baseDir;
+ IndexFileManager(Path baseDir) throws IOException {
+ indexFilesDir = baseDir.resolve("index");
+
+ Files.createDirectories(indexFilesDir);
+ }
+
+ Path indexFilesDir() {
+ return indexFilesDir;
}
/**
@@ -124,7 +130,7 @@ class IndexFileManager {
Path saveIndexMemtable(ReadModeIndexMemTable indexMemTable) throws
IOException {
String fileName = indexFileName(curFileOrdinal, 0);
- Path tmpFilePath = baseDir.resolve(fileName + ".tmp");
+ Path tmpFilePath = indexFilesDir.resolve(fileName + ".tmp");
try (var os = new
BufferedOutputStream(Files.newOutputStream(tmpFilePath, CREATE_NEW, WRITE))) {
byte[] headerBytes = serializeHeaderAndFillMetadata(indexMemTable);
@@ -161,7 +167,7 @@ class IndexFileManager {
return null;
}
- Path indexFile =
baseDir.resolve(indexFileName(indexFileMeta.indexFileOrdinal(), 0));
+ Path indexFile =
indexFilesDir.resolve(indexFileName(indexFileMeta.indexFileOrdinal(), 0));
// Index file payload is a 0-based array, which indices correspond to
the [fileMeta.firstLogIndex, fileMeta.lastLogIndex) range.
long payloadArrayIndex = logIndex -
indexFileMeta.firstLogIndexInclusive();
diff --git
a/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManager.java
b/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManager.java
index b17a07cc61f..8e52851a9d8 100644
---
a/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManager.java
+++
b/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManager.java
@@ -22,6 +22,7 @@ import static
org.apache.ignite.lang.ErrorGroups.Common.NODE_STOPPING_ERR;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.internal.close.ManuallyCloseable;
@@ -90,7 +91,7 @@ class SegmentFileManager implements ManuallyCloseable {
*/
static final byte[] SWITCH_SEGMENT_RECORD = new byte[8]; // 8 zero bytes.
- private final Path baseDir;
+ private final Path segmentFilesDir;
/** Configured size of a segment file. */
private final long fileSize;
@@ -124,12 +125,15 @@ class SegmentFileManager implements ManuallyCloseable {
*/
private boolean isStopped;
- SegmentFileManager(String nodeName, Path baseDir, long fileSize, int
stripes, FailureProcessor failureProcessor) {
+ SegmentFileManager(String nodeName, Path baseDir, long fileSize, int
stripes, FailureProcessor failureProcessor) throws IOException {
if (fileSize <= HEADER_RECORD.length) {
throw new IllegalArgumentException("File size must be greater than
the header size: " + fileSize);
}
- this.baseDir = baseDir;
+ this.segmentFilesDir = baseDir.resolve("segments");
+
+ Files.createDirectories(segmentFilesDir);
+
this.fileSize = fileSize;
this.stripes = stripes;
@@ -144,8 +148,16 @@ class SegmentFileManager implements ManuallyCloseable {
currentSegmentFile.set(allocateNewSegmentFile(0));
}
+ Path segmentFilesDir() {
+ return segmentFilesDir;
+ }
+
+ Path indexFilesDir() {
+ return indexFileManager.indexFilesDir();
+ }
+
private SegmentFileWithMemtable allocateNewSegmentFile(int fileOrdinal)
throws IOException {
- Path path = baseDir.resolve(segmentFileName(fileOrdinal, 0));
+ Path path = segmentFilesDir.resolve(segmentFileName(fileOrdinal, 0));
SegmentFile segmentFile = SegmentFile.createNew(path, fileSize);
@@ -385,7 +397,7 @@ class SegmentFileManager implements ManuallyCloseable {
return null;
}
- Path path =
baseDir.resolve(segmentFileName(segmentFilePointer.fileOrdinal(), 0));
+ Path path =
segmentFilesDir.resolve(segmentFileName(segmentFilePointer.fileOrdinal(), 0));
// TODO: Add a cache for recently accessed segment files, see
https://issues.apache.org/jira/browse/IGNITE-26622.
SegmentFile segmentFile = SegmentFile.openExisting(path);
diff --git
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/IndexFileManagerTest.java
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/IndexFileManagerTest.java
index f8fa068148b..d4ad1f22a7b 100644
---
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/IndexFileManagerTest.java
+++
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/IndexFileManagerTest.java
@@ -36,7 +36,7 @@ class IndexFileManagerTest extends IgniteAbstractTest {
private IndexFileManager indexFileManager;
@BeforeEach
- void setUp() {
+ void setUp() throws IOException {
indexFileManager = new IndexFileManager(workDir);
}
@@ -48,9 +48,9 @@ class IndexFileManagerTest extends IgniteAbstractTest {
Path path1 = indexFileManager.saveIndexMemtable(memtable);
Path path2 = indexFileManager.saveIndexMemtable(memtable);
- assertThat(path0,
is(workDir.resolve("index-0000000000-0000000000.bin")));
- assertThat(path1,
is(workDir.resolve("index-0000000001-0000000000.bin")));
- assertThat(path2,
is(workDir.resolve("index-0000000002-0000000000.bin")));
+ assertThat(path0,
is(indexFileManager.indexFilesDir().resolve("index-0000000000-0000000000.bin")));
+ assertThat(path1,
is(indexFileManager.indexFilesDir().resolve("index-0000000001-0000000000.bin")));
+ assertThat(path2,
is(indexFileManager.indexFilesDir().resolve("index-0000000002-0000000000.bin")));
}
@Test
diff --git
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerTest.java
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerTest.java
index 6ac85e32815..63829a826db 100644
---
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerTest.java
+++
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerTest.java
@@ -376,21 +376,18 @@ class SegmentFileManagerTest extends IgniteAbstractTest {
}
private List<Path> segmentFiles() throws IOException {
- try (Stream<Path> files = Files.list(workDir)) {
- return files
- .filter(p ->
p.getFileName().toString().startsWith("segment"))
- .sorted()
- .collect(toList());
+ try (Stream<Path> files = Files.list(fileManager.segmentFilesDir())) {
+ return files.sorted().collect(toList());
}
}
private List<Path> indexFiles() throws IOException {
- try (Stream<Path> files = Files.list(workDir)) {
+ try (Stream<Path> files = Files.list(fileManager.indexFilesDir())) {
return files
.filter(p -> {
String fileName = p.getFileName().toString();
- return fileName.startsWith("index") &&
!fileName.endsWith(".tmp");
+ return !fileName.endsWith(".tmp");
})
.sorted()
.collect(toList());
diff --git
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageTest.java
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageTest.java
index 1187a1113bb..d84f3fe3d68 100644
---
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageTest.java
+++
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageTest.java
@@ -42,11 +42,11 @@ class SegstoreLogStorageTest extends BaseLogStorageTest {
@Override
protected LogStorage newLogStorage() {
- segmentFileManager = new SegmentFileManager(NODE_NAME, path,
SEGMENT_SIZE, 1, new NoOpFailureManager());
+ try {
+ segmentFileManager = new SegmentFileManager(NODE_NAME, path,
SEGMENT_SIZE, 1, new NoOpFailureManager());
- logStorage = new SegstoreLogStorage(GROUP_ID, segmentFileManager);
+ logStorage = new SegstoreLogStorage(GROUP_ID, segmentFileManager);
- try {
segmentFileManager.start();
} catch (IOException e) {
throw new RuntimeException(e);
