This is an automated email from the ASF dual-hosted git repository.
apolovtsev pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new b7ba2f3f571 IGNITE-26476 Add new log storage configuration (#7225)
b7ba2f3f571 is described below
commit b7ba2f3f571e1de2d86b521dc1e8297dccb70f83
Author: Alexander Polovtcev <[email protected]>
AuthorDate: Mon Dec 15 11:46:48 2025 +0200
IGNITE-26476 Add new log storage configuration (#7225)
---
.../LogStorageConfigurationSchema.java | 43 ++++++++++++++++++++++
.../configuration/RaftConfigurationSchema.java | 4 ++
.../raft/storage/segstore/RaftLogCheckpointer.java | 13 ++++---
.../raft/storage/segstore/SegmentFileManager.java | 23 ++++++++----
.../storage/segstore/RaftLogCheckpointerTest.java | 5 ++-
.../segstore/SegmentFileManagerGetEntryTest.java | 17 ++++++++-
.../storage/segstore/SegmentFileManagerTest.java | 22 +++++++----
.../SegstoreLogStorageConcurrencyTest.java | 18 ++++++++-
.../storage/segstore/SegstoreLogStorageTest.java | 16 +++++++-
9 files changed, 134 insertions(+), 27 deletions(-)
diff --git
a/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/configuration/LogStorageConfigurationSchema.java
b/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/configuration/LogStorageConfigurationSchema.java
new file mode 100644
index 00000000000..46173b44353
--- /dev/null
+++
b/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/configuration/LogStorageConfigurationSchema.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.configuration;
+
+import static org.apache.ignite.internal.util.Constants.KiB;
+
+import org.apache.ignite.configuration.annotation.Config;
+import org.apache.ignite.configuration.annotation.Value;
+import org.apache.ignite.configuration.validation.Range;
+
+/** Configuration of the Raft log storage. */
+@Config
+public class LogStorageConfigurationSchema {
+ /**
+ * Maximum size of the log storage checkpoint queue.
+ */
+ @Value(hasDefault = true)
+ @Range(min = 1)
+ public int maxCheckpointQueueSize = 10;
+
+ /**
+ * Size of a segment file in bytes.
+ */
+ @SuppressWarnings("PointlessArithmeticExpression") // Suppressed for
better readability.
+ @Value(hasDefault = true)
+ @Range(min = 1 * KiB, max = Integer.MAX_VALUE)
+ public long segmentFileSizeBytes = Integer.MAX_VALUE;
+}
diff --git
a/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/configuration/RaftConfigurationSchema.java
b/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/configuration/RaftConfigurationSchema.java
index 49972858aa9..5e7400eb129 100644
---
a/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/configuration/RaftConfigurationSchema.java
+++
b/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/configuration/RaftConfigurationSchema.java
@@ -105,4 +105,8 @@ public class RaftConfigurationSchema {
/** Configuration for RAFT disruptor's. */
@ConfigValue
public DisruptorConfigurationSchema disruptor;
+
+ // TODO: Uncomment when the new log storage is released, see
https://issues.apache.org/jira/browse/IGNITE-26300.
+ // @ConfigValue
+ // public LogStorageConfigurationSchema logStorage;
}
diff --git
a/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/RaftLogCheckpointer.java
b/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/RaftLogCheckpointer.java
index e3400628b4a..debe9f4b686 100644
---
a/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/RaftLogCheckpointer.java
+++
b/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/RaftLogCheckpointer.java
@@ -54,10 +54,7 @@ import org.apache.ignite.internal.thread.IgniteThread;
* </ol>
*/
class RaftLogCheckpointer {
- // TODO: Move to configuration, see
https://issues.apache.org/jira/browse/IGNITE-26476.
- static final int MAX_QUEUE_SIZE = 10;
-
- private final CheckpointQueue queue = new CheckpointQueue(MAX_QUEUE_SIZE);
+ private final CheckpointQueue queue;
private final Thread checkpointThread;
@@ -65,10 +62,16 @@ class RaftLogCheckpointer {
private final FailureProcessor failureProcessor;
- RaftLogCheckpointer(String nodeName, IndexFileManager indexFileManager,
FailureProcessor failureProcessor) {
+ RaftLogCheckpointer(
+ String nodeName,
+ IndexFileManager indexFileManager,
+ FailureProcessor failureProcessor,
+ int maxQueueSize
+ ) {
this.indexFileManager = indexFileManager;
this.failureProcessor = failureProcessor;
+ queue = new CheckpointQueue(maxQueueSize);
checkpointThread = new IgniteThread(nodeName, "segstore-checkpoint",
new CheckpointTask());
}
diff --git
a/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManager.java
b/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManager.java
index 33b8f2e0948..cb7ed6a88cd 100644
---
a/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManager.java
+++
b/modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManager.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.raft.storage.segstore;
+import static java.lang.Math.toIntExact;
import static
org.apache.ignite.internal.raft.storage.segstore.SegmentInfo.MISSING_SEGMENT_FILE_OFFSET;
import static
org.apache.ignite.internal.raft.storage.segstore.SegmentPayload.RESET_RECORD_SIZE;
import static
org.apache.ignite.internal.raft.storage.segstore.SegmentPayload.TRUNCATE_PREFIX_RECORD_SIZE;
@@ -38,6 +39,7 @@ import org.apache.ignite.internal.failure.FailureProcessor;
import org.apache.ignite.internal.lang.IgniteInternalException;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.raft.configuration.LogStorageConfiguration;
import
org.apache.ignite.internal.raft.storage.segstore.EntrySearchResult.SearchOutcome;
import
org.apache.ignite.internal.raft.storage.segstore.SegmentFile.WriteBuffer;
import org.apache.ignite.raft.jraft.entity.LogEntry;
@@ -146,20 +148,27 @@ class SegmentFileManager implements ManuallyCloseable {
*/
private boolean isStopped;
- SegmentFileManager(String nodeName, Path baseDir, long fileSize, int
stripes, FailureProcessor failureProcessor) throws IOException {
- if (fileSize <= HEADER_RECORD.length) {
- throw new IllegalArgumentException("File size must be greater than
the header size: " + fileSize);
- }
-
+ SegmentFileManager(
+ String nodeName,
+ Path baseDir,
+ int stripes,
+ FailureProcessor failureProcessor,
+ LogStorageConfiguration storageConfiguration
+ ) throws IOException {
this.segmentFilesDir = baseDir.resolve("segments");
Files.createDirectories(segmentFilesDir);
- this.fileSize = fileSize;
+ this.fileSize =
toIntExact(storageConfiguration.segmentFileSizeBytes().value());
this.stripes = stripes;
indexFileManager = new IndexFileManager(baseDir);
- checkpointer = new RaftLogCheckpointer(nodeName, indexFileManager,
failureProcessor);
+ checkpointer = new RaftLogCheckpointer(
+ nodeName,
+ indexFileManager,
+ failureProcessor,
+ storageConfiguration.maxCheckpointQueueSize().value()
+ );
}
void start() throws IOException {
diff --git
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/RaftLogCheckpointerTest.java
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/RaftLogCheckpointerTest.java
index f3bd0972190..c8d56c432ea 100644
---
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/RaftLogCheckpointerTest.java
+++
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/RaftLogCheckpointerTest.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.raft.storage.segstore;
import static java.util.concurrent.CompletableFuture.runAsync;
-import static
org.apache.ignite.internal.raft.storage.segstore.RaftLogCheckpointer.MAX_QUEUE_SIZE;
import static
org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willTimeoutFast;
import static
org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
import static org.awaitility.Awaitility.await;
@@ -53,6 +52,8 @@ import org.mockito.junit.jupiter.MockitoExtension;
class RaftLogCheckpointerTest extends BaseIgniteAbstractTest {
private static final String NODE_NAME = "test";
+ private static final int MAX_QUEUE_SIZE = 10;
+
private RaftLogCheckpointer checkpointer;
@Mock
@@ -60,7 +61,7 @@ class RaftLogCheckpointerTest extends BaseIgniteAbstractTest {
@BeforeEach
void setUp() {
- checkpointer = new RaftLogCheckpointer(NODE_NAME, indexFileManager,
new NoOpFailureManager());
+ checkpointer = new RaftLogCheckpointer(NODE_NAME, indexFileManager,
new NoOpFailureManager(), MAX_QUEUE_SIZE);
checkpointer.start();
}
diff --git
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerGetEntryTest.java
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerGetEntryTest.java
index e565f4ab163..dbe22480cff 100644
---
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerGetEntryTest.java
+++
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerGetEntryTest.java
@@ -40,8 +40,11 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.IntStream;
+import
org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
+import
org.apache.ignite.internal.configuration.testframework.InjectConfiguration;
import org.apache.ignite.internal.failure.NoOpFailureManager;
import org.apache.ignite.internal.lang.RunnableX;
+import org.apache.ignite.internal.raft.configuration.LogStorageConfiguration;
import org.apache.ignite.internal.testframework.IgniteAbstractTest;
import org.apache.ignite.raft.jraft.entity.LogEntry;
import org.apache.ignite.raft.jraft.entity.LogId;
@@ -56,6 +59,7 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
+@ExtendWith(ConfigurationExtension.class)
class SegmentFileManagerGetEntryTest extends IgniteAbstractTest {
private static final String NODE_NAME = "test";
@@ -72,8 +76,17 @@ class SegmentFileManagerGetEntryTest extends
IgniteAbstractTest {
private LogEntryDecoder decoder;
@BeforeEach
- void setUp() throws IOException {
- fileManager = new SegmentFileManager(NODE_NAME, workDir, FILE_SIZE,
STRIPES, new NoOpFailureManager());
+ void setUp(
+ @InjectConfiguration("mock.segmentFileSizeBytes=" + FILE_SIZE)
+ LogStorageConfiguration storageConfiguration
+ ) throws IOException {
+ fileManager = new SegmentFileManager(
+ NODE_NAME,
+ workDir,
+ STRIPES,
+ new NoOpFailureManager(),
+ storageConfiguration
+ );
fileManager.start();
}
diff --git
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerTest.java
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerTest.java
index 5283f0bad2d..39d69494f65 100644
---
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerTest.java
+++
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegmentFileManagerTest.java
@@ -61,10 +61,13 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.IntStream;
import java.util.stream.Stream;
+import
org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
+import
org.apache.ignite.internal.configuration.testframework.InjectConfiguration;
import org.apache.ignite.internal.failure.FailureManager;
import org.apache.ignite.internal.failure.NoOpFailureManager;
import org.apache.ignite.internal.lang.IgniteInternalException;
import org.apache.ignite.internal.lang.RunnableX;
+import org.apache.ignite.internal.raft.configuration.LogStorageConfiguration;
import org.apache.ignite.internal.testframework.ExecutorServiceExtension;
import org.apache.ignite.internal.testframework.IgniteAbstractTest;
import org.apache.ignite.internal.testframework.InjectExecutorService;
@@ -80,6 +83,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
@ExtendWith(ExecutorServiceExtension.class)
+@ExtendWith(ConfigurationExtension.class)
class SegmentFileManagerTest extends IgniteAbstractTest {
private static final int FILE_SIZE = 100;
@@ -91,6 +95,9 @@ class SegmentFileManagerTest extends IgniteAbstractTest {
private final FailureManager failureManager = new NoOpFailureManager();
+ @InjectConfiguration("mock.segmentFileSizeBytes=" + FILE_SIZE)
+ private LogStorageConfiguration storageConfiguration;
+
private SegmentFileManager fileManager;
@BeforeEach
@@ -101,7 +108,13 @@ class SegmentFileManagerTest extends IgniteAbstractTest {
}
private SegmentFileManager createFileManager() throws IOException {
- return new SegmentFileManager(NODE_NAME, workDir, FILE_SIZE, STRIPES,
failureManager);
+ return new SegmentFileManager(
+ NODE_NAME,
+ workDir,
+ STRIPES,
+ failureManager,
+ storageConfiguration
+ );
}
@AfterEach
@@ -109,13 +122,6 @@ class SegmentFileManagerTest extends IgniteAbstractTest {
closeAllManually(fileManager);
}
- @SuppressWarnings("ResultOfObjectAllocationIgnored")
- @Test
- void testConstructorInvariants() {
- assertThrows(IllegalArgumentException.class, () -> new
SegmentFileManager(NODE_NAME, workDir, 0, 1, failureManager));
- assertThrows(IllegalArgumentException.class, () -> new
SegmentFileManager(NODE_NAME, workDir, 1, 1, failureManager));
- }
-
@Test
void segmentFileIsInitializedAfterStart() throws IOException {
Path segmentFile = findSoleSegmentFile();
diff --git
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageConcurrencyTest.java
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageConcurrencyTest.java
index 7d10cefdc54..e6da6d236df 100644
---
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageConcurrencyTest.java
+++
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageConcurrencyTest.java
@@ -25,8 +25,11 @@ import static org.hamcrest.Matchers.is;
import java.io.IOException;
import java.util.List;
import java.util.function.IntFunction;
+import
org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
+import
org.apache.ignite.internal.configuration.testframework.InjectConfiguration;
import org.apache.ignite.internal.failure.NoOpFailureManager;
import org.apache.ignite.internal.lang.RunnableX;
+import org.apache.ignite.internal.raft.configuration.LogStorageConfiguration;
import org.apache.ignite.internal.testframework.IgniteAbstractTest;
import org.apache.ignite.raft.jraft.conf.ConfigurationManager;
import org.apache.ignite.raft.jraft.entity.LogEntry;
@@ -36,7 +39,9 @@ import org.apache.ignite.raft.jraft.test.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+@ExtendWith(ConfigurationExtension.class)
class SegstoreLogStorageConcurrencyTest extends IgniteAbstractTest {
private static final int SEGMENT_SIZE = 100;
@@ -45,8 +50,17 @@ class SegstoreLogStorageConcurrencyTest extends
IgniteAbstractTest {
private SegmentFileManager segmentFileManager;
@BeforeEach
- void setUp() throws IOException {
- segmentFileManager = new SegmentFileManager(NODE_NAME, workDir,
SEGMENT_SIZE, 1, new NoOpFailureManager());
+ void setUp(
+ @InjectConfiguration("mock.segmentFileSizeBytes=" + SEGMENT_SIZE)
+ LogStorageConfiguration storageConfiguration
+ ) throws IOException {
+ segmentFileManager = new SegmentFileManager(
+ NODE_NAME,
+ workDir,
+ 1,
+ new NoOpFailureManager(),
+ storageConfiguration
+ );
segmentFileManager.start();
}
diff --git
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageTest.java
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageTest.java
index 553d7fd4412..b48f14ad667 100644
---
a/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageTest.java
+++
b/modules/raft/src/test/java/org/apache/ignite/internal/raft/storage/segstore/SegstoreLogStorageTest.java
@@ -22,14 +22,19 @@ import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import java.io.IOException;
+import
org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
+import
org.apache.ignite.internal.configuration.testframework.InjectConfiguration;
import org.apache.ignite.internal.failure.NoOpFailureManager;
+import org.apache.ignite.internal.raft.configuration.LogStorageConfiguration;
import org.apache.ignite.raft.jraft.storage.LogStorage;
import org.apache.ignite.raft.jraft.storage.impl.BaseLogStorageTest;
import org.apache.ignite.raft.jraft.test.TestUtils;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+@ExtendWith(ConfigurationExtension.class)
class SegstoreLogStorageTest extends BaseLogStorageTest {
private static final int SEGMENT_SIZE = 512 * 1024; // Same as in JRaft
tests.
@@ -39,6 +44,9 @@ class SegstoreLogStorageTest extends BaseLogStorageTest {
private SegmentFileManager segmentFileManager;
+ @InjectConfiguration("mock.segmentFileSizeBytes=" + SEGMENT_SIZE)
+ private LogStorageConfiguration storageConfiguration;
+
@AfterEach
void tearDown() throws Exception {
closeAllManually(segmentFileManager);
@@ -47,7 +55,13 @@ class SegstoreLogStorageTest extends BaseLogStorageTest {
@Override
protected LogStorage newLogStorage() {
try {
- segmentFileManager = new SegmentFileManager(NODE_NAME, path,
SEGMENT_SIZE, 1, new NoOpFailureManager());
+ segmentFileManager = new SegmentFileManager(
+ NODE_NAME,
+ path,
+ 1,
+ new NoOpFailureManager(),
+ storageConfiguration
+ );
segmentFileManager.start();
