This is an automated email from the ASF dual-hosted git repository.
tkalkirill pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 3fd4ed2076 IGNITE-23363 Add support for working with snapshots in
SimpleInMemoryKeyValueStorage (#4503)
3fd4ed2076 is described below
commit 3fd4ed20767315df1a6e93833fda13e15fc39d4e
Author: Kirill Tkalenko <[email protected]>
AuthorDate: Fri Oct 4 13:45:56 2024 +0300
IGNITE-23363 Add support for working with snapshots in
SimpleInMemoryKeyValueStorage (#4503)
---
.../metastorage/server/KeyValueStorage.java | 1 +
.../server/persistence/RocksDbKeyValueStorage.java | 2 +-
.../AbstractCompactionKeyValueStorageTest.java | 61 ++++---------
.../server/BasicOperationsKeyValueStorageTest.java | 52 ++++++++++++
.../RocksDbCompactionKeyValueStorageTest.java | 26 +-----
.../server/RocksDbKeyValueStorageTest.java | 9 --
...impleInMemoryCompactionKeyValueStorageTest.java | 14 ---
.../server/SimpleInMemoryKeyValueStorageTest.java | 1 -
.../server/TestRocksDbKeyValueStorageTest.java | 13 ++-
.../server/AbstractKeyValueStorageTest.java | 11 +++
.../server/SimpleInMemoryKeyValueStorage.java | 99 +++++++++++++++++++++-
.../SimpleInMemoryKeyValueStorageSnapshot.java | 68 +++++++++++++++
.../metastorage/server/ValueSnapshot.java} | 35 ++++----
13 files changed, 271 insertions(+), 121 deletions(-)
diff --git
a/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java
b/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java
index f4be043a14..06dc9f7178 100644
---
a/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java
+++
b/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java
@@ -290,6 +290,7 @@ public interface KeyValueStorage extends ManuallyCloseable {
* Restores a state of the storage which was previously captured with a
{@link #snapshot(Path)}.
*
* @param snapshotPath Path to the snapshot's directory.
+ * @throws MetaStorageException If there was an error while restoring from
a snapshot.
*/
void restoreSnapshot(Path snapshotPath);
diff --git
a/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/persistence/RocksDbKeyValueStorage.java
b/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/persistence/RocksDbKeyValueStorage.java
index f7fd2d4846..31d50fab0d 100644
---
a/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/persistence/RocksDbKeyValueStorage.java
+++
b/modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/server/persistence/RocksDbKeyValueStorage.java
@@ -219,7 +219,7 @@ public class RocksDbKeyValueStorage implements
KeyValueStorage {
private long updCntr;
/**
- * Last revision of a compact that was set or restored from a snapshot.
+ * Last compaction revision that was set or restored from a snapshot.
*
* <p>This field is used by metastorage read methods to determine whether
{@link CompactedException} should be thrown.</p>
*
diff --git
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/AbstractCompactionKeyValueStorageTest.java
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/AbstractCompactionKeyValueStorageTest.java
index e1a460ecc8..6a6c6c2a0f 100644
---
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/AbstractCompactionKeyValueStorageTest.java
+++
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/AbstractCompactionKeyValueStorageTest.java
@@ -26,7 +26,6 @@ import static
org.apache.ignite.internal.testframework.matchers.CompletableFutur
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assumptions.assumeTrue;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -43,8 +42,6 @@ import
org.apache.ignite.internal.testframework.WorkDirectoryExtension;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
/** Compaction tests. */
@ExtendWith(WorkDirectoryExtension.class)
@@ -93,10 +90,6 @@ public abstract class AbstractCompactionKeyValueStorageTest
extends AbstractKeyV
assertEquals(List.of(4, 6), collectRevisions(SOME_KEY));
}
- abstract boolean isPersistent();
-
- abstract void restartStorage(boolean clear) throws Exception;
-
@Test
void testCompactRevision1() {
storage.compact(1);
@@ -161,19 +154,14 @@ public abstract class
AbstractCompactionKeyValueStorageTest extends AbstractKeyV
testCompactRevision6();
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- void testRevisionsAfterRestart(boolean clearStorage) throws Exception {
- assumeTrue(isPersistent());
-
+ @Test
+ void testRevisionsAfterRestart() {
storage.compact(6);
Path snapshotDir = workDir.resolve("snapshot");
assertThat(storage.snapshot(snapshotDir), willCompleteSuccessfully());
- restartStorage(clearStorage);
-
storage.restoreSnapshot(snapshotDir);
assertEquals(List.of(5), collectRevisions(FOO_KEY));
@@ -203,19 +191,16 @@ public abstract class
AbstractCompactionKeyValueStorageTest extends AbstractKeyV
assertEquals(1, storage.getCompactionRevision());
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- void testSetAndGetCompactionRevisionAndRestart(boolean clearStorage)
throws Exception {
+ @Test
+ void testSetAndGetCompactionRevisionAndRestart() throws Exception {
storage.setCompactionRevision(1);
- restartStorage(clearStorage);
+ restartStorage();
assertEquals(-1, storage.getCompactionRevision());
}
@Test
void testSaveCompactionRevisionDoesNotChangeRevisionInMemory() {
- assumeTrue(isPersistent());
-
storage.saveCompactionRevision(0);
assertEquals(-1, storage.getCompactionRevision());
@@ -223,22 +208,17 @@ public abstract class
AbstractCompactionKeyValueStorageTest extends AbstractKeyV
assertEquals(-1, storage.getCompactionRevision());
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- void testSaveCompactionRevisionAndRestart(boolean clearStorage) throws
Exception {
- assumeTrue(isPersistent());
-
+ @Test
+ void testSaveCompactionRevisionAndRestart() throws Exception {
storage.saveCompactionRevision(1);
- restartStorage(clearStorage);
+ restartStorage();
assertEquals(-1, storage.getCompactionRevision());
}
@Test
void testSaveCompactionRevisionInSnapshot() {
- assumeTrue(isPersistent());
-
storage.saveCompactionRevision(1);
Path snapshotDir = workDir.resolve("snapshot");
@@ -250,41 +230,34 @@ public abstract class
AbstractCompactionKeyValueStorageTest extends AbstractKeyV
assertEquals(1, storage.getCompactionRevision());
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- void testSaveCompactionRevisionInSnapshotAndRestart(boolean clearStorage)
throws Exception {
- assumeTrue(isPersistent());
-
+ @Test
+ void testSaveCompactionRevisionInSnapshotAndRestart() throws Exception {
storage.saveCompactionRevision(1);
Path snapshotDir = workDir.resolve("snapshot");
assertThat(storage.snapshot(snapshotDir), willCompleteSuccessfully());
- restartStorage(clearStorage);
+ restartStorage();
storage.restoreSnapshot(snapshotDir);
assertEquals(1, storage.getCompactionRevision());
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- void testCompactDontSetAndSaveCompactionRevision(boolean clearStorage)
throws Exception {
+ @Test
+ void testCompactDontSetAndSaveCompactionRevision() throws Exception {
storage.compact(1);
assertEquals(-1, storage.getCompactionRevision());
- restartStorage(clearStorage);
+ restartStorage();
assertEquals(-1, storage.getCompactionRevision());
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- void testRestoreFromSnapshotWithoutSaveCompactionRevision(boolean
clearStorage) throws Exception {
- assumeTrue(isPersistent());
-
+ @Test
+ void testRestoreFromSnapshotWithoutSaveCompactionRevision() throws
Exception {
Path snapshotDir = workDir.resolve("snapshot");
assertThat(storage.snapshot(snapshotDir), willCompleteSuccessfully());
- restartStorage(clearStorage);
+ restartStorage();
storage.restoreSnapshot(snapshotDir);
assertEquals(-1, storage.getCompactionRevision());
diff --git
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/BasicOperationsKeyValueStorageTest.java
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/BasicOperationsKeyValueStorageTest.java
index 4de7a84701..f1e6f7b57d 100644
---
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/BasicOperationsKeyValueStorageTest.java
+++
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/BasicOperationsKeyValueStorageTest.java
@@ -48,6 +48,7 @@ import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -69,14 +70,21 @@ import org.apache.ignite.internal.metastorage.dsl.Operation;
import org.apache.ignite.internal.metastorage.dsl.StatementResult;
import org.apache.ignite.internal.metastorage.impl.CommandIdGenerator;
import org.apache.ignite.internal.metastorage.server.ValueCondition.Type;
+import org.apache.ignite.internal.testframework.WorkDirectory;
+import org.apache.ignite.internal.testframework.WorkDirectoryExtension;
import org.apache.ignite.internal.util.Cursor;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
/**
* Tests for key-value storage implementations.
*/
+@ExtendWith(WorkDirectoryExtension.class)
public abstract class BasicOperationsKeyValueStorageTest extends
AbstractKeyValueStorageTest {
+ @WorkDirectory
+ Path workDir;
+
@Test
public void testPut() {
byte[] key = key(1);
@@ -2287,6 +2295,50 @@ public abstract class BasicOperationsKeyValueStorageTest
extends AbstractKeyValu
checkEntriesTimestamp(keys, storage.revision(), timestamp2,
timestamp0, timestamp1);
}
+ @Test
+ void testSnapshot() throws Exception {
+ byte[] key = key(0);
+ byte[] value = keyValue(0, 0);
+
+ storage.put(key, value, hybridTimestamp(10));
+
+ Path snapshotDir = workDir.resolve("snapshotDir");
+ assertThat(storage.snapshot(snapshotDir), willCompleteSuccessfully());
+
+ restartStorage();
+
+ assertEquals(0L, storage.revision());
+ assertEquals(0L, storage.updateCounter());
+ assertTrue(storage.get(key).empty());
+
+ storage.restoreSnapshot(snapshotDir);
+
+ assertEquals(1L, storage.revision());
+ assertEquals(1L, storage.updateCounter());
+ assertFalse(storage.get(key).empty());
+ }
+
+ @Test
+ void testClearDataBeforeRestoreFromSnapshot() {
+ byte[] key0 = key(0);
+ byte[] key1 = key(1);
+ byte[] value = keyValue(0, 0);
+
+ storage.put(key0, value, hybridTimestamp(10));
+
+ Path snapshotDir = workDir.resolve("snapshotDir");
+ assertThat(storage.snapshot(snapshotDir), willCompleteSuccessfully());
+
+ storage.put(key1, value, hybridTimestamp(10));
+
+ storage.restoreSnapshot(snapshotDir);
+
+ assertEquals(1L, storage.revision());
+ assertEquals(1L, storage.updateCounter());
+ assertFalse(storage.get(key0).empty());
+ assertTrue(storage.get(key1).empty());
+ }
+
private CompletableFuture<Void> watchExact(
byte[] key, long revision, int expectedNumCalls,
BiConsumer<WatchEvent, Integer> testCondition
) {
diff --git
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/RocksDbCompactionKeyValueStorageTest.java
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/RocksDbCompactionKeyValueStorageTest.java
index 186d21a15b..c224a25654 100644
---
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/RocksDbCompactionKeyValueStorageTest.java
+++
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/RocksDbCompactionKeyValueStorageTest.java
@@ -17,37 +17,13 @@
package org.apache.ignite.internal.metastorage.server;
-import java.nio.file.Path;
import org.apache.ignite.internal.failure.NoOpFailureManager;
import
org.apache.ignite.internal.metastorage.server.persistence.RocksDbKeyValueStorage;
-import org.apache.ignite.internal.util.IgniteUtils;
/** Compaction test for the RocksDB implementation of {@link KeyValueStorage}.
*/
public class RocksDbCompactionKeyValueStorageTest extends
AbstractCompactionKeyValueStorageTest {
@Override
public KeyValueStorage createStorage() {
- return new RocksDbKeyValueStorage("test", storageDir(), new
NoOpFailureManager());
- }
-
- @Override
- boolean isPersistent() {
- return true;
- }
-
- @Override
- void restartStorage(boolean clear) throws Exception {
- storage.close();
-
- if (clear) {
- IgniteUtils.deleteIfExists(storageDir());
- }
-
- storage = createStorage();
-
- storage.start();
- }
-
- private Path storageDir() {
- return workDir.resolve("storage");
+ return new RocksDbKeyValueStorage("test", workDir.resolve("storage"),
new NoOpFailureManager());
}
}
diff --git
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/RocksDbKeyValueStorageTest.java
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/RocksDbKeyValueStorageTest.java
index 6c7cd4c6ef..aafb437601 100644
---
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/RocksDbKeyValueStorageTest.java
+++
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/RocksDbKeyValueStorageTest.java
@@ -17,22 +17,13 @@
package org.apache.ignite.internal.metastorage.server;
-import java.nio.file.Path;
import org.apache.ignite.internal.failure.NoOpFailureManager;
import
org.apache.ignite.internal.metastorage.server.persistence.RocksDbKeyValueStorage;
-import org.apache.ignite.internal.testframework.WorkDirectory;
-import org.apache.ignite.internal.testframework.WorkDirectoryExtension;
-import org.junit.jupiter.api.extension.ExtendWith;
/**
* Tests for RocksDB key-value storage implementation.
*/
-@ExtendWith(WorkDirectoryExtension.class)
public class RocksDbKeyValueStorageTest extends
BasicOperationsKeyValueStorageTest {
- @WorkDirectory
- private Path workDir;
-
- /** {@inheritDoc} */
@Override
public KeyValueStorage createStorage() {
return new RocksDbKeyValueStorage("test", workDir.resolve("storage"),
new NoOpFailureManager());
diff --git
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryCompactionKeyValueStorageTest.java
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryCompactionKeyValueStorageTest.java
index 62930a332e..d7adf966d6 100644
---
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryCompactionKeyValueStorageTest.java
+++
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryCompactionKeyValueStorageTest.java
@@ -23,18 +23,4 @@ public class SimpleInMemoryCompactionKeyValueStorageTest
extends AbstractCompact
public KeyValueStorage createStorage() {
return new SimpleInMemoryKeyValueStorage("test");
}
-
- @Override
- boolean isPersistent() {
- return false;
- }
-
- @Override
- void restartStorage(boolean clear) throws Exception {
- storage.close();
-
- storage = createStorage();
-
- storage.start();
- }
}
diff --git
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java
index d9c5f9256c..6761656c5c 100644
---
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java
+++
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java
@@ -21,7 +21,6 @@ package org.apache.ignite.internal.metastorage.server;
* Tests for in-memory key-value storage implementation.
*/
class SimpleInMemoryKeyValueStorageTest extends
BasicOperationsKeyValueStorageTest {
- /** {@inheritDoc} */
@Override
public KeyValueStorage createStorage() {
return new SimpleInMemoryKeyValueStorage("test");
diff --git
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/TestRocksDbKeyValueStorageTest.java
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/TestRocksDbKeyValueStorageTest.java
index b6ad007e46..49540a170a 100644
---
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/TestRocksDbKeyValueStorageTest.java
+++
b/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/TestRocksDbKeyValueStorageTest.java
@@ -21,23 +21,15 @@ import static
org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import java.nio.file.Path;
import org.apache.ignite.internal.metastorage.Entry;
-import org.apache.ignite.internal.testframework.WorkDirectory;
-import org.apache.ignite.internal.testframework.WorkDirectoryExtension;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
/**
* Tests for {@link TestRocksDbKeyValueStorage} key-value storage
implementation.
*/
-@ExtendWith(WorkDirectoryExtension.class)
public class TestRocksDbKeyValueStorageTest extends
BasicOperationsKeyValueStorageTest {
private TestRocksDbKeyValueStorage testRocksDbKeyValueStorage;
- @WorkDirectory
- private Path workDir;
-
@Override
protected KeyValueStorage createStorage() {
testRocksDbKeyValueStorage = new TestRocksDbKeyValueStorage("test",
workDir.resolve("storage"));
@@ -76,4 +68,9 @@ public class TestRocksDbKeyValueStorageTest extends
BasicOperationsKeyValueStora
assertArrayEquals(key, e.key());
assertArrayEquals(val, e.value());
}
+
+ @Override
+ void testSnapshot() {
+ // TestRocksDbKeyValueStorage does not clean up the storage on startup.
+ }
}
diff --git
a/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/AbstractKeyValueStorageTest.java
b/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/AbstractKeyValueStorageTest.java
index 5abbcd0104..3f0cd7d5b0 100644
---
a/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/AbstractKeyValueStorageTest.java
+++
b/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/AbstractKeyValueStorageTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.metastorage.server;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
import org.junit.jupiter.api.AfterEach;
@@ -46,6 +47,16 @@ public abstract class AbstractKeyValueStorageTest extends
BaseIgniteAbstractTest
*/
protected abstract KeyValueStorage createStorage();
+ protected void restartStorage() throws Exception {
+ assertNotNull(storage);
+
+ storage.close();
+
+ storage = createStorage();
+
+ storage.start();
+ }
+
protected static byte[] key(int k) {
return ("key" + k).getBytes(UTF_8);
}
diff --git
a/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java
b/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java
index ab165e418e..a0ff4be9ad 100644
---
a/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java
+++
b/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java
@@ -17,8 +17,11 @@
package org.apache.ignite.internal.metastorage.server;
+import static java.nio.file.StandardOpenOption.WRITE;
+import static java.util.concurrent.CompletableFuture.failedFuture;
import static java.util.stream.Collectors.collectingAndThen;
import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
import static
org.apache.ignite.internal.metastorage.server.KeyValueStorageUtils.NOTHING_TO_COMPACT_INDEX;
import static
org.apache.ignite.internal.metastorage.server.KeyValueStorageUtils.assertCompactionRevisionLessThanCurrent;
import static
org.apache.ignite.internal.metastorage.server.KeyValueStorageUtils.indexToCompact;
@@ -28,8 +31,12 @@ import static
org.apache.ignite.internal.metastorage.server.raft.MetaStorageWrit
import static org.apache.ignite.internal.rocksdb.RocksUtils.incrementPrefix;
import static org.apache.ignite.internal.util.ArrayUtils.LONG_EMPTY_ARRAY;
import static org.apache.ignite.internal.util.ByteUtils.toByteArray;
+import static
org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
import static org.apache.ignite.lang.ErrorGroups.MetaStorage.OP_EXECUTION_ERR;
+import static
org.apache.ignite.lang.ErrorGroups.MetaStorage.RESTORING_STORAGE_ERR;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
@@ -63,7 +70,9 @@ import
org.apache.ignite.internal.metastorage.exceptions.CompactedException;
import org.apache.ignite.internal.metastorage.exceptions.MetaStorageException;
import org.apache.ignite.internal.metastorage.impl.EntryImpl;
import org.apache.ignite.internal.metastorage.impl.MetaStorageManagerImpl;
+import org.apache.ignite.internal.util.ByteUtils;
import org.apache.ignite.internal.util.Cursor;
+import org.apache.ignite.internal.util.IgniteUtils;
import org.jetbrains.annotations.Nullable;
/**
@@ -112,7 +121,7 @@ public class SimpleInMemoryKeyValueStorage implements
KeyValueStorage {
private long updCntr;
/**
- * Last revision of a compact that was set or restored from a snapshot.
+ * Last compaction revision that was set or restored from a snapshot.
*
* <p>This field is used by metastorage read methods to determine whether
{@link CompactedException} should be thrown.</p>
*
@@ -120,6 +129,13 @@ public class SimpleInMemoryKeyValueStorage implements
KeyValueStorage {
*/
private long compactionRevision = -1;
+ /**
+ * Last {@link #saveCompactionRevision saved} compaction revision.
+ *
+ * <p>Used only when working with snapshots.</p>
+ */
+ private long savedCompactionRevision = -1;
+
/** All operations are queued on this lock. */
private final Object mux = new Object();
@@ -568,12 +584,81 @@ public class SimpleInMemoryKeyValueStorage implements
KeyValueStorage {
@Override
public CompletableFuture<Void> snapshot(Path snapshotPath) {
- throw new UnsupportedOperationException();
+ synchronized (mux) {
+ try {
+ Files.createDirectories(snapshotPath);
+
+ Path snapshotFile =
snapshotPath.resolve(SimpleInMemoryKeyValueStorageSnapshot.FILE_NAME);
+
+ assertTrue(IgniteUtils.deleteIfExists(snapshotFile),
snapshotFile.toString());
+
+ Files.createFile(snapshotFile);
+
+ Map<Long, Map<byte[], ValueSnapshot>> revsIdxCopy =
revsIdx.entrySet().stream()
+ .collect(toMap(
+ Map.Entry::getKey,
+ revIdxEntry -> revIdxEntry.getValue()
+ .entrySet()
+ .stream()
+ .collect(toMap(Map.Entry::getKey, e ->
new ValueSnapshot(e.getValue())))
+ ));
+
+ var snapshot = new SimpleInMemoryKeyValueStorageSnapshot(
+ Map.copyOf(keysIdx),
+ Map.copyOf(tsToRevMap),
+ Map.copyOf(revToTsMap),
+ revsIdxCopy,
+ rev,
+ updCntr,
+ savedCompactionRevision
+ );
+
+ byte[] snapshotBytes = ByteUtils.toBytes(snapshot);
+
+ Files.write(snapshotFile, snapshotBytes, WRITE);
+
+ return nullCompletedFuture();
+ } catch (Throwable t) {
+ return failedFuture(t);
+ }
+ }
}
@Override
public void restoreSnapshot(Path snapshotPath) {
- throw new UnsupportedOperationException();
+ synchronized (mux) {
+ try {
+ keysIdx.clear();
+ tsToRevMap.clear();
+ revToTsMap.clear();
+ revsIdx.clear();
+
+ Path snapshotFile =
snapshotPath.resolve(SimpleInMemoryKeyValueStorageSnapshot.FILE_NAME);
+
+ assertTrue(Files.exists(snapshotPath),
snapshotFile.toString());
+
+ byte[] snapshotBytes = Files.readAllBytes(snapshotFile);
+
+ var snapshot = (SimpleInMemoryKeyValueStorageSnapshot)
ByteUtils.fromBytes(snapshotBytes);
+
+ keysIdx.putAll(snapshot.keysIdx);
+ tsToRevMap.putAll(snapshot.tsToRevMap);
+ revToTsMap.putAll(snapshot.revToTsMap);
+ snapshot.revsIdx.forEach((revision, entries) -> {
+ TreeMap<byte[], Value> entries0 = new TreeMap<>(CMP);
+ entries.forEach((keyBytes, valueSnapshot) ->
entries0.put(keyBytes, valueSnapshot.toValue()));
+
+ revsIdx.put(revision, entries0);
+ });
+
+ rev = snapshot.rev;
+ updCntr = snapshot.updCntr;
+ compactionRevision = snapshot.savedCompactionRevision;
+ savedCompactionRevision = snapshot.savedCompactionRevision;
+ } catch (Throwable t) {
+ throw new MetaStorageException(RESTORING_STORAGE_ERR, t);
+ }
+ }
}
private boolean doRemove(byte[] key, long curRev, HybridTimestamp opTs) {
@@ -869,7 +954,13 @@ public class SimpleInMemoryKeyValueStorage implements
KeyValueStorage {
@Override
public void saveCompactionRevision(long revision) {
- throw new UnsupportedOperationException();
+ assert revision >= 0;
+
+ synchronized (mux) {
+ assertCompactionRevisionLessThanCurrent(revision, rev);
+
+ savedCompactionRevision = revision;
+ }
}
@Override
diff --git
a/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageSnapshot.java
b/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageSnapshot.java
new file mode 100644
index 0000000000..16a726e691
--- /dev/null
+++
b/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageSnapshot.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.metastorage.server;
+
+import java.io.Serializable;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+
+/**
+ * {@link SimpleInMemoryKeyValueStorage} container for creating and restoring
from a snapshot.
+ *
+ * @see SimpleInMemoryKeyValueStorage#snapshot(Path)
+ * @see SimpleInMemoryKeyValueStorage#restoreSnapshot(Path)
+ */
+class SimpleInMemoryKeyValueStorageSnapshot implements Serializable {
+ private static final long serialVersionUID = -4263291644707737634L;
+
+ static final String FILE_NAME = "snapshot.bin";
+
+ final Map<byte[], List<Long>> keysIdx;
+
+ final Map<Long, Long> tsToRevMap;
+
+ final Map<Long, HybridTimestamp> revToTsMap;
+
+ final Map<Long, Map<byte[], ValueSnapshot>> revsIdx;
+
+ final long rev;
+
+ final long updCntr;
+
+ final long savedCompactionRevision;
+
+ SimpleInMemoryKeyValueStorageSnapshot(
+ Map<byte[], List<Long>> keysIdx,
+ Map<Long, Long> tsToRevMap,
+ Map<Long, HybridTimestamp> revToTsMap,
+ Map<Long, Map<byte[], ValueSnapshot>> revsIdx,
+ long rev,
+ long updCntr,
+ long savedCompactionRevision
+ ) {
+ this.keysIdx = keysIdx;
+ this.tsToRevMap = tsToRevMap;
+ this.revToTsMap = revToTsMap;
+ this.revsIdx = revsIdx;
+ this.rev = rev;
+ this.updCntr = updCntr;
+ this.savedCompactionRevision = savedCompactionRevision;
+ }
+}
diff --git
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryCompactionKeyValueStorageTest.java
b/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/ValueSnapshot.java
similarity index 52%
copy from
modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryCompactionKeyValueStorageTest.java
copy to
modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/ValueSnapshot.java
index 62930a332e..0d1865aa3d 100644
---
a/modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryCompactionKeyValueStorageTest.java
+++
b/modules/metastorage/src/testFixtures/java/org/apache/ignite/internal/metastorage/server/ValueSnapshot.java
@@ -17,24 +17,29 @@
package org.apache.ignite.internal.metastorage.server;
-/** Compaction test for the simple in-memory implementation of {@link
KeyValueStorage}. */
-public class SimpleInMemoryCompactionKeyValueStorageTest extends
AbstractCompactionKeyValueStorageTest {
- @Override
- public KeyValueStorage createStorage() {
- return new SimpleInMemoryKeyValueStorage("test");
- }
+import java.io.Serializable;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
- @Override
- boolean isPersistent() {
- return false;
- }
+/** {@link Value} container for creating and restoring from a snapshot. */
+class ValueSnapshot implements Serializable {
+ private static final long serialVersionUID = -435898107081479568L;
+
+ private final byte[] bytes;
- @Override
- void restartStorage(boolean clear) throws Exception {
- storage.close();
+ private final long updCntr;
- storage = createStorage();
+ private final boolean tombstone;
+
+ private final HybridTimestamp operationTimestamp;
+
+ ValueSnapshot(Value value) {
+ bytes = value.bytes();
+ updCntr = value.updateCounter();
+ tombstone = value.tombstone();
+ operationTimestamp = value.operationTimestamp();
+ }
- storage.start();
+ Value toValue() {
+ return tombstone ? new Value(Value.TOMBSTONE, updCntr,
operationTimestamp) : new Value(bytes, updCntr, operationTimestamp);
}
}
