This is an automated email from the ASF dual-hosted git repository. agura pushed a commit to branch ignite-14389 in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 8ff3571bce120ea41bf5f1123c6c52b51c36a983 Author: Andrey Gura <[email protected]> AuthorDate: Tue Apr 13 21:34:37 2021 +0300 IGNITE-14389 getAll and tests (WIP) --- .../metastorage/client/MetaStorageService.java | 4 +- modules/metastorage-server/pom.xml | 6 + .../metastorage/server/KeyValueStorage.java | 8 + .../server/SimpleInMemoryKeyValueStorage.java | 36 +++- .../server/SimpleInMemoryKeyValueStorageTest.java | 208 +++++++++++++++++++-- 5 files changed, 244 insertions(+), 18 deletions(-) diff --git a/modules/metastorage-client/src/main/java/org/apache/ignite/metastorage/client/MetaStorageService.java b/modules/metastorage-client/src/main/java/org/apache/ignite/metastorage/client/MetaStorageService.java index dfb4868..d4ded44 100644 --- a/modules/metastorage-client/src/main/java/org/apache/ignite/metastorage/client/MetaStorageService.java +++ b/modules/metastorage-client/src/main/java/org/apache/ignite/metastorage/client/MetaStorageService.java @@ -52,7 +52,7 @@ public interface MetaStorageService { * Retrieves an entry for the given key and the revision upper bound. * * @param key The key. Couldn't be {@code null}. - * @param revUpperBound The upper bound for entry revisions. Must be positive. + * @param revUpperBound The upper bound for entry revisions. Must be positive. * @return An entry for the given key and maximum revision limited by {@code revUpperBound}. * Couldn't be {@code null}. * @throws OperationTimeoutException If the operation is timed out. Will be thrown on getting future result. @@ -82,7 +82,7 @@ public interface MetaStorageService { * * @param keys The collection of keys. Couldn't be {@code null} or empty. * Collection elements couldn't be {@code null}. - * @param revUpperBound The upper bound for entry revisions. Must be positive. + * @param revUpperBound The upper bound for entry revisions. Must be positive. * @return A map of entries for given keys and maximum revision limited by {@code revUpperBound}. * Couldn't be {@code null}. * @throws OperationTimeoutException If the operation is timed out. Will be thrown on getting future result. diff --git a/modules/metastorage-server/pom.xml b/modules/metastorage-server/pom.xml index 3c51fc5..d73d080 100644 --- a/modules/metastorage-server/pom.xml +++ b/modules/metastorage-server/pom.xml @@ -40,6 +40,12 @@ </dependency> <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>metastorage-common</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> <groupId>org.jetbrains</groupId> <artifactId>annotations</artifactId> </dependency> diff --git a/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java b/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java index ead1043..0596c4a 100644 --- a/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java +++ b/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java @@ -2,7 +2,9 @@ package org.apache.ignite.internal.metastorage.server; import org.jetbrains.annotations.NotNull; +import java.util.Collection; import java.util.Iterator; +import java.util.List; public interface KeyValueStorage { @@ -16,6 +18,12 @@ public interface KeyValueStorage { @NotNull Entry get(byte[] key, long rev); + @NotNull + Collection<Entry> getAll(List<byte[]> keys); + + @NotNull + Collection<Entry> getAll(List<byte[]> keys, long revUpperBound); + void put(byte[] key, byte[] value); @NotNull diff --git a/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java b/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java index 3700f4a..8523f51 100644 --- a/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java +++ b/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java @@ -2,6 +2,7 @@ package org.apache.ignite.internal.metastorage.server; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -78,6 +79,16 @@ public class SimpleInMemoryKeyValueStorage implements KeyValueStorage { } @Override + public @NotNull Collection<Entry> getAll(List<byte[]> keys) { + return doGetAll(keys, LATEST_REV); + } + + @Override + public @NotNull Collection<Entry> getAll(List<byte[]> keys, long revUpperBound) { + return doGetAll(keys, revUpperBound); + } + + @Override public void remove(byte[] key) { synchronized (mux) { Entry e = doGet(key, LATEST_REV, false); @@ -197,17 +208,17 @@ public class SimpleInMemoryKeyValueStorage implements KeyValueStorage { NavigableMap<byte[], List<Long>> compactedKeysIdx, NavigableMap<Long, NavigableMap<byte[], Value>> compactedRevsIdx ) { - Long lrev = lastRevision(revs); + Long lastRev = lastRevision(revs); - NavigableMap<byte[], Value> kv = revsIdx.get(lrev); + NavigableMap<byte[], Value> kv = revsIdx.get(lastRev); Value lastVal = kv.get(key); if (!lastVal.tombstone()) { - compactedKeysIdx.put(key, listOf(lrev)); + compactedKeysIdx.put(key, listOf(lastRev)); NavigableMap<byte[], Value> compactedKv = compactedRevsIdx.computeIfAbsent( - lrev, + lastRev, k -> new TreeMap<>(LEXICOGRAPHIC_COMPARATOR) ); @@ -215,6 +226,23 @@ public class SimpleInMemoryKeyValueStorage implements KeyValueStorage { } } + @NotNull + private Collection<Entry> doGetAll(List<byte[]> keys, long rev) { + assert keys != null : "keys list can't be null."; + assert !keys.isEmpty() : "keys list can't be empty."; + assert rev > 0 : "Revision must be positive."; + + Collection<Entry> res = new ArrayList<>(keys.size()); + + synchronized (mux) { + for (byte[] key : keys) { + res.add(doGet(key, rev, false)); + } + } + + return res; + } + /** * Returns entry for given key. * diff --git a/modules/metastorage-server/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java b/modules/metastorage-server/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java index 5b797fc..fa130e6 100644 --- a/modules/metastorage-server/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java +++ b/modules/metastorage-server/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java @@ -1,17 +1,18 @@ package org.apache.ignite.internal.metastorage.server; -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - +import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.ignite.metastorage.common.Key; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; class SimpleInMemoryKeyValueStorageTest { private KeyValueStorage storage; @@ -56,6 +57,193 @@ class SimpleInMemoryKeyValueStorageTest { } @Test + void getAll() { + byte[] key1 = k(1); + byte[] val1 = kv(1, 1); + + byte[] key2 = k(2); + byte[] val2_1 = kv(2, 21); + byte[] val2_2 = kv(2, 22); + + byte[] key3 = k(3); + byte[] val3 = kv(3, 3); + + byte[] key4 = k(4); + + assertEquals(0, storage.revision()); + assertEquals(0, storage.updateCounter()); + + // Regular put. + storage.put(key1, val1); + + // Rewrite. + storage.put(key2, val2_1); + storage.put(key2, val2_2); + + // Remove. + storage.put(key3, val3); + storage.remove(key3); + + assertEquals(5, storage.revision()); + assertEquals(5, storage.updateCounter()); + + Collection<Entry> entries = storage.getAll(List.of(key1, key2, key3, key4)); + + assertEquals(4, entries.size()); + + Map<Key, Entry> map = entries.stream().collect(Collectors.toMap(e -> new Key(e.key()), Function.identity())); + + // Test regular put value. + Entry e1 = map.get(new Key(key1)); + + assertNotNull(e1); + assertEquals(1, e1.revision()); + assertEquals(1, e1.updateCounter()); + assertFalse(e1.tombstone()); + assertFalse(e1.empty()); + assertArrayEquals(val1, e1.value()); + + // Test rewritten value. + Entry e2 = map.get(new Key(key2)); + + assertNotNull(e2); + assertEquals(3, e2.revision()); + assertEquals(3, e2.updateCounter()); + assertFalse(e2.tombstone()); + assertFalse(e2.empty()); + assertArrayEquals(val2_2, e2.value()); + + // Test removed value. + Entry e3 = map.get(new Key(key3)); + + assertNotNull(e3); + assertEquals(5, e3.revision()); + assertEquals(5, e3.updateCounter()); + assertTrue(e3.tombstone()); + assertFalse(e3.empty()); + + // Test empty value. + Entry e4 = map.get(new Key(key4)); + + assertNotNull(e4); + assertFalse(e4.tombstone()); + assertTrue(e4.empty()); + } + + @Test + void getAllWithRevisionBound() { + byte[] key1 = k(1); + byte[] val1 = kv(1, 1); + + byte[] key2 = k(2); + byte[] val2_1 = kv(2, 21); + byte[] val2_2 = kv(2, 22); + + byte[] key3 = k(3); + byte[] val3 = kv(3, 3); + + byte[] key4 = k(4); + + assertEquals(0, storage.revision()); + assertEquals(0, storage.updateCounter()); + + // Regular put. + storage.put(key1, val1); + + // Rewrite. + storage.put(key2, val2_1); + storage.put(key2, val2_2); + + // Remove. + storage.put(key3, val3); + storage.remove(key3); + + assertEquals(5, storage.revision()); + assertEquals(5, storage.updateCounter()); + + // Bounded by revision 2. + Collection<Entry> entries = storage.getAll(List.of(key1, key2, key3, key4), 2); + + assertEquals(4, entries.size()); + + Map<Key, Entry> map = entries.stream().collect(Collectors.toMap(e -> new Key(e.key()), Function.identity())); + + // Test regular put value. + Entry e1 = map.get(new Key(key1)); + + assertNotNull(e1); + assertEquals(1, e1.revision()); + assertEquals(1, e1.updateCounter()); + assertFalse(e1.tombstone()); + assertFalse(e1.empty()); + assertArrayEquals(val1, e1.value()); + + // Test while not rewritten value. + Entry e2 = map.get(new Key(key2)); + + assertNotNull(e2); + assertEquals(2, e2.revision()); + assertEquals(2, e2.updateCounter()); + assertFalse(e2.tombstone()); + assertFalse(e2.empty()); + assertArrayEquals(val2_1, e2.value()); + + // Values with larger revision don't exist yet. + Entry e3 = map.get(new Key(key3)); + + assertNotNull(e3); + assertTrue(e3.empty()); + + Entry e4 = map.get(new Key(key4)); + + assertNotNull(e4); + assertTrue(e4.empty()); + + // Bounded by revision 4. + entries = storage.getAll(List.of(key1, key2, key3, key4), 4); + + assertEquals(4, entries.size()); + + map = entries.stream().collect(Collectors.toMap(e -> new Key(e.key()), Function.identity())); + + // Test regular put value. + e1 = map.get(new Key(key1)); + + assertNotNull(e1); + assertEquals(1, e1.revision()); + assertEquals(1, e1.updateCounter()); + assertFalse(e1.tombstone()); + assertFalse(e1.empty()); + assertArrayEquals(val1, e1.value()); + + // Test rewritten value. + e2 = map.get(new Key(key2)); + + assertNotNull(e2); + assertEquals(3, e2.revision()); + assertEquals(3, e2.updateCounter()); + assertFalse(e2.tombstone()); + assertFalse(e2.empty()); + assertArrayEquals(val2_2, e2.value()); + + // Test not removed value. + e3 = map.get(new Key(key3)); + + assertNotNull(e3); + assertEquals(4, e3.revision()); + assertEquals(4, e3.updateCounter()); + assertFalse(e3.tombstone()); + assertFalse(e3.empty()); + assertArrayEquals(val3, e3.value()); + + // Value with larger revision doesn't exist yet. + e4 = map.get(new Key(key4)); + + assertNotNull(e4); + assertTrue(e4.empty()); + } + + @Test public void getAndPut() { byte[] key = k(1); byte[] val = kv(1, 1); @@ -208,7 +396,6 @@ class SimpleInMemoryKeyValueStorageTest { @Test public void getAndPutAfterRemove() { byte[] key = k(1); - byte[] val = kv(1, 1); storage.getAndPut(key, val); @@ -218,11 +405,8 @@ class SimpleInMemoryKeyValueStorageTest { Entry e = storage.getAndPut(key, val); assertEquals(3, storage.revision()); - assertEquals(3, storage.updateCounter()); - assertEquals(2, e.revision()); - assertTrue(e.tombstone()); }
