This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch ignite-16004 in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit f67db6b3216929e9fc4bbd8bf6a6b3604a7129e5 Author: Andrew Mashenkov <andrey.mashen...@gmail.com> AuthorDate: Fri Dec 10 17:25:48 2021 +0300 Add getAll/putAll tests. --- .../ignite/internal/table/KeyValueViewImpl.java | 61 +++++++++-- .../internal/table/RecordBinaryViewImpl.java | 2 +- .../table/KeyValueBinaryViewOperationsTest.java | 114 ++++++++++----------- .../KeyValueViewOperationsSimpleSchemaTest.java | 19 ++++ .../table/RecordBinaryViewOperationsTest.java | 46 ++++++--- 5 files changed, 156 insertions(+), 86 deletions(-) diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/KeyValueViewImpl.java b/modules/table/src/main/java/org/apache/ignite/internal/table/KeyValueViewImpl.java index 5441314..37fd4ad 100644 --- a/modules/table/src/main/java/org/apache/ignite/internal/table/KeyValueViewImpl.java +++ b/modules/table/src/main/java/org/apache/ignite/internal/table/KeyValueViewImpl.java @@ -18,11 +18,14 @@ package org.apache.ignite.internal.table; import java.io.Serializable; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.ignite.internal.schema.BinaryRow; import org.apache.ignite.internal.schema.SchemaDescriptor; import org.apache.ignite.internal.schema.SchemaRegistry; @@ -30,6 +33,7 @@ import org.apache.ignite.internal.schema.marshaller.KvMarshaller; import org.apache.ignite.internal.schema.marshaller.MarshallerException; import org.apache.ignite.internal.schema.marshaller.reflection.KvMarshallerImpl; import org.apache.ignite.internal.schema.row.Row; +import org.apache.ignite.internal.util.Pair; import org.apache.ignite.lang.IgniteException; import org.apache.ignite.table.InvokeProcessor; import org.apache.ignite.table.KeyValueView; @@ -81,7 +85,7 @@ public class KeyValueViewImpl<K, V> extends AbstractTableView implements KeyValu BinaryRow keyRow = marshal(Objects.requireNonNull(key), null); return tbl.get(keyRow, tx) - .thenApply(this::unmarshalValue); // row -> deserialized obj. + .thenApply(this::unmarshalValue); // row -> deserialized obj. } /** @@ -98,7 +102,20 @@ public class KeyValueViewImpl<K, V> extends AbstractTableView implements KeyValu @Override public @NotNull CompletableFuture<Map<K, V>> getAllAsync(@NotNull Collection<K> keys) { - throw new UnsupportedOperationException("Not implemented yet."); + Objects.requireNonNull(keys); + + List<BinaryRow> keyRows = new ArrayList<>(keys.size()); + + for (K keyRec : keys) { + final BinaryRow keyRow = marshal(keyRec, null); + + keyRows.add(keyRow); + } + + return tbl.getAll(keyRows, tx) + .thenApply(ts -> ts.stream().filter(Objects::nonNull) + .map(this::unmarshalPair) + .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond))); } /** @@ -151,7 +168,17 @@ public class KeyValueViewImpl<K, V> extends AbstractTableView implements KeyValu @Override public @NotNull CompletableFuture<Void> putAllAsync(@NotNull Map<K, V> pairs) { - throw new UnsupportedOperationException("Not implemented yet."); + Objects.requireNonNull(pairs); + + List<BinaryRow> rows = new ArrayList<>(pairs.size()); + + for (Map.Entry<K, V> pair : pairs.entrySet()) { + final BinaryRow row = marshal(pair.getKey(), pair.getValue()); + + rows.add(row); + } + + return tbl.upsertAll(rows, tx); } /** @@ -336,8 +363,7 @@ public class KeyValueViewImpl<K, V> extends AbstractTableView implements KeyValu * {@inheritDoc} */ @Override - public @NotNull - <R extends Serializable> CompletableFuture<R> invokeAsync( + public @NotNull <R extends Serializable> CompletableFuture<R> invokeAsync( @NotNull K key, InvokeProcessor<K, V, R> proc, Serializable... args @@ -361,8 +387,7 @@ public class KeyValueViewImpl<K, V> extends AbstractTableView implements KeyValu * {@inheritDoc} */ @Override - public @NotNull - <R extends Serializable> CompletableFuture<Map<K, R>> invokeAllAsync( + public @NotNull <R extends Serializable> CompletableFuture<Map<K, R>> invokeAllAsync( @NotNull Collection<K> keys, InvokeProcessor<K, V, R> proc, Serializable... args ) { @@ -417,6 +442,28 @@ public class KeyValueViewImpl<K, V> extends AbstractTableView implements KeyValu } /** + * Unmarshal value object from given binary row. + * + * @param binaryRow Binary row. + * @return Value object. + */ + private Pair<K, V> unmarshalPair(BinaryRow binaryRow) { + if (binaryRow == null) { + return null; + } + + Row row = schemaReg.resolve(binaryRow); + + KvMarshaller<K, V> marshaller = marshaller(row.schemaVersion()); + + try { + return new Pair<>(marshaller.unmarshalKey(row), marshaller.unmarshalValue(row)); + } catch (MarshallerException e) { + throw new IgniteException(e); + } + } + + /** * Marshal key-value pair to a row. * * @param key Key object. diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/RecordBinaryViewImpl.java b/modules/table/src/main/java/org/apache/ignite/internal/table/RecordBinaryViewImpl.java index b354a0c..4e1ea66 100644 --- a/modules/table/src/main/java/org/apache/ignite/internal/table/RecordBinaryViewImpl.java +++ b/modules/table/src/main/java/org/apache/ignite/internal/table/RecordBinaryViewImpl.java @@ -384,7 +384,7 @@ public class RecordBinaryViewImpl extends AbstractTableView implements RecordVie return null; } - return schemaReg.resolve(rows).stream().map(TableRow::tuple).collect(toList()); + return schemaReg.resolve(rows).stream().filter(Objects::nonNull).map(TableRow::tuple).collect(toList()); } /** diff --git a/modules/table/src/test/java/org/apache/ignite/internal/table/KeyValueBinaryViewOperationsTest.java b/modules/table/src/test/java/org/apache/ignite/internal/table/KeyValueBinaryViewOperationsTest.java index 152e4dc..c4f3c5a 100644 --- a/modules/table/src/test/java/org/apache/ignite/internal/table/KeyValueBinaryViewOperationsTest.java +++ b/modules/table/src/test/java/org/apache/ignite/internal/table/KeyValueBinaryViewOperationsTest.java @@ -17,12 +17,15 @@ package org.apache.ignite.internal.table; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import java.util.List; +import java.util.Map; import org.apache.ignite.internal.schema.Column; import org.apache.ignite.internal.schema.NativeTypes; import org.apache.ignite.internal.schema.SchemaDescriptor; @@ -30,12 +33,14 @@ import org.apache.ignite.internal.storage.basic.ConcurrentHashMapPartitionStorag import org.apache.ignite.internal.table.distributed.storage.VersionedRowStore; import org.apache.ignite.internal.table.impl.DummyInternalTableImpl; import org.apache.ignite.internal.table.impl.DummySchemaManagerImpl; +import org.apache.ignite.internal.tx.LockManager; +import org.apache.ignite.internal.tx.TxManager; import org.apache.ignite.internal.tx.impl.HeapLockManager; import org.apache.ignite.internal.tx.impl.TxManagerImpl; import org.apache.ignite.network.ClusterService; import org.apache.ignite.table.KeyValueView; import org.apache.ignite.table.Tuple; -import org.junit.jupiter.api.Assertions; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -46,27 +51,6 @@ import org.mockito.Mockito; * ignored for value or exception is thrown? */ public class KeyValueBinaryViewOperationsTest { - /** - * Cluster service. - */ - private ClusterService clusterService; - - /** - * Creates a table for tests. - * - * @return The test table. - */ - private InternalTable createTable() { - clusterService = Mockito.mock(ClusterService.class, RETURNS_DEEP_STUBS); - Mockito.when(clusterService.topologyService().localMember().address()) - .thenReturn(DummyInternalTableImpl.ADDR); - - TxManagerImpl txManager = new TxManagerImpl(clusterService, new HeapLockManager()); - - return new DummyInternalTableImpl( - new VersionedRowStore(new ConcurrentHashMapPartitionStorage(), txManager), txManager); - } - @Test public void put() { SchemaDescriptor schema = new SchemaDescriptor( @@ -75,8 +59,7 @@ public class KeyValueBinaryViewOperationsTest { new Column[]{new Column("val", NativeTypes.INT64, false)} ); - KeyValueView<Tuple, Tuple> tbl = - new KeyValueBinaryViewImpl(createTable(), new DummySchemaManagerImpl(schema), null, null); + KeyValueView<Tuple, Tuple> tbl = createTable(schema).keyValueView(); final Tuple key = Tuple.create().set("id", 1L); final Tuple val = Tuple.create().set("val", 11L); @@ -115,8 +98,7 @@ public class KeyValueBinaryViewOperationsTest { new Column[]{new Column("val", NativeTypes.INT64, false)} ); - KeyValueView<Tuple, Tuple> tbl = - new KeyValueBinaryViewImpl(createTable(), new DummySchemaManagerImpl(schema), null, null); + KeyValueView<Tuple, Tuple> tbl = createTable(schema).keyValueView(); final Tuple key = Tuple.create().set("id", 1L); final Tuple val = Tuple.create().set("val", 11L); @@ -145,8 +127,7 @@ public class KeyValueBinaryViewOperationsTest { new Column[]{new Column("val", NativeTypes.INT64, false)} ); - KeyValueView<Tuple, Tuple> tbl = - new KeyValueBinaryViewImpl(createTable(), new DummySchemaManagerImpl(schema), null, null); + KeyValueView<Tuple, Tuple> tbl = createTable(schema).keyValueView(); final Tuple key = Tuple.create().set("id", 1L); final Tuple val = Tuple.create().set("val", 11L); @@ -176,8 +157,7 @@ public class KeyValueBinaryViewOperationsTest { new Column[]{new Column("val", NativeTypes.INT64, false)} ); - KeyValueView<Tuple, Tuple> tbl = - new KeyValueBinaryViewImpl(createTable(), new DummySchemaManagerImpl(schema), null, null); + KeyValueView<Tuple, Tuple> tbl = createTable(schema).keyValueView(); final Tuple key = Tuple.create().set("id", 1L); final Tuple val = Tuple.create().set("val", 11L); @@ -212,8 +192,7 @@ public class KeyValueBinaryViewOperationsTest { new Column[]{new Column("val", NativeTypes.INT64, false)} ); - KeyValueView<Tuple, Tuple> tbl = - new KeyValueBinaryViewImpl(createTable(), new DummySchemaManagerImpl(schema), null, null); + KeyValueView<Tuple, Tuple> tbl = createTable(schema).keyValueView(); final Tuple key = Tuple.create().set("id", 1L); final Tuple key2 = Tuple.create().set("id", 2L); @@ -252,8 +231,7 @@ public class KeyValueBinaryViewOperationsTest { new Column[]{new Column("val", NativeTypes.INT64, false)} ); - final KeyValueView<Tuple, Tuple> tbl = - new KeyValueBinaryViewImpl(createTable(), new DummySchemaManagerImpl(schema), null, null); + final KeyValueView<Tuple, Tuple> tbl = createTable(schema).keyValueView(); final Tuple key = Tuple.create().set("id", 1L); final Tuple key2 = Tuple.create().set("id", 2L); @@ -304,8 +282,7 @@ public class KeyValueBinaryViewOperationsTest { new Column[]{new Column("val", NativeTypes.INT64, false)} ); - KeyValueView<Tuple, Tuple> tbl = - new KeyValueBinaryViewImpl(createTable(), new DummySchemaManagerImpl(schema), null, null); + KeyValueView<Tuple, Tuple> tbl = createTable(schema).keyValueView(); final Tuple key = Tuple.create().set("id", 1L); final Tuple key2 = Tuple.create().set("id", 2L); @@ -339,8 +316,7 @@ public class KeyValueBinaryViewOperationsTest { new Column[]{new Column("val", NativeTypes.INT64, false)} ); - KeyValueView<Tuple, Tuple> tbl = - new KeyValueBinaryViewImpl(createTable(), new DummySchemaManagerImpl(schema), null, null); + KeyValueView<Tuple, Tuple> tbl = createTable(schema).keyValueView(); final Tuple key = Tuple.create().set("id", 1L); final Tuple key2 = Tuple.create().set("id", 2L); @@ -358,39 +334,55 @@ public class KeyValueBinaryViewOperationsTest { assertEqualsValues(schema, val2, tbl.get(key)); } - /** - * Check key columns equality. - * - * @param schema Schema. - * @param expected Expected tuple. - * @param actual Actual tuple. - */ - void assertEqualsKeys(SchemaDescriptor schema, Tuple expected, Tuple actual) { - int nonNullKey = 0; + @Test + public void getAll() { + SchemaDescriptor schema = new SchemaDescriptor( + 1, + new Column[]{new Column("id", NativeTypes.INT64, false)}, + new Column[]{new Column("val", NativeTypes.INT64, false)} + ); - for (int i = 0; i < schema.keyColumns().length(); i++) { - final Column col = schema.keyColumns().column(i); + KeyValueView<Tuple, Tuple> tbl = createTable(schema).keyValueView(); - final Object val1 = expected.value(col.name()); - final Object val2 = actual.value(col.name()); + Tuple key1 = Tuple.create().set("id", 1L); + Tuple key2 = Tuple.create().set("id", 2L); + Tuple key3 = Tuple.create().set("id", 3L); - Assertions.assertEquals(val1, val2, - "Value columns equality check failed: colIdx=" + col.schemaIndex()); + tbl.putAll(Map.of( + key1, Tuple.create().set("val", 11L), + key3, Tuple.create().set("val", 33L) + )); - if (schema.isKeyColumn(i) && val1 != null) { - nonNullKey++; - } - } + Map<Tuple, Tuple> res = tbl.getAll(List.of(key1, key2, key3)); + + assertEquals(2, res.size()); + assertEquals(Tuple.create().set("val", 11L), res.get(key1)); + assertEquals(Tuple.create().set("val", 33L), res.get(key3)); + assertNull(res.get(key2)); + } + + @NotNull + private TableImpl createTable(SchemaDescriptor schema) { + ClusterService clusterService = Mockito.mock(ClusterService.class, RETURNS_DEEP_STUBS); + Mockito.when(clusterService.topologyService().localMember().address()).thenReturn(DummyInternalTableImpl.ADDR); + + LockManager lockManager = new HeapLockManager(); + + TxManager txManager = new TxManagerImpl(clusterService, lockManager); + + DummyInternalTableImpl table = new DummyInternalTableImpl( + new VersionedRowStore(new ConcurrentHashMapPartitionStorage(), txManager), + txManager); - assertTrue(nonNullKey > 0, "At least one non-null key column must exist."); + return new TableImpl(table, new DummySchemaManagerImpl(schema), null); } /** * Check value columns equality. * - * @param schema Schema. + * @param schema Schema. * @param expected Expected tuple. - * @param actual Actual tuple. + * @param actual Actual tuple. */ void assertEqualsValues(SchemaDescriptor schema, Tuple expected, Tuple actual) { for (int i = 0; i < schema.valueColumns().length(); i++) { @@ -399,7 +391,7 @@ public class KeyValueBinaryViewOperationsTest { final Object val1 = expected.value(col.name()); final Object val2 = actual.value(col.name()); - Assertions.assertEquals(val1, val2, "Key columns equality check failed: colIdx=" + col.schemaIndex()); + assertEquals(val1, val2, "Key columns equality check failed: colIdx=" + col.schemaIndex()); } } } diff --git a/modules/table/src/test/java/org/apache/ignite/internal/table/KeyValueViewOperationsSimpleSchemaTest.java b/modules/table/src/test/java/org/apache/ignite/internal/table/KeyValueViewOperationsSimpleSchemaTest.java index e118a7b..0e7d86c 100644 --- a/modules/table/src/test/java/org/apache/ignite/internal/table/KeyValueViewOperationsSimpleSchemaTest.java +++ b/modules/table/src/test/java/org/apache/ignite/internal/table/KeyValueViewOperationsSimpleSchemaTest.java @@ -26,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.Set; import java.util.stream.Collectors; @@ -44,6 +45,7 @@ import org.apache.ignite.internal.tx.impl.HeapLockManager; import org.apache.ignite.internal.tx.impl.TxManagerImpl; import org.apache.ignite.network.ClusterService; import org.apache.ignite.table.KeyValueView; +import org.apache.ignite.table.Tuple; import org.apache.ignite.table.mapper.Mapper; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -314,6 +316,23 @@ public class KeyValueViewOperationsSimpleSchemaTest { } } + @Test + public void getAll() { + KeyValueView<Long, Long> kvView = kvView(); + + kvView.putAll(Map.of( + 1L, 11L, + 3L, 33L + )); + + Map<Long, Long> res = kvView.getAll(List.of(1L, 2L, 3L)); + + assertEquals(11L, res.get(1L)); + assertNull(res.get(22L)); + assertEquals(33L, res.get(3L)); + } + + /** * Creates key-value view. * diff --git a/modules/table/src/test/java/org/apache/ignite/internal/table/RecordBinaryViewOperationsTest.java b/modules/table/src/test/java/org/apache/ignite/internal/table/RecordBinaryViewOperationsTest.java index cc7d1cb..b8fbc7c 100644 --- a/modules/table/src/test/java/org/apache/ignite/internal/table/RecordBinaryViewOperationsTest.java +++ b/modules/table/src/test/java/org/apache/ignite/internal/table/RecordBinaryViewOperationsTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.table; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -24,6 +25,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import java.util.Collection; +import java.util.List; import org.apache.ignite.internal.schema.Column; import org.apache.ignite.internal.schema.InvalidTypeException; import org.apache.ignite.internal.schema.NativeTypes; @@ -54,23 +57,6 @@ import org.mockito.Mockito; * <p>TODO: IGNITE-14486 Add tests for invoke operations. Add tests for bulk operations. Add tests for async operations. */ public class RecordBinaryViewOperationsTest { - /** Cluster service. */ - private ClusterService clusterService; - - /** - * Creates a table for tests. - * - * @return The table instance for test. - */ - private InternalTable createTable() { - clusterService = Mockito.mock(ClusterService.class, RETURNS_DEEP_STUBS); - Mockito.when(clusterService.topologyService().localMember().address()).thenReturn(DummyInternalTableImpl.ADDR); - - TxManagerImpl txManager = new TxManagerImpl(clusterService, new HeapLockManager()); - - return new DummyInternalTableImpl(new VersionedRowStore(new ConcurrentHashMapPartitionStorage(), txManager), txManager); - } - @Test public void insert() { SchemaDescriptor schema = new SchemaDescriptor( @@ -349,6 +335,32 @@ public class RecordBinaryViewOperationsTest { assertEqualsRows(schema, tuple1, tbl.get(keyTuple1)); } + @Test + public void getAll() { + SchemaDescriptor schema = new SchemaDescriptor( + 1, + new Column[]{new Column("id", NativeTypes.INT64, false)}, + new Column[]{new Column("val", NativeTypes.INT64, false)} + ); + + RecordView<Tuple> tbl = createTableImpl(schema).recordView(); + + Tuple rec1 = Tuple.create().set("id", 1L).set("val", 11L); + Tuple rec3 = Tuple.create().set("id", 3L).set("val", 33L); + + tbl.upsertAll(List.of(rec1, rec3)); + + Collection<Tuple> res = tbl.getAll(List.of(Tuple.create().set("id", 1L), + Tuple.create().set("id", 2L), + Tuple.create().set("id", 3L) + )); + + assertEquals(2, res.size()); + assertTrue(res.contains(rec1)); + assertTrue(res.contains(rec1)); + } + + /** * Check tuples equality. *