This is an automated email from the ASF dual-hosted git repository.
zhouxj pushed a commit to branch support/1.15
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/support/1.15 by this push:
new 8f887e6777 GEODE-10290: GII requester should remove departed members
(#7670)
8f887e6777 is described below
commit 8f887e6777297543cb4ab0abed85a88fe4e71e4c
Author: Xiaojian Zhou <[email protected]>
AuthorDate: Wed May 18 11:55:15 2022 -0700
GEODE-10290: GII requester should remove departed members (#7670)
(cherry picked from commit 3d6354cb6b182d54531a8103a357f03754cf5165)
---
...PartitionedRegionRestartRebalanceDUnitTest.java | 133 +++++++++++++++++
.../internal/cache/InitialImageOperation.java | 43 ++++--
.../cache/versions/RegionVersionHolder.java | 2 +
.../cache/versions/RegionVersionVector.java | 4 +
.../internal/cache/InitialImageOperationTest.java | 163 +++++++++++++++++++++
5 files changed, 332 insertions(+), 13 deletions(-)
diff --git
a/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/PartitionedRegionRestartRebalanceDUnitTest.java
b/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/PartitionedRegionRestartRebalanceDUnitTest.java
new file mode 100644
index 0000000000..8f8a37758b
--- /dev/null
+++
b/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/PartitionedRegionRestartRebalanceDUnitTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional
information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
+ */
+package org.apache.geode.internal.cache;
+
+import static org.apache.geode.test.dunit.VM.getVM;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.Serializable;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import org.apache.geode.cache.DataPolicy;
+import org.apache.geode.cache.PartitionAttributesFactory;
+import org.apache.geode.cache.RegionFactory;
+import org.apache.geode.cache.control.RebalanceOperation;
+import org.apache.geode.cache.control.RebalanceResults;
+import org.apache.geode.internal.cache.versions.VersionSource;
+import org.apache.geode.test.dunit.VM;
+import org.apache.geode.test.dunit.rules.CacheRule;
+import org.apache.geode.test.dunit.rules.DistributedRule;
+
+public class PartitionedRegionRestartRebalanceDUnitTest implements
Serializable {
+ private static final int REDUNDANT_COPIES = 2;
+ private static final int TOTAL_NUM_BUCKETS = 12;
+ private static final Logger logger = LogManager.getLogger();
+
+ private String REGION_NAME = getClass().getSimpleName();;
+ private VM[] datastores;
+
+ @Rule
+ public DistributedRule distributedRule = new DistributedRule();
+
+ @Rule
+ public CacheRule cacheRule = new CacheRule();
+
+ @Before
+ public void setUp() throws Exception {
+ datastores = new VM[4];
+ for (int i = 0; i < datastores.length; i++) {
+ datastores[i] = getVM(i);
+ datastores[i].invoke(() -> cacheRule.createCache());
+ datastores[i].invoke(() -> createRegion());
+ }
+ datastores[0].invoke(() -> feedData());
+ }
+
+ private void createRegion() {
+ PartitionAttributesFactory<String, Integer> paf = new
PartitionAttributesFactory();
+ paf.setRedundantCopies(REDUNDANT_COPIES);
+ paf.setTotalNumBuckets(TOTAL_NUM_BUCKETS);
+
+ RegionFactory<String, Integer> rf =
cacheRule.getCache().createRegionFactory();
+ rf.setDataPolicy(DataPolicy.PARTITION);
+ rf.setPartitionAttributes(paf.create());
+ LocalRegion region = (LocalRegion) rf.create(REGION_NAME);
+ }
+
+ private void feedData() throws InterruptedException {
+ PartitionedRegion pr = (PartitionedRegion)
cacheRule.getCache().getRegion(REGION_NAME);
+ for (int i = 0; i < TOTAL_NUM_BUCKETS * 2; i++) {
+ pr.put(i, "VALUE-" + i);
+ if (i < TOTAL_NUM_BUCKETS) {
+ pr.destroy(i);
+ }
+ }
+
cacheRule.getCache().getTombstoneService().forceBatchExpirationForTests(TOTAL_NUM_BUCKETS);
+ }
+
+ private void rebalance() throws InterruptedException {
+ RebalanceOperation op =
+
cacheRule.getCache().getResourceManager().createRebalanceFactory().start();
+ RebalanceResults results = op.getResults();
+ logger.info("Rebalance total time is " + results.getTotalTime());
+ }
+
+ private void verify() {
+ PartitionedRegion pr = (PartitionedRegion)
cacheRule.getCache().getRegion(REGION_NAME);
+ for (BucketRegion br : pr.getDataStore().getAllLocalBucketRegions()) {
+ Set<VersionSource> departedMemberSet =
br.getVersionVector().getDepartedMembersSet();
+ for (Object key : br.getRegionKeysForIteration()) {
+ RegionEntry entry = br.getRegionEntry(key);
+ departedMemberSet.remove(entry.getVersionStamp().getMemberID());
+ if (departedMemberSet.isEmpty()) {
+ break;
+ }
+ }
+ Map map = br.getVersionVector().getMemberToVersion();
+ for (Object key : br.getVersionVector().getMemberToVersion().keySet()) {
+ logger.info(br.getFullPath() + ":" + key + ":"
+ + br.getVersionVector().getMemberToVersion().get(key));
+ }
+ // The test proved that departedMemberSet is not growing
+
assertThat(departedMemberSet.size()).isLessThanOrEqualTo(datastores.length);
+ }
+ }
+
+ @Test
+ public void restartAndRebalanceShouldNotIncreaseMemberToVersionMap() {
+ for (int i = 0; i < datastores.length * 10; i++) {
+ datastores[i % datastores.length].invoke(() -> {
+ cacheRule.getCache().close();
+ });
+ datastores[(i + 1) % datastores.length].invoke(() -> {
+ rebalance();
+ verify();
+ });
+ datastores[i % datastores.length].invoke(() -> {
+ cacheRule.createCache();
+ createRegion();
+ rebalance();
+ verify();
+ });
+ }
+ }
+}
diff --git
a/geode-core/src/main/java/org/apache/geode/internal/cache/InitialImageOperation.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/InitialImageOperation.java
index 3c90a4ef14..42f746088a 100644
---
a/geode-core/src/main/java/org/apache/geode/internal/cache/InitialImageOperation.java
+++
b/geode-core/src/main/java/org/apache/geode/internal/cache/InitialImageOperation.java
@@ -358,7 +358,7 @@ public class InitialImageOperation {
// remote_rvv will be filled with the versions of unfinished keys
// then if recoveredRVV is still newer than the filled remote_rvv, do
fullGII
remote_rvv = received_rvv.getCloneForTransmission();
- keysOfUnfinishedOps = processReceivedRVV(remote_rvv, recoveredRVV);
+ keysOfUnfinishedOps = processReceivedRVV(remote_rvv, recoveredRVV,
received_rvv);
if (internalAfterCalculatedUnfinishedOps != null
&&
internalAfterCalculatedUnfinishedOps.getRegionName().equals(region.getName())) {
internalAfterCalculatedUnfinishedOps.run();
@@ -1052,21 +1052,31 @@ public class InitialImageOperation {
/**
* Compare the received RVV with local RVV and return a set of keys for
unfinished operations.
*
- * @param remoteRVV RVV from provider
+ * @param remoteRVV RVV from provider to be filled with unfinished operations
* @param localRVV RVV recovered from disk
+ * @param receivedRVV original RVV from provider to remove departed members
* @return set for keys of unfinished operations.
*/
protected Set<Object> processReceivedRVV(RegionVersionVector remoteRVV,
- RegionVersionVector localRVV) {
+ RegionVersionVector localRVV, RegionVersionVector receivedRVV) {
if (remoteRVV == null) {
return null;
}
// calculate keys for unfinished ops
HashSet<Object> keys = new HashSet<>();
- if (region.getDataPolicy().withPersistence()
- && localRVV.isNewerThanOrCanFillExceptionsFor(remoteRVV)) {
- // only search for unfinished keys when localRVV has something newer
- // and the region is persistent region
+ Set<VersionSource> departedMemberSet = receivedRVV.getDepartedMembersSet();
+ boolean isPersistentRegion = region.getDataPolicy().withPersistence();
+ Set<VersionSource> foundIds;
+ if (!isPersistentRegion) {
+ foundIds = new HashSet<>();
+ } else {
+ foundIds = Collections.emptySet();
+ }
+ if ((isPersistentRegion &&
localRVV.isNewerThanOrCanFillExceptionsFor(remoteRVV))
+ || !departedMemberSet.isEmpty()) {
+ // Only search for unfinished keys when localRVV has something newer
+ // and the region is persistent region.
+ // Search for departed members if region is not persistent region
Iterator<RegionEntry> it = region.getBestIterator(false);
int count = 0;
VersionSource<?> myId = region.getVersionMember();
@@ -1077,7 +1087,9 @@ public class InitialImageOperation {
if (id == null) {
id = myId;
}
- if (!remoteRVV.contains(id, stamp.getRegionVersion())) {
+ if (!isPersistentRegion) {
+ foundIds.add(id);
+ } else if (!remoteRVV.contains(id, stamp.getRegionVersion())) {
// found an unfinished operation
keys.add(mapEntry.getKey());
remoteRVV.recordVersion(id, stamp.getRegionVersion());
@@ -1100,6 +1112,13 @@ public class InitialImageOperation {
}
}
}
+ if (!departedMemberSet.isEmpty()) {
+ if (localRVV != null) {
+ localRVV.removeOldMembers(foundIds);
+ }
+ receivedRVV.removeOldMembers(foundIds);
+ remoteRVV.removeOldMembers(foundIds);
+ }
return keys;
}
@@ -2042,11 +2061,9 @@ public class InitialImageOperation {
// if this region is destroyed while we are sending data, then abort.
} while (keepGoing && it.hasNext());
- if (foundIds.size() > 0) {
- RegionVersionVector vv = rgn.getVersionVector();
- if (vv != null) {
- vv.removeOldMembers(foundIds);
- }
+ RegionVersionVector vv = rgn.getVersionVector();
+ if (vv != null) {
+ vv.removeOldMembers(foundIds);
}
// return false if we were told to abort
return sentLastChunk;
diff --git
a/geode-core/src/main/java/org/apache/geode/internal/cache/versions/RegionVersionHolder.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/versions/RegionVersionHolder.java
index 5910f79f53..f81beaef92 100755
---
a/geode-core/src/main/java/org/apache/geode/internal/cache/versions/RegionVersionHolder.java
+++
b/geode-core/src/main/java/org/apache/geode/internal/cache/versions/RegionVersionHolder.java
@@ -207,6 +207,8 @@ public class RegionVersionHolder<T> implements Cloneable,
DataSerializable {
sb.append(exceptions);
}
sb.append("}");
+ sb.append("id=" + id);
+ sb.append(",departed?" + isDepartedMember);
return sb.toString();
}
diff --git
a/geode-core/src/main/java/org/apache/geode/internal/cache/versions/RegionVersionVector.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/versions/RegionVersionVector.java
index 3e0627cc9b..de6e48b477 100644
---
a/geode-core/src/main/java/org/apache/geode/internal/cache/versions/RegionVersionVector.java
+++
b/geode-core/src/main/java/org/apache/geode/internal/cache/versions/RegionVersionVector.java
@@ -196,6 +196,7 @@ public abstract class RegionVersionVector<T extends
VersionSource<?>>
isLiveVector = true;
region = owner;
localExceptions = new RegionVersionHolder<>(0);
+ localExceptions.id = myId;
memberToVersion =
new ConcurrentHashMap<>(INITIAL_CAPACITY, LOAD_FACTOR,
CONCURRENCY_LEVEL);
memberToGCVersion =
@@ -609,6 +610,9 @@ public abstract class RegionVersionVector<T extends
VersionSource<?>>
if (!mbr.equals(myId)) {
h = otherHolder.clone();
h.makeReadyForRecording();
+ if (h.id == null) {
+ h.id = mbr;
+ }
memberToVersion.put(mbr, h);
} else {
RegionVersionHolder<T> vh = otherHolder;
diff --git
a/geode-core/src/test/java/org/apache/geode/internal/cache/InitialImageOperationTest.java
b/geode-core/src/test/java/org/apache/geode/internal/cache/InitialImageOperationTest.java
index 0108cea69b..8eb694a787 100644
---
a/geode-core/src/test/java/org/apache/geode/internal/cache/InitialImageOperationTest.java
+++
b/geode-core/src/test/java/org/apache/geode/internal/cache/InitialImageOperationTest.java
@@ -15,6 +15,7 @@
package org.apache.geode.internal.cache;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
@@ -25,17 +26,24 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import org.junit.Before;
import org.junit.Test;
import org.apache.geode.cache.CacheClosedException;
+import org.apache.geode.cache.DataPolicy;
import org.apache.geode.cache.Scope;
import org.apache.geode.distributed.internal.ClusterDistributionManager;
import
org.apache.geode.distributed.internal.membership.InternalDistributedMember;
+import org.apache.geode.internal.cache.persistence.DiskStoreID;
+import org.apache.geode.internal.cache.versions.DiskRegionVersionVector;
import org.apache.geode.internal.cache.versions.DiskVersionTag;
+import org.apache.geode.internal.cache.versions.RegionVersionVector;
+import org.apache.geode.internal.cache.versions.VMRegionVersionVector;
import org.apache.geode.internal.cache.versions.VersionSource;
+import org.apache.geode.internal.cache.versions.VersionStamp;
public class InitialImageOperationTest {
@@ -125,4 +133,159 @@ public class InitialImageOperationTest {
verify(versionTag).replaceNullIDs(member);
}
+
+ @Test
+ public void shouldRemoveDepartedMembersFromRVVForNonPersistentRegion() {
+ InternalDistributedMember server1 = new InternalDistributedMember("host1",
101);
+ InternalDistributedMember server2 = new InternalDistributedMember("host2",
102);
+ InternalDistributedMember server3 = new InternalDistributedMember("host3",
103);
+ InternalDistributedMember server4 = new InternalDistributedMember("host4",
104);
+ when(distributedRegion.getDataPolicy()).thenReturn(DataPolicy.REPLICATE);
+ when(distributedRegion.getVersionMember()).thenReturn(server1);
+
+ RegionEntry re1 = mock(RegionEntry.class);
+ RegionEntry re2 = mock(RegionEntry.class);
+ RegionEntry re3 = mock(RegionEntry.class);
+ ArrayList<RegionEntry> entries = new ArrayList<>();
+ entries.add(re1);
+ entries.add(re2);
+ entries.add(re3);
+ Iterator<RegionEntry> iterator = entries.iterator();
+ when(distributedRegion.getBestIterator(false)).thenReturn(iterator);
+ VersionStamp stamp1 = mock(VersionStamp.class);
+ VersionStamp stamp2 = mock(VersionStamp.class);
+ VersionStamp stamp3 = mock(VersionStamp.class);
+ when(re1.getVersionStamp()).thenReturn(stamp1);
+ when(re2.getVersionStamp()).thenReturn(stamp2);
+ when(re3.getVersionStamp()).thenReturn(stamp3);
+ when(stamp1.getMemberID()).thenReturn(server1);
+ when(stamp2.getMemberID()).thenReturn(server2);
+ when(stamp3.getMemberID()).thenReturn(server3);
+
+ RegionMap regionMap = mock(RegionMap.class);
+ InitialImageOperation operation = spy(new
InitialImageOperation(distributedRegion, regionMap));
+
+ RegionVersionVector recoveredRVV = new VMRegionVersionVector(server1);
+ recoveredRVV.recordVersion(server1, 1);
+ recoveredRVV.recordVersion(server2, 1);
+ recoveredRVV.recordVersion(server3, 1);
+ recoveredRVV.recordVersion(server4, 1);
+ recoveredRVV.recordGCVersion(server2, 1);
+ recoveredRVV.recordGCVersion(server3, 1);
+ recoveredRVV.recordGCVersion(server4, 1);
+ recoveredRVV.memberDeparted(null, server3, true);
+ recoveredRVV.memberDeparted(null, server4, true);
+ assertThat(recoveredRVV.isDepartedMember(server3)).isTrue();
+ assertThat(recoveredRVV.isDepartedMember(server4)).isTrue();
+ assertThat(recoveredRVV.getMemberToVersion().size()).isEqualTo(4);
+ assertThat(recoveredRVV.getMemberToGCVersion().size()).isEqualTo(3);
+
+ RegionVersionVector receivedRVV = new VMRegionVersionVector(server2);
+ receivedRVV.recordVersion(server1, 1);
+ receivedRVV.recordVersion(server2, 1);
+ receivedRVV.recordVersion(server2, 2);
+ receivedRVV.recordVersion(server3, 1);
+ receivedRVV.recordVersion(server4, 1);
+ receivedRVV.recordGCVersion(server2, 1);
+ receivedRVV.recordGCVersion(server3, 1);
+ receivedRVV.recordGCVersion(server4, 1);
+ receivedRVV.memberDeparted(null, server3, true);
+ receivedRVV.memberDeparted(null, server4, true);
+ assertThat(receivedRVV.isDepartedMember(server3)).isTrue();
+ assertThat(receivedRVV.isDepartedMember(server4)).isTrue();
+ assertThat(receivedRVV.getMemberToVersion().size()).isEqualTo(4);
+ assertThat(receivedRVV.getMemberToGCVersion().size()).isEqualTo(3);
+
+ RegionVersionVector remoteRVV = receivedRVV.getCloneForTransmission();
+
+ operation.processReceivedRVV(remoteRVV, recoveredRVV, receivedRVV);
+ assertThat(receivedRVV.getMemberToVersion().size()).isEqualTo(3);
+ assertThat(receivedRVV.getMemberToGCVersion().size()).isEqualTo(2);
+ assertThat(recoveredRVV.getMemberToVersion().size()).isEqualTo(3);
+ assertThat(recoveredRVV.getMemberToGCVersion().size()).isEqualTo(2);
+ assertThat(remoteRVV.getMemberToVersion().size()).isEqualTo(3);
+ assertThat(remoteRVV.getMemberToGCVersion().size()).isEqualTo(2);
+
assertThat(recoveredRVV.getMemberToVersion().containsKey(server3)).isTrue();
+
assertThat(recoveredRVV.getMemberToVersion().containsKey(server4)).isFalse();
+
assertThat(recoveredRVV.getMemberToGCVersion().containsKey(server3)).isTrue();
+
assertThat(recoveredRVV.getMemberToGCVersion().containsKey(server4)).isFalse();
+ assertThat(receivedRVV.getMemberToVersion().containsKey(server3)).isTrue();
+
assertThat(receivedRVV.getMemberToVersion().containsKey(server4)).isFalse();
+
assertThat(receivedRVV.getMemberToGCVersion().containsKey(server3)).isTrue();
+
assertThat(receivedRVV.getMemberToGCVersion().containsKey(server4)).isFalse();
+ }
+
+ @Test
+ public void shouldNotRemoveDepartedMembersFromRVVForPersistentRegion() {
+ InternalDistributedMember idm = new InternalDistributedMember("host1",
101);
+ DiskStoreID server1 = new DiskStoreID(0, 0);
+ DiskStoreID server2 = new DiskStoreID(0, 1);
+ DiskStoreID server3 = new DiskStoreID(0, 2);
+ DiskStoreID server4 = new DiskStoreID(0, 3);
+
when(distributedRegion.getDataPolicy()).thenReturn(DataPolicy.PERSISTENT_REPLICATE);
+ when(distributedRegion.getVersionMember()).thenReturn(server1);
+
+ RegionEntry re1 = mock(RegionEntry.class);
+ RegionEntry re2 = mock(RegionEntry.class);
+ RegionEntry re3 = mock(RegionEntry.class);
+ ArrayList<RegionEntry> entries = new ArrayList<>();
+ entries.add(re1);
+ entries.add(re2);
+ entries.add(re3);
+ Iterator<RegionEntry> iterator = entries.iterator();
+ when(distributedRegion.getBestIterator(false)).thenReturn(iterator);
+ VersionStamp stamp1 = mock(VersionStamp.class);
+ VersionStamp stamp2 = mock(VersionStamp.class);
+ VersionStamp stamp3 = mock(VersionStamp.class);
+ when(re1.getVersionStamp()).thenReturn(stamp1);
+ when(re2.getVersionStamp()).thenReturn(stamp2);
+ when(re3.getVersionStamp()).thenReturn(stamp3);
+ when(stamp1.getMemberID()).thenReturn(server1);
+ when(stamp2.getMemberID()).thenReturn(server2);
+ when(stamp3.getMemberID()).thenReturn(server3);
+
+ RegionMap regionMap = mock(RegionMap.class);
+ InitialImageOperation operation = spy(new
InitialImageOperation(distributedRegion, regionMap));
+
+ RegionVersionVector recoveredRVV = new DiskRegionVersionVector(server1);
+ recoveredRVV.recordVersion(server1, 1);
+ recoveredRVV.recordVersion(server2, 1);
+ recoveredRVV.recordVersion(server3, 1);
+ recoveredRVV.recordVersion(server4, 1);
+ recoveredRVV.recordGCVersion(server2, 1);
+ recoveredRVV.recordGCVersion(server3, 1);
+ recoveredRVV.recordGCVersion(server4, 1);
+ recoveredRVV.memberDeparted(null, idm, true);
+ assertThat(recoveredRVV.getMemberToVersion().size()).isEqualTo(4);
+ assertThat(recoveredRVV.getMemberToGCVersion().size()).isEqualTo(3);
+
+ RegionVersionVector receivedRVV = new DiskRegionVersionVector(server2);
+ receivedRVV.recordVersion(server1, 1);
+ receivedRVV.recordVersion(server2, 1);
+ receivedRVV.recordVersion(server2, 2);
+ receivedRVV.recordVersion(server3, 1);
+ receivedRVV.recordVersion(server4, 1);
+ receivedRVV.recordGCVersion(server2, 1);
+ receivedRVV.recordGCVersion(server3, 1);
+ receivedRVV.recordGCVersion(server4, 1);
+ receivedRVV.memberDeparted(null, idm, true);
+ assertThat(receivedRVV.getMemberToVersion().size()).isEqualTo(4);
+ assertThat(receivedRVV.getMemberToGCVersion().size()).isEqualTo(3);
+
+ RegionVersionVector remoteRVV = receivedRVV.getCloneForTransmission();
+ receivedRVV = spy(receivedRVV);
+ recoveredRVV = spy(recoveredRVV);
+ remoteRVV = spy(remoteRVV);
+
+ operation.processReceivedRVV(remoteRVV, recoveredRVV, receivedRVV);
+ assertThat(receivedRVV.getMemberToVersion().size()).isEqualTo(4);
+ assertThat(receivedRVV.getMemberToGCVersion().size()).isEqualTo(3);
+ assertThat(recoveredRVV.getMemberToVersion().size()).isEqualTo(4);
+ assertThat(recoveredRVV.getMemberToGCVersion().size()).isEqualTo(3);
+ assertThat(remoteRVV.getMemberToVersion().size()).isEqualTo(4);
+ assertThat(remoteRVV.getMemberToGCVersion().size()).isEqualTo(3);
+ verify(receivedRVV, never()).removeOldMembers(any());
+ verify(recoveredRVV, never()).removeOldMembers(any());
+ verify(remoteRVV, never()).removeOldMembers(any());
+ }
}
