This is an automated email from the ASF dual-hosted git repository.
CRZbulabula pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9f0bc18d700 Retry peer-list reset during AddRegionPeer rollback
(#19118)
9f0bc18d700 is described below
commit 9f0bc18d700f87f81b15d8e041edc1f1c8a65699
Author: d-wang-commit <[email protected]>
AuthorDate: Thu Nov 5 07:56:14 2026 +0800
Retry peer-list reset during AddRegionPeer rollback (#19118)
Wait five seconds between up to three additional reset RPC batches.
Cover retry success, exhaustion, early completion and interruption.
---
.../impl/region/AddRegionPeerProcedure.java | 17 +++
.../impl/region/AddRegionPeerProcedureTest.java | 122 +++++++++++++++++++++
2 files changed, 139 insertions(+)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/AddRegionPeerProcedure.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/AddRegionPeerProcedure.java
index 78538a4d57d..cecd8b5608a 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/AddRegionPeerProcedure.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/AddRegionPeerProcedure.java
@@ -55,6 +55,8 @@ import static
org.apache.iotdb.rpc.TSStatusCode.SUCCESS_STATUS;
public class AddRegionPeerProcedure extends
RegionOperationProcedure<AddRegionPeerState> {
private static final Logger LOGGER =
LoggerFactory.getLogger(AddRegionPeerProcedure.class);
+ private static final int MAX_RESET_PEER_LIST_RETRY = 3;
+ private static final long RESET_PEER_LIST_RETRY_INTERVAL_MS = 5_000;
private TDataNodeLocation coordinator;
@@ -252,6 +254,21 @@ public class AddRegionPeerProcedure extends
RegionOperationProcedure<AddRegionPe
Map<Integer, TSStatus> resultMap =
handler.resetPeerList(regionId, correctDataNodeLocations,
relatedDataNodeLocationMap);
+ for (int retry = 0; retry < MAX_RESET_PEER_LIST_RETRY; retry++) {
+ if (resultMap.keySet().containsAll(relatedDataNodeLocationMap.keySet())
+ && resultMap.values().stream()
+ .allMatch(status -> status.getCode() ==
SUCCESS_STATUS.getStatusCode())) {
+ break;
+ }
+ try {
+ Thread.sleep(RESET_PEER_LIST_RETRY_INTERVAL_MS);
+ } catch (InterruptedException interruptedException) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ resultMap =
+ handler.resetPeerList(regionId, correctDataNodeLocations,
relatedDataNodeLocationMap);
+ }
resultMap.forEach(
(dataNodeId, resetResult) -> {
diff --git
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/AddRegionPeerProcedureTest.java
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/AddRegionPeerProcedureTest.java
index 895bd6483ce..fdbb0fe57d9 100644
---
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/AddRegionPeerProcedureTest.java
+++
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/region/AddRegionPeerProcedureTest.java
@@ -23,16 +23,66 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.partition.PartitionManager;
+import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+import org.apache.iotdb.confignode.procedure.env.RegionMaintainHandler;
+import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure.Flow;
+import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState;
import org.apache.iotdb.confignode.procedure.store.ProcedureFactory;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.utils.PublicBAOS;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import java.io.DataOutputStream;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class AddRegionPeerProcedureTest {
+ private final TConsensusGroupId regionId =
+ new TConsensusGroupId(TConsensusGroupType.DataRegion, 10);
+ private final TDataNodeLocation coordinator = location(1);
+ private final TDataNodeLocation target = location(2);
+ private RegionMaintainHandler handler;
+ private ConfigNodeProcedureEnv env;
+
+ @Before
+ public void setUp() throws Exception {
+ ConfigManager configManager = mock(ConfigManager.class);
+ PartitionManager partitionManager = mock(PartitionManager.class);
+ handler = mock(RegionMaintainHandler.class);
+ env = mock(ConfigNodeProcedureEnv.class);
+ when(env.getConfigManager()).thenReturn(configManager);
+ when(env.getRegionMaintainHandler()).thenReturn(handler);
+ when(configManager.getPartitionManager()).thenReturn(partitionManager);
+ when(partitionManager.getAllReplicaSets())
+ .thenReturn(
+ Collections.singletonList(
+ new TRegionReplicaSet(
+ regionId, new
ArrayList<>(Collections.singletonList(coordinator)))));
+ when(handler.createNewRegionPeer(regionId, target))
+ .thenReturn(new
TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()));
+ when(handler.removeRegionLocation(regionId, target))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+ }
+
@Test
public void serDeTest() throws Exception {
AddRegionPeerProcedure procedure =
@@ -60,4 +110,76 @@ public class AddRegionPeerProcedureTest {
Assert.assertEquals(procedure,
ProcedureFactory.getInstance().create(buffer));
}
}
+
+ @Test
+ public void rollbackRetriesUnavailablePeerAfterDelay() throws Exception {
+ when(handler.resetPeerList(eq(regionId), anyList(), anyMap()))
+ .thenReturn(
+ responses(TSStatusCode.CAN_NOT_CONNECT_DATANODE),
+ responses(TSStatusCode.SUCCESS_STATUS));
+
+ long start = System.nanoTime();
+ executeFailedCreate();
+ Assert.assertTrue(System.nanoTime() - start >=
TimeUnit.SECONDS.toNanos(5));
+ verify(handler, times(2)).resetPeerList(eq(regionId), anyList(), anyMap());
+ }
+
+ @Test
+ public void rollbackStopsAfterFourFailedRpcBatches() throws Exception {
+ when(handler.resetPeerList(eq(regionId), anyList(), anyMap()))
+ .thenReturn(responses(TSStatusCode.CAN_NOT_CONNECT_DATANODE));
+
+ long start = System.nanoTime();
+ executeFailedCreate();
+ Assert.assertTrue(System.nanoTime() - start >=
TimeUnit.SECONDS.toNanos(15));
+ verify(handler, times(4)).resetPeerList(eq(regionId), anyList(), anyMap());
+ }
+
+ @Test
+ public void rollbackDoesNotRetrySuccessfulRpcBatch() throws Exception {
+ when(handler.resetPeerList(eq(regionId), anyList(), anyMap()))
+ .thenReturn(responses(TSStatusCode.SUCCESS_STATUS));
+
+ executeFailedCreate();
+ verify(handler, times(1)).resetPeerList(eq(regionId), anyList(), anyMap());
+ }
+
+ @Test
+ public void interruptedRollbackWaitDoesNotRetry() throws Exception {
+ when(handler.resetPeerList(eq(regionId), anyList(), anyMap()))
+ .thenReturn(responses(TSStatusCode.CAN_NOT_CONNECT_DATANODE));
+
+ try {
+ Thread.currentThread().interrupt();
+ executeFailedCreate();
+ Assert.assertTrue(Thread.currentThread().isInterrupted());
+ verify(handler, times(1)).resetPeerList(eq(regionId), anyList(),
anyMap());
+ } finally {
+ Thread.interrupted();
+ }
+ }
+
+ private void executeFailedCreate() throws Exception {
+ AddRegionPeerProcedure procedure = new AddRegionPeerProcedure(regionId,
coordinator, target);
+ Assert.assertEquals(
+ Flow.NO_MORE_STATE,
+ procedure.executeFromState(env,
AddRegionPeerState.CREATE_NEW_REGION_PEER));
+ Assert.assertFalse(procedure.isFailed());
+ verify(handler, times(1)).createNewRegionPeer(regionId, target);
+ verify(handler, times(1)).removeRegionLocation(regionId, target);
+ }
+
+ private Map<Integer, TSStatus> responses(TSStatusCode targetStatus) {
+ Map<Integer, TSStatus> responses = new HashMap<>();
+ responses.put(
+ coordinator.getDataNodeId(), new
TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+ responses.put(target.getDataNodeId(), new
TSStatus(targetStatus.getStatusCode()));
+ return responses;
+ }
+
+ private static TDataNodeLocation location(int id) {
+ return new TDataNodeLocation()
+ .setDataNodeId(id)
+ .setInternalEndPoint(new TEndPoint("127.0.0.1", 10730 + id));
+ }
}