This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new ebbdd8bb81 Make it possible to abort all kinds of multi step operations
ebbdd8bb81 is described below
commit ebbdd8bb81cff1f120ae86a59a3934f26c30b6b7
Author: Marcus Eriksson <marc...@apache.org>
AuthorDate: Wed Jan 15 14:15:47 2025 +0100
Make it possible to abort all kinds of multi step operations
Patch by marcuse; reviewed by Sam Tunnicliffe for CASSANDRA-20217
---
CHANGES.txt | 1 +
.../apache/cassandra/service/StorageService.java | 14 +++++-
.../cassandra/service/StorageServiceMBean.java | 5 +-
.../tcm/sequences/SingleNodeSequences.java | 57 +++++++++++++++++-----
src/java/org/apache/cassandra/tools/NodeProbe.java | 14 +++++-
src/java/org/apache/cassandra/tools/NodeTool.java | 3 ++
.../cassandra/tools/nodetool/Decommission.java | 14 +++++-
.../org/apache/cassandra/tools/nodetool/Move.java | 24 ++++-----
.../cassandra/tools/nodetool/RemoveNode.java | 20 +++++---
.../distributed/test/FailingMoveTest.java | 44 ++++++++++++++++-
.../cassandra/distributed/test/RemoveNodeTest.java | 2 +-
.../distributed/test/ring/DecommissionTest.java | 56 +++++++++++++++++++++
12 files changed, 216 insertions(+), 38 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index c0b984e5ca..4202555d11 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
5.1
+ * Make it possible to abort all kinds of multi step operations
(CASSANDRA-20217)
* Do not leak non-Java exceptions when calling snapshot operations via JMX
(CASSANDRA-20335)
* Implement NOT_NULL constraint (CASSANDRA-20276)
* Improve error messages for constraints (CASSANDRA-20266)
diff --git a/src/java/org/apache/cassandra/service/StorageService.java
b/src/java/org/apache/cassandra/service/StorageService.java
index 5e16bae944..252f451d85 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -3381,6 +3381,11 @@ public class StorageService extends
NotificationBroadcasterSupport implements IE
SingleNodeSequences.decommission(true, force);
}
+ public void abortDecommission(String nodeId)
+ {
+ SingleNodeSequences.abortDecommission(nodeId);
+ }
+
public void shutdownNetworking()
{
shutdownClientServers();
@@ -3461,9 +3466,9 @@ public class StorageService extends
NotificationBroadcasterSupport implements IE
}
@Override
- public void abortMove()
+ public void abortMove(String nodeId)
{
- SingleNodeSequences.abortMove();
+ SingleNodeSequences.abortMove(nodeId);
}
public String getRemovalStatus()
@@ -3533,6 +3538,11 @@ public class StorageService extends
NotificationBroadcasterSupport implements IE
SingleNodeSequences.removeNode(toRemove, force);
}
+ public void abortRemoveNode(String nodeId)
+ {
+ SingleNodeSequences.abortRemoveNode(nodeId);
+ }
+
public void assassinateEndpoint(String address)
{
try
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index f2b75195c1..57dfcea673 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -523,7 +523,7 @@ public interface StorageServiceMBean extends
NotificationEmitter
* @param force Decommission even if this will reduce N to be less than RF.
*/
public void decommission(boolean force) throws InterruptedException;
-
+ public void abortDecommission(String nodeId);
/**
* Returns whether a node has failed to decommission.
*
@@ -547,7 +547,7 @@ public interface StorageServiceMBean extends
NotificationEmitter
*/
public void move(String newToken) throws IOException;
public void resumeMove();
- public void abortMove();
+ public void abortMove(String nodeId);
/**
* removeToken removes token (and all data associated with
@@ -555,6 +555,7 @@ public interface StorageServiceMBean extends
NotificationEmitter
*/
public void removeNode(String token);
public void removeNode(String token, boolean force);
+ public void abortRemoveNode(String nodeId);
public void assassinateEndpoint(String addr);
diff --git
a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java
b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java
index 7813fb1492..58c5f024f9 100644
--- a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java
+++ b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java
@@ -21,6 +21,8 @@ package org.apache.cassandra.tcm.sequences;
import java.util.Collections;
import java.util.EnumSet;
+import javax.annotation.Nullable;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +40,7 @@ import org.apache.cassandra.tcm.transformations.PrepareLeave;
import org.apache.cassandra.tcm.transformations.PrepareMove;
import static org.apache.cassandra.service.StorageService.Mode.LEAVING;
+import static org.apache.cassandra.service.StorageService.Mode.MOVE_FAILED;
import static org.apache.cassandra.service.StorageService.Mode.NORMAL;
import static
org.apache.cassandra.service.StorageService.Mode.DECOMMISSION_FAILED;
import static
org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
@@ -96,6 +99,11 @@ public interface SingleNodeSequences
StorageService.instance.shutdownNetworking();
}
+ static void abortDecommission(String nodeId)
+ {
+ abortHelper(nodeId, MultiStepOperation.Kind.LEAVE,
DECOMMISSION_FAILED);
+ }
+
/**
* Entrypoint to begin node removal process
*
@@ -134,6 +142,11 @@ public interface SingleNodeSequences
InProgressSequences.finishInProgressSequences(toRemove);
}
+ static void abortRemoveNode(String nodeId)
+ {
+ abortHelper(nodeId, MultiStepOperation.Kind.REMOVE, null);
+ }
+
/**
* move the node to new token or find a new token to boot to according to
load
*
@@ -184,7 +197,7 @@ public interface SingleNodeSequences
logger.info(msg);
throw new IllegalStateException(msg);
}
- if (StorageService.instance.operationMode() !=
StorageService.Mode.MOVE_FAILED)
+ if (StorageService.instance.operationMode() != MOVE_FAILED)
{
String msg = "Can't resume a move operation unless it has failed";
logger.info(msg);
@@ -194,28 +207,48 @@ public interface SingleNodeSequences
InProgressSequences.finishInProgressSequences(self);
}
- static void abortMove()
+ static void abortMove(String nodeId)
+ {
+ abortHelper(nodeId, MultiStepOperation.Kind.MOVE, MOVE_FAILED);
+ }
+
+ /**
+ *
+ * @param nodeId node id to abort the MSO for, null for local node
+ * @param kind the expected kind of the multi step operation to abolt
+ * @param ssMode the legacy mode we want storage service to be in, null
for any
+ */
+ private static void abortHelper(@Nullable String nodeId,
MultiStepOperation.Kind kind, @Nullable StorageService.Mode ssMode)
{
if (ClusterMetadataService.instance().isMigrating() ||
ClusterMetadataService.state() == ClusterMetadataService.State.GOSSIP)
- throw new IllegalStateException("This cluster is migrating to
cluster metadata, can't move until that is done.");
+ throw new IllegalStateException(String.format("This cluster is
migrating to cluster metadata, can't abort %s until that is done.", kind));
ClusterMetadata metadata = ClusterMetadata.current();
- NodeId self = metadata.myNodeId();
- MultiStepOperation<?> sequence =
metadata.inProgressSequences.get(self);
- if (sequence == null || sequence.kind() !=
MultiStepOperation.Kind.MOVE)
+ NodeId toAbort = nodeId == null ? metadata.myNodeId() :
NodeId.fromString(nodeId);
+ MultiStepOperation<?> sequence =
metadata.inProgressSequences.get(toAbort);
+ if (sequence == null || sequence.kind() != kind)
{
- String msg = "No move operation in progress, can't abort";
+ String msg = String.format("No %s operation in progress for %s,
can't abort (%s)", kind, toAbort, sequence);
logger.info(msg);
throw new IllegalStateException(msg);
}
- if (StorageService.instance.operationMode() !=
StorageService.Mode.MOVE_FAILED)
+ if (toAbort.equals(metadata.myNodeId()))
+ {
+ if (ssMode != null && StorageService.instance.operationMode() !=
ssMode)
+ {
+ String msg = String.format("Can't abort a %s operation unless
it has failed", kind);
+ logger.info(msg);
+ throw new IllegalStateException(msg);
+ }
+ StorageService.instance.clearTransientMode();
+ }
+ else if
(Gossiper.instance.isAlive(metadata.directory.endpoint(toAbort)))
{
- String msg = "Can't abort a move operation unless it has failed";
+ String msg = String.format("Can't abort a %s operation for a node
%s (%s) that is UP - run abortdecommission on that instance",
+ kind, toAbort,
metadata.directory.endpoint(toAbort));
logger.info(msg);
throw new IllegalStateException(msg);
}
- StorageService.instance.clearTransientMode();
- ClusterMetadataService.instance().commit(new
CancelInProgressSequence(self));
+ ClusterMetadataService.instance().commit(new
CancelInProgressSequence(toAbort));
}
-
}
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java
b/src/java/org/apache/cassandra/tools/NodeProbe.java
index f7ac9eff3b..5a6e8b3598 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -1026,6 +1026,11 @@ public class NodeProbe implements AutoCloseable
ssProxy.decommission(force);
}
+ public void abortDecommission(String nodeId)
+ {
+ ssProxy.abortDecommission(nodeId);
+ }
+
public void move(String newToken) throws IOException
{
ssProxy.move(newToken);
@@ -1036,9 +1041,9 @@ public class NodeProbe implements AutoCloseable
ssProxy.resumeMove();
}
- public void abortMove()
+ public void abortMove(String nodeId)
{
- ssProxy.abortMove();
+ ssProxy.abortMove(nodeId);
}
public void removeNode(String token)
@@ -1051,6 +1056,11 @@ public class NodeProbe implements AutoCloseable
ssProxy.removeNode(token, force);
}
+ public void abortRemoveNode(String nodeId)
+ {
+ ssProxy.abortRemoveNode(nodeId);
+ }
+
public String getRemovalStatus(boolean withPort)
{
return withPort ? ssProxy.getRemovalStatusWithPort() :
ssProxy.getRemovalStatus();
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java
b/src/java/org/apache/cassandra/tools/NodeTool.java
index bd1e302ba0..49be441b89 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -106,6 +106,7 @@ public class NodeTool
CompactionStats.class,
DataPaths.class,
Decommission.class,
+ Decommission.Abort.class,
DescribeCluster.class,
DescribeRing.class,
DisableAuditLog.class,
@@ -172,6 +173,7 @@ public class NodeTool
ListPendingHints.class,
ListSnapshots.class,
Move.class,
+ Move.Abort.class,
NetStats.class,
PauseHandoff.class,
ProfileLoad.class,
@@ -188,6 +190,7 @@ public class NodeTool
ReloadTriggers.class,
RelocateSSTables.class,
RemoveNode.class,
+ RemoveNode.Abort.class,
Repair.class,
ReplayBatchlog.class,
ResetFullQueryLog.class,
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
b/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
index de70932c53..2c326c9b29 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
@@ -26,7 +26,6 @@ import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@Command(name = "decommission", description = "Decommission the *node I am
connecting to*")
public class Decommission extends NodeToolCmd
{
-
@Option(title = "force",
name = {"-f", "--force"},
description = "Force decommission of this node even when it reduces the
number of replicas to below configured RF")
@@ -56,4 +55,17 @@ public class Decommission extends NodeToolCmd
throw new IllegalStateException("Unsupported operation: " +
e.getMessage(), e);
}
}
+
+ @Command(name = "abortdecommission", description = "Abort an ongoing,
failed decommission")
+ public static class Abort extends NodeToolCmd
+ {
+ @Option(title = "node id", name = "--node")
+ private String nodeId;
+
+ @Override
+ protected void execute(NodeProbe probe)
+ {
+ probe.abortDecommission(nodeId);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Move.java
b/src/java/org/apache/cassandra/tools/nodetool/Move.java
index 87c085b86b..ef05bf89b6 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Move.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Move.java
@@ -36,9 +36,6 @@ public class Move extends NodeToolCmd
@Option(title = "Resume an ongoing move operation", name = "--resume")
private boolean resume;
- @Option(title = "Abort an ongoing move operation", name = "--abort")
- private boolean abort;
-
@Override
public void execute(NodeProbe probe)
{
@@ -46,20 +43,12 @@ public class Move extends NodeToolCmd
{
if (!newToken.isEmpty())
{
- if (resume || abort)
- throw new IllegalArgumentException("Can't give both a
token and --resume/--abort");
-
probe.move(newToken);
}
else
{
- if (abort && resume)
- throw new IllegalArgumentException("Can't both resume and
abort");
-
if (resume)
probe.resumeMove();
- else if (abort)
- probe.abortMove();
else
throw new IllegalArgumentException("Need to give either a
token for a new move operation, or --resume/--abort for an existing one");
}
@@ -68,4 +57,17 @@ public class Move extends NodeToolCmd
throw new RuntimeException("Error during moving node", e);
}
}
+
+ @Command(name = "abortmove", description = "Abort a failed move operation
for this or a remote node")
+ public static class Abort extends NodeToolCmd
+ {
+ @Option(title = "node id", name = "--node")
+ private String nodeId;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.abortMove(nodeId);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
b/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
index 89912ec640..94b3262058 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
@@ -22,13 +22,14 @@ import java.util.List;
import io.airlift.airline.Arguments;
import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@Command(name = "removenode", description = "Show status of current node
removal, abort removal or remove provided ID")
public class RemoveNode extends NodeToolCmd
{
- @Arguments(title = "remove_operation", usage = "<status>|<abort>
<ID>|<ID>|<ID> --force", description = "Show status of current node removal,
abort removal, or remove provided ID", required = true)
+ @Arguments(title = "remove_operation", usage = "<status>|<ID>|<ID>
--force", description = "Show status of current node removal, or remove
provided ID", required = true)
private List<String> removeOperation = null;
@Override
@@ -41,15 +42,22 @@ public class RemoveNode extends NodeToolCmd
break;
case "force":
throw new IllegalArgumentException("Can't force a nodetool
removenode. Instead abort the ongoing removenode and retry.");
- case "abort":
- if (removeOperation.size() < 2)
- probe.output().err.print("Abort requires the node id to
abort the removal for.");
-
probe.getCMSOperationsProxy().cancelInProgressSequences(removeOperation.get(1),
"REMOVE");
- break;
default:
boolean force = removeOperation.size() > 1 &&
removeOperation.get(1).equals("--force");
probe.removeNode(removeOperation.get(0), force);
break;
}
}
+
+ @Command(name = "abortremovenode", description = "Abort a removenode
command")
+ public static class Abort extends NodeToolCmd
+ {
+ @Option(title = "node id", name="--node", description = "The node
being removed", required = true)
+ private String nodeId;
+
+ public void execute(NodeProbe probe)
+ {
+ probe.abortRemoveNode(nodeId);
+ }
+ }
}
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/FailingMoveTest.java
b/test/distributed/org/apache/cassandra/distributed/test/FailingMoveTest.java
index ac60f90cd3..65e0c28c3a 100644
---
a/test/distributed/org/apache/cassandra/distributed/test/FailingMoveTest.java
+++
b/test/distributed/org/apache/cassandra/distributed/test/FailingMoveTest.java
@@ -20,8 +20,11 @@ package org.apache.cassandra.distributed.test;
import java.io.IOException;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
import net.bytebuddy.ByteBuddy;
@@ -32,6 +35,7 @@ import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamResultFuture;
@@ -41,6 +45,7 @@ import org.apache.cassandra.tcm.membership.NodeId;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
public class FailingMoveTest extends TestBaseImpl
{
@@ -94,12 +99,49 @@ public class FailingMoveTest extends TestBaseImpl
BB.shouldFail.set(false);
});
- cluster.get(3).nodetoolResult("move",
"--abort").asserts().success();
+ cluster.get(3).nodetoolResult("abortmove").asserts().success();
cluster.get(3).runOnInstance(() ->
assertEquals(StorageService.Mode.NORMAL,
StorageService.instance.operationMode()));
assertNotEquals(moveToToken, getToken(cluster.get(3)));
}
}
+ @Test
+ public void testAbortMoveRemote() throws IOException, ExecutionException,
InterruptedException
+ {
+ try (Cluster cluster = init(Cluster.build(3)
+ .withoutVNodes()
+ .withConfig(c ->
c.with(Feature.GOSSIP, Feature.NETWORK))
+
.withInstanceInitializer(BB::install)
+ .start()))
+ {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl(id int
primary key);"));
+ for (int i=0; i<30; i++)
+ cluster.coordinator(1).execute(withKeyspace("INSERT INTO
%s.tbl (id) VALUES (?)"),
+ ConsistencyLevel.ALL, i);
+ String oldToken = getToken(cluster.get(3));
+ String moveToToken = "2305843009213693949";
+ assertNotEquals(oldToken, moveToToken);
+ cluster.get(3).nodetoolResult("move",
moveToToken).asserts().failure();
+ int nodeId = cluster.get(3).callOnInstance(() -> {
+ assertEquals(StorageService.Mode.MOVE_FAILED,
StorageService.instance.operationMode());
+ BB.shouldFail.set(false);
+ return ClusterMetadata.current().myNodeId().id();
+ });
+ cluster.get(3).shutdown().get();
+ cluster.get(2).runOnInstance(() -> {
+ while
(Gossiper.instance.isAlive(ClusterMetadata.current().directory.endpoint(new
NodeId(nodeId))))
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+ });
+ cluster.get(2).nodetoolResult("abortmove", "--node",
String.valueOf(nodeId)).asserts().success();
+ cluster.get(3).startup();
+ assertNotEquals(moveToToken, getToken(cluster.get(3)));
+ cluster.get(3).runOnInstance(() -> {
+ assertEquals(StorageService.Mode.NORMAL,
StorageService.instance.operationMode());
+
assertTrue(ClusterMetadata.current().inProgressSequences.isEmpty());
+ });
+ }
+ }
+
private String getToken(IInvokableInstance instance)
{
return instance.callsOnInstance(() -> {
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/RemoveNodeTest.java
b/test/distributed/org/apache/cassandra/distributed/test/RemoveNodeTest.java
index 2710fdc832..cb1514b2c8 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RemoveNodeTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RemoveNodeTest.java
@@ -79,7 +79,7 @@ public class RemoveNodeTest extends TestBaseImpl
// Now abort the removal. This should succeed in committing a
cancellation of the removal sequence before
// it can be completed, as non-CMS instance is still paused.
- cmsInstance.nodetoolResult("removenode", "abort",
nodeId).asserts().success();
+ cmsInstance.nodetoolResult("abortremovenode", "--node",
nodeId).asserts().success();
// Resume processing on the non-CMS instance. It will enact the
MID_LEAVE step followed by the cancellation
// of the removal process.
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java
b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java
index 9826d013d5..6f67dac03b 100644
---
a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java
+++
b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java
@@ -22,9 +22,12 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
import net.bytebuddy.ByteBuddy;
@@ -37,9 +40,12 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.tcm.transformations.Startup;
@@ -94,6 +100,56 @@ public class DecommissionTest extends TestBaseImpl
}
}
+ @Test
+ public void testAbortDecom() throws IOException
+ {
+ try (Cluster cluster = builder().withNodes(3)
+ .withConfig(config ->
config.with(NETWORK, GOSSIP))
+ .withInstanceInitializer(BB::install)
+ .start())
+ {
+ populate(cluster, 0, 100, 1, 2, ConsistencyLevel.QUORUM);
+ cluster.get(2).nodetoolResult("decommission",
"--force").asserts().failure();
+
cluster.get(2).nodetoolResult("abortdecommission").asserts().success();
+ cluster.get(2).runOnInstance(() -> {
+ assertEquals(StorageService.Mode.NORMAL,
StorageService.instance.operationMode());
+
assertTrue(ClusterMetadata.current().inProgressSequences.isEmpty());
+ });
+ cluster.get(2).nodetoolResult("decommission",
"--force").asserts().success();
+ }
+ }
+
+ @Test
+ public void testAbortDecomRemote() throws IOException, ExecutionException,
InterruptedException
+ {
+ try (Cluster cluster = builder().withNodes(3)
+ .withConfig(config ->
config.with(NETWORK, GOSSIP))
+ .withInstanceInitializer(BB::install)
+ .start())
+ {
+ populate(cluster, 0, 100, 1, 2, ConsistencyLevel.QUORUM);
+ int nodeId = cluster.get(2).callOnInstance(() -> {
+ return ClusterMetadata.current().myNodeId().id();
+ });
+ cluster.get(2).nodetoolResult("decommission",
"--force").asserts().failure();
+ cluster.get(2).shutdown().get();
+ cluster.get(3).runOnInstance(() -> {
+ while
(Gossiper.instance.isAlive(ClusterMetadata.current().directory.endpoint(new
NodeId(nodeId))))
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+ });
+ cluster.get(3).nodetoolResult("abortdecommission", "--node",
String.valueOf(nodeId)).asserts().success();
+ cluster.get(2).startup();
+ cluster.get(2).runOnInstance(() -> {
+ assertEquals(StorageService.Mode.NORMAL,
StorageService.instance.operationMode());
+
assertTrue(ClusterMetadata.current().inProgressSequences.isEmpty());
+ });
+ cluster.get(2).runOnInstance(() -> {
+ BB.first.set(true);
+ });
+ cluster.get(2).nodetoolResult("decommission",
"--force").asserts().success();
+ }
+ }
+
@Test
public void testDecomDirectoryMinMaxVersions() throws IOException {
try (Cluster cluster = builder()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
