This is an automated email from the ASF dual-hosted git repository.
dcapwell pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2d80e99 Unable to ALTER KEYSPACE while decommissioned/assassinated
nodes are in gossip
2d80e99 is described below
commit 2d80e9904b46474b3a7cc135690c27511a234713
Author: Jon Meredith <[email protected]>
AuthorDate: Thu Feb 11 14:20:57 2021 -0800
Unable to ALTER KEYSPACE while decommissioned/assassinated nodes are in
gossip
patch by Jon Meredith; reviewed by Brandon Williams, David Capwell for
CASSANDRA-16422
---
CHANGES.txt | 1 +
.../statements/schema/AlterKeyspaceStatement.java | 6 +++++-
src/java/org/apache/cassandra/gms/Gossiper.java | 21 ++++++++++++++++++++-
.../AssassinateAbruptDownedNodeTest.java | 20 ++++++++++++++++++++
.../test/hostreplacement/BaseAssassinatedCase.java | 7 +++++++
5 files changed, 53 insertions(+), 2 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 0883a8f..9f6ee1b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.0-beta5
+ * Unable to ALTER KEYSPACE while decommissioned/assassinated nodes are in
gossip (CASSANDRA-16422)
* Metrics backward compatibility restored after CASSANDRA-15066
(CASSANDRA-16083)
* Reduce new reserved keywords introduced since 3.0 (CASSANDRA-16439)
* Improve system tables handling in case of disk failures (CASSANDRA-14793)
diff --git
a/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
index 2f0c188..2ef890e 100644
---
a/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
+++
b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java
@@ -112,7 +112,11 @@ public final class AlterKeyspaceStatement extends
AlterSchemaStatement
if (allow_alter_rf_during_range_movement)
return;
- Stream<InetAddressAndPort> endpoints =
Stream.concat(Gossiper.instance.getLiveMembers().stream(),
Gossiper.instance.getUnreachableMembers().stream());
+ Stream<InetAddressAndPort> unreachableNotAdministrativelyInactive =
+ Gossiper.instance.getUnreachableMembers().stream().filter(endpoint
-> !FBUtilities.getBroadcastAddressAndPort().equals(endpoint) &&
+
!Gossiper.instance.isAdministrativelyInactiveState(endpoint));
+ Stream<InetAddressAndPort> endpoints =
Stream.concat(Gossiper.instance.getLiveMembers().stream(),
+
unreachableNotAdministrativelyInactive);
List<InetAddressAndPort> notNormalEndpoints =
endpoints.filter(endpoint ->
!FBUtilities.getBroadcastAddressAndPort().equals(endpoint) &&
!Gossiper.instance.getEndpointStateForEndpoint(endpoint).isNormalState())
.collect(Collectors.toList());
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java
b/src/java/org/apache/cassandra/gms/Gossiper.java
index 74abea3..1129c29 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -105,7 +105,9 @@ public class Gossiper implements
IFailureDetectionEventListener, GossiperMBean
SILENT_SHUTDOWN_STATES.add(VersionedValue.STATUS_BOOTSTRAPPING);
SILENT_SHUTDOWN_STATES.add(VersionedValue.STATUS_BOOTSTRAPPING_REPLACE);
}
-
+ private static List<String> ADMINISTRATIVELY_INACTIVE_STATES =
Arrays.asList(VersionedValue.HIBERNATE,
+
VersionedValue.REMOVED_TOKEN,
+
VersionedValue.STATUS_LEFT);
private volatile ScheduledFuture<?> scheduledGossipTask;
private static final ReentrantLock taskLock = new ReentrantLock();
public final static int intervalInMillis = 1000;
@@ -1301,6 +1303,23 @@ public class Gossiper implements
IFailureDetectionEventListener, GossiperMBean
return SILENT_SHUTDOWN_STATES.contains(status);
}
+ public boolean isAdministrativelyInactiveState(EndpointState epState)
+ {
+ String status = getGossipStatus(epState);
+ if (status.isEmpty())
+ return false;
+
+ return ADMINISTRATIVELY_INACTIVE_STATES.contains(status);
+ }
+
+ public boolean isAdministrativelyInactiveState(InetAddressAndPort endpoint)
+ {
+ EndpointState epState = getEndpointStateForEndpoint(endpoint);
+ if (epState == null)
+ return true; // if the end point cannot be found, treat as inactive
+ return isAdministrativelyInactiveState(epState);
+ }
+
private static String getGossipStatus(EndpointState epState)
{
if (epState == null)
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateAbruptDownedNodeTest.java
b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateAbruptDownedNodeTest.java
index 95fa767..d377300 100644
---
a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateAbruptDownedNodeTest.java
+++
b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateAbruptDownedNodeTest.java
@@ -18,8 +18,12 @@
package org.apache.cassandra.distributed.test.hostreplacement;
+import java.net.InetSocketAddress;
+
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.locator.InetAddressAndPort;
import static org.apache.cassandra.distributed.shared.ClusterUtils.stopAbrupt;
@@ -36,4 +40,20 @@ public class AssassinateAbruptDownedNodeTest extends
BaseAssassinatedCase
{
stopAbrupt(cluster, nodeToRemove);
}
+
+ @Override
+ protected void afterNodeStatusIsLeft(Cluster cluster, IInvokableInstance
removedNode)
+ {
+ // Check it is possible to alter keyspaces (see CASSANDRA-16422)
+
+ // First, make sure the node is convicted so the gossiper considers it
unreachable
+ InetSocketAddress socketAddress =
removedNode.config().broadcastAddress();
+ InetAddressAndPort removedEndpoint =
InetAddressAndPort.getByAddressOverrideDefaults(socketAddress.getAddress(),
+
socketAddress.getPort());
+ cluster.get(BaseAssassinatedCase.SEED_NUM).runOnInstance(() ->
Gossiper.instance.convict(removedEndpoint, 1.0));
+
+ // Second, try and alter the keyspace. Before the bug was fixed, this
would fail as the check includes
+ // unreachable nodes that could have LEFT status.
+ cluster.schemaChangeIgnoringStoppedInstances(String.format("ALTER
KEYSPACE %s WITH DURABLE_WRITES = false", KEYSPACE));
+ }
}
diff --git
a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/BaseAssassinatedCase.java
b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/BaseAssassinatedCase.java
index 4d4768e..353732c 100644
---
a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/BaseAssassinatedCase.java
+++
b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/BaseAssassinatedCase.java
@@ -46,6 +46,10 @@ public abstract class BaseAssassinatedCase extends
TestBaseImpl
abstract void consume(Cluster cluster, IInvokableInstance nodeToRemove);
+ protected void afterNodeStatusIsLeft(Cluster cluster, IInvokableInstance
nodeToRemove)
+ {
+ }
+
protected String expectedMessage(IInvokableInstance nodeToRemove)
{
return "Cannot replace token " + getTokens(nodeToRemove).get(0) + "
which does not exist!";
@@ -77,6 +81,9 @@ public abstract class BaseAssassinatedCase extends
TestBaseImpl
// wait until the peer sees this assassination
awaitGossipStatus(seed, nodeToRemove, "LEFT");
+ // Any extra checks to run after the node has been as LEFT
+ afterNodeStatusIsLeft(cluster, nodeToRemove);
+
// allow replacing nodes with the LEFT state, this should fail
since the token isn't in the ring
assertThatThrownBy(() ->
replaceHostAndStart(cluster, nodeToRemove,
properties -> {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
