Repository: cassandra Updated Branches: refs/heads/trunk a07977900 -> a33733601
Make assassinate a first class command. Patch by brandonwilliams, reviewed by thobbs for CASSANDRA-7935 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a3373360 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a3373360 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a3373360 Branch: refs/heads/trunk Commit: a337336013eb1efabe709844f1a7755d54b746a8 Parents: a079779 Author: Brandon Williams <[email protected]> Authored: Wed Sep 17 15:25:01 2014 +0000 Committer: Brandon Williams <[email protected]> Committed: Wed Sep 17 15:25:01 2014 +0000 ---------------------------------------------------------------------- CHANGES.txt | 1 + src/java/org/apache/cassandra/gms/Gossiper.java | 16 ++++++++++++---- .../org/apache/cassandra/gms/GossiperMBean.java | 2 ++ .../org/apache/cassandra/tools/NodeProbe.java | 10 ++++++++++ .../org/apache/cassandra/tools/NodeTool.java | 20 ++++++++++++++++++++ 5 files changed, 45 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/a3373360/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 5b80e07..6f9e681 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0 + * Make assassinate a first class command (CASSANDRA-7935) * Support IN clause on any clustering column (CASSANDRA-4762) * Improve compaction logging (CASSANDRA-7818) * Remove YamlFileNetworkTopologySnitch (CASSANDRA-7917) http://git-wip-us.apache.org/repos/asf/cassandra/blob/a3373360/src/java/org/apache/cassandra/gms/Gossiper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index 01af8cd..b9576f0 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -63,7 +63,7 @@ import com.google.common.collect.ImmutableList; public class Gossiper implements IFailureDetectionEventListener, GossiperMBean { - private static final String MBEAN_NAME = "org.apache.cassandra.net:type=Gossiper"; + public static final String MBEAN_NAME = "org.apache.cassandra.net:type=Gossiper"; private static final DebuggableScheduledThreadPoolExecutor executor = new DebuggableScheduledThreadPoolExecutor("GossipTasks"); @@ -474,16 +474,21 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean Uninterruptibles.sleepUninterruptibly(intervalInMillis * 2, TimeUnit.MILLISECONDS); } + public void unsafeAssassinateEndpoint(String address) throws UnknownHostException + { + logger.warn("Gossiper.unsafeAssassinateEndpoint is deprecated and will be removed in the next release; use assassinateEndpoint instead"); + assassinateEndpoint(address); + } + /** * Do not call this method unless you know what you are doing. * It will try extremely hard to obliterate any endpoint from the ring, * even if it does not know about it. - * This should only ever be called by human via JMX. * * @param address * @throws UnknownHostException */ - public void unsafeAssassinateEndpoint(String address) throws UnknownHostException + public void assassinateEndpoint(String address) throws UnknownHostException { InetAddress endpoint = InetAddress.getByName(address); EndpointState epState = endpointStateMap.get(endpoint); @@ -507,6 +512,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean tokens = Collections.singletonList(StorageService.getPartitioner().getRandomToken()); } int generation = epState.getHeartBeatState().getGeneration(); + int heartbeat = epState.getHeartBeatState().getHeartBeatVersion(); logger.info("Sleeping for {}ms to ensure {} does not change", StorageService.RING_DELAY, endpoint); Uninterruptibles.sleepUninterruptibly(StorageService.RING_DELAY, TimeUnit.MILLISECONDS); // make sure it did not change @@ -514,7 +520,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean if (newState == null) logger.warn("Endpoint {} disappeared while trying to assassinate, continuing anyway", endpoint); else if (newState.getHeartBeatState().getGeneration() != generation) - throw new RuntimeException("Endpoint " + endpoint + " generation changed while trying to remove it"); + throw new RuntimeException("Endpoint still alive: " + endpoint + " generation changed while trying to assassinate it"); + else if (newState.getHeartBeatState().getHeartBeatVersion() != heartbeat) + throw new RuntimeException("Endpoint still alive: " + endpoint + " heartbeat changed while trying to assassinate it"); epState.updateTimestamp(); // make sure we don't evict it too soon epState.getHeartBeatState().forceNewerGenerationUnsafe(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a3373360/src/java/org/apache/cassandra/gms/GossiperMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/gms/GossiperMBean.java b/src/java/org/apache/cassandra/gms/GossiperMBean.java index 521fd21..c4b244c 100644 --- a/src/java/org/apache/cassandra/gms/GossiperMBean.java +++ b/src/java/org/apache/cassandra/gms/GossiperMBean.java @@ -27,4 +27,6 @@ public interface GossiperMBean public void unsafeAssassinateEndpoint(String address) throws UnknownHostException; + public void assassinateEndpoint(String address) throws UnknownHostException; + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/a3373360/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 0ad1186..b2ade5a 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -52,6 +52,8 @@ import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.CompactionManagerMBean; import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.gms.FailureDetectorMBean; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.gms.GossiperMBean; import org.apache.cassandra.locator.EndpointSnitchInfoMBean; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.MessagingServiceMBean; @@ -78,6 +80,7 @@ public class NodeProbe implements AutoCloseable private MBeanServerConnection mbeanServerConn; private CompactionManagerMBean compactionProxy; private StorageServiceMBean ssProxy; + private GossiperMBean gossProxy; private MemoryMXBean memProxy; private GCInspectorMXBean gcProxy; private RuntimeMXBean runtimeProxy; @@ -172,6 +175,8 @@ public class NodeProbe implements AutoCloseable hhProxy = JMX.newMBeanProxy(mbeanServerConn, name, HintedHandOffManagerMBean.class); name = new ObjectName(GCInspector.MBEAN_NAME); gcProxy = JMX.newMBeanProxy(mbeanServerConn, name, GCInspectorMXBean.class); + name = new ObjectName(Gossiper.MBEAN_NAME); + gossProxy = JMX.newMBeanProxy(mbeanServerConn, name, GossiperMBean.class); } catch (MalformedObjectNameException e) { @@ -530,6 +535,11 @@ public class NodeProbe implements AutoCloseable ssProxy.forceRemoveCompletion(); } + public void assassinateEndpoint(String address) throws UnknownHostException + { + gossProxy.assassinateEndpoint(address); + } + public Iterator<Map.Entry<String, JMXEnabledThreadPoolExecutorMBean>> getThreadPoolMBeanProxies() { try http://git-wip-us.apache.org/repos/asf/cassandra/blob/a3373360/src/java/org/apache/cassandra/tools/NodeTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index f7baa8b..db75305 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -114,6 +114,7 @@ public class NodeTool Rebuild.class, Refresh.class, RemoveNode.class, + Assassinate.class, Repair.class, SetCacheCapacity.class, SetHintedHandoffThrottleInKB.class, @@ -1626,6 +1627,25 @@ public class NodeTool } } + @Command(name = "assassinate", description = "Forcefully remove a dead node without re-replicating any data. Use as a last resort if you cannot removenode") + public static class Assassinate extends NodeToolCmd + { + @Arguments(title = "ip address", usage = "<ip_address>", description = "IP address of the endpoint to assassinate", required = true) + private String endpoint = EMPTY; + + @Override + public void execute(NodeProbe probe) + { + try { + probe.assassinateEndpoint(endpoint); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + } + } + @Command(name = "repair", description = "Repair one or more tables") public static class Repair extends NodeToolCmd {
