Repository: cassandra Updated Branches: refs/heads/trunk 69db2359e -> bfecdf520
Add nodetool getseeds and reloadseeds commands patch by Samuel Fink; reviewed by jasobrown for CASSANDRA-14190 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bfecdf52 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bfecdf52 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bfecdf52 Branch: refs/heads/trunk Commit: bfecdf52054a4da472af22b0c35c5db5f1132bbc Parents: 69db235 Author: Samuel Fink <[email protected]> Authored: Wed Jan 24 13:28:54 2018 -0500 Committer: Jason Brown <[email protected]> Committed: Tue Jan 30 16:32:09 2018 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/config/DatabaseDescriptor.java | 10 ++ src/java/org/apache/cassandra/gms/Gossiper.java | 83 +++++++++-- .../org/apache/cassandra/gms/GossiperMBean.java | 5 + .../org/apache/cassandra/tools/NodeProbe.java | 10 ++ .../org/apache/cassandra/tools/NodeTool.java | 2 + .../cassandra/tools/nodetool/GetSeeds.java | 44 ++++++ .../cassandra/tools/nodetool/ReloadSeeds.java | 47 +++++++ .../org/apache/cassandra/gms/GossiperTest.java | 140 ++++++++++++++++++- 9 files changed, 332 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index e28ffd9..a2e3654 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Non-disruptive seed node list reload (CASSANDRA-14190) * Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185) * Migrate dtests to use pytest and python3 (CASSANDRA-14134) * Allow storage port to be configurable per node (CASSANDRA-7544) http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 9012e3a..8e831cf 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -1661,6 +1661,16 @@ public class DatabaseDescriptor return ImmutableSet.<InetAddressAndPort>builder().addAll(seedProvider.getSeeds()).build(); } + public static SeedProvider getSeedProvider() + { + return seedProvider; + } + + public static void setSeedProvider(SeedProvider newSeedProvider) + { + seedProvider = newSeedProvider; + } + public static InetAddress getListenAddress() { return listenAddress; http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/gms/Gossiper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index eb6c500..a4e46f2 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Uninterruptibles; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.SeedProvider; import org.apache.cassandra.utils.CassandraVersion; import org.apache.cassandra.utils.Pair; import org.slf4j.Logger; @@ -89,7 +90,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean public final static int intervalInMillis = 1000; public final static int QUARANTINE_DELAY = StorageService.RING_DELAY * 2; private static final Logger logger = LoggerFactory.getLogger(Gossiper.class); - public static final Gossiper instance = new Gossiper(); + public static final Gossiper instance = new Gossiper(true); // Timestamp to prevent processing any in-flight messages for we've not send any SYN yet, see CASSANDRA-12653. volatile long firstSynSendAt = 0L; @@ -199,7 +200,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean } } - private Gossiper() + Gossiper(boolean registerJmx) { // half of QUARATINE_DELAY, to ensure justRemovedEndpoints has enough leeway to prevent re-gossip fatClientTimeout = (QUARANTINE_DELAY / 2); @@ -207,14 +208,17 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean FailureDetector.instance.registerFailureDetectionEventListener(this); // Register this instance with JMX - try - { - MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); - mbs.registerMBean(this, new ObjectName(MBEAN_NAME)); - } - catch (Exception e) + if (registerJmx) { - throw new RuntimeException(e); + try + { + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + mbs.registerMBean(this, new ObjectName(MBEAN_NAME)); + } + catch (Exception e) + { + throw new RuntimeException(e); + } } } @@ -1468,6 +1472,67 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean } } + /** + * JMX interface for triggering an update of the seed node list. + */ + public List<String> reloadSeeds() + { + logger.trace("Triggering reload of seed node list"); + + // Get the new set in the same that buildSeedsList does + Set<InetAddressAndPort> tmp = new HashSet<>(); + try + { + for (InetAddressAndPort seed : DatabaseDescriptor.getSeeds()) + { + if (seed.equals(FBUtilities.getBroadcastAddressAndPort())) + continue; + tmp.add(seed); + } + } + // If using the SimpleSeedProvider invalid yaml added to the config since startup could + // cause this to throw. Additionally, third party seed providers may throw exceptions. + // Handle the error and return a null to indicate that there was a problem. + catch (Throwable e) + { + JVMStabilityInspector.inspectThrowable(e); + logger.warn("Error while getting seed node list: {}", e.getLocalizedMessage()); + return null; + } + + if (tmp.size() == 0) + { + logger.trace("New seed node list is empty. Not updating seed list."); + return getSeeds(); + } + + if (tmp.equals(seeds)) + { + logger.trace("New seed node list matches the existing list."); + return getSeeds(); + } + + // Add the new entries + seeds.addAll(tmp); + // Remove the old entries + seeds.retainAll(tmp); + logger.trace("New seed node list after reload {}", seeds); + return getSeeds(); + } + + /** + * JMX endpoint for getting the list of seeds from the node + */ + public List<String> getSeeds() + { + List<String> seedList = new ArrayList<String>(); + for (InetAddressAndPort seed : seeds) + { + seedList.add(seed.toString()); + } + return seedList; + } + // initialize local HB state if needed, i.e., if gossiper has never been started before. public void maybeInitializeLocalState(int generationNbr) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/gms/GossiperMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/gms/GossiperMBean.java b/src/java/org/apache/cassandra/gms/GossiperMBean.java index c4b244c..1b116e1 100644 --- a/src/java/org/apache/cassandra/gms/GossiperMBean.java +++ b/src/java/org/apache/cassandra/gms/GossiperMBean.java @@ -18,6 +18,7 @@ package org.apache.cassandra.gms; import java.net.UnknownHostException; +import java.util.List; public interface GossiperMBean { @@ -29,4 +30,8 @@ public interface GossiperMBean public void assassinateEndpoint(String address) throws UnknownHostException; + public List<String> reloadSeeds(); + + public List<String> getSeeds(); + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index d330ed4..ec8f7ba 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -711,6 +711,16 @@ public class NodeProbe implements AutoCloseable gossProxy.assassinateEndpoint(address); } + public List<String> reloadSeeds() + { + return gossProxy.reloadSeeds(); + } + + public List<String> getSeeds() + { + return gossProxy.getSeeds(); + } + /** * Set the compaction threshold * http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/NodeTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index d707499..f7b7f76 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -87,6 +87,7 @@ public class NodeTool GetTraceProbability.class, GetInterDCStreamThroughput.class, GetEndpoints.class, + GetSeeds.class, GetSSTables.class, GetMaxHintWindow.class, GossipInfo.class, @@ -102,6 +103,7 @@ public class NodeTool Refresh.class, RemoveNode.class, Assassinate.class, + ReloadSeeds.class, ResetFullQueryLog.class, Repair.class, RepairAdmin.class, http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java b/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java new file mode 100644 index 0000000..207363c --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools.nodetool; + +import java.util.List; + +import io.airlift.airline.Command; + +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool.NodeToolCmd; + +@Command(name = "getseeds", description = "Get the currently in use seed node IP list excluding the node IP") +public class GetSeeds extends NodeToolCmd +{ + @Override + public void execute(NodeProbe probe) + { + List<String> seedList = probe.getSeeds(); + if (seedList.isEmpty()) + { + System.out.println("Seed node list does not contain any remote node IPs"); + } + else + { + System.out.println("Current list of seed node IPs, excluding the current node's IP: " + String.join(" ", seedList)); + } + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java b/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java new file mode 100644 index 0000000..b9682cf --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools.nodetool; + +import java.util.List; + +import io.airlift.airline.Command; + +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool.NodeToolCmd; + +@Command(name = "reloadseeds", description = "Reload the seed node list from the seed node provider") +public class ReloadSeeds extends NodeToolCmd +{ + @Override + public void execute(NodeProbe probe) + { + List<String> seedList = probe.reloadSeeds(); + if (seedList == null) + { + System.out.println("Failed to reload the seed node list."); + } + else if (seedList.isEmpty()) + { + System.out.println("Seed node list does not contain any remote node IPs"); + } + else + { + System.out.println("Updated seed node IP list, excluding the current node's IP: " + String.join(" ", seedList)); + } + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/test/unit/org/apache/cassandra/gms/GossiperTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/gms/GossiperTest.java b/test/unit/org/apache/cassandra/gms/GossiperTest.java index 8c65cb4..b856983 100644 --- a/test/unit/org/apache/cassandra/gms/GossiperTest.java +++ b/test/unit/org/apache/cassandra/gms/GossiperTest.java @@ -18,12 +18,16 @@ package org.apache.cassandra.gms; +import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.UUID; import com.google.common.collect.ImmutableMap; +import com.google.common.net.InetAddresses; +import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -33,6 +37,7 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.SeedProvider; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.service.StorageService; @@ -44,6 +49,7 @@ public class GossiperTest { DatabaseDescriptor.daemonInitialization(); } + static final IPartitioner partitioner = new RandomPartitioner(); StorageService ss = StorageService.instance; TokenMetadata tmd = StorageService.instance.getTokenMetadata(); @@ -52,11 +58,20 @@ public class GossiperTest List<InetAddressAndPort> hosts = new ArrayList<>(); List<UUID> hostIds = new ArrayList<>(); + private SeedProvider originalSeedProvider; + @Before public void setup() { tmd.clearUnsafe(); - }; + originalSeedProvider = DatabaseDescriptor.getSeedProvider(); + } + + @After + public void tearDown() + { + DatabaseDescriptor.setSeedProvider(originalSeedProvider); + } @Test public void testLargeGenerationJump() throws UnknownHostException, InterruptedException @@ -90,4 +105,127 @@ public class GossiperTest //The generation should not have been updated because it is over Gossiper.MAX_GENERATION_DIFFERENCE in the future assertEquals(proposedRemoteHeartBeat.getGeneration(), actualRemoteHeartBeat.getGeneration()); } + + // Note: This test might fail if for some reason the node broadcast address is in 127.99.0.0/16 + @Test + public void testReloadSeeds() throws UnknownHostException + { + Gossiper gossiper = new Gossiper(false); + List<String> loadedList; + + // Initialize the seed list directly to a known set to start with + gossiper.seeds.clear(); + InetAddressAndPort addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.1.1")); + int nextSize = 4; + List<InetAddressAndPort> nextSeeds = new ArrayList<>(nextSize); + for (int i = 0; i < nextSize; i ++) + { + gossiper.seeds.add(addr); + nextSeeds.add(addr); + addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address)); + } + Assert.assertEquals(nextSize, gossiper.seeds.size()); + + // Add another unique address to the list + addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address)); + nextSeeds.add(addr); + nextSize++; + DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds)); + loadedList = gossiper.reloadSeeds(); + + // Check that the new entry was added + Assert.assertEquals(nextSize, loadedList.size()); + for (InetAddressAndPort a : nextSeeds) + Assert.assertTrue(loadedList.contains(a.toString())); + + // Check that the return value of the reloadSeeds matches the content of the getSeeds call + // and that they both match the internal contents of the Gossiper seeds list + Assert.assertEquals(loadedList.size(), gossiper.getSeeds().size()); + for (InetAddressAndPort a : gossiper.seeds) + { + Assert.assertTrue(loadedList.contains(a.toString())); + Assert.assertTrue(gossiper.getSeeds().contains(a.toString())); + } + + // Add a duplicate of the last address to the seed provider list + int uniqueSize = nextSize; + nextSeeds.add(addr); + nextSize++; + DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds)); + loadedList = gossiper.reloadSeeds(); + + // Check that the number of seed nodes reported hasn't increased + Assert.assertEquals(uniqueSize, loadedList.size()); + for (InetAddressAndPort a : nextSeeds) + Assert.assertTrue(loadedList.contains(a.toString())); + + // Create a new list that has no overlaps with the previous list + addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.2.1")); + int disjointSize = 3; + List<InetAddressAndPort> disjointSeeds = new ArrayList<>(disjointSize); + for (int i = 0; i < disjointSize; i ++) + { + disjointSeeds.add(addr); + addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address)); + } + DatabaseDescriptor.setSeedProvider(new TestSeedProvider(disjointSeeds)); + loadedList = gossiper.reloadSeeds(); + + // Check that the list now contains exactly the new other list. + Assert.assertEquals(disjointSize, gossiper.getSeeds().size()); + Assert.assertEquals(disjointSize, loadedList.size()); + for (InetAddressAndPort a : disjointSeeds) + { + Assert.assertTrue(gossiper.getSeeds().contains(a.toString())); + Assert.assertTrue(loadedList.contains(a.toString())); + } + + // Set the seed node provider to return an empty list + DatabaseDescriptor.setSeedProvider(new TestSeedProvider(new ArrayList<InetAddressAndPort>())); + loadedList = gossiper.reloadSeeds(); + + // Check that the in memory seed node list was not modified + Assert.assertEquals(disjointSize, loadedList.size()); + for (InetAddressAndPort a : disjointSeeds) + Assert.assertTrue(loadedList.contains(a.toString())); + + // Change the seed provider to one that throws an unchecked exception + DatabaseDescriptor.setSeedProvider(new ErrorSeedProvider()); + loadedList = gossiper.reloadSeeds(); + + // Check for the expected null response from a reload error + Assert.assertNull(loadedList); + + // Check that the in memory seed node list was not modified and the exception was caught + Assert.assertEquals(disjointSize, gossiper.getSeeds().size()); + for (InetAddressAndPort a : disjointSeeds) + Assert.assertTrue(gossiper.getSeeds().contains(a.toString())); + } + + static class TestSeedProvider implements SeedProvider + { + private List<InetAddressAndPort> seeds; + + TestSeedProvider(List<InetAddressAndPort> seeds) + { + this.seeds = seeds; + } + + @Override + public List<InetAddressAndPort> getSeeds() + { + return seeds; + } + } + + // A seed provider for testing which throws assertion errors when queried + static class ErrorSeedProvider implements SeedProvider + { + @Override + public List<InetAddressAndPort> getSeeds() + { + assert(false); + return new ArrayList<>(); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
