Fix bootstrap instajoin, take three. Patch by jbellis and brandonwilliams, reviewed by jbellis for CASSANDRA-4427.
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/988c4132 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/988c4132 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/988c4132 Branch: refs/heads/trunk Commit: 988c4132d77b40740b93a9eaca8fb54318bf8a97 Parents: 953bcb3 Author: Brandon Williams <[email protected]> Authored: Tue Jul 31 15:33:08 2012 -0500 Committer: Brandon Williams <[email protected]> Committed: Tue Jul 31 15:33:08 2012 -0500 ---------------------------------------------------------------------- .../org/apache/cassandra/dht/BootStrapper.java | 23 ++++- .../apache/cassandra/service/StorageService.java | 69 +++++++-------- 2 files changed, 51 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/988c4132/src/java/org/apache/cassandra/dht/BootStrapper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/dht/BootStrapper.java b/src/java/org/apache/cassandra/dht/BootStrapper.java index 2ea150e..8afcdee 100644 --- a/src/java/org/apache/cassandra/dht/BootStrapper.java +++ b/src/java/org/apache/cassandra/dht/BootStrapper.java @@ -26,7 +26,8 @@ package org.apache.cassandra.dht; import com.google.common.base.Charsets; import org.apache.cassandra.config.Schema; - import org.apache.cassandra.gms.Gossiper; + import org.apache.cassandra.gms.*; + import org.apache.commons.lang.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +35,6 @@ package org.apache.cassandra.dht; import org.apache.cassandra.config.ConfigurationException; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Table; - import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.net.IAsyncCallback; @@ -92,6 +92,7 @@ public class BootStrapper */ public static Token getBootstrapToken(final TokenMetadata metadata, final Map<InetAddress, Double> load) throws IOException, ConfigurationException { + // if user specified a token, use that if (DatabaseDescriptor.getInitialToken() != null) { logger.debug("token manually specified as " + DatabaseDescriptor.getInitialToken()); @@ -101,7 +102,23 @@ public class BootStrapper return token; } - return getBalancedToken(metadata, load); + // if there is a schema, then we're joining an existing cluster so get a "balanced" token + for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.getEndpointStates()) + { + if (entry.getKey().equals(FBUtilities.getBroadcastAddress())) + { + // skip ourselves to avoid confusing the tests, which always load a schema first thing + continue; + } + + VersionedValue schemaValue = entry.getValue().getApplicationState(ApplicationState.SCHEMA); + if (schemaValue != null && !schemaValue.value.equals(Schema.emptyVersion.toString())) + return getBalancedToken(metadata, load); + } + + // no schema; pick a random token (so multiple non-seeds starting up simultaneously in a new cluster + // don't get the same token; see CASSANDRA-3219) + return StorageService.getPartitioner().getRandomToken(); } public static Token getBalancedToken(TokenMetadata metadata, Map<InetAddress, Double> load) http://git-wip-us.apache.org/repos/asf/cassandra/blob/988c4132/src/java/org/apache/cassandra/service/StorageService.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 4bbffc8..b1eaa1e 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -557,41 +557,25 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe HintedHandOffManager.instance.start(); - boolean schemaPresent = false; - if (DatabaseDescriptor.isAutoBootstrap() && !SystemTable.bootstrapComplete() && delay > 0) - { - // wait a couple gossip rounds so our schema check has something to go by - FBUtilities.sleep(2 * Gossiper.intervalInMillis); - } - for (Entry<InetAddress, EndpointState> entry : Gossiper.instance.getEndpointStates()) - { - if (entry.getKey().equals(FBUtilities.getBroadcastAddress())) - { - // skip ourselves to avoid confusing the tests, which always load a schema first thing - continue; - } - - VersionedValue schemaValue = entry.getValue().getApplicationState(ApplicationState.SCHEMA); - if (schemaValue != null && !schemaValue.value.equals(Schema.emptyVersion.toString())) - { - schemaPresent = true; - break; - } - } - - // We can bootstrap at startup, or if we detect a previous attempt that failed. Either way, if the user - // manually sets auto_bootstrap to false, we'll skip streaming data from other nodes and jump directly - // into the ring. + // We bootstrap if we haven't successfully bootstrapped before, as long as we are not a seed. + // If we are a seed, or if the user manually sets auto_bootstrap to false, + // we'll skip streaming data from other nodes and jump directly into the ring. // - // The one exception is if after the above sleep we still have no schema information, we'll assume - // we're part of a fresh cluster start, and also skip bootstrap. This is less confusing for new users, - // as well as avoiding the nonsensical state of trying to stream from cluster with no active peers. + // The seed check allows us to skip the RING_DELAY sleep for the single-node cluster case, + // which is useful for both new users and testing. + // + // We attempted to replace this with a schema-presence check, but you need a meaningful sleep + // to get schema info from gossip which defeats the purpose. See CASSANDRA-4427 for the gory details. Token<?> token; InetAddress current = null; - logger_.debug(String.format("Bootstrap variables: %s %s %s %s", - DatabaseDescriptor.isAutoBootstrap(), SystemTable.bootstrapInProgress(), SystemTable.bootstrapComplete(), schemaPresent)); + logger_.debug("Bootstrap variables: {} {} {} {}", + new Object[]{ DatabaseDescriptor.isAutoBootstrap(), + SystemTable.bootstrapInProgress(), + SystemTable.bootstrapComplete(), + DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddress())}); if (DatabaseDescriptor.isAutoBootstrap() - && (SystemTable.bootstrapInProgress() || (!SystemTable.bootstrapComplete() && schemaPresent))) + && !SystemTable.bootstrapComplete() + && !DatabaseDescriptor.getSeeds().contains(FBUtilities.getBroadcastAddress())) { if (SystemTable.bootstrapInProgress()) logger_.warn("Detected previous bootstrap failure; retrying"); @@ -599,13 +583,22 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe SystemTable.setBootstrapState(SystemTable.BootstrapState.IN_PROGRESS); setMode(Mode.JOINING, "waiting for ring information", true); // first sleep the delay to make sure we see all our peers - try + for (int i = 0; i < delay; i += 1000) { - Thread.sleep(delay); - } - catch (InterruptedException e) - { - throw new AssertionError(e); + // if we see schema, we can proceed to the next check directly + if (!Schema.instance.getVersion().equals(Schema.emptyVersion)) + { + logger_.debug("got schema: {}", Schema.instance.getVersion()); + break; + } + try + { + Thread.sleep(1000); + } + catch (InterruptedException e) + { + throw new AssertionError(e); + } } // if our schema hasn't matched yet, keep sleeping until it does // (post CASSANDRA-1391 we don't expect this to be necessary very often, but it doesn't hurt to be careful) @@ -614,7 +607,7 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe setMode(Mode.JOINING, "waiting for schema information to complete", true); try { - Thread.sleep(delay); + Thread.sleep(1000); } catch (InterruptedException e) {
