David Capwell created CASSANDRA-20263:
-----------------------------------------
Summary: Accord bootstrap stuck during host replacement
Key: CASSANDRA-20263
URL: https://issues.apache.org/jira/browse/CASSANDRA-20263
Project: Apache Cassandra
Issue Type: Bug
Components: Accord
Reporter: David Capwell
Assignee: Benedict Elliott Smith
This was found by the fuzz tests, and took the history to create a non-fuzzed
repo test.
The issue we are seeing is that streaming runs and completes, but the fetch
never notifies that it completed, so the bootstrap is hung unable to make
progress.
{code}
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.fuzz.topology;
import java.io.IOException;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import org.junit.Test;
import accord.utils.Invariants;
import accord.utils.async.TimeoutUtils;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.Constants;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.test.TestBaseImpl;
public class RepoTest extends TestBaseImpl
{
private static final int TARGET_RF = 3;
static
{
// enable most expensive debugging checks
CassandraRelevantProperties.ACCORD_KEY_PARANOIA_CPU.setString(Invariants.Paranoia.QUADRATIC.name());
CassandraRelevantProperties.ACCORD_KEY_PARANOIA_MEMORY.setString(Invariants.Paranoia.QUADRATIC.name());
CassandraRelevantProperties.ACCORD_KEY_PARANOIA_COSTFACTOR.setString(Invariants.ParanoiaCostFactor.HIGH.name());
}
@Test
public void test() throws IOException, ExecutionException,
InterruptedException, TimeoutException
{
Map<Integer, Long> nodeToToken = new HashMap<>();
nodeToToken.put(1, -1257885350L);
nodeToToken.put(2, 449717126L);
nodeToToken.put(3, 1959554097L);
nodeToToken.put(4, 1959554097L); // replaces node3
TokenSupplier tokenSupplier = n ->
Collections.singletonList(Long.toString(nodeToToken.get(n)));
try (Cluster cluster = Cluster.build(2)
.withTokenSupplier(tokenSupplier)
.withConfig(c -> c.with(Feature.values())
.set("write_request_timeout", "10s")
.set("read_request_timeout", "10s")
.set("range_request_timeout", "20s")
.set("request_timeout",
"20s")
.set("transaction_timeout", "15s")
.set("native_transport_timeout", "30s")
.set(Constants.KEY_DTEST_STARTUP_TIMEOUT, "4m")
.set(Constants.KEY_DTEST_API_STARTUP_FAILURE_AS_SHUTDOWN, false)
.set("metadata_snapshot_frequency", 5)
.set("accord.command_store_shard_count", 1)
.set("accord.queue_shard_count", 1))
.start())
{
fixDistributedSchemas(cluster);
init(cluster, TARGET_RF);
// fix TCM
cluster.get(1).nodetoolResult("cms", "reconfigure",
"2").asserts().success();
cluster.schemaChange("CREATE KEYSPACE harry WITH replication =
{'class': 'SimpleStrategy', 'replication_factor' : 3};");
cluster.schemaChange("CREATE TABLE harry.table0 (pk0 uuid,pk1
timestamp,pk2 int,pk3 double,pk4 bigint,pk5 timestamp,pk6 float,pk7 ascii,ck0
ascii,ck1 uuid,static0 timestamp static,static1 text static,static2 ascii
static,static3 tinyint static,static4 smallint static,static5 tinyint
static,static6 uuid static,regular0 ascii,regular1 ascii, PRIMARY KEY
((pk0,pk1,pk2,pk3,pk4,pk5,pk6,pk7), ck0, ck1)) WITH transactional_mode = 'full'
AND CLUSTERING ORDER BY (ck0 DESC,ck1 ASC);");
long waitForEpoch =
TopologyMixupTestBase.HackSerialization.tcmEpoch(cluster.get(1));
ClusterUtils.awaitAccordEpochReady(cluster, waitForEpoch);
// nodetool repair harry table0 from node2
cluster.get(2).nodetoolResult("repair", "harry",
"table0").asserts().success();
// Add Node3;
IInvokableInstance node3 = ClusterUtils.addInstance(cluster,
cluster.get(1).config(), c -> c.set("auto_bootstrap", true));
node3.startup(cluster);
ClusterUtils.assertModeJoined(node3);
ClusterUtils.waitForCMSToQuiesce(cluster,
ClusterUtils.maxEpoch(cluster, new int[] {1, 2, 3}), true);
// fix TCM
fixDistributedSchemas(cluster);
for (String ks : Arrays.asList("system_auth", "system_traces"))
cluster.get(1).nodetoolResult("repair", ks);
cluster.get(1).nodetoolResult("cms", "reconfigure",
"3").asserts().success();
IInvokableInstance toReplace = cluster.get(3);
// Stop Node3 for Normal Stop
ClusterUtils.stopUnchecked(toReplace);
// Start Node3
toReplace.startup();
// start host replace of node3
ClusterUtils.stopUnchecked(toReplace);
TimeoutUtils.runBlocking(Duration.ofMinutes(3), "host replace", ()
-> {
IInvokableInstance node4 =
ClusterUtils.replaceHostAndStart(cluster, toReplace);
ClusterUtils.assertModeJoined(node4);
});
}
}
}
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
