This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new b657ec0c217 SOLR-17453: Leverage waitForState() instead of busy
waiting (#2737)
b657ec0c217 is described below
commit b657ec0c2176ef58c224147b71a91fa5a8267e51
Author: Pierre Salagnac <[email protected]>
AuthorDate: Thu Nov 7 08:14:40 2024 +0100
SOLR-17453: Leverage waitForState() instead of busy waiting (#2737)
Leverage waitForState() instead of busy waiting in CREATE, MIGRATE,
REINDEXCOLLECTION, MOVEREPLICA commands, and in some tests.
---
solr/CHANGES.txt | 4 +-
.../cloud/api/collections/CreateCollectionCmd.java | 19 +-
.../solr/cloud/api/collections/MigrateCmd.java | 39 ++-
.../solr/cloud/api/collections/MoveReplicaCmd.java | 45 ++-
.../api/collections/ReindexCollectionCmd.java | 23 +-
.../java/org/apache/solr/servlet/HttpSolrCall.java | 25 +-
...aosMonkeyNothingIsSafeWithPullReplicasTest.java | 2 +-
.../ChaosMonkeySafeLeaderWithPullReplicasTest.java | 2 +-
.../org/apache/solr/cloud/DeleteReplicaTest.java | 12 +-
.../org/apache/solr/cloud/HttpPartitionTest.java | 39 ++-
.../cloud/LeaderFailureAfterFreshStartTest.java | 8 +-
.../apache/solr/cloud/PeerSyncReplicationTest.java | 8 +-
.../org/apache/solr/cloud/TestPullReplica.java | 21 +-
.../solr/cloud/TestPullReplicaErrorHandling.java | 18 +-
.../apache/solr/cloud/TestRebalanceLeaders.java | 312 +++++++++------------
.../org/apache/solr/cloud/TestTlogReplica.java | 65 ++---
.../solr/cloud/api/collections/ShardSplitTest.java | 4 +-
.../SharedFileSystemAutoReplicaFailoverTest.java | 71 ++---
.../org/apache/solr/hdfs/cloud/StressHdfsTest.java | 12 +-
.../AbstractChaosMonkeyNothingIsSafeTestBase.java | 2 +-
.../AbstractChaosMonkeySafeLeaderTestBase.java | 2 +-
.../solr/cloud/AbstractDistribZkTestBase.java | 87 ++----
.../solr/cloud/AbstractFullDistribZkTestBase.java | 128 ++++-----
.../cloud/AbstractUnloadDistributedZkTestBase.java | 69 ++---
24 files changed, 419 insertions(+), 598 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3a03a54985a..907827a6fd0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -143,7 +143,7 @@ Improvements
when PKI is used between nodes. (Jason Gerlowski)
* SOLR-17383: Resolved overlapping arguments in the Solr CLI. Removed
duplicative but differing arguments,
- consolidated use of short form arguments -v to not have differing meanings
based on tool. Provide deprecation warning
+ consolidated use of short form arguments -v to not have differing meanings
based on tool. Provide deprecation warning
in command line when deprecated arguments are used. (Eric Pugh, Christos
Malliaridis)
* SOLR-17256: Deprecate SolrRequest `setBasePath` and `getBasePath` methods.
SolrJ users wishing to temporarily
@@ -181,6 +181,8 @@ Optimizations
* SOLR-16503: Switched from HTTP1 to HTTP2 in SolrClientCloudManager by
replacing CloudLegacySolrClient with CloudHttp2SolrClient. (Sanjay Dutt, David
Smiley)
+* SOLR-17453: Leverage waitForState() instead of busy waiting in CREATE,
MIGRATE, REINDEXCOLLECTION, MOVEREPLICA commands, and in some tests. (Pierre
Salagnac)
+
Bug Fixes
---------------------
* SOLR-12429: Uploading a configset with a symbolic link produces a
IOException. Now a error message to user generated instead. (Eric Pugh)
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index ccfd4a1e5e2..5592303580a 100644
---
a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++
b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -36,9 +36,11 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.Objects;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.solr.client.solrj.cloud.AlreadyExistsException;
import org.apache.solr.client.solrj.cloud.BadVersionException;
import org.apache.solr.client.solrj.cloud.DelegatingCloudManager;
@@ -221,24 +223,19 @@ public class CreateCollectionCmd implements
CollApiCmds.CollectionApiCommand {
}
// wait for a while until we see the collection
- TimeOut waitUntil =
- new TimeOut(30, TimeUnit.SECONDS,
ccc.getSolrCloudManager().getTimeSource());
- boolean created = false;
- while (!waitUntil.hasTimedOut()) {
- waitUntil.sleep(100);
- created =
ccc.getSolrCloudManager().getClusterState().hasCollection(collectionName);
- if (created) break;
- }
- if (!created) {
+ try {
+ newColl =
+ zkStateReader.waitForState(collectionName, 30, TimeUnit.SECONDS,
Objects::nonNull);
+ } catch (TimeoutException e) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
- "Could not fully create collection: " + collectionName);
+ "Could not fully create collection: " + collectionName,
+ e);
}
// refresh cluster state (value read below comes from Zookeeper watch
firing following the
// update done previously, be it by Overseer or by this thread when
updates are distributed)
clusterState = ccc.getSolrCloudManager().getClusterState();
- newColl = clusterState.getCollection(collectionName);
}
final List<ReplicaPosition> replicaPositions;
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java
index cdcb3c27449..95d1eb2ac11 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java
@@ -31,6 +31,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.cloud.DistributedClusterStateUpdater;
import org.apache.solr.cloud.Overseer;
@@ -52,7 +53,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.handler.component.ShardHandler;
-import org.apache.solr.util.TimeOut;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -272,27 +272,26 @@ public class MigrateCmd implements
CollApiCmds.CollectionApiCommand {
// wait for a while until we see the new rule
log.info("Waiting to see routing rule updated in clusterstate");
- TimeOut waitUntil =
- new TimeOut(60, TimeUnit.SECONDS,
ccc.getSolrCloudManager().getTimeSource());
- boolean added = false;
- while (!waitUntil.hasTimedOut()) {
- waitUntil.sleep(100);
- sourceCollection =
zkStateReader.getClusterState().getCollection(sourceCollection.getName());
- sourceSlice = sourceCollection.getSlice(sourceSlice.getName());
- Map<String, RoutingRule> rules = sourceSlice.getRoutingRules();
- if (rules != null) {
- RoutingRule rule =
rules.get(sourceRouter.getRouteKeyNoSuffix(splitKey) + "!");
- if (rule != null && rule.getRouteRanges().contains(splitRange)) {
- added = true;
- break;
- }
- }
- }
- if (!added) {
+
+ try {
+ sourceCollection =
+ zkStateReader.waitForState(
+ sourceCollection.getName(),
+ 60,
+ TimeUnit.SECONDS,
+ c -> {
+ Slice s = c.getSlice(sourceSlice.getName());
+ Map<String, RoutingRule> rules = s.getRoutingRules();
+ if (rules != null) {
+ RoutingRule rule =
rules.get(sourceRouter.getRouteKeyNoSuffix(splitKey) + "!");
+ return rule != null &&
rule.getRouteRanges().contains(splitRange);
+ }
+ return false;
+ });
+ } catch (TimeoutException e) {
throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Could not add routing rule: "
+ m);
+ SolrException.ErrorCode.SERVER_ERROR, "Could not add routing rule: "
+ m, e);
}
-
log.info("Routing rule added successfully");
// Create temp core on source shard
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
index cd5097993b4..48f065f537e 100644
---
a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
+++
b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
@@ -33,6 +33,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.solr.cloud.ActiveReplicaWatcher;
import org.apache.solr.common.SolrCloseableLatch;
import org.apache.solr.common.SolrException;
@@ -46,7 +47,6 @@ import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
-import org.apache.solr.util.TimeOut;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -161,9 +161,7 @@ public class MoveReplicaCmd implements
CollApiCmds.CollectionApiCommand {
dataDir.toString(),
targetNode,
async,
- coll,
replica,
- slice,
timeout,
waitForFinalState);
} else {
@@ -187,9 +185,7 @@ public class MoveReplicaCmd implements
CollApiCmds.CollectionApiCommand {
String dataDir,
String targetNode,
String async,
- DocCollection coll,
Replica replica,
- Slice slice,
int timeout,
boolean waitForFinalState)
throws Exception {
@@ -198,8 +194,8 @@ public class MoveReplicaCmd implements
CollApiCmds.CollectionApiCommand {
skipCreateReplicaInClusterState = "false";
ZkNodeProps removeReplicasProps =
new ZkNodeProps(
- COLLECTION_PROP, coll.getName(),
- SHARD_ID_PROP, slice.getName(),
+ COLLECTION_PROP, replica.getCollection(),
+ SHARD_ID_PROP, replica.getShard(),
REPLICA_PROP, replica.getName());
removeReplicasProps.getProperties().put(CoreAdminParams.DELETE_DATA_DIR,
false);
removeReplicasProps.getProperties().put(CoreAdminParams.DELETE_INDEX,
false);
@@ -217,8 +213,8 @@ public class MoveReplicaCmd implements
CollApiCmds.CollectionApiCommand {
String.format(
Locale.ROOT,
"Failed to cleanup replica collection=%s shard=%s name=%s,
failure=%s",
- coll.getName(),
- slice.getName(),
+ replica.getCollection(),
+ replica.getShard(),
replica.getName(),
deleteResult.get("failure"));
log.warn(errorString);
@@ -226,17 +222,14 @@ public class MoveReplicaCmd implements
CollApiCmds.CollectionApiCommand {
return;
}
- TimeOut timeOut =
- new TimeOut(20L, TimeUnit.SECONDS,
ccc.getSolrCloudManager().getTimeSource());
- while (!timeOut.hasTimedOut()) {
- coll =
ccc.getZkStateReader().getClusterState().getCollection(coll.getName());
- if (coll.getReplica(replica.getName()) != null) {
- timeOut.sleep(100);
- } else {
- break;
- }
- }
- if (timeOut.hasTimedOut()) {
+ try {
+ ccc.getZkStateReader()
+ .waitForState(
+ replica.getCollection(),
+ 20L,
+ TimeUnit.SECONDS,
+ c -> c.getReplica(replica.getName()) != null);
+ } catch (TimeoutException e) {
results.add("failure", "Still see deleted replica in clusterstate!");
return;
}
@@ -246,9 +239,9 @@ public class MoveReplicaCmd implements
CollApiCmds.CollectionApiCommand {
ZkNodeProps addReplicasProps =
new ZkNodeProps(
COLLECTION_PROP,
- coll.getName(),
+ replica.getCollection(),
SHARD_ID_PROP,
- slice.getName(),
+ replica.getShard(),
CoreAdminParams.NODE,
targetNode,
CoreAdminParams.CORE_NODE_NAME,
@@ -277,8 +270,8 @@ public class MoveReplicaCmd implements
CollApiCmds.CollectionApiCommand {
String.format(
Locale.ROOT,
"Failed to create replica for collection=%s shard=%s" + " on
node=%s, failure=%s",
- coll.getName(),
- slice.getName(),
+ replica.getCollection(),
+ replica.getShard(),
targetNode,
addResult.get("failure"));
results.add("failure", errorString);
@@ -302,8 +295,8 @@ public class MoveReplicaCmd implements
CollApiCmds.CollectionApiCommand {
String.format(
Locale.ROOT,
"Failed to create replica for collection=%s shard=%s" + " on
node=%s, failure=%s",
- coll.getName(),
- slice.getName(),
+ replica.getCollection(),
+ replica.getShard(),
targetNode,
addResult.get("failure"));
log.warn(errorString);
diff --git
a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
index 91299b3c259..15324ec61aa 100644
---
a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
+++
b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
@@ -25,8 +25,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -360,22 +362,17 @@ public class ReindexCollectionCmd implements
CollApiCmds.CollectionApiCommand {
CollectionHandlingUtils.checkResults(
"creating checkpoint collection " + chkCollection, cmdResults, true);
// wait for a while until we see both collections
- TimeOut waitUntil =
- new TimeOut(30, TimeUnit.SECONDS,
ccc.getSolrCloudManager().getTimeSource());
- boolean created = false;
- while (!waitUntil.hasTimedOut()) {
- waitUntil.sleep(100);
- // this also refreshes our local var clusterState
- clusterState = ccc.getSolrCloudManager().getClusterState();
- created =
- clusterState.hasCollection(targetCollection)
- && clusterState.hasCollection(chkCollection);
- if (created) break;
- }
- if (!created) {
+ try {
+ for (String col : List.of(targetCollection, chkCollection)) {
+ ccc.getZkStateReader().waitForState(col, 30, TimeUnit.SECONDS,
Objects::nonNull);
+ }
+ } catch (TimeoutException e) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Could not fully create
temporary collection(s)");
}
+
+ clusterState = ccc.getSolrCloudManager().getClusterState();
+
if (maybeAbort(collection)) {
aborted = true;
return;
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index cc645476aef..99291fc6ad0 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -53,6 +53,7 @@ import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -96,7 +97,6 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
import org.apache.solr.common.util.ValidatingJsonMap;
import org.apache.solr.core.CoreContainer;
@@ -125,7 +125,6 @@ import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
import org.apache.solr.servlet.cache.Method;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.RTimerTree;
-import org.apache.solr.util.TimeOut;
import org.apache.solr.util.tracing.TraceUtils;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -388,18 +387,16 @@ public class HttpSolrCall {
+ " collection: "
+ Utils.toJSONString(rsp.getValues()));
}
- TimeOut timeOut = new TimeOut(3, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- for (; ; ) {
- if
(cores.getZkController().getClusterState().getCollectionOrNull(SYSTEM_COLL) !=
null) {
- break;
- } else {
- if (timeOut.hasTimedOut()) {
- throw new SolrException(
- ErrorCode.SERVER_ERROR,
- "Could not find " + SYSTEM_COLL + " collection even after 3
seconds");
- }
- timeOut.sleep(50);
- }
+
+ try {
+ cores
+ .getZkController()
+ .getZkStateReader()
+ .waitForState(SYSTEM_COLL, 3, TimeUnit.SECONDS, Objects::nonNull);
+ } catch (TimeoutException e) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Could not find " + SYSTEM_COLL + " collection even after 3
seconds");
}
action = RETRY;
diff --git
a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java
b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java
index ddb72aad118..6652009774f 100644
---
a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java
+++
b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPullReplicasTest.java
@@ -343,7 +343,7 @@ public class ChaosMonkeyNothingIsSafeWithPullReplicasTest
extends AbstractFullDi
List<Integer> numShardsNumReplicas = new ArrayList<>(2);
numShardsNumReplicas.add(1);
numShardsNumReplicas.add(1 + getPullReplicaCount());
- checkForCollection("testcollection", numShardsNumReplicas, null);
+ checkForCollection("testcollection", numShardsNumReplicas);
testSuccessful = true;
} finally {
diff --git
a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java
b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java
index 764035dc043..b097ebea968 100644
---
a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java
+++
b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPullReplicasTest.java
@@ -255,7 +255,7 @@ public class ChaosMonkeySafeLeaderWithPullReplicasTest
extends AbstractFullDistr
List<Integer> numShardsNumReplicas = new ArrayList<>(2);
numShardsNumReplicas.add(1);
numShardsNumReplicas.add(1 + getPullReplicaCount());
- checkForCollection("testcollection", numShardsNumReplicas, null);
+ checkForCollection("testcollection", numShardsNumReplicas);
}
private void tryDelete() throws Exception {
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
index 686a0a2554a..16242bcc5eb 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
@@ -488,11 +488,13 @@ public class DeleteReplicaTest extends SolrCloudTestCase {
}
private void waitForNodeLeave(String lostNodeName) throws
InterruptedException {
- ZkStateReader reader = cluster.getZkStateReader();
- TimeOut timeOut = new TimeOut(20, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while (reader.getClusterState().getLiveNodes().contains(lostNodeName)) {
- Thread.sleep(100);
- if (timeOut.hasTimedOut()) fail("Wait for " + lostNodeName + " to leave
failed!");
+
+ try {
+ cluster
+ .getZkStateReader()
+ .waitForLiveNodes(20, TimeUnit.SECONDS, (o, n) ->
!n.contains(lostNodeName));
+ } catch (TimeoutException e) {
+ fail("Wait for " + lostNodeName + " to leave failed!");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
index 2f3f62ad763..0d4f1c2b722 100644
--- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
@@ -30,6 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.solr.JSONTestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
@@ -48,13 +49,11 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.embedded.JettySolrRunner;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.TestInjection;
-import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -291,27 +290,23 @@ public class HttpPartitionTest extends
AbstractFullDistribZkTestBase {
protected void waitForState(String collection, String replicaName,
Replica.State state, long ms)
throws KeeperException, InterruptedException {
- TimeOut timeOut = new TimeOut(ms, TimeUnit.MILLISECONDS,
TimeSource.NANO_TIME);
- Replica.State replicaState = Replica.State.ACTIVE;
- while (!timeOut.hasTimedOut()) {
- ZkStateReader zkr = ZkStateReader.from(cloudClient);
- zkr.forceUpdateCollection(collection); // force the state to be fresh
- ClusterState cs = zkr.getClusterState();
- Collection<Slice> slices =
cs.getCollection(collection).getActiveSlices();
- Slice slice = slices.iterator().next();
- Replica partitionedReplica = slice.getReplica(replicaName);
- replicaState = partitionedReplica.getState();
- if (replicaState == state) return;
+ ZkStateReader zkr = ZkStateReader.from(cloudClient);
+
+ try {
+ zkr.waitForState(
+ collection,
+ ms,
+ TimeUnit.MILLISECONDS,
+ c -> {
+ Collection<Slice> slices = c.getActiveSlices();
+ Slice slice = slices.iterator().next();
+ Replica partitionedReplica = slice.getReplica(replicaName);
+ Replica.State replicaState = partitionedReplica.getState();
+ return replicaState == state;
+ });
+ } catch (TimeoutException e) {
+ fail("Timeout waiting for state " + state + " of replica " +
replicaName);
}
- assertEquals(
- "Timeout waiting for state "
- + state
- + " of replica "
- + replicaName
- + ", current state "
- + replicaState,
- state,
- replicaState);
}
protected void testRf3() throws Exception {
diff --git
a/solr/core/src/test/org/apache/solr/cloud/LeaderFailureAfterFreshStartTest.java
b/solr/core/src/test/org/apache/solr/cloud/LeaderFailureAfterFreshStartTest.java
index 5b7ff3f0428..1bc87e4f4e1 100644
---
a/solr/core/src/test/org/apache/solr/cloud/LeaderFailureAfterFreshStartTest.java
+++
b/solr/core/src/test/org/apache/solr/cloud/LeaderFailureAfterFreshStartTest.java
@@ -39,8 +39,6 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.util.TimeOut;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -154,11 +152,7 @@ public class LeaderFailureAfterFreshStartTest extends
AbstractFullDistribZkTestB
// shutdown the original leader
log.info("Now shutting down initial leader");
forceNodeFailures(singletonList(initialLeaderJetty));
- waitForNewLeader(
- cloudClient,
- "shard1",
- (Replica) initialLeaderJetty.client.info,
- new TimeOut(15, TimeUnit.SECONDS, TimeSource.NANO_TIME));
+ waitForNewLeader(cloudClient, "shard1", (Replica)
initialLeaderJetty.client.info);
waitTillNodesActive();
log.info("Updating mappings from zk");
updateMappingsFromZk(jettys, clients, true);
diff --git
a/solr/core/src/test/org/apache/solr/cloud/PeerSyncReplicationTest.java
b/solr/core/src/test/org/apache/solr/cloud/PeerSyncReplicationTest.java
index 7042c3018e2..d9e501e69b6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/PeerSyncReplicationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/PeerSyncReplicationTest.java
@@ -44,10 +44,8 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.util.TimeOut;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -154,11 +152,7 @@ public class PeerSyncReplicationTest extends
AbstractFullDistribZkTestBase {
log.info("Now shutting down initial leader");
forceNodeFailures(singletonList(initialLeaderJetty));
log.info("Updating mappings from zk");
- waitForNewLeader(
- cloudClient,
- "shard1",
- (Replica) initialLeaderJetty.client.info,
- new TimeOut(15, TimeUnit.SECONDS, TimeSource.NANO_TIME));
+ waitForNewLeader(cloudClient, "shard1", (Replica)
initialLeaderJetty.client.info);
updateMappingsFromZk(jettys, clients, true);
assertEquals(
"PeerSynced node did not become leader",
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
index f5cf82c701e..ec0564c249d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
@@ -749,20 +749,13 @@ public class TestPullReplica extends SolrCloudTestCase {
}
}
- static void waitForDeletion(String collection) throws InterruptedException,
KeeperException {
- TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while
(cluster.getSolrClient().getClusterState().hasCollection(collection)) {
- log.info("Collection not yet deleted");
- try {
- Thread.sleep(100);
- if (t.hasTimedOut()) {
- fail("Timed out waiting for collection " + collection + " to be
deleted.");
- }
- cluster.getZkStateReader().forceUpdateCollection(collection);
- } catch (SolrException e) {
- return;
- }
- }
+ static void waitForDeletion(String collection) {
+ waitForState(
+ "Waiting for collection " + collection + " to be deleted",
+ collection,
+ (n, c) -> c == null,
+ 10,
+ TimeUnit.SECONDS);
}
private DocCollection assertNumberOfReplicas(
diff --git
a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
index f4af183434c..0d68f2f1993 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
@@ -120,7 +120,7 @@ public class TestPullReplicaErrorHandling extends
SolrCloudTestCase {
log.info("tearDown deleting collection");
CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
log.info("Collection deleted");
- waitForDeletion(collectionName);
+ TestPullReplica.waitForDeletion(collectionName);
}
collectionName = null;
super.tearDown();
@@ -347,22 +347,6 @@ public class TestPullReplicaErrorHandling extends
SolrCloudTestCase {
return proxy;
}
- private void waitForDeletion(String collection) throws InterruptedException,
KeeperException {
- TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while
(cluster.getSolrClient().getClusterState().hasCollection(collection)) {
- log.info("Collection not yet deleted");
- try {
- Thread.sleep(100);
- if (t.hasTimedOut()) {
- fail("Timed out waiting for collection " + collection + " to be
deleted.");
- }
- cluster.getZkStateReader().forceUpdateCollection(collection);
- } catch (SolrException e) {
- return;
- }
- }
- }
-
private CollectionStatePredicate activeReplicaCount(
int numWriter, int numActive, int numPassive) {
return (liveNodes, collectionState) -> {
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java
b/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java
index 0b59d5d74ab..20343487523 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java
@@ -111,8 +111,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase
{
// shardUnique=true for the special property preferredLeader. That was
removed at one point, so
// we're explicitly testing that as well.
@Test
- public void testSetArbitraryPropertySliceUnique()
- throws IOException, SolrServerException, InterruptedException {
+ public void testSetArbitraryPropertySliceUnique() throws IOException,
SolrServerException {
// Check both special (preferredLeader) and something arbitrary.
doTestSetArbitraryPropertySliceUnique("foo" + random().nextInt(1_000_000));
removeAllProperties();
@@ -124,8 +123,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase
{
// individual properties on individual nodes. This one relies on Solr to
pick which replicas to
// set the property on
@Test
- public void testBalancePropertySliceUnique()
- throws InterruptedException, IOException, SolrServerException {
+ public void testBalancePropertySliceUnique() throws IOException,
SolrServerException {
// Check both cases of "special" property preferred(Ll)eader
doTestBalancePropertySliceUnique("foo" + random().nextInt(1_000_000));
removeAllProperties();
@@ -160,7 +158,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase
{
// on an individual
// replica.
private void doTestSetArbitraryPropertySliceUnique(String propIn)
- throws InterruptedException, IOException, SolrServerException {
+ throws IOException, SolrServerException {
final String prop = (random().nextBoolean()) ? propIn :
propIn.toUpperCase(Locale.ROOT);
// First set the property in some replica in some slice
forceUpdateCollectionStatus();
@@ -176,43 +174,29 @@ public class TestRebalanceLeaders extends
SolrCloudTestCase {
Replica rep = reps[random().nextInt(reps.length)];
// Set the property on a particular replica
setProp(slice, rep, prop);
- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS,
TimeSource.NANO_TIME);
-
- long count = 0;
- boolean rightRep = false;
- Slice modSlice;
- DocCollection modColl = null; // keeps IDE happy
// ensure that no other replica in that slice has the property when we
return.
- while (timeout.hasTimedOut() == false) {
- forceUpdateCollectionStatus();
- modColl =
cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);
- modSlice = modColl.getSlice(slice.getName());
- rightRep =
- modSlice
- .getReplica(rep.getName())
- .getBool("property." + prop.toLowerCase(Locale.ROOT), false);
- count =
- modSlice.getReplicas().stream()
- .filter(
- thisRep -> thisRep.getBool("property." +
prop.toLowerCase(Locale.ROOT), false))
- .count();
-
- if (count == 1 && rightRep) {
- break;
- }
-
- TimeUnit.MILLISECONDS.sleep(50);
- }
- if (count != 1 || rightRep == false) {
- fail(
- "The property "
- + prop
- + " was not uniquely distributed in slice "
- + slice.getName()
- + " "
- + modColl.toString());
- }
+ waitForState(
+ "Check property is uniquely distributed in slice: " + prop,
+ COLLECTION_NAME,
+ (n, c) -> {
+ forceUpdateCollectionStatus();
+ Slice modSlice = c.getSlice(slice.getName());
+ boolean rightRep =
+ modSlice
+ .getReplica(rep.getName())
+ .getBool("property." + prop.toLowerCase(Locale.ROOT),
false);
+ long count =
+ modSlice.getReplicas().stream()
+ .filter(
+ thisRep ->
+ thisRep.getBool("property." +
prop.toLowerCase(Locale.ROOT), false))
+ .count();
+
+ return count == 1 && rightRep;
+ },
+ timeoutMs,
+ TimeUnit.MILLISECONDS);
}
}
@@ -332,7 +316,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase
{
// re-assigned with the
// BALANCESHARDUNIQUE command.
private void doTestBalancePropertySliceUnique(String propIn)
- throws InterruptedException, IOException, SolrServerException {
+ throws IOException, SolrServerException {
final String prop = (random().nextBoolean()) ? propIn :
propIn.toUpperCase(Locale.ROOT);
// Concentrate the properties on as few replicas a possible
@@ -350,63 +334,55 @@ public class TestRebalanceLeaders extends
SolrCloudTestCase {
private void verifyPropCorrectlyDistributed(String prop) {
- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS,
TimeSource.NANO_TIME);
-
String propLC = prop.toLowerCase(Locale.ROOT);
- DocCollection docCollection = null;
- while (timeout.hasTimedOut() == false) {
- forceUpdateCollectionStatus();
- docCollection =
cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);
- int maxPropCount = Integer.MAX_VALUE;
- int minPropCount = Integer.MIN_VALUE;
- for (Slice slice : docCollection.getSlices()) {
- int repCount = 0;
- for (Replica rep : slice.getReplicas()) {
- if (rep.getBool("property." + propLC, false)) {
- repCount++;
+ waitForState(
+ "Check property is distributed evenly: " + prop,
+ COLLECTION_NAME,
+ (liveNodes, docCollection) -> {
+ int maxPropCount = 0;
+ int minPropCount = Integer.MAX_VALUE;
+ for (Slice slice : docCollection.getSlices()) {
+ int repCount = 0;
+ for (Replica rep : slice.getReplicas()) {
+ if (rep.getBool("property." + propLC, false)) {
+ repCount++;
+ }
+ }
+ maxPropCount = Math.max(maxPropCount, repCount);
+ minPropCount = Math.min(minPropCount, repCount);
}
- }
- maxPropCount = Math.max(maxPropCount, repCount);
- minPropCount = Math.min(minPropCount, repCount);
- }
- if (Math.abs(maxPropCount - minPropCount) < 2) return;
- }
- log.error("Property {} is not distributed evenly. {}", prop,
docCollection);
- fail("Property is not distributed evenly " + prop);
+ return Math.abs(maxPropCount - minPropCount) < 2;
+ },
+ timeoutMs,
+ TimeUnit.MILLISECONDS);
}
// Used when we concentrate the leader on a few nodes.
private void verifyPropDistributedAsExpected(
- Map<String, String> expectedShardReplicaMap, String prop) throws
InterruptedException {
+ Map<String, String> expectedShardReplicaMap, String prop) {
// Make sure that the shard unique are where you expect.
- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS,
TimeSource.NANO_TIME);
-
String propLC = prop.toLowerCase(Locale.ROOT);
- boolean failure = false;
- DocCollection docCollection = null;
- while (timeout.hasTimedOut() == false) {
- forceUpdateCollectionStatus();
- docCollection =
cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);
- failure = false;
- for (Map.Entry<String, String> ent : expectedShardReplicaMap.entrySet())
{
- Replica rep =
docCollection.getSlice(ent.getKey()).getReplica(ent.getValue());
- if (rep.getBool("property." + propLC, false) == false) {
- failure = true;
- }
- }
- if (failure == false) {
- return;
- }
- TimeUnit.MILLISECONDS.sleep(100);
- }
+ String message =
+ String.format(
+ Locale.ROOT,
+ "Checking properties are on the expected replicas. Props:%s
Expected:%s",
+ prop,
+ expectedShardReplicaMap.toString());
- fail(
- prop
- + " properties are not on the expected replicas: "
- + docCollection.toString()
- + System.lineSeparator()
- + "Expected "
- + expectedShardReplicaMap.toString());
+ waitForState(
+ message,
+ COLLECTION_NAME,
+ (liveNodes, docCollection) -> {
+ for (Map.Entry<String, String> ent :
expectedShardReplicaMap.entrySet()) {
+ Replica rep =
docCollection.getSlice(ent.getKey()).getReplica(ent.getValue());
+ if (rep.getBool("property." + propLC, false) == false) {
+ return false;
+ }
+ }
+ return true;
+ },
+ timeoutMs,
+ TimeUnit.MILLISECONDS);
}
// Just check that the property is distributed as expectecd. This does _not_
rebalance the leaders
@@ -558,14 +534,6 @@ public class TestRebalanceLeaders extends
SolrCloudTestCase {
// then start them again to concentrate the leaders. It's not necessary
that all shards have a
// leader.
- ExecutorService executorService =
ExecutorUtil.newMDCAwareCachedThreadPool("Start Jetty");
-
- for (JettySolrRunner jetty : jettys) {
- cluster.stopJettySolrRunner(jetty);
- }
-
- ExecutorUtil.shutdownAndAwaitTermination(executorService);
-
for (JettySolrRunner jetty : jettys) {
cluster.stopJettySolrRunner(jetty);
}
@@ -575,7 +543,7 @@ public class TestRebalanceLeaders extends SolrCloudTestCase
{
}
checkReplicasInactive(jettys);
- executorService = ExecutorUtil.newMDCAwareCachedThreadPool("Start Jetty");
+ ExecutorService executorService =
ExecutorUtil.newMDCAwareCachedThreadPool("Start Jetty");
for (int idx = 0; idx < jettys.size(); ++idx) {
int finalIdx = idx;
@@ -604,74 +572,61 @@ public class TestRebalanceLeaders extends
SolrCloudTestCase {
// Since we have to restart jettys, we don't want to try re-balancing etc.
until we're sure all
// jettys that should be up are and all replicas are active.
- private void checkReplicasInactive(List<JettySolrRunner> downJettys) throws
InterruptedException {
- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS,
TimeSource.NANO_TIME);
- DocCollection docCollection = null;
- Set<String> liveNodes = null;
+ private void checkReplicasInactive(List<JettySolrRunner> downJettys) {
Set<String> downJettyNodes = new TreeSet<>();
for (JettySolrRunner jetty : downJettys) {
downJettyNodes.add(
jetty.getBaseUrl().getHost() + ":" + jetty.getBaseUrl().getPort() +
"_solr");
}
- while (timeout.hasTimedOut() == false) {
- forceUpdateCollectionStatus();
- docCollection =
cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);
- liveNodes = cluster.getSolrClient().getClusterState().getLiveNodes();
- boolean expectedInactive = true;
-
- for (Slice slice : docCollection.getSlices()) {
- for (Replica rep : slice.getReplicas()) {
- if (downJettyNodes.contains(rep.getNodeName()) == false) {
- continue; // We are on a live node
- }
- // A replica on an allegedly down node is reported as active.
- if (rep.isActive(liveNodes)) {
- expectedInactive = false;
+
+ waitForState(
+ "Waiting for all replicas to become inactive",
+ COLLECTION_NAME,
+ (liveNodes, docCollection) -> {
+ boolean expectedInactive = true;
+
+ for (Slice slice : docCollection.getSlices()) {
+ for (Replica rep : slice.getReplicas()) {
+ if (downJettyNodes.contains(rep.getNodeName()) == false) {
+ continue; // We are on a live node
+ }
+ // A replica on an allegedly down node is reported as active.
+ if (rep.isActive(liveNodes)) {
+ expectedInactive = false;
+ }
+ }
}
- }
- }
- if (expectedInactive) {
- return;
- }
- TimeUnit.MILLISECONDS.sleep(100);
- }
- fail(
- "timed out waiting for all replicas to become inactive: livenodes: "
- + liveNodes
- + " Collection state: "
- + docCollection.toString());
+ return expectedInactive;
+ },
+ timeoutMs,
+ TimeUnit.MILLISECONDS);
}
// We need to wait around until all replicas are active before expecting
rebalancing or
// distributing shard-unique properties to work.
- private void checkAllReplicasActive() throws InterruptedException {
- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS,
TimeSource.NANO_TIME);
- while (timeout.hasTimedOut() == false) {
- forceUpdateCollectionStatus();
- DocCollection docCollection =
-
cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);
- Set<String> liveNodes =
cluster.getSolrClient().getClusterState().getLiveNodes();
- boolean allActive = true;
- for (Slice slice : docCollection.getSlices()) {
- for (Replica rep : slice.getReplicas()) {
- if (rep.isActive(liveNodes) == false) {
- allActive = false;
+ private void checkAllReplicasActive() {
+ waitForState(
+ "Waiting for all replicas to become active",
+ COLLECTION_NAME,
+ (liveNodes, docCollection) -> {
+ boolean allActive = true;
+ for (Slice slice : docCollection.getSlices()) {
+ for (Replica rep : slice.getReplicas()) {
+ if (rep.isActive(liveNodes) == false) {
+ allActive = false;
+ }
+ }
}
- }
- }
- if (allActive) {
- return;
- }
- TimeUnit.MILLISECONDS.sleep(100);
- }
- fail("timed out waiting for all replicas to become active");
+ return allActive;
+ },
+ timeoutMs,
+ TimeUnit.MILLISECONDS);
}
// use a simple heuristic to put as many replicas with the property on as
few nodes as possible.
// The point is that then we can execute BALANCESHARDUNIQUE and be sure it
worked correctly
- private void concentrateProp(String prop)
- throws InterruptedException, IOException, SolrServerException {
+ private void concentrateProp(String prop) throws IOException,
SolrServerException {
// find all the live nodes for each slice, assign the leader to the first
replica that is in the
// lowest position on live_nodes
List<String> liveNodes =
@@ -704,43 +659,26 @@ public class TestRebalanceLeaders extends
SolrCloudTestCase {
}
// make sure that the property in question is unique per shard.
- private Map<String, String> verifyPropUniquePerShard(String prop) throws
InterruptedException {
- Map<String, String> uniquePropMaps = new TreeMap<>();
-
- TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS,
TimeSource.NANO_TIME);
- while (timeout.hasTimedOut() == false) {
- uniquePropMaps.clear();
- if (checkUniquePropPerShard(uniquePropMaps, prop)) {
- return uniquePropMaps;
- }
- TimeUnit.MILLISECONDS.sleep(10);
- }
- fail(
- "There should be exactly one replica with value "
- + prop
- + " set to true per shard: "
- +
cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME).toString());
- return null; // keeps IDE happy.
- }
-
- // return true if every shard has exactly one replica with the unique
property set to "true"
- private boolean checkUniquePropPerShard(Map<String, String> uniques, String
prop) {
- forceUpdateCollectionStatus();
- DocCollection docCollection =
-
cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);
+ private void verifyPropUniquePerShard(String prop) {
- for (Slice slice : docCollection.getSlices()) {
- int propCount = 0;
- for (Replica rep : slice.getReplicas()) {
- if (rep.getBool("property." + prop.toLowerCase(Locale.ROOT), false)) {
- propCount++;
- uniques.put(slice.getName(), rep.getName());
- }
- }
- if (1 != propCount) {
- return false;
- }
- }
- return true;
+ waitForState(
+ "Waiting to have exactly one replica with " + prop + "set per shard",
+ COLLECTION_NAME,
+ (liveNodes, docCollection) -> {
+ for (Slice slice : docCollection.getSlices()) {
+ int propCount = 0;
+ for (Replica rep : slice.getReplicas()) {
+ if (rep.getBool("property." + prop.toLowerCase(Locale.ROOT),
false)) {
+ propCount++;
+ }
+ }
+ if (1 != propCount) {
+ return false;
+ }
+ }
+ return true;
+ },
+ timeoutMs,
+ TimeUnit.MILLISECONDS);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java
b/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java
index 563ed686298..d559de6d333 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java
@@ -815,17 +815,17 @@ public class TestTlogReplica extends SolrCloudTestCase {
cloudClient.request(request);
// Wait until a preferredleader flag is set to the new leader candidate
- TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while (!timeout.hasTimedOut()) {
- Map<String, Slice> slices =
-
cloudClient.getClusterState().getCollection(collectionName).getSlicesMap();
- Replica me = slices.get(slice.getName()).getReplica(newLeader.getName());
- if (me.getBool("property.preferredleader", false)) {
- break;
- }
- Thread.sleep(100);
- }
- assertFalse("Timeout waiting for setting preferredleader flag",
timeout.hasTimedOut());
+ String newLeaderName = newLeader.getName();
+ waitForState(
+ "Waiting for setting preferredleader flag",
+ collectionName,
+ (n, c) -> {
+ Map<String, Slice> slices = c.getSlicesMap();
+ Replica me = slices.get(slice.getName()).getReplica(newLeaderName);
+ return me.getBool("property.preferredleader", false);
+ },
+ 10,
+ TimeUnit.SECONDS);
// Rebalance leaders
params = new ModifiableSolrParams();
@@ -837,18 +837,17 @@ public class TestTlogReplica extends SolrCloudTestCase {
cloudClient.request(request);
// Wait until a new leader is elected
- timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while (!timeout.hasTimedOut()) {
- docCollection = getCollectionState(collectionName);
- Replica leader = docCollection.getSlice(slice.getName()).getLeader();
- if (leader != null
- && leader.getName().equals(newLeader.getName())
- && leader.isActive(cloudClient.getClusterState().getLiveNodes())) {
- break;
- }
- Thread.sleep(100);
- }
- assertFalse("Timeout waiting for a new leader to be elected",
timeout.hasTimedOut());
+ waitForState(
+ "Waiting for a new leader to be elected",
+ collectionName,
+ (n, c) -> {
+ Replica leader = c.getSlice(slice.getName()).getLeader();
+ return leader != null
+ && leader.getName().equals(newLeaderName)
+ && leader.isActive(cloudClient.getClusterState().getLiveNodes());
+ },
+ 30,
+ TimeUnit.SECONDS);
new UpdateRequest()
.add(sdoc("id", "1"))
@@ -1027,19 +1026,13 @@ public class TestTlogReplica extends SolrCloudTestCase {
}
}
- private void waitForDeletion(String collection) throws InterruptedException,
KeeperException {
- TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while
(cluster.getSolrClient().getClusterState().hasCollection(collection)) {
- try {
- Thread.sleep(100);
- if (t.hasTimedOut()) {
- fail("Timed out waiting for collection " + collection + " to be
deleted.");
- }
- cluster.getZkStateReader().forceUpdateCollection(collection);
- } catch (SolrException e) {
- return;
- }
- }
+ private void waitForDeletion(String collection) {
+ waitForState(
+ "Waiting for collection " + collection + " to be deleted",
+ collection,
+ (n, c) -> c == null,
+ 10,
+ TimeUnit.SECONDS);
}
private DocCollection assertNumberOfReplicas(
diff --git
a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
index 43f3367f489..28335c920eb 100644
---
a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
+++
b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
@@ -960,7 +960,7 @@ public class ShardSplitTest extends BasicDistributedZkTest {
}
List<Integer> list = collectionInfos.get(collectionName);
- checkForCollection(collectionName, list, null);
+ checkForCollection(collectionName, list);
waitForRecoveriesToFinish(false);
@@ -1031,7 +1031,7 @@ public class ShardSplitTest extends
BasicDistributedZkTest {
}
List<Integer> list = collectionInfos.get(collectionName);
- checkForCollection(collectionName, list, null);
+ checkForCollection(collectionName, list);
waitForRecoveriesToFinish(false);
diff --git
a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java
b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java
index 329cc7fee11..34e3b2a1492 100644
---
a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java
+++
b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java
@@ -31,6 +31,7 @@ import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.tests.util.LuceneTestCase;
@@ -57,13 +58,11 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.SolrNamedThreadFactory;
-import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.embedded.JettySolrRunner;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
import org.apache.solr.util.LogLevel;
import org.apache.solr.util.TestInjection;
-import org.apache.solr.util.TimeOut;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -425,37 +424,43 @@ public class SharedFileSystemAutoReplicaFailoverTest
extends AbstractFullDistrib
private void assertSliceAndReplicaCount(
String collection, int numSlices, int numReplicas, int timeOutInMs)
- throws InterruptedException, IOException {
- TimeOut timeOut = new TimeOut(timeOutInMs, TimeUnit.MILLISECONDS,
TimeSource.NANO_TIME);
- while (!timeOut.hasTimedOut()) {
- ClusterState clusterState = cloudClient.getClusterState();
- Collection<Slice> slices =
clusterState.getCollection(collection).getActiveSlices();
- if (slices.size() == numSlices) {
- boolean isMatch = true;
- for (Slice slice : slices) {
- int count = 0;
- for (Replica replica : slice.getReplicas()) {
- if (replica.getState() == Replica.State.ACTIVE
- && clusterState.liveNodesContain(replica.getNodeName())) {
- count++;
- }
- }
- if (count < numReplicas) {
- isMatch = false;
- }
- }
- if (isMatch) return;
- }
- Thread.sleep(200);
+ throws InterruptedException {
+
+ try {
+ ZkStateReader.from(cloudClient)
+ .waitForState(
+ collection,
+ timeOutInMs,
+ TimeUnit.MILLISECONDS,
+ (liveNodes, c) -> {
+ Collection<Slice> slices = c.getActiveSlices();
+ if (slices.size() == numSlices) {
+ for (Slice slice : slices) {
+ int count = 0;
+ for (Replica replica : slice.getReplicas()) {
+ if (replica.getState() == Replica.State.ACTIVE
+ && liveNodes.contains(replica.getNodeName())) {
+ count++;
+ }
+ }
+ if (count < numReplicas) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+ });
+ } catch (TimeoutException e) {
+ fail(
+ "Expected numSlices="
+ + numSlices
+ + " numReplicas="
+ + numReplicas
+ + " but found "
+ + cloudClient.getClusterState().getCollection(collection)
+ + " with /live_nodes: "
+ + cloudClient.getClusterState().getLiveNodes());
}
- fail(
- "Expected numSlices="
- + numSlices
- + " numReplicas="
- + numReplicas
- + " but found "
- + cloudClient.getClusterState().getCollection(collection)
- + " with /live_nodes: "
- + cloudClient.getClusterState().getLiveNodes());
}
}
diff --git
a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java
b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java
index 7bb98b8d97b..326daaddc03 100644
--- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java
+++ b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/StressHdfsTest.java
@@ -48,9 +48,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TimeSource;
import org.apache.solr.hdfs.util.BadHdfsThreadsFilter;
-import org.apache.solr.util.TimeOut;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -232,14 +230,8 @@ public class StressHdfsTest extends
AbstractBasicDistributedZkTestBase {
request.setPath("/admin/collections");
solrClient.request(request);
- final TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS,
TimeSource.NANO_TIME);
- while
(cloudClient.getClusterState().hasCollection(DELETE_DATA_DIR_COLLECTION)) {
- if (timeout.hasTimedOut()) {
- throw new AssertionError("Timeout waiting to see removed collection
leave clusterstate");
- }
-
- Thread.sleep(200);
- }
+ waitForCollectionToDisappear(
+ DELETE_DATA_DIR_COLLECTION, ZkStateReader.from(cloudClient), true, 10);
// check that all dirs are gone
for (String dataDir : dataDirs) {
diff --git
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java
index fb138876b30..8c8a115636e 100644
---
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java
+++
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java
@@ -303,7 +303,7 @@ public abstract class
AbstractChaosMonkeyNothingIsSafeTestBase
List<Integer> numShardsNumReplicas = new ArrayList<>(2);
numShardsNumReplicas.add(1);
numShardsNumReplicas.add(1);
- checkForCollection("testcollection", numShardsNumReplicas, null);
+ checkForCollection("testcollection", numShardsNumReplicas);
testSuccessful = true;
} finally {
diff --git
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java
index ab765d32699..ef371efcad4 100644
---
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java
+++
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java
@@ -203,7 +203,7 @@ public abstract class AbstractChaosMonkeySafeLeaderTestBase
extends AbstractFull
List<Integer> numShardsNumReplicas = new ArrayList<>(2);
numShardsNumReplicas.add(1);
numShardsNumReplicas.add(1);
- checkForCollection("testcollection", numShardsNumReplicas, null);
+ checkForCollection("testcollection", numShardsNumReplicas);
}
private void tryDelete() throws Exception {
diff --git
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
index 37c90d98db9..f256a5fb937 100644
---
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
+++
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.cloud;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import java.io.File;
@@ -41,7 +40,6 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.Diagnostics;
import org.apache.solr.core.MockDirectoryFactory;
import org.apache.solr.embedded.JettySolrRunner;
-import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
import org.slf4j.Logger;
@@ -242,65 +240,42 @@ public abstract class AbstractDistribZkTestBase extends
BaseDistributedSearchTes
log.info("Collection has disappeared - collection:{}", collection);
}
- static void waitForNewLeader(
- CloudSolrClient cloudClient, String shardName, Replica oldLeader,
TimeOut timeOut)
+ static void waitForNewLeader(CloudSolrClient cloudClient, String shardName,
Replica oldLeader)
throws Exception {
- log.info("Will wait for a node to become leader for {} secs",
timeOut.timeLeft(SECONDS));
+ log.info("Will wait for a node to become leader for 15 secs");
ZkStateReader zkStateReader = ZkStateReader.from(cloudClient);
- zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION);
-
- for (; ; ) {
- ClusterState clusterState = zkStateReader.getClusterState();
- DocCollection coll = clusterState.getCollection("collection1");
- Slice slice = coll.getSlice(shardName);
- if (slice.getLeader() != null
- && !slice.getLeader().equals(oldLeader)
- && slice.getLeader().getState() == Replica.State.ACTIVE) {
- if (log.isInfoEnabled()) {
- log.info(
- "Old leader {}, new leader {}. New leader got elected in {} ms",
- oldLeader,
- slice.getLeader(),
- timeOut.timeElapsed(MILLISECONDS));
- }
- break;
- }
-
- if (timeOut.hasTimedOut()) {
- Diagnostics.logThreadDumps("Could not find new leader in specified
timeout");
- zkStateReader.getZkClient().printLayoutToStream(System.out);
- fail(
- "Could not find new leader even after waiting for "
- + timeOut.timeElapsed(MILLISECONDS)
- + "ms");
- }
- Thread.sleep(100);
- }
+ long startNs = System.nanoTime();
+ try {
+ zkStateReader.waitForState(
+ "collection1",
+ 15,
+ TimeUnit.SECONDS,
+ (docCollection) -> {
+ if (docCollection == null) return false;
- zkStateReader.waitForState(
- "collection1",
- timeOut.timeLeft(SECONDS),
- TimeUnit.SECONDS,
- (docCollection) -> {
- if (docCollection == null) return false;
-
- Slice slice = docCollection.getSlice(shardName);
- if (slice != null
- && slice.getLeader() != null
- && !slice.getLeader().equals(oldLeader)
- && slice.getLeader().getState() == Replica.State.ACTIVE) {
- if (log.isInfoEnabled()) {
- log.info(
- "Old leader {}, new leader {}. New leader got elected in {}
ms",
- oldLeader,
- slice.getLeader(),
- timeOut.timeElapsed(MILLISECONDS));
+ Slice slice = docCollection.getSlice(shardName);
+ if (slice != null
+ && slice.getLeader() != null
+ && !slice.getLeader().equals(oldLeader)
+ && slice.getLeader().getState() == Replica.State.ACTIVE) {
+ if (log.isInfoEnabled()) {
+ log.info(
+ "Old leader {}, new leader {}. New leader got elected in
{} ms",
+ oldLeader,
+ slice.getLeader(),
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() -
startNs));
+ }
+ return true;
}
- return true;
- }
- return false;
- });
+ return false;
+ });
+ } catch (TimeoutException e) {
+ // If we failed to get a new leader, print some diagnotics before the
test fails
+ Diagnostics.logThreadDumps("Could not find new leader in specified
timeout");
+ zkStateReader.getZkClient().printLayoutToStream(System.out);
+ fail("Could not find new leader even after waiting for 15s");
+ }
}
public static void verifyReplicaStatus(
diff --git
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index dcdab4e86d1..003ef951b0f 100644
---
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -44,6 +44,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
@@ -2217,83 +2218,60 @@ public abstract class AbstractFullDistribZkTestBase
extends AbstractDistribZkTes
return sdoc(fields);
}
- private String checkCollectionExpectations(
- String collectionName,
- List<Integer> numShardsNumReplicaList,
- List<String> nodesAllowedToRunShards)
- throws IOException {
- getCommonCloudSolrClient();
- ClusterState clusterState = cloudClient.getClusterState();
- int expectedSlices = numShardsNumReplicaList.get(0);
- // The Math.min thing is here, because we expect replication-factor to be
reduced to if there
- // are not enough live nodes to spread all shards of a collection over
different nodes
- int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
- int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
-
- // Map<String,DocCollection> collections = clusterState
- // .getCollectionStates();
- if (clusterState.hasCollection(collectionName)) {
- Map<String, Slice> slices =
clusterState.getCollection(collectionName).getSlicesMap();
- // did we find expectedSlices slices/shards?
- if (slices.size() != expectedSlices) {
- return "Found new collection "
- + collectionName
- + ", but mismatch on number of slices. Expected: "
- + expectedSlices
- + ", actual: "
- + slices.size();
- }
- int totalShards = 0;
- for (String sliceName : slices.keySet()) {
- for (Replica replica : slices.get(sliceName).getReplicas()) {
- if (nodesAllowedToRunShards != null
- &&
!nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP)))
{
- return "Shard "
- + replica.getName()
- + " created on node "
- + replica.getNodeName()
- + " not allowed to run shards for the created collection "
- + collectionName;
- }
- }
- totalShards += slices.get(sliceName).getReplicas().size();
- }
- if (totalShards != expectedTotalShards) {
- return "Found new collection "
- + collectionName
- + " with correct number of slices, but mismatch on number of
shards. Expected: "
- + expectedTotalShards
- + ", actual: "
- + totalShards;
- }
- return null;
- } else {
- return "Could not find new collection " + collectionName;
- }
- }
-
- protected void checkForCollection(
- String collectionName,
- List<Integer> numShardsNumReplicaList,
- List<String> nodesAllowedToRunShards)
+ protected void checkForCollection(String collectionName, List<Integer>
numShardsNumReplicaList)
throws Exception {
// check for an expectedSlices new collection - we poll the state
- final TimeOut timeout = new TimeOut(120, TimeUnit.SECONDS,
TimeSource.NANO_TIME);
- boolean success = false;
- String checkResult = "Didnt get to perform a single check";
- while (!timeout.hasTimedOut()) {
- checkResult =
- checkCollectionExpectations(
- collectionName, numShardsNumReplicaList,
nodesAllowedToRunShards);
- if (checkResult == null) {
- success = true;
- break;
- }
- Thread.sleep(500);
- }
- if (!success) {
- super.printLayout();
- fail(checkResult);
+ ZkStateReader reader = ZkStateReader.from(cloudClient);
+
+ AtomicReference<String> message = new AtomicReference<>();
+ try {
+ reader.waitForState(
+ collectionName,
+ 120,
+ TimeUnit.SECONDS,
+ c -> {
+ int expectedSlices = numShardsNumReplicaList.get(0);
+ // The Math.min thing is here, because we expect
replication-factor to be reduced to if
+ // there are not enough live nodes to spread all shards of a
collection over different
+ // nodes.
+ int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
+ int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
+
+ if (c != null) {
+ Collection<Slice> slices = c.getSlices();
+ // did we find expectedSlices slices/shards?
+ if (slices.size() != expectedSlices) {
+ message.set(
+ "Found new collection "
+ + collectionName
+ + ", but mismatch on number of slices. Expected: "
+ + expectedSlices
+ + ", actual: "
+ + slices.size());
+ return false;
+ }
+ int totalShards = 0;
+ for (Slice slice : slices) {
+ totalShards += slice.getReplicas().size();
+ }
+ if (totalShards != expectedTotalShards) {
+ message.set(
+ "Found new collection "
+ + collectionName
+ + " with correct number of slices, but mismatch on
number of shards. Expected: "
+ + expectedTotalShards
+ + ", actual: "
+ + totalShards);
+ return false;
+ }
+ return true;
+ } else {
+ message.set("Could not find new collection " + collectionName);
+ return false;
+ }
+ });
+ } catch (TimeoutException e) {
+ fail(message.get());
}
}
diff --git
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java
index 2f77b69b27b..ee7fac68736 100644
---
a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java
+++
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java
@@ -26,6 +26,7 @@ import java.util.Set;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
@@ -33,7 +34,6 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -41,12 +41,10 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.SolrNamedThreadFactory;
-import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrPaths;
import org.apache.solr.embedded.JettySolrRunner;
import org.apache.solr.util.TestInjection;
-import org.apache.solr.util.TimeOut;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,42 +95,37 @@ public abstract class AbstractUnloadDistributedZkTestBase
extends AbstractFullDi
private void checkCoreNamePresenceAndSliceCount(
String collectionName, String coreName, boolean shouldBePresent, int
expectedSliceCount)
throws Exception {
- final TimeOut timeout = new TimeOut(45, TimeUnit.SECONDS,
TimeSource.NANO_TIME);
- Boolean isPresent = null; // null meaning "don't know"
- while (null == isPresent || shouldBePresent != isPresent) {
- getCommonCloudSolrClient();
- final DocCollection docCollection =
- cloudClient.getClusterState().getCollectionOrNull(collectionName);
- final Collection<Slice> slices =
- (docCollection != null) ? docCollection.getSlices() :
Collections.emptyList();
- if (timeout.hasTimedOut()) {
- printLayout();
- fail(
- "checkCoreNamePresenceAndSliceCount failed:"
- + " collection="
- + collectionName
- + " CoreName="
- + coreName
- + " shouldBePresent="
- + shouldBePresent
- + " isPresent="
- + isPresent
- + " expectedSliceCount="
- + expectedSliceCount
- + " actualSliceCount="
- + slices.size());
- }
- if (expectedSliceCount == slices.size()) {
- isPresent = false;
- for (Slice slice : slices) {
- for (Replica replica : slice.getReplicas()) {
- if (coreName.equals(replica.get("core"))) {
- isPresent = true;
+ ZkStateReader reader = ZkStateReader.from(cloudClient);
+ try {
+ reader.waitForState(
+ collectionName,
+ 45,
+ TimeUnit.SECONDS,
+ c -> {
+ final Collection<Slice> slices = (c != null) ? c.getSlices() :
Collections.emptyList();
+ if (expectedSliceCount == slices.size()) {
+ for (Slice slice : slices) {
+ for (Replica replica : slice.getReplicas()) {
+ if (coreName.equals(replica.get("core"))) {
+ return shouldBePresent;
+ }
+ }
+ }
+ return !shouldBePresent;
+ } else {
+ return false;
}
- }
- }
- }
- Thread.sleep(1000);
+ });
+ } catch (TimeoutException e) {
+ printLayout();
+ fail(
+ "checkCoreNamePresenceAndSliceCount failed:"
+ + " collection="
+ + collectionName
+ + " CoreName="
+ + coreName
+ + " shouldBePresent="
+ + shouldBePresent);
}
}
