This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_10x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 52bf8dfa62ac55fafa6bc2ca82116b89ee961172 Author: sarkaramrit1993 <[email protected]> AuthorDate: Wed Oct 21 02:00:37 2026 -0700 SOLR-17971: Fix REBALANCELEADERS wait when the Overseer is disabled (#5127) (cherry picked from commit 036103c962663b7af520219c0a69a16d29f54e46) --- .../solr-17971-rebalance-leaders-distributed.yml | 7 +++ .../solr/cloud/DistributedApiAsyncTracker.java | 5 +++ .../java/org/apache/solr/cloud/ZkController.java | 18 ++++++++ ...istributedCollectionConfigSetCommandRunner.java | 8 ++++ .../solr/handler/admin/RebalanceLeaders.java | 15 ++----- .../solr/handler/admin/api/ClusterCommands.java | 15 +------ .../apache/solr/cloud/TestRebalanceLeaders.java | 50 ++++++++++++++++++++++ 7 files changed, 92 insertions(+), 26 deletions(-) diff --git a/changelog/unreleased/solr-17971-rebalance-leaders-distributed.yml b/changelog/unreleased/solr-17971-rebalance-leaders-distributed.yml new file mode 100644 index 00000000000..a4c4fb79316 --- /dev/null +++ b/changelog/unreleased/solr-17971-rebalance-leaders-distributed.yml @@ -0,0 +1,7 @@ +title: "REBALANCELEADERS no longer waits until timeout or leaves async ids behind when the Overseer is disabled." +type: fixed +authors: + - name: Amrit Sarkar +links: + - name: SOLR-17971 + url: https://issues.apache.org/jira/browse/SOLR-17971 diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.java b/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.java index 556ea22211d..c10a6870728 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.java @@ -286,6 +286,11 @@ public class DistributedApiAsyncTracker { && trackedAsyncTasks.remove(asyncId); } + @VisibleForTesting + public Collection<String> getTrackedAsyncIds() throws Exception { + return trackedAsyncTasks.keys(); + } + /** * Deletes all async id's for completed or failed async jobs. Does not touch id's for submitted or * running jobs. diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 2a16644ab2c..8dc73f07e35 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -2510,6 +2510,24 @@ public class ZkController implements Closeable { } } + /** + * Removes the stored response of a completed or failed async operation and releases its asyncId, + * in both Overseer and distributed modes. An asyncId with no stored response is left claimed, + * since its task may still be running or pending. + * + * @return True if a stored response was found and removed. + */ + public boolean removeStoredAsyncResponse(String asyncId) throws Exception { + if (distributedCommandRunner.isPresent()) { + return distributedCommandRunner.get().deleteSingleAsyncId(asyncId); + } + if (overseerCompletedMap.remove(asyncId) || overseerFailureMap.remove(asyncId)) { + clearAsyncId(asyncId); + return true; + } + return false; + } + public Overseer getOverseer() { return overseer; } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java index 80797e0e91f..77f9481b196 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java @@ -24,8 +24,10 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; import static org.apache.solr.common.params.CommonParams.NAME; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.Collection; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; @@ -152,6 +154,12 @@ public class DistributedCollectionConfigSetCommandRunner { return asyncTaskTracker.deleteSingleAsyncId(asyncId); } + /** See {@link DistributedApiAsyncTracker#getTrackedAsyncIds()} */ + @VisibleForTesting + public Collection<String> getTrackedAsyncIds() throws Exception { + return asyncTaskTracker.getTrackedAsyncIds(); + } + /** See {@link DistributedApiAsyncTracker#deleteAllAsyncIds()} */ public void deleteAllAsyncIds() throws Exception { asyncTaskTracker.deleteAllAsyncIds(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java index 1a435a1ad97..eb27f99f931 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java @@ -122,7 +122,7 @@ class RebalanceLeaders { coreContainer = collectionsHandler.getCoreContainer(); } - void execute() throws KeeperException, InterruptedException { + void execute() throws Exception { DocCollection dc = checkParams(); int max = req.getParams().getInt(MAX_AT_ONCE_PROP, Integer.MAX_VALUE); @@ -471,8 +471,7 @@ class RebalanceLeaders { // mean failure! // - private boolean waitAsyncRequests(final int maxWaitSecs, Boolean waitForAll) - throws KeeperException, InterruptedException { + private boolean waitAsyncRequests(final int maxWaitSecs, Boolean waitForAll) throws Exception { if (asyncRequests.size() == 0) { return true; @@ -482,15 +481,7 @@ class RebalanceLeaders { Iterator<String> iter = asyncRequests.iterator(); boolean foundChange = false; while (iter.hasNext()) { - String asyncId = iter.next(); - if (coreContainer.getZkController().getOverseerFailureMap().contains(asyncId)) { - coreContainer.getZkController().getOverseerFailureMap().remove(asyncId); - coreContainer.getZkController().clearAsyncId(asyncId); - iter.remove(); - foundChange = true; - } else if (coreContainer.getZkController().getOverseerCompletedMap().contains(asyncId)) { - coreContainer.getZkController().getOverseerCompletedMap().remove(asyncId); - coreContainer.getZkController().clearAsyncId(asyncId); + if (coreContainer.getZkController().removeStoredAsyncResponse(iter.next())) { iter.remove(); foundChange = true; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterCommands.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterCommands.java index 60cc18e483e..4094f373205 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterCommands.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterCommands.java @@ -129,20 +129,7 @@ public class ClusterCommands extends AdminAPIBase implements ClusterCommandsApi final DeleteClusterCommandStatusResponse response = instantiateJerseyResponse(DeleteClusterCommandStatusResponse.class); - final ZkController zkController = coreContainer.getZkController(); - - if (zkController.getDistributedCommandRunner().isEmpty()) { - if (zkController.getOverseerCompletedMap().remove(requestId)) { - zkController.clearAsyncId(requestId); - response.status = "successfully removed stored response for [" + requestId + "]"; - } else if (zkController.getOverseerFailureMap().remove(requestId)) { - zkController.clearAsyncId(requestId); - response.status = "successfully removed stored response for [" + requestId + "]"; - } else { - // Don't call zkController.clearAsyncId for this, since it could be a running/pending task - response.status = "[" + requestId + "] not found in stored responses"; - } - } else if (zkController.getDistributedCommandRunner().get().deleteSingleAsyncId(requestId)) { + if (coreContainer.getZkController().removeStoredAsyncResponse(requestId)) { response.status = "successfully removed stored response for [" + requestId + "]"; } else { response.status = "[" + requestId + "] not found in stored responses"; diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java b/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java index de4f130fecb..7015e77c916 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java @@ -24,6 +24,7 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -36,6 +37,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.SimpleSolrResponse; +import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; @@ -157,6 +159,54 @@ public class TestRebalanceLeaders extends SolrCloudTestCase { checkPreferredsAreLeaders(); } + @Test + public void testRebalanceLeadersClearsAsyncIds() throws Exception { + DocCollection docCollection = + cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME); + for (Slice slice : docCollection.getSlices()) { + Replica nonLeader = + slice.getReplicas().stream() + .filter(rep -> rep.getName().equals(slice.getLeader().getName()) == false) + .findFirst() + .orElseThrow(); + setProp(slice, nonLeader, "preferredLeader"); + } + + ZkController zkController = + cluster.getRandomJetty(random()).getCoreContainer().getZkController(); + Set<String> idsBefore = asyncIds(zkController); + + int maxWaitSeconds = 60; + long start = System.nanoTime(); + CollectionAdminResponse resp = + CollectionAdminRequest.rebalanceLeaders(COLLECTION_NAME) + .setMaxWaitSeconds(maxWaitSeconds) + .process(cluster.getSolrClient()); + long elapsedSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); + assertTrue( + resp.getResponse().toString(), + resp.getResponse().get("Summary").toString().contains("Success")); + assertNotNull("No leader change was requested", resp.getResponse().get("successes")); + assertTrue( + "Waited " + elapsedSeconds + "s, hit maxWaitSeconds", elapsedSeconds < maxWaitSeconds); + + Set<String> leftover = asyncIds(zkController); + leftover.removeAll(idsBefore); + assertTrue("Async ids not cleared: " + leftover, leftover.isEmpty()); + } + + private Set<String> asyncIds(ZkController zkController) throws Exception { + Optional<DistributedCollectionConfigSetCommandRunner> runner = + zkController.getDistributedCommandRunner(); + if (runner.isPresent()) { + return new HashSet<>(runner.get().getTrackedAsyncIds()); + } + Set<String> ids = new HashSet<>(zkController.getOverseerCompletedMap().keys()); + ids.addAll(zkController.getOverseerFailureMap().keys()); + ids.addAll(Overseer.getAsyncIdsMap(zkController.getZkClient()).keys()); + return ids; + } + // Ensure that the property is set on only one replica per slice when changing a unique property // on an individual // replica.
