This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_10x in repository https://gitbox.apache.org/repos/asf/solr.git
commit b9502c72be3f738edbd3bc4d9232cc23a165a479 Author: Jason Gerlowski <[email protected]> AuthorDate: Mon Oct 26 12:05:19 2026 -0400 SOLR-18569: Wait on init for ephemeral node-role cleanup (#5211) Solr has long had code in ZkController to check whether the ephemeral /live_node entry from a previous process is still around, and wait for ZK to deleted it before proceeding. But /live_node entries aren't the only ephemeral nodes that Solr expects to create on startup and that can linger after process exit. The /node_roles/ tree also contains ephemeral nodes. Currently, if any of these exist at startup-time, Solr will hit a NodeExists exception from ZK and fail to start up. This commit changes ZkController's init code so that it now waits for these nodes as well. In practice, this prevents Solr from potentially stalling on start up because of an unexpected or unhandled NodeExists exception. Co-authored-by: Nick Shanin <[email protected]> (cherry picked from commit 0d6b2e32f598d772aed094038022a6a588fdfc5a) --- ...8569-wait-for-node-role-deletion-on-startup.yml | 9 ++ .../java/org/apache/solr/cloud/ZkController.java | 80 +++++++++++----- .../org/apache/solr/cloud/ZkControllerTest.java | 103 +++++++++++++++++++++ 3 files changed, 167 insertions(+), 25 deletions(-) diff --git a/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml new file mode 100644 index 00000000000..686f6fb8c81 --- /dev/null +++ b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml @@ -0,0 +1,9 @@ +title: > + Solr startup no longer hangs if ephemeral "node-role" znodes from a previous process are still present. +type: fixed +authors: + - name: Jason Gerlowski + - name: Nick Shanin +links: + - name: SOLR-18569 + url: https://issues.apache.org/jira/browse/SOLR-18569 diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 8dc73f07e35..d34a25f9a9e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -1171,39 +1171,69 @@ public class ZkController implements Closeable { } } - private void checkForExistingEphemeralNode() throws KeeperException, InterruptedException { + /** + * Waits for any ephemeral znodes left by a previous process with this node name to disappear. + * + * <p>Covers both the {@code /live_nodes} entry and this node's entries under any {@code + * /node_roles/<role>/<mode>}. Throws if any are still present after twice the ZK session timeout. + * + * @throws SolrException if ephemeral nodes were not all removed by the end of the waiting period + */ + // package-private for testing + void checkForExistingEphemeralNode() throws KeeperException, InterruptedException { String nodeName = getNodeName(); - String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName; - if (!zkClient.exists(nodePath)) { - return; + List<String> candidatePaths = new ArrayList<>(); + candidatePaths.add(ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName); + for (NodeRoles.Role role : NodeRoles.Role.values()) { + for (String mode : role.supportedModes()) { + candidatePaths.add(NodeRoles.getZNodeForRoleMode(role, mode) + "/" + nodeName); + } } - final CountDownLatch deletedLatch = new CountDownLatch(1); - Stat stat = - zkClient.exists( - nodePath, - event -> { - if (Watcher.Event.EventType.None.equals(event.getType())) { - return; - } - if (Watcher.Event.EventType.NodeDeleted.equals(event.getType())) { - deletedLatch.countDown(); - } - }); + // Set a deletion watch on each znode that exists; the latch counts down as each is deleted + final List<String> watchedPaths = new ArrayList<>(); + final List<CountDownLatch> latches = new ArrayList<>(); + for (String path : candidatePaths) { + if (!zkClient.exists(path)) { + continue; + } + final CountDownLatch deletedLatch = new CountDownLatch(1); + Stat stat = + zkClient.exists( + path, + event -> { + if (Watcher.Event.EventType.None.equals(event.getType())) { + return; + } + if (Watcher.Event.EventType.NodeDeleted.equals(event.getType())) { + deletedLatch.countDown(); + } + }); + if (stat == null) { + // znode suddenly disappeared but that's okay + continue; + } + watchedPaths.add(path); + latches.add(deletedLatch); + } - if (stat == null) { - // znode suddenly disappeared but that's okay + if (watchedPaths.isEmpty()) { return; } - boolean deleted = - deletedLatch.await(zkClient.getZkSessionTimeout() * 2L, TimeUnit.MILLISECONDS); - if (!deleted) { - throw new SolrException( - ErrorCode.SERVER_ERROR, - "A previous ephemeral live node still exists. " - + "Solr cannot continue. Please ensure that no other Solr process using the same port is running already."); + // A single overall deadline, shared by all of the znodes + final long deadlineNanos = + System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(zkClient.getZkSessionTimeout() * 2L); + for (int i = 0; i < latches.size(); i++) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (!latches.get(i).await(Math.max(remainingNanos, 0), TimeUnit.NANOSECONDS)) { + throw new SolrException( + ErrorCode.SERVER_ERROR, + "An ephemeral node from a previous process representing this Solr instance still exists: " + + watchedPaths.get(i) + + ". Solr cannot continue. Please ensure that no other Solr process using the same port is running already."); + } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java index 0d993da70ac..ea339d7126d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java @@ -59,6 +59,7 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.core.CloudConfig; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.core.NodeRoles; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrXmlConfig; import org.apache.solr.handler.admin.CoreAdminHandler; @@ -222,6 +223,108 @@ public class ZkControllerTest extends SolrCloudTestCase { } } + @Test + public void testStartupWaitsForLeftoverEphemeralNodesToDisappear() throws Exception { + final String staleRoleEntry = ephemeralNodeRolePath(NodeRoles.Role.OVERSEER); + runWithLeftoverEphemeralNodes( + "testStartupWaitsForLeftoverEphemeralNodes", + staleRoleEntry, + (zkController, otherClient, leftoverPaths) -> { + // Remove the leftovers shortly after the check starts waiting on them + AtomicReference<Exception> deleterFailure = new AtomicReference<>(); + Thread deleter = + new Thread( + () -> { + try { + Thread.sleep(1000); + for (String path : leftoverPaths) { + otherClient.delete(path, -1); + } + } catch (Exception e) { + deleterFailure.set(e); + } + }); + deleter.start(); + try { + // Returns normally once everything is gone + zkController.checkForExistingEphemeralNode(); + for (String path : leftoverPaths) { + assertFalse( + path + " should be gone when the check returns", otherClient.exists(path)); + } + } finally { + deleter.join(TimeUnit.SECONDS.toMillis(30)); + } + assertFalse("Deleter thread did not finish", deleter.isAlive()); + assertNull("Deleter thread failed", deleterFailure.get()); + }); + } + + @Test + public void testStartupFailsIfLeftoverEphemeralNodeNeverDisappears() throws Exception { + final String staleRoleEntry = ephemeralNodeRolePath(NodeRoles.Role.OVERSEER); + runWithLeftoverEphemeralNodes( + "testStartupFailsIfLeftoverEphemeralNode", + staleRoleEntry, + (zkController, otherClient, leftoverPaths) -> { + // Only the role entry is left behind, so the message must name it specifically + SolrException e = + expectThrows(SolrException.class, zkController::checkForExistingEphemeralNode); + assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, e.code()); + assertTrue( + "Message should name the leftover path: " + e.getMessage(), + e.getMessage().contains(staleRoleEntry)); + }); + } + + /** The path of this test node's entry under a mode of 'role' that the node isn't using itself */ + private static String ephemeralNodeRolePath(NodeRoles.Role role) { + String unusedMode = NodeRoles.MODE_PREFERRED; // default roles use 'allowed' + assertTrue(role.supportedModes().contains(unusedMode)); + return NodeRoles.getZNodeForRoleMode(role, unusedMode) + "/" + TEST_NODE_NAME; + } + + private static final String TEST_NODE_NAME = "127.0.0.1:8983_solr"; + + @FunctionalInterface + private interface LeftoverNodesCheck { + void run(ZkController zkController, SolrZkClient otherClient, List<String> leftoverPaths) + throws Exception; + } + + /** + * Starts a ZkController, removes its own ephemeral entries, then has a different ZK client create + * the given ephemeral paths (simulating a previous process with the same node name that hasn't + * fully gone away) before running 'check'. + */ + private void runWithLeftoverEphemeralNodes( + String dirName, String leftoverPath, LeftoverNodesCheck check) throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir(dirName)); + try { + server.run(); + CoreContainer cc = getCoreContainer(); + // Short session timeout, as the check waits for twice the session timeout + CloudConfig cloudConfig = + new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983).setZkClientTimeout(3000).build(); + ZkController zkController = new ZkController(cc, server.getZkAddress(), 3000, cloudConfig); + try (SolrZkClient otherClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + assertEquals(TEST_NODE_NAME, zkController.getNodeName()); + zkController.removeEphemeralLiveNode(); + otherClient.create(leftoverPath, null, CreateMode.EPHEMERAL); + check.run(zkController, otherClient, List.of(leftoverPath)); + } finally { + zkController.close(); + cc.shutdown(); + } + } finally { + server.shutdown(); + } + } + @LogLevel(value = "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.overseer=DEBUG") @Test public void testPublishAndWaitForDownStates() throws Exception {
