This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new daa4b301350 SOLR-18569: Cleanup ephem /node_roles/... on process exit
(#5237)
daa4b301350 is described below
commit daa4b3013503d8ae628b118e9c2a763a58751499
Author: Jason Gerlowski <[email protected]>
AuthorDate: Fri Oct 30 11:26:16 2026 -0400
SOLR-18569: Cleanup ephem /node_roles/... on process exit (#5237)
Co-authored-by: Nick Shanin <[email protected]>
---
...8569-wait-for-node-role-deletion-on-startup.yml | 9 ++
.../java/org/apache/solr/cloud/ZkController.java | 96 ++++++++++++++-----
.../org/apache/solr/cloud/ZkControllerTest.java | 105 +++++++++++++++++++++
3 files changed, 184 insertions(+), 26 deletions(-)
diff --git
a/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml
b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml
new file mode 100644
index 00000000000..686f6fb8c81
--- /dev/null
+++ b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml
@@ -0,0 +1,9 @@
+title: >
+ Solr startup no longer hangs if ephemeral "node-role" znodes from a previous
process are still present.
+type: fixed
+authors:
+ - name: Jason Gerlowski
+ - name: Nick Shanin
+links:
+ - name: SOLR-18569
+ url: https://issues.apache.org/jira/browse/SOLR-18569
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 72467e0fc51..05157a55f4c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1092,43 +1092,74 @@ public class ZkController implements Closeable {
}
}
- private void checkForExistingEphemeralNode() throws KeeperException,
InterruptedException {
+ /**
+ * Waits for any ephemeral znodes left by a previous process with this node
name to disappear.
+ *
+ * <p>Covers both the {@code /live_nodes} entry and this node's entries
under any {@code
+ * /node_roles/<role>/<mode>}. Throws if any are still present after twice
the ZK session timeout.
+ *
+ * @throws SolrException if ephemeral nodes were not all removed by the end
of the waiting period
+ */
+ // package-private for testing
+ void checkForExistingEphemeralNode() throws KeeperException,
InterruptedException {
if (zkRunOnly) {
return;
}
String nodeName = getNodeName();
- String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
- if (!zkClient.exists(nodePath, true)) {
- return;
+ List<String> candidatePaths = new ArrayList<>();
+ candidatePaths.add(ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName);
+ for (NodeRoles.Role role : NodeRoles.Role.values()) {
+ for (String mode : role.supportedModes()) {
+ candidatePaths.add(NodeRoles.getZNodeForRoleMode(role, mode) + "/" +
nodeName);
+ }
}
- final CountDownLatch deletedLatch = new CountDownLatch(1);
- Stat stat =
- zkClient.exists(
- nodePath,
- event -> {
- if (Watcher.Event.EventType.None.equals(event.getType())) {
- return;
- }
- if (Watcher.Event.EventType.NodeDeleted.equals(event.getType()))
{
- deletedLatch.countDown();
- }
- },
- true);
+ // Set a deletion watch on each znode that exists; the latch counts down
as each is deleted
+ final List<String> watchedPaths = new ArrayList<>();
+ final List<CountDownLatch> latches = new ArrayList<>();
+ for (String path : candidatePaths) {
+ if (!zkClient.exists(path, true)) {
+ continue;
+ }
+ final CountDownLatch deletedLatch = new CountDownLatch(1);
+ Stat stat =
+ zkClient.exists(
+ path,
+ event -> {
+ if (Watcher.Event.EventType.None.equals(event.getType())) {
+ return;
+ }
+ if
(Watcher.Event.EventType.NodeDeleted.equals(event.getType())) {
+ deletedLatch.countDown();
+ }
+ },
+ true);
+ if (stat == null) {
+ // znode suddenly disappeared but that's okay
+ continue;
+ }
+ watchedPaths.add(path);
+ latches.add(deletedLatch);
+ }
- if (stat == null) {
- // znode suddenly disappeared but that's okay
+ if (watchedPaths.isEmpty()) {
return;
}
- boolean deleted =
- deletedLatch.await(zkClient.getZooKeeper().getSessionTimeout() * 2L,
TimeUnit.MILLISECONDS);
- if (!deleted) {
- throw new SolrException(
- ErrorCode.SERVER_ERROR,
- "A previous ephemeral live node still exists. "
- + "Solr cannot continue. Please ensure that no other Solr
process using the same port is running already.");
+ // A single overall deadline, shared by all of the znodes
+ final long deadlineNanos =
+ System.nanoTime()
+ +
TimeUnit.MILLISECONDS.toNanos(zkClient.getZooKeeper().getSessionTimeout() * 2L);
+ for (int i = 0; i < latches.size(); i++) {
+ long remainingNanos = deadlineNanos - System.nanoTime();
+ if (!latches.get(i).await(Math.max(remainingNanos, 0),
TimeUnit.NANOSECONDS)) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "An ephemeral node from a previous process representing this Solr
instance still exists: "
+ + watchedPaths.get(i)
+ + ". Solr cannot continue. Please ensure that no other Solr
process using the same port is running already.");
+ }
}
}
@@ -1289,6 +1320,19 @@ public class ZkController implements Closeable {
} catch (NoNodeException e) {
}
+ cc.nodeRoles
+ .getRoles()
+ .forEach(
+ (role, mode) -> {
+ try {
+ zkClient.delete(
+ NodeRoles.getZNodeForRoleMode(role, mode) + "/" +
nodeName, -1, true);
+ } catch (KeeperException e) {
+
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
}
public String getNodeName() {
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
index 2d45de8ebb3..6599a12a0bd 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
@@ -56,6 +56,7 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CloudConfig;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.NodeRoles;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrXmlConfig;
import org.apache.solr.handler.admin.CoreAdminHandler;
@@ -255,6 +256,110 @@ public class ZkControllerTest extends SolrCloudTestCase {
}
}
+ @Test
+ public void testStartupWaitsForLeftoverEphemeralNodesToDisappear() throws
Exception {
+ final String staleRoleEntry =
ephemeralNodeRolePath(NodeRoles.Role.OVERSEER);
+ runWithLeftoverEphemeralNodes(
+ "testStartupWaitsForLeftoverEphemeralNodes",
+ staleRoleEntry,
+ (zkController, otherClient, leftoverPaths) -> {
+ // Remove the leftovers shortly after the check starts waiting on
them
+ AtomicReference<Exception> deleterFailure = new AtomicReference<>();
+ Thread deleter =
+ new Thread(
+ () -> {
+ try {
+ Thread.sleep(1000);
+ for (String path : leftoverPaths) {
+ otherClient.delete(path, -1, true);
+ }
+ } catch (Exception e) {
+ deleterFailure.set(e);
+ }
+ });
+ deleter.start();
+ try {
+ // Returns normally once everything is gone
+ zkController.checkForExistingEphemeralNode();
+ for (String path : leftoverPaths) {
+ assertFalse(
+ path + " should be gone when the check returns",
otherClient.exists(path, true));
+ }
+ } finally {
+ deleter.join(TimeUnit.SECONDS.toMillis(30));
+ }
+ assertFalse("Deleter thread did not finish", deleter.isAlive());
+ assertNull("Deleter thread failed", deleterFailure.get());
+ });
+ }
+
+ @Test
+ public void testStartupFailsIfLeftoverEphemeralNodeNeverDisappears() throws
Exception {
+ final String staleRoleEntry =
ephemeralNodeRolePath(NodeRoles.Role.OVERSEER);
+ runWithLeftoverEphemeralNodes(
+ "testStartupFailsIfLeftoverEphemeralNode",
+ staleRoleEntry,
+ (zkController, otherClient, leftoverPaths) -> {
+ // Only the role entry is left behind, so the message must name it
specifically
+ SolrException e =
+ expectThrows(SolrException.class,
zkController::checkForExistingEphemeralNode);
+ assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, e.code());
+ assertTrue(
+ "Message should name the leftover path: " + e.getMessage(),
+ e.getMessage().contains(staleRoleEntry));
+ });
+ }
+
+ /** The path of this test node's entry under a mode of 'role' that the node
isn't using itself */
+ private static String ephemeralNodeRolePath(NodeRoles.Role role) {
+ String unusedMode = NodeRoles.MODE_PREFERRED; // default roles use
'allowed'
+ assertTrue(role.supportedModes().contains(unusedMode));
+ return NodeRoles.getZNodeForRoleMode(role, unusedMode) + "/" +
TEST_NODE_NAME;
+ }
+
+ private static final String TEST_NODE_NAME = "127.0.0.1:8983_solr";
+
+ @FunctionalInterface
+ private interface LeftoverNodesCheck {
+ void run(ZkController zkController, SolrZkClient otherClient, List<String>
leftoverPaths)
+ throws Exception;
+ }
+
+ /**
+ * Starts a ZkController, removes its own ephemeral entries, then has a
different ZK client create
+ * the given ephemeral paths (simulating a previous process with the same
node name that hasn't
+ * fully gone away) before running 'check'.
+ */
+ private void runWithLeftoverEphemeralNodes(
+ String dirName, String leftoverPath, LeftoverNodesCheck check) throws
Exception {
+ ZkTestServer server = new ZkTestServer(createTempDir(dirName));
+ try {
+ server.run();
+ CoreContainer cc = getCoreContainer();
+ // Short session timeout, as the check waits for twice the session
timeout
+ CloudConfig cloudConfig =
+ new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "solr")
+ .setZkClientTimeout(3000)
+ .build();
+ ZkController zkController = new ZkController(cc, server.getZkAddress(),
3000, cloudConfig);
+ try (SolrZkClient otherClient =
+ new SolrZkClient.Builder()
+ .withUrl(server.getZkAddress())
+ .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS)
+ .build()) {
+ assertEquals(TEST_NODE_NAME, zkController.getNodeName());
+ zkController.removeEphemeralLiveNode();
+ otherClient.create(leftoverPath, null, CreateMode.EPHEMERAL, true);
+ check.run(zkController, otherClient, List.of(leftoverPath));
+ } finally {
+ zkController.close();
+ cc.shutdown();
+ }
+ } finally {
+ server.shutdown();
+ }
+ }
+
@LogLevel(value =
"org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.overseer=DEBUG")
@Test
public void testPublishAndWaitForDownStates() throws Exception {
