This is an automated email from the ASF dual-hosted git repository.
hossman pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new fe7fe7966a6 SOLR-17744: Solr now enables Jetty's Graceful Shutdown
features to prevent client connections from being abruptly terminated on
orderly shutdown
fe7fe7966a6 is described below
commit fe7fe7966a654802f70f6853095ff427ee17d318
Author: Chris Hostetter <[email protected]>
AuthorDate: Mon Apr 28 09:32:32 2025 -0700
SOLR-17744: Solr now enables Jetty's Graceful Shutdown features to prevent
client connections from being abruptly terminated on orderly shutdown
(cherry picked from commit d0d71a7b38f87f3c70551678b2eab62d925483ef)
---
solr/CHANGES.txt | 2 +
solr/bin/solr | 5 +
.../test/org/apache/solr/cloud/SplitShardTest.java | 1 +
.../solr/cloud/TestGracefulJettyShutdown.java | 161 +++++++++++++++++++++
solr/server/etc/jetty-graceful.xml | 22 +++
solr/server/modules/graceful.mod | 8 +
.../apache/solr/cloud/MiniSolrCloudCluster.java | 2 +
.../org/apache/solr/embedded/JettySolrRunner.java | 7 +
8 files changed, 208 insertions(+)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3a6ad50b554..7fc58f71438 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -49,6 +49,8 @@ Improvements
* SOLR-17732: Score-based return fields other than "score" can now be returned
in distributed queries. (Houston Putman)
+* SOLR-17744: Solr now enables Jetty's Graceful Shutdown features to prevent
client connections from being abruptly terminated on orderly shutdown (hossman)
+
Optimizations
---------------------
* SOLR-17578: Remove ZkController internal core supplier, for slightly faster
reconnection after Zookeeper session loss. (Pierre Salagnac)
diff --git a/solr/bin/solr b/solr/bin/solr
index 0759d8ed7db..a49cc54e537 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -304,6 +304,11 @@ else
fi
export SOLR_URL_SCHEME
+# Gracefully wait for existing requests on shutdown
+if [ "${SOLR_JETTY_GRACEFUL:-true}" == "true" ]; then
+ SOLR_JETTY_CONFIG+=("--module=graceful")
+fi
+
# Requestlog options
if [ "${SOLR_REQUESTLOG_ENABLED:-true}" == "true" ]; then
SOLR_JETTY_CONFIG+=("--module=requestlog")
diff --git a/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java
b/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java
index 1210b5169ed..b8585de0c87 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java
@@ -390,6 +390,7 @@ public class SplitShardTest extends SolrCloudTestCase {
(s, replica) -> {
if (replica.getNodeName().equals(jetty.getNodeName())
&& !replica.isLeader()
+ && replica.getState().equals(Replica.State.ACTIVE)
&& set.contains(replica.shard)) {
set.remove(replica.shard);
}
diff --git
a/solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.java
b/solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.java
new file mode 100644
index 00000000000..44caa313c52
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.embedded.JettyConfig;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.apache.solr.handler.component.SearchHandler;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestGracefulJettyShutdown extends SolrTestCaseJ4 {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public void testSingleShardInFlightRequestsDuringShutDown() throws Exception
{
+ final String collection = getSaferTestName();
+ final String handler = "/foo";
+
+ final Semaphore handlerGate = new Semaphore(0);
+ final Semaphore handlerSignal = new Semaphore(0);
+
+ final ExecutorService exec =
ExecutorUtil.newMDCAwareCachedThreadPool("client-requests");
+ final MiniSolrCloudCluster cluster =
+ new MiniSolrCloudCluster(1, createTempDir(),
JettyConfig.builder().build());
+ try {
+ assertTrue(
+ CollectionAdminRequest.createCollection(collection, "_default", 1, 1)
+ .process(cluster.getSolrClient())
+ .isSuccess());
+
+ final JettySolrRunner nodeToStop = cluster.getJettySolrRunner(0);
+
+ // register our custom handler with "all" (one) of our SolrCores
+ for (String coreName :
nodeToStop.getCoreContainer().getLoadedCoreNames()) {
+ try (SolrCore core = nodeToStop.getCoreContainer().getCore(coreName)) {
+ final BlockingSearchHandler h = new
BlockingSearchHandler(handlerGate, handlerSignal);
+ h.inform(core);
+ core.registerRequestHandler(handler, h);
+ }
+ }
+
+ final CloudSolrClient cloudClient = cluster.getSolrClient();
+
+ // add a few docs...
+ cloudClient.add(collection, sdoc("id", "xxx", "foo_s", "aaa"));
+ cloudClient.add(collection, sdoc("id", "yyy", "foo_s", "bbb"));
+ cloudClient.add(collection, sdoc("id", "zzz", "foo_s", "aaa"));
+ cloudClient.commit(collection);
+
+ final List<Future<QueryResponse>> results = new ArrayList<>(13);
+
+ try (SolrClient jettyClient = nodeToStop.newClient()) {
+ final QueryRequest req = new QueryRequest(params("q", "foo_s:aaa"));
+ req.setPath(handler);
+
+ // check inflight requests using both clients...
+ for (SolrClient client : Arrays.asList(cloudClient, jettyClient)) {
+ results.add(
+ exec.submit(
+ () -> {
+ return req.process(client, collection);
+ }));
+ }
+
+ // wait for our handlers to indicate they have recieved the requests
and started processing
+ log.info("Waiting for signals from both requests");
+ assertTrue(handlerSignal.tryAcquire(2, 300, TimeUnit.SECONDS)); //
safety valve
+
+ // stop our node (via executor so it doesn't block) and open the gate
for our handlers
+ log.info("Stopping jetty node");
+ final Future<Boolean> stopped =
+ exec.submit(
+ () -> {
+ nodeToStop.stop();
+ return true;
+ });
+ log.info("Releasing gate for requests");
+ handlerGate.release(2);
+ log.info("Released gate for requests");
+
+ // confirm success of requests
+ assertEquals(2, results.size());
+ for (Future<QueryResponse> f : results) {
+ final QueryResponse rsp = f.get(300, TimeUnit.SECONDS); // safety
valve
+ assertEquals(2, rsp.getResults().getNumFound());
+ }
+ assertTrue(stopped.get(300, TimeUnit.SECONDS)); // safety valve
+ }
+
+ } finally {
+ handlerGate.release(9999999); // safety valve
+ handlerSignal.release(9999999); // safety valve
+ cluster.shutdown();
+ ExecutorUtil.shutdownAndAwaitTermination(exec);
+ }
+ }
+
+ /**
+ * On every request, prior to handling, releases a ticket to it's signal
Semaphre, and then blocks
+ * until it can acquire a ticket from it's gate semaphore. Has a safety
valve that gives up on the
+ * gate after 300 seconds
+ */
+ private static final class BlockingSearchHandler extends SearchHandler {
+ private final Semaphore gate;
+ private final Semaphore signal;
+
+ public BlockingSearchHandler(final Semaphore gate, final Semaphore signal)
{
+ super();
+ this.gate = gate;
+ this.signal = signal;
+ super.init(new NamedList<>());
+ }
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
+ log.info("Starting request");
+ signal.release();
+ if (gate.tryAcquire(300, TimeUnit.SECONDS)) {
+ super.handleRequestBody(req, rsp);
+ } else {
+ log.error("Gate safety valve timeout");
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Gate
timeout");
+ }
+ log.info("Finishing request");
+ }
+ }
+}
diff --git a/solr/server/etc/jetty-graceful.xml
b/solr/server/etc/jetty-graceful.xml
new file mode 100644
index 00000000000..34ec9316445
--- /dev/null
+++ b/solr/server/etc/jetty-graceful.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN"
"https://www.eclipse.org/jetty/configure_10_0.dtd";>
+
+<!-- =============================================================== -->
+<!-- Configure Jetty to finish existing requests on STOP. -->
+<!-- -->
+<!-- This is inspired by the "official" Jetty 12 graceful module, -->
+<!-- adapted to use StatisticsHandler for Jetty 10 and 11. -->
+<!-- -->
+<!-- IF THIS IS MODIFIED, JettySolrRunner MUST BE MODIFIED! -->
+<!-- =============================================================== -->
+
+<Configure id="Server" class="org.eclipse.jetty.server.Server">
+ <Call name="insertHandler">
+ <Arg>
+ <New id="GracefulHandler"
class="org.eclipse.jetty.server.handler.StatisticsHandler">
+ <Set name="gracefulShutdownWaitsForRequests">true</Set>
+ </New>
+ </Arg>
+ </Call>
+ <Set name="stopTimeout"><Property name="solr.jetty.stop.timeout"
default="15000"/></Set>
+</Configure>
diff --git a/solr/server/modules/graceful.mod b/solr/server/modules/graceful.mod
new file mode 100644
index 00000000000..581ebca898f
--- /dev/null
+++ b/solr/server/modules/graceful.mod
@@ -0,0 +1,8 @@
+[description]
+Enables Graceful processing of requests on STOP. Inspired by graceful.mod for
Jetty-12, this works with Jetty 10 and 11.
+
+[depend]
+server
+
+[xml]
+etc/jetty-graceful.xml
diff --git
a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index 5789b6b42f2..f17ec1d0fe5 100644
---
a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++
b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -853,9 +853,11 @@ public class MiniSolrCloudCluster {
try {
future.get();
} catch (ExecutionException e) {
+ log.error(message, e);
parsed.addSuppressed(e.getCause());
ok = false;
} catch (InterruptedException e) {
+ log.error(message, e);
Thread.interrupted();
throw e;
}
diff --git
a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java
b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java
index 92d8e4b2efc..76dd1add5f7 100644
--- a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java
+++ b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java
@@ -87,6 +87,7 @@ import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.handler.HandlerWrapper;
+import org.eclipse.jetty.server.handler.StatisticsHandler;
import org.eclipse.jetty.server.handler.gzip.GzipHandler;
import org.eclipse.jetty.server.session.DefaultSessionIdManager;
import org.eclipse.jetty.servlet.FilterHolder;
@@ -438,6 +439,12 @@ public class JettySolrRunner {
gzipHandler.setIncludedMethods("GET");
server.setHandler(gzipHandler);
+
+ // Mimic "graceful.mod"
+ final StatisticsHandler graceful = new StatisticsHandler();
+ graceful.setGracefulShutdownWaitsForRequests(true);
+ server.insertHandler(graceful);
+ server.setStopTimeout(15 * 1000);
}
/**
