This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new ee614dc3439 Add /status/ready endpoint for service health. (#19148)
ee614dc3439 is described below
commit ee614dc3439835e05cd1670bd5f145e675b98f8a
Author: Gian Merlino <[email protected]>
AuthorDate: Fri Mar 13 01:05:21 2026 -0700
Add /status/ready endpoint for service health. (#19148)
Currently the most natural endpoint to use for service health (e.g. if
adding Druid services to a load balancer) is /status/health. However,
this does not play nicely with graceful shutdown mechanisms.
When druid.server.http.unannouncePropagationDelay is used, there is a
delay between unannounce and server shutdown, which allows Druid's
internal service discovery to stop sending traffic to a service before
it shuts down its server. However, /status/health continues to return OK
until the server is shut down, so external load balancers cannot take
advantage of this.
This patch adds /status/ready, an endpoint that is tied to announcement.
It allows external load balancers to take advantage of this graceful
shutdown mechanism.
---
.../druid/curator/discovery/DiscoveryModule.java | 3 +
.../druid/server/ServiceAnnouncementState.java | 44 +++++++++++++++
.../org/apache/druid/server/StatusResource.java | 23 +++++++-
.../druid/server/ServiceAnnouncementStateTest.java | 64 ++++++++++++++++++++++
.../apache/druid/server/StatusResourceTest.java | 22 ++++++++
.../java/org/apache/druid/cli/CliOverlord.java | 3 +-
.../cli/CoordinatorJettyServerInitializer.java | 1 +
.../cli/MiddleManagerJettyServerInitializer.java | 3 +-
.../druid/cli/QueryJettyServerInitializer.java | 1 +
.../druid/cli/RouterJettyServerInitializer.java | 1 +
.../java/org/apache/druid/cli/ServerRunnable.java | 8 +++
11 files changed, 169 insertions(+), 4 deletions(-)
diff --git
a/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java
b/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java
index fec67cb3412..bd1ad64aaca 100644
---
a/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java
+++
b/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java
@@ -59,6 +59,7 @@ import org.apache.druid.guice.PolyBind;
import org.apache.druid.guice.annotations.Self;
import org.apache.druid.java.util.common.lifecycle.Lifecycle;
import org.apache.druid.server.DruidNode;
+import org.apache.druid.server.ServiceAnnouncementState;
import org.apache.druid.server.initialization.CuratorDiscoveryConfig;
import org.apache.druid.server.initialization.ZkPathsConfig;
@@ -163,6 +164,8 @@ public class DiscoveryModule implements Module
binder.bind(CuratorServiceAnnouncer.class).in(LazySingleton.class);
+ binder.bind(ServiceAnnouncementState.class).in(LazySingleton.class);
+
// Build the binder so that it will at a minimum inject an empty set.
DruidBinders.discoveryAnnouncementBinder(binder);
diff --git
a/server/src/main/java/org/apache/druid/server/ServiceAnnouncementState.java
b/server/src/main/java/org/apache/druid/server/ServiceAnnouncementState.java
new file mode 100644
index 00000000000..3e8d804583e
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/server/ServiceAnnouncementState.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server;
+
+/**
+ * Tracks whether this service has announced itself to service discovery and
is ready to receive traffic.
+ * Used by the {@code /status/ready} endpoint to report readiness to load
balancers.
+ */
+public class ServiceAnnouncementState
+{
+ private volatile boolean ready = false;
+
+ public void markReady()
+ {
+ ready = true;
+ }
+
+ public void markNotReady()
+ {
+ ready = false;
+ }
+
+ public boolean isReady()
+ {
+ return ready;
+ }
+}
diff --git a/server/src/main/java/org/apache/druid/server/StatusResource.java
b/server/src/main/java/org/apache/druid/server/StatusResource.java
index 29aaae98c25..4c971eae6b7 100644
--- a/server/src/main/java/org/apache/druid/server/StatusResource.java
+++ b/server/src/main/java/org/apache/druid/server/StatusResource.java
@@ -40,6 +40,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -59,19 +60,22 @@ public class StatusResource
private final DruidServerConfig druidServerConfig;
private final ExtensionsLoader extnLoader;
private final RuntimeInfo runtimeInfo;
+ private final ServiceAnnouncementState serviceAnnouncementState;
@Inject
public StatusResource(
final Properties properties,
final DruidServerConfig druidServerConfig,
final ExtensionsLoader extnLoader,
- final RuntimeInfo runtimeInfo
+ final RuntimeInfo runtimeInfo,
+ final ServiceAnnouncementState serviceAnnouncementState
)
{
this.properties = properties;
this.druidServerConfig = druidServerConfig;
this.extnLoader = extnLoader;
this.runtimeInfo = runtimeInfo;
+ this.serviceAnnouncementState = serviceAnnouncementState;
}
@GET
@@ -135,6 +139,23 @@ public class StatusResource
return true;
}
+ /**
+ * This is an unsecured endpoint, defined as such in UNSECURED_PATHS in the
service initialization files.
+ * Returns 200 when the service has announced itself and is ready to receive
traffic,
+ * or 503 when the service has not yet announced or has begun unannouncing
(e.g. during shutdown).
+ */
+ @GET
+ @Path("/ready")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getReady()
+ {
+ final boolean ready = serviceAnnouncementState.isReady();
+ if (ready) {
+ return Response.ok(true).build();
+ }
+ return
Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(false).build();
+ }
+
public static class Status
{
final String version;
diff --git
a/server/src/test/java/org/apache/druid/server/ServiceAnnouncementStateTest.java
b/server/src/test/java/org/apache/druid/server/ServiceAnnouncementStateTest.java
new file mode 100644
index 00000000000..cf60882b5be
--- /dev/null
+++
b/server/src/test/java/org/apache/druid/server/ServiceAnnouncementStateTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ServiceAnnouncementStateTest
+{
+ @Test
+ public void testInitialStateIsNotReady()
+ {
+ final ServiceAnnouncementState state = new ServiceAnnouncementState();
+ Assert.assertFalse(state.isReady());
+ }
+
+ @Test
+ public void testMarkReady()
+ {
+ final ServiceAnnouncementState state = new ServiceAnnouncementState();
+ state.markReady();
+ Assert.assertTrue(state.isReady());
+ }
+
+ @Test
+ public void testMarkNotReady()
+ {
+ final ServiceAnnouncementState state = new ServiceAnnouncementState();
+ state.markReady();
+ Assert.assertTrue(state.isReady());
+ state.markNotReady();
+ Assert.assertFalse(state.isReady());
+ }
+
+ @Test
+ public void testMultipleTransitions()
+ {
+ final ServiceAnnouncementState state = new ServiceAnnouncementState();
+ state.markReady();
+ state.markReady();
+ Assert.assertTrue(state.isReady());
+ state.markNotReady();
+ Assert.assertFalse(state.isReady());
+ state.markNotReady();
+ Assert.assertFalse(state.isReady());
+ }
+}
diff --git
a/server/src/test/java/org/apache/druid/server/StatusResourceTest.java
b/server/src/test/java/org/apache/druid/server/StatusResourceTest.java
index eb96b769f73..54dab17a40a 100644
--- a/server/src/test/java/org/apache/druid/server/StatusResourceTest.java
+++ b/server/src/test/java/org/apache/druid/server/StatusResourceTest.java
@@ -32,6 +32,7 @@ import org.apache.druid.utils.JvmUtils;
import org.junit.Assert;
import org.junit.Test;
+import javax.ws.rs.core.Response;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -78,6 +79,27 @@ public class StatusResourceTest
testHiddenPropertiesWithPropertyFileName("status.resource.test.runtime.hpc.properties");
}
+ @Test
+ public void testGetReadyReturns200WhenReady()
+ {
+ final ServiceAnnouncementState state = new ServiceAnnouncementState();
+ state.markReady();
+ final StatusResource resource = new StatusResource(new Properties(), null,
null, null, state);
+ final Response response = resource.getReady();
+ Assert.assertEquals(200, response.getStatus());
+ Assert.assertEquals(true, response.getEntity());
+ }
+
+ @Test
+ public void testGetReadyReturns503WhenNotReady()
+ {
+ final ServiceAnnouncementState state = new ServiceAnnouncementState();
+ final StatusResource resource = new StatusResource(new Properties(), null,
null, null, state);
+ final Response response = resource.getReady();
+ Assert.assertEquals(503, response.getStatus());
+ Assert.assertEquals(false, response.getEntity());
+ }
+
private void testHiddenPropertiesWithPropertyFileName(String fileName)
throws Exception
{
Injector injector = new StartupInjectorBuilder()
diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java
b/services/src/main/java/org/apache/druid/cli/CliOverlord.java
index 6b82029055e..1ae9b827f9e 100644
--- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java
+++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java
@@ -174,7 +174,8 @@ public class CliOverlord extends ServerRunnable
protected static final List<String> UNSECURED_PATHS = ImmutableList.of(
"/druid/indexer/v1/isLeader",
- "/status/health"
+ "/status/health",
+ "/status/ready"
);
private Properties properties;
diff --git
a/services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java
b/services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java
index 3fd6b41031b..ba32c3f666f 100644
---
a/services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java
+++
b/services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java
@@ -53,6 +53,7 @@ class CoordinatorJettyServerInitializer implements
JettyServerInitializer
"/coordinator/false",
"/overlord/false",
"/status/health",
+ "/status/ready",
"/druid/coordinator/v1/isLeader"
);
diff --git
a/services/src/main/java/org/apache/druid/cli/MiddleManagerJettyServerInitializer.java
b/services/src/main/java/org/apache/druid/cli/MiddleManagerJettyServerInitializer.java
index b338aa805fd..caa971c4efa 100644
---
a/services/src/main/java/org/apache/druid/cli/MiddleManagerJettyServerInitializer.java
+++
b/services/src/main/java/org/apache/druid/cli/MiddleManagerJettyServerInitializer.java
@@ -40,7 +40,6 @@ import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.handler.DefaultHandler;
-import java.util.Collections;
import java.util.List;
/**
@@ -59,7 +58,7 @@ class MiddleManagerJettyServerInitializer implements
JettyServerInitializer
this.authConfig = authConfig;
}
- private static List<String> UNSECURED_PATHS =
Collections.singletonList("/status/health");
+ private static final List<String> UNSECURED_PATHS =
List.of("/status/health", "/status/ready");
@Override
public void initialize(Server server, Injector injector)
diff --git
a/services/src/main/java/org/apache/druid/cli/QueryJettyServerInitializer.java
b/services/src/main/java/org/apache/druid/cli/QueryJettyServerInitializer.java
index a236e887e57..565b0dcf278 100644
---
a/services/src/main/java/org/apache/druid/cli/QueryJettyServerInitializer.java
+++
b/services/src/main/java/org/apache/druid/cli/QueryJettyServerInitializer.java
@@ -59,6 +59,7 @@ public class QueryJettyServerInitializer implements
JettyServerInitializer
private static final Logger log = new
Logger(QueryJettyServerInitializer.class);
private static List<String> UNSECURED_PATHS = Lists.newArrayList(
"/status/health",
+ "/status/ready",
"/druid/historical/v1/readiness",
"/druid/broker/v1/readiness"
);
diff --git
a/services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java
b/services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java
index 6ec4a38548c..3e657d49460 100644
---
a/services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java
+++
b/services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java
@@ -55,6 +55,7 @@ public class RouterJettyServerInitializer implements
JettyServerInitializer
{
private static final List<String> UNSECURED_PATHS = ImmutableList.of(
"/status/health",
+ "/status/ready",
// JDBC authentication uses the JDBC connection context instead of HTTP
headers, skip the normal auth checks.
// The router will keep the connection context in the forwarded message,
and the broker is responsible for
// performing the auth checks.
diff --git a/services/src/main/java/org/apache/druid/cli/ServerRunnable.java
b/services/src/main/java/org/apache/druid/cli/ServerRunnable.java
index 3dc1711a50e..0c4143c48ee 100644
--- a/services/src/main/java/org/apache/druid/cli/ServerRunnable.java
+++ b/services/src/main/java/org/apache/druid/cli/ServerRunnable.java
@@ -41,6 +41,7 @@ import org.apache.druid.java.util.common.lifecycle.Lifecycle;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.server.DruidNode;
+import org.apache.druid.server.ServiceAnnouncementState;
import java.lang.annotation.Annotation;
import java.util.Collections;
@@ -135,6 +136,9 @@ public abstract class ServerRunnable extends GuiceRunnable
@Inject
private Map<NodeRole, Set<Class<? extends DruidService>>> serviceClasses;
+ @Inject
+ private ServiceAnnouncementState serviceAnnouncementState;
+
private final boolean useLegacyAnnouncer;
public static DiscoverySideEffectsProvider create()
@@ -181,11 +185,15 @@ public abstract class ServerRunnable extends GuiceRunnable
if (useLegacyAnnouncer) {
legacyAnnouncer.announce(discoveryDruidNode.getDruidNode());
}
+
+ serviceAnnouncementState.markReady();
}
@Override
public void stop()
{
+ serviceAnnouncementState.markNotReady();
+
// Reverse order vs. start().
if (useLegacyAnnouncer) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
