This is an automated email from the ASF dual-hosted git repository.
jojochuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 11ba2a1f13c HDDS-16360. Add liveness and readiness health endpoints to
S3 Gateway (#11224)
11ba2a1f13c is described below
commit 11ba2a1f13c8d4fd9fb1f030f1fd7bad425fcfb3
Author: Andrey Yarovoy <[email protected]>
AuthorDate: Thu Sep 17 18:00:55 2026 -0400
HDDS-16360. Add liveness and readiness health endpoints to S3 Gateway
(#11224)
Generated-by: Claude Code (Claude Opus 4.8)
---
hadoop-hdds/docs/content/interface/S3.md | 24 +++
.../ozone/conf/S3GatewayHealthCheckConfig.java | 95 ++++++++++++
.../dist/src/main/compose/common/s3-haproxy.cfg | 11 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
.../src/main/compose/ozonesecure-ha/s3-haproxy.cfg | 11 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 13 ++
.../src/main/k8s/definitions/ozone/s3g-ss.yaml | 6 +-
.../examples/getting-started/s3g-statefulset.yaml | 6 +-
.../k8s/examples/minikube/s3g-statefulset.yaml | 6 +-
.../k8s/examples/ozone-dev/s3g-statefulset.yaml | 6 +-
.../k8s/examples/ozone-ha/s3g-statefulset.yaml | 6 +-
.../main/k8s/examples/ozone/s3g-statefulset.yaml | 6 +-
.../dist/src/main/smoketest/s3/health.robot | 43 ++++++
.../hadoop/ozone/s3/TestS3GatewayHealthCheck.java | 126 ++++++++++++++++
hadoop-ozone/s3gateway/pom.xml | 4 +
.../hadoop/ozone/s3/S3GatewayReadinessProbe.java | 164 +++++++++++++++++++++
.../hadoop/ozone/s3/S3GatewayWebAdminServer.java | 81 ++++++++++
.../ozone/s3/TestS3GatewayLivenessServlet.java | 126 ++++++++++++++++
18 files changed, 724 insertions(+), 12 deletions(-)
diff --git a/hadoop-hdds/docs/content/interface/S3.md
b/hadoop-hdds/docs/content/interface/S3.md
index b43abb13db5..5fb5f4edbd0 100644
--- a/hadoop-hdds/docs/content/interface/S3.md
+++ b/hadoop-hdds/docs/content/interface/S3.md
@@ -161,6 +161,30 @@ ozone s3 revokesecret -u om/[email protected] -y
S3 secret revoked.
```
+## Health check
+
+The S3 Gateway serves two unauthenticated health endpoints on its web admin
server (default port `19878`), separate from the S3 listener (`9878`). Because
they are served on the admin port, they do not collide with S3 bucket names and
do not require AWS SigV4 signing, so a load balancer can poll them directly.
+
+### Liveness
+
+```bash
+curl http://localhost:19878/health/live
+OK
+```
+
+Returns `200 OK` while the gateway process is up and its web admin server can
serve requests. It does not check OM reachability.
+
+### Readiness
+
+```bash
+curl http://localhost:19878/health/ready
+READY
+```
+
+Returns `200 READY` only when the gateway can reach OM, and `503 NOT READY`
during startup or while OM is unreachable. A background probe refreshes OM
reachability off the request path, so the endpoint always responds immediately
and cannot be blocked by a slow OM. The probe cadence and its timeout are
tunable with `ozone.s3g.health-check.probe.interval` and
`ozone.s3g.health-check.probe.timeout`.
+
+When S3 Gateway is horizontally scaled, point the load balancer's HTTP health
check at the readiness endpoint so it only routes S3 traffic to gateways that
can serve it; see the bundled `compose/common/s3-haproxy.cfg` for a worked
HAProxy example. Both endpoints can be turned off with
`ozone.s3g.health-check.enabled=false`, which removes them entirely. Note that
the bundled Kubernetes manifests and HAProxy configs probe `/health/live` and
`/health/ready` unconditionally, so if you disabl [...]
+
## Expose any volume
Ozone has one more element in the name-space hierarchy compared to S3: the
volumes. By default, all the buckets of the `/s3v` volume can be accessed with
S3 interface but only the (Ozone) buckets of the `/s3v` volumes are exposed.
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/S3GatewayHealthCheckConfig.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/S3GatewayHealthCheckConfig.java
new file mode 100644
index 00000000000..3ced4a9ff53
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/conf/S3GatewayHealthCheckConfig.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.conf;
+
+import static org.apache.hadoop.hdds.conf.ConfigTag.MANAGEMENT;
+import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
+import static org.apache.hadoop.hdds.conf.ConfigTag.S3GATEWAY;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigType;
+
+/**
+ * Config for the S3 Gateway health check endpoints served on the web admin
+ * server (default port 19878).
+ */
+@ConfigGroup(prefix = "ozone.s3g.health-check")
+public class S3GatewayHealthCheckConfig {
+
+ @Config(key = "ozone.s3g.health-check.enabled",
+ defaultValue = "true",
+ type = ConfigType.BOOLEAN,
+ tags = {OZONE, S3GATEWAY, MANAGEMENT},
+ description = "If enabled, the S3 Gateway web admin server exposes " +
+ "unauthenticated health endpoints that a load balancer can poll: " +
+ "a liveness endpoint at /health/live (process is up) and a " +
+ "readiness endpoint at /health/ready (the gateway can reach OM). " +
+ "Disable to remove the endpoints entirely; the bundled Kubernetes " +
+ "manifests and HAProxy configs probe these paths unconditionally, " +
+ "so disabling here also requires removing those probes from the " +
+ "deployment, or liveness checks will restart the pods and HAProxy " +
+ "will mark the backends down.")
+ private boolean enabled = true;
+
+ @Config(key = "ozone.s3g.health-check.probe.interval",
+ defaultValue = "10s",
+ type = ConfigType.TIME,
+ timeUnit = TimeUnit.MILLISECONDS,
+ tags = {OZONE, S3GATEWAY, MANAGEMENT},
+ description = "How often the readiness endpoint refreshes its cached " +
+ "view of OM reachability in the background. The probe runs off the "
+
+ "request path so /health/ready always responds immediately.")
+ private long probeInterval = 10 * 1000;
+
+ @Config(key = "ozone.s3g.health-check.probe.timeout",
+ defaultValue = "10s",
+ type = ConfigType.TIME,
+ timeUnit = TimeUnit.MILLISECONDS,
+ tags = {OZONE, S3GATEWAY, MANAGEMENT},
+ description = "Upper bound on a single background readiness probe to " +
+ "OM. If a probe does not complete within this time the gateway is " +
+ "reported as not ready, so a stuck OM cannot leave readiness " +
+ "stuck at ready.")
+ private long probeTimeout = 10 * 1000;
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ public long getProbeInterval() {
+ return probeInterval;
+ }
+
+ public void setProbeInterval(long probeInterval) {
+ this.probeInterval = probeInterval;
+ }
+
+ public long getProbeTimeout() {
+ return probeTimeout;
+ }
+
+ public void setProbeTimeout(long probeTimeout) {
+ this.probeTimeout = probeTimeout;
+ }
+}
diff --git a/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg
b/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg
index 6e8728945f2..9f82f51c7f8 100644
--- a/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg
+++ b/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg
@@ -33,9 +33,14 @@ frontend http-in
backend servers
balance roundrobin
- server server1 s3g1:9878 maxconn 32
- server server2 s3g2:9878 maxconn 32
- server server3 s3g3:9878 maxconn 32
+ # Gate S3 traffic (9878) on the readiness endpoint served by the web admin
+ # server (19878): it reports ready only when the gateway can reach OM. The
+ # endpoint is unauthenticated, so no SigV4 is needed.
+ option httpchk GET /health/ready
+ http-check expect status 200
+ server server1 s3g1:9878 maxconn 32 check port 19878 inter 2s rise 1
+ server server2 s3g2:9878 maxconn 32 check port 19878 inter 2s rise 1
+ server server3 s3g3:9878 maxconn 32 check port 19878 inter 2s rise 1
frontend webadmin
bind *:19878
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index 980d1487f80..e4e27ede99d 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -57,6 +57,8 @@ execute_robot_test scm -v SCHEME:ofs -N ozonefs-obs
ozonefs/ozonefs-obs.robot
execute_robot_test s3g grpc/grpc-om-s3-metrics.robot
+execute_robot_test s3g s3/health.robot
+
execute_robot_test scm --exclude om_filesystem --exclude
pre-finalized-snapshot-tests snapshot
# snapshot-defrag.robot reads OmSnapshot local YAML under the OM data
directory; Robot must run in the om container.
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.cfg
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.cfg
index c93d1b16b64..441486af58f 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.cfg
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3-haproxy.cfg
@@ -33,9 +33,14 @@ frontend http-in
backend servers
balance roundrobin
- server server1 172.25.0.121:9878 maxconn 32
- server server2 172.25.0.122:9878 maxconn 32
- server server3 172.25.0.123:9878 maxconn 32
+ # Gate S3 traffic (9878) on the readiness endpoint served by the web admin
+ # server (19878): it reports ready only when the gateway can reach OM. The
+ # endpoint is unauthenticated, so no SigV4 is needed.
+ option httpchk GET /health/ready
+ http-check expect status 200
+ server server1 172.25.0.121:9878 maxconn 32 check port 19878 inter 2s rise
1
+ server server2 172.25.0.122:9878 maxconn 32 check port 19878 inter 2s rise
1
+ server server3 172.25.0.123:9878 maxconn 32 check port 19878 inter 2s rise
1
frontend webadmin
bind *:19878
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh
b/hadoop-ozone/dist/src/main/compose/testlib.sh
index c541a234b71..d9f3868bcdd 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -115,6 +115,18 @@ find_tests(){
}
## @description wait until safemode exit (or 240 seconds)
+## @description wait until S3 Gateway reports ready via /health/ready (port
19878)
+## Required when HAProxy health-checks gate S3 traffic: without a wait, tests
+## start while all backends are still in DOWN state and HAProxy returns 503.
+## Only waits when the multi-instance HAProxy setup (s3g1/s3g2/s3g3) is
present.
+wait_for_s3g_ready() {
+ if ! docker-compose config --services 2>/dev/null | grep -q "^s3g1$"; then
+ return 0
+ fi
+ echo "Waiting for S3 Gateway (s3g1) to report ready..."
+ wait_for_execute_command s3g1 120 "curl -sf
http://localhost:19878/health/ready";
+}
+
wait_for_safemode_exit(){
local cmd="ozone admin safemode wait -t 240"
if [[ "${SECURITY_ENABLED}" == 'true' ]]; then
@@ -207,6 +219,7 @@ start_docker_env(){
wait_for_safemode_exit
wait_for_om_leader
+ wait_for_s3g_ready
}
has_scalable_datanode() {
diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss.yaml
b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss.yaml
index deb9cfe7b16..2ee375a75a1 100644
--- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss.yaml
@@ -38,6 +38,10 @@ spec:
args: ["ozone","s3g"]
livenessProbe:
httpGet:
- path: /
+ path: /health/live
port: 19878
initialDelaySeconds: 30
+ readinessProbe:
+ httpGet:
+ path: /health/ready
+ port: 19878
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/s3g-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/s3g-statefulset.yaml
index ecfd85801ba..319a04f9caf 100644
---
a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/s3g-statefulset.yaml
+++
b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/s3g-statefulset.yaml
@@ -41,9 +41,13 @@ spec:
- s3g
livenessProbe:
httpGet:
- path: /
+ path: /health/live
port: 19878
initialDelaySeconds: 30
+ readinessProbe:
+ httpGet:
+ path: /health/ready
+ port: 19878
envFrom:
- configMapRef:
name: config
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-statefulset.yaml
index ecfd85801ba..319a04f9caf 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-statefulset.yaml
@@ -41,9 +41,13 @@ spec:
- s3g
livenessProbe:
httpGet:
- path: /
+ path: /health/live
port: 19878
initialDelaySeconds: 30
+ readinessProbe:
+ httpGet:
+ path: /health/ready
+ port: 19878
envFrom:
- configMapRef:
name: config
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml
index 7e405922fd8..37dc42f8a97 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml
@@ -41,9 +41,13 @@ spec:
- s3g
livenessProbe:
httpGet:
- path: /
+ path: /health/live
port: 19878
initialDelaySeconds: 30
+ readinessProbe:
+ httpGet:
+ path: /health/ready
+ port: 19878
envFrom:
- configMapRef:
name: config
diff --git
a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/s3g-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/s3g-statefulset.yaml
index 1ed0deb3099..b7bc8f525a2 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/s3g-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/s3g-statefulset.yaml
@@ -41,9 +41,13 @@ spec:
- s3g
livenessProbe:
httpGet:
- path: /
+ path: /health/live
port: 19878
initialDelaySeconds: 30
+ readinessProbe:
+ httpGet:
+ path: /health/ready
+ port: 19878
envFrom:
- configMapRef:
name: config
diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml
b/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml
index 1ed0deb3099..b7bc8f525a2 100644
--- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml
+++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml
@@ -41,9 +41,13 @@ spec:
- s3g
livenessProbe:
httpGet:
- path: /
+ path: /health/live
port: 19878
initialDelaySeconds: 30
+ readinessProbe:
+ httpGet:
+ path: /health/ready
+ port: 19878
envFrom:
- configMapRef:
name: config
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/health.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/health.robot
new file mode 100644
index 00000000000..35cdb34749e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/health.robot
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation S3 gateway health endpoints test
+Library OperatingSystem
+Library String
+Resource ../commonlib.robot
+Test Timeout 5 minutes
+Default Tags no-bucket-type
+
+*** Variables ***
+
+${S3G_HEALTH} http://s3g:19878
+
+*** Keywords ***
+# The health endpoints are unauthenticated, so a plain curl (no SPNEGO) must
+# succeed even when security is enabled.
+Readiness reports ready
+ ${result} = Execute curl -sS -i
${S3G_HEALTH}/health/ready
+ Should contain ${result} 200
+ Should contain ${result} READY
+
+*** Test Cases ***
+Liveness endpoint returns OK
+ ${result} = Execute curl -sS -i
${S3G_HEALTH}/health/live
+ Should contain ${result} 200
+ Should contain ${result} OK
+
+Readiness endpoint reports ready when OM is reachable
+ Wait Until Keyword Succeeds 30sec 5sec Readiness reports
ready
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayHealthCheck.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayHealthCheck.java
new file mode 100644
index 00000000000..562daef81c4
--- /dev/null
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayHealthCheck.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3;
+
+import static java.net.HttpURLConnection.HTTP_OK;
+import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.ozone.test.ClusterForTests;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+/**
+ * Integration test for the S3 Gateway health endpoints against a live
+ * {@link MiniOzoneCluster}. Covers both the ready path (OM reachable) and the
+ * not-ready path (OM stopped). The OM-stop step is destructive and the
cluster
+ * is shared across the class, so the checks are ordered.
+ */
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class TestS3GatewayHealthCheck extends ClusterForTests<MiniOzoneCluster> {
+
+ private final S3GatewayService s3g = new S3GatewayService();
+
+ @Override
+ protected OzoneConfiguration createOzoneConfig() {
+ OzoneConfiguration conf = createBaseConfiguration();
+ // Poll OM often and time out quickly so readiness flips fast in the test.
+ conf.setTimeDuration("ozone.s3g.health-check.probe.interval", 500,
TimeUnit.MILLISECONDS);
+ conf.setTimeDuration("ozone.s3g.health-check.probe.timeout", 2,
TimeUnit.SECONDS);
+ return conf;
+ }
+
+ @Override
+ protected MiniOzoneCluster createCluster() throws Exception {
+ return newClusterBuilder()
+ .addService(s3g)
+ .build();
+ }
+
+ @Test
+ @Order(1)
+ void livenessReturnsOk() throws Exception {
+ assertEquals(HTTP_OK, responseCode(healthUrl("/health/live")));
+ assertEquals("OK", body(healthUrl("/health/live")));
+ }
+
+ @Test
+ @Order(2)
+ void readinessReturnsReadyWhenOmReachable() throws Exception {
+ GenericTestUtils.waitFor(() -> responseCode(healthUrl("/health/ready")) ==
HTTP_OK,
+ 200, 30_000);
+ assertEquals("READY", body(healthUrl("/health/ready")));
+ }
+
+ @Test
+ @Order(3)
+ void readinessReturnsNotReadyWhenOmUnreachable() throws Exception {
+ getCluster().getOzoneManager().stop();
+
+ GenericTestUtils.waitFor(() -> responseCode(healthUrl("/health/ready")) ==
HTTP_UNAVAILABLE,
+ 200, 30_000);
+ assertEquals("NOT READY", body(healthUrl("/health/ready")));
+
+ // Liveness stays up regardless of OM reachability.
+ assertEquals(HTTP_OK, responseCode(healthUrl("/health/live")));
+ }
+
+ private String healthUrl(String path) {
+ String address = s3g.getConf()
+ .get(S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_ADDRESS_KEY);
+ return "http://"; + address + path;
+ }
+
+ private static int responseCode(String url) {
+ HttpURLConnection connection = null;
+ try {
+ connection = (HttpURLConnection) new URL(url).openConnection();
+ connection.setRequestMethod("GET");
+ return connection.getResponseCode();
+ } catch (Exception e) {
+ return -1;
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
+ }
+
+ private static String body(String url) throws Exception {
+ HttpURLConnection connection = (HttpURLConnection) new
URL(url).openConnection();
+ connection.setRequestMethod("GET");
+ int code = connection.getResponseCode();
+ try (InputStream in = code < 400
+ ? connection.getInputStream() : connection.getErrorStream()) {
+ return IOUtils.toString(in, StandardCharsets.UTF_8).trim();
+ } finally {
+ connection.disconnect();
+ }
+ }
+}
diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml
index 7a27e1d4b53..007e399f929 100644
--- a/hadoop-ozone/s3gateway/pom.xml
+++ b/hadoop-ozone/s3gateway/pom.xml
@@ -171,6 +171,10 @@
<groupId>org.apache.ratis</groupId>
<artifactId>ratis-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayReadinessProbe.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayReadinessProbe.java
new file mode 100644
index 00000000000..65b4fff043a
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayReadinessProbe.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.io.Closeable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.conf.S3GatewayHealthCheckConfig;
+import org.apache.hadoop.ozone.om.protocol.S3Auth;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Backs the S3 Gateway readiness endpoint (/health/ready).
+ *
+ * <p>A single background thread periodically probes OM reachability and stores
+ * the result in a volatile flag. The servlet only reads that flag, so
+ * /health/ready responds immediately (no OM RPC on the request path) and a
+ * load balancer polling it can never be blocked by a slow or unreachable OM.
+ *
+ * <p>Readiness starts as {@code false} and flips to {@code true} only after a
+ * probe succeeds, so the endpoint reports "not ready" during startup (before
+ * the first successful probe) without waiting for the first S3 request. The
+ * probe owns a dedicated {@link OzoneClient} created through
+ * {@link OzoneClientCache#createClient}, so it exercises the same OM transport
+ * the gateway serves with, independent of the lazily-created CDI client on the
+ * S3 listener.
+ */
+public class S3GatewayReadinessProbe implements Closeable {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(S3GatewayReadinessProbe.class);
+
+ private final OzoneConfiguration conf;
+ private final long intervalMillis;
+ private final long timeoutMillis;
+
+ private final ScheduledExecutorService scheduler;
+ private final ExecutorService probeExecutor;
+
+ private volatile boolean ready = false;
+ private volatile OzoneClient client;
+
+ /**
+ * The probe task submitted on the previous tick, accessed only from the
+ * single scheduler thread. Used to keep at most one probe in flight: if the
+ * task is still running (e.g. an OM call that did not unblock on
+ * interruption), the next tick skips submitting another rather than queueing
+ * work on the single probe thread.
+ */
+ private Future<Void> inFlight;
+
+ S3GatewayReadinessProbe(OzoneConfiguration conf,
+ S3GatewayHealthCheckConfig healthConfig) {
+ this.conf = conf;
+ this.intervalMillis = healthConfig.getProbeInterval();
+ this.timeoutMillis = healthConfig.getProbeTimeout();
+ this.scheduler = Executors.newSingleThreadScheduledExecutor(
+ new ThreadFactoryBuilder()
+ .setNameFormat("s3g-readiness-probe-%d")
+ .setDaemon(true)
+ .build());
+ this.probeExecutor = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder()
+ .setNameFormat("s3g-readiness-om-%d")
+ .setDaemon(true)
+ .build());
+ }
+
+ public void start() {
+ scheduler.scheduleWithFixedDelay(this::probe, 0, intervalMillis,
+ TimeUnit.MILLISECONDS);
+ }
+
+ public boolean isReady() {
+ return ready;
+ }
+
+ /**
+ * Runs one probe on the scheduler thread, bounded by the configured timeout.
+ * The OM call runs on a separate single-threaded executor so a hung call
+ * cannot block the scheduler and gets an enforced deadline: on timeout the
+ * gateway is reported not ready rather than staying stuck at ready. At most
+ * one probe is kept in flight, so a call that ignores interruption cannot
pile
+ * up queued work on the probe thread during a prolonged OM outage.
+ */
+ private void probe() {
+ if (inFlight != null && !inFlight.isDone()) {
+ // The previous probe's OM call has not returned yet (e.g. it did not
+ // unblock on interruption). Report not ready and retry cancelling rather
+ // than submitting another task onto the single probe thread.
+ ready = false;
+ inFlight.cancel(true);
+ LOG.debug("Previous readiness probe still in flight; reporting not
ready");
+ return;
+ }
+ Future<Void> future = probeExecutor.submit(() -> {
+ OzoneClient c = client;
+ if (c == null) {
+ c = OzoneClientCache.createClient(probeClientConf());
+ client = c;
+ }
+ c.getObjectStore().getClientProxy().getOzoneManagerClient()
+ .getServiceInfo();
+ return null;
+ });
+ inFlight = future;
+ try {
+ future.get(timeoutMillis, TimeUnit.MILLISECONDS);
+ ready = true;
+ } catch (TimeoutException e) {
+ ready = false;
+ future.cancel(true);
+ LOG.debug("Readiness probe timed out after {} ms", timeoutMillis);
+ } catch (Exception e) {
+ ready = false;
+ LOG.debug("Readiness probe failed; reporting not ready", e);
+ }
+ }
+
+ /**
+ * A copy of the gateway config tuned for probing: S3 auth is disabled
+ * (getServiceInfo does not need S3 credentials) and the OM RPC timeout is
+ * bounded so a stuck probe thread eventually unblocks.
+ */
+ private OzoneConfiguration probeClientConf() {
+ OzoneConfiguration probeConf = new OzoneConfiguration(conf);
+ probeConf.setBoolean(S3Auth.S3_AUTH_CHECK, false);
+ probeConf.setBoolean("ipc.client.ping", false);
+ probeConf.setTimeDuration("ozone.om.client.rpc.timeout", timeoutMillis,
+ TimeUnit.MILLISECONDS);
+ return probeConf;
+ }
+
+ @Override
+ public void close() {
+ scheduler.shutdownNow();
+ probeExecutor.shutdownNow();
+ IOUtils.close(LOG, client);
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayWebAdminServer.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayWebAdminServer.java
index f87343cfb92..d60228d87fc 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayWebAdminServer.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayWebAdminServer.java
@@ -43,8 +43,10 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.server.http.BaseHttpServer;
import org.apache.hadoop.hdds.server.http.ServletElementsFactory;
+import org.apache.hadoop.ozone.conf.S3GatewayHealthCheckConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
@@ -62,12 +64,48 @@ class S3GatewayWebAdminServer extends BaseHttpServer {
private static final Logger LOG =
LoggerFactory.getLogger(S3GatewayWebAdminServer.class);
+ static final String READINESS_PROBE_ATTRIBUTE =
+ "ozone.s3g.readiness.probe";
+
+ private final S3GatewayReadinessProbe readinessProbe;
+
S3GatewayWebAdminServer(MutableConfigurationSource conf, String name) throws
IOException {
super(conf, name);
addServlet("icon", "/favicon.ico", IconServlet.class);
+ S3GatewayHealthCheckConfig healthConfig =
+ conf.getObject(S3GatewayHealthCheckConfig.class);
+ if (healthConfig.isEnabled()) {
+ // Register via addInternalServlet so no authentication filter is mapped
+ // to the paths: the endpoints must be reachable by a load balancer
+ // without Kerberos/SPNEGO, including in secure mode.
+ addInternalServlet("liveness", "/health/live", LivenessServlet.class);
+ readinessProbe =
+ new S3GatewayReadinessProbe(OzoneConfiguration.of(conf),
healthConfig);
+ getWebAppContext().getServletContext()
+ .setAttribute(READINESS_PROBE_ATTRIBUTE, readinessProbe);
+ addInternalServlet("readiness", "/health/ready", ReadinessServlet.class);
+ } else {
+ readinessProbe = null;
+ }
addSecretAuthentication(conf);
}
+ @Override
+ public void start() throws IOException {
+ super.start();
+ if (readinessProbe != null) {
+ readinessProbe.start();
+ }
+ }
+
+ @Override
+ public void stop() throws Exception {
+ if (readinessProbe != null) {
+ readinessProbe.close();
+ }
+ super.stop();
+ }
+
private void addSecretAuthentication(MutableConfigurationSource conf)
throws IOException {
@@ -183,4 +221,47 @@ public void doGet(HttpServletRequest request,
HttpServletResponse response)
response.sendRedirect("/images/ozone.ico");
}
}
+
+ /**
+ * Liveness probe: returns 200 as long as the gateway process is up and its
+ * web admin server can serve requests. It does not check OM reachability;
+ * that is the responsibility of the readiness endpoint.
+ */
+ public static class LivenessServlet extends HttpServlet {
+ private static final long serialVersionUID = -1L;
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws IOException {
+ response.setStatus(HttpServletResponse.SC_OK);
+ response.setContentType("text/plain; charset=utf-8");
+ response.getWriter().println("OK");
+ }
+ }
+
+ /**
+ * Readiness probe: returns 200 only when a background probe has confirmed
the
+ * gateway can reach OM, and 503 otherwise (during startup or while OM is
+ * unreachable). The check reads a cached flag maintained off the request
+ * path, so the endpoint always responds immediately and a load balancer
+ * polling it cannot be blocked by a slow OM.
+ */
+ public static class ReadinessServlet extends HttpServlet {
+ private static final long serialVersionUID = -1L;
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws IOException {
+ S3GatewayReadinessProbe probe = (S3GatewayReadinessProbe)
+ getServletContext().getAttribute(READINESS_PROBE_ATTRIBUTE);
+ response.setContentType("text/plain; charset=utf-8");
+ if (probe != null && probe.isReady()) {
+ response.setStatus(HttpServletResponse.SC_OK);
+ response.getWriter().println("READY");
+ } else {
+ response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+ response.getWriter().println("NOT READY");
+ }
+ }
+ }
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayLivenessServlet.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayLivenessServlet.java
new file mode 100644
index 00000000000..4add70246b3
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayLivenessServlet.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.conf.S3GatewayHealthCheckConfig;
+import org.apache.hadoop.ozone.s3.S3GatewayWebAdminServer.LivenessServlet;
+import org.apache.hadoop.ozone.s3.S3GatewayWebAdminServer.ReadinessServlet;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for the S3 Gateway health endpoints and their config toggle.
+ */
+public class TestS3GatewayLivenessServlet {
+
+ @Test
+ public void livenessReturnsOk() throws Exception {
+ HttpServletRequest request = mock(HttpServletRequest.class);
+ HttpServletResponse response = mock(HttpServletResponse.class);
+ StringWriter body = new StringWriter();
+ when(response.getWriter()).thenReturn(new PrintWriter(body));
+
+ new LivenessServlet().doGet(request, response);
+
+ verify(response).setStatus(HttpServletResponse.SC_OK);
+ assertEquals("OK", body.toString().trim());
+ }
+
+ @Test
+ public void readinessReturnsOkWhenReady() throws Exception {
+ StringWriter body = new StringWriter();
+ HttpServletResponse response = invokeReadiness(readyProbe(true), body);
+ verify(response).setStatus(HttpServletResponse.SC_OK);
+ assertEquals("READY", body.toString().trim());
+ }
+
+ @Test
+ public void readinessReturns503WhenNotReady() throws Exception {
+ StringWriter body = new StringWriter();
+ HttpServletResponse response = invokeReadiness(readyProbe(false), body);
+ verify(response).setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+ assertEquals("NOT READY", body.toString().trim());
+ }
+
+ @Test
+ public void readinessReturns503WhenProbeMissing() throws Exception {
+ HttpServletResponse response = invokeReadiness(null, new StringWriter());
+ verify(response).setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+ }
+
+ @Test
+ public void readinessStartsNotReady() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ S3GatewayReadinessProbe probe = new S3GatewayReadinessProbe(conf,
+ conf.getObject(S3GatewayHealthCheckConfig.class));
+ try {
+ assertFalse(probe.isReady());
+ } finally {
+ probe.close();
+ }
+ }
+
+ @Test
+ public void healthCheckEnabledByDefault() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ assertTrue(conf.getObject(S3GatewayHealthCheckConfig.class).isEnabled());
+ }
+
+ @Test
+ public void healthCheckCanBeDisabled() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean("ozone.s3g.health-check.enabled", false);
+ assertFalse(conf.getObject(S3GatewayHealthCheckConfig.class).isEnabled());
+ }
+
+ private static S3GatewayReadinessProbe readyProbe(boolean ready) {
+ S3GatewayReadinessProbe probe = mock(S3GatewayReadinessProbe.class);
+ when(probe.isReady()).thenReturn(ready);
+ return probe;
+ }
+
+ private static HttpServletResponse invokeReadiness(
+ S3GatewayReadinessProbe probe, StringWriter body) throws Exception {
+ ReadinessServlet servlet = spy(new ReadinessServlet());
+ ServletContext context = mock(ServletContext.class);
+
when(context.getAttribute(S3GatewayWebAdminServer.READINESS_PROBE_ATTRIBUTE))
+ .thenReturn(probe);
+ doReturn(context).when(servlet).getServletContext();
+
+ HttpServletRequest request = mock(HttpServletRequest.class);
+ HttpServletResponse response = mock(HttpServletResponse.class);
+ when(response.getWriter()).thenReturn(new PrintWriter(body));
+
+ servlet.doGet(request, response);
+ return response;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
