This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6ef19acbbc2 HDDS-15552. Ratis events should not be published as
metrics (#10523)
6ef19acbbc2 is described below
commit 6ef19acbbc288080748d87c140c9d8a84d94bdbc
Author: Wei-Chiu Chuang <[email protected]>
AuthorDate: Mon Jul 6 23:40:16 2026 -0700
HDDS-15552. Ratis events should not be published as metrics (#10523)
---
.../container/placement/metrics/SCMMetrics.java | 4 +-
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 2 +
.../hdds/scm/server/StorageContainerManager.java | 5 ++
.../src/main/resources/webapps/scm/scm.js | 4 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
.../src/main/smoketest/prometheus/prometheus.robot | 29 +++++++++
.../main/smoketest/prometheus/prometheus_check.py | 76 ++++++++++++++++++++++
.../org/apache/hadoop/hdds/scm/TestSCMMXBean.java | 3 +
.../java/org/apache/hadoop/ozone/om/OMMXBean.java | 2 +
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 5 ++
.../resources/webapps/ozoneManager/ozoneManager.js | 4 +-
12 files changed, 134 insertions(+), 6 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java
index d5dab7800c4..b746f7eaea1 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java
@@ -180,7 +180,9 @@ public void addRatisEvent(String event) {
}
}
- @Metric("Ratis state machine events")
+ // Ratis state machine events are multi-line logs, which should not be
+ // published as time-series metrics to metrics systems like Prometheus.
+ // Instead, they are exposed via JMX / MXBean endpoints.
public String getRatisEvents() {
synchronized (ratisEvents) {
return String.join("\n", ratisEvents);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java
index 947484864e4..0fcc4625387 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java
@@ -82,4 +82,6 @@ public interface SCMMXBean extends ServiceRuntimeInfo {
* @return the SCM hostname for the datanode.
*/
String getHostname();
+
+ String getRatisEvents();
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 59a04d6a9a5..13e297b9cae 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -2243,6 +2243,11 @@ public String getHostname() {
return scmHostName;
}
+ @Override
+ public String getRatisEvents() {
+ return metrics != null ? metrics.getRatisEvents() : "";
+ }
+
public Collection<String> getScmAdminUsernames() {
return scmAdmins.getAdminUsernames();
}
diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
index 27ecc7f8155..73ecda3d49c 100644
--- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
+++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
@@ -30,10 +30,10 @@
templateUrl: 'ratis-events.html',
controller: function ($http) {
var ctrl = this;
-
$http.get("jmx?qry=Hadoop:service=StorageContainerManager,name=SCMMetrics")
+
$http.get("jmx?qry=Hadoop:service=StorageContainerManager,name=StorageContainerManagerInfo,component=ServerRuntime")
.then(function (result) {
var metrics = result.data.beans[0];
- var rawEvents = metrics['tag.RatisEvents'] ?
metrics['tag.RatisEvents'].split('\n') : [];
+ var rawEvents = (metrics && metrics['RatisEvents']) ?
metrics['RatisEvents'].split('\n') : [];
ctrl.events = rawEvents.map(function(e) {
var parts = e.split('|');
return {
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index 800f1c41f28..980d1487f80 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -24,6 +24,7 @@ export COMPOSE_DIR
export SECURITY_ENABLED=false
export OZONE_REPLICATION_FACTOR=3
+export COMPOSE_FILE=docker-compose.yaml:monitoring.yaml
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
@@ -40,6 +41,7 @@ execute_robot_test scm gdpr
execute_robot_test scm security/ozone-secure-token.robot
execute_robot_test scm recon
+execute_robot_test scm prometheus
execute_robot_test scm om-ratis
diff --git a/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus.robot
b/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus.robot
new file mode 100644
index 00000000000..11cc7d6031a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus.robot
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test Prometheus monitoring integration
+Library OperatingSystem
+Library BuiltIn
+Resource ../commonlib.robot
+
+*** Test Cases ***
+Verify Prometheus targets are healthy
+ Wait Until Keyword Succeeds 90sec 10sec Check Prometheus
Targets Health
+
+*** Keywords ***
+Check Prometheus Targets Health
+ ${result} = Execute python3
${OZONE_DIR}/smoketest/prometheus/prometheus_check.py
+ Should Contain ${result} Successfully verified
diff --git
a/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus_check.py
b/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus_check.py
new file mode 100644
index 00000000000..6fe85be8341
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus_check.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import urllib.request
+import json
+import sys
+import socket
+
+def is_running(host, port):
+ try:
+ with socket.create_connection((host, int(port)), timeout=2):
+ return True
+ except Exception:
+ return False
+
+def main():
+ try:
+ res = urllib.request.urlopen("http://prometheus:9090/api/v1/targets";)
+ data = json.loads(res.read().decode())
+ targets = data.get("data", {}).get("activeTargets", [])
+ if not targets:
+ print("No active targets found in Prometheus")
+ sys.exit(1)
+
+ failed = False
+ checked = 0
+ for t in targets:
+ url = t.get("scrapeUrl", "")
+ # scrapeUrl is like "http://scm:9876/prom";
+ try:
+ host_port = url.split("//")[1].split("/")[0]
+ if ":" in host_port:
+ host, port = host_port.split(":")
+ else:
+ host = host_port
+ port = 80
+ except Exception:
+ continue
+
+ if is_running(host, port):
+ checked += 1
+ health = t.get("health", "")
+ print(f"Target {host}:{port} is running. Prometheus health:
{health}")
+ if health != "up":
+ print(f"Error: Target {host}:{port} is running but
Prometheus health is '{health}'. Last error: {t.get('lastError')}")
+ failed = True
+ else:
+ print(f"Target {host}:{port} is not running. Skipping check.")
+
+ if checked == 0:
+ print("Error: No running targets were checked!")
+ sys.exit(1)
+
+ if failed:
+ sys.exit(1)
+
+ print(f"Successfully verified {checked} running targets.")
+ except Exception as e:
+ print(f"Exception during health check: {e}")
+ sys.exit(1)
+
+if __name__ == "__main__":
+ main()
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMMXBean.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMMXBean.java
index bde82b2ab95..f2ae925dcbd 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMMXBean.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMMXBean.java
@@ -86,6 +86,9 @@ public void testSCMMXBean() throws Exception {
double containerThreshold = (double) mbs.getAttribute(bean,
"SafeModeCurrentContainerThreshold");
assertEquals(scm.getCurrentContainerThreshold(), containerThreshold, 0);
+
+ String ratisEvents = (String) mbs.getAttribute(bean, "RatisEvents");
+ assertEquals(scm.getMetrics().getRatisEvents(), ratisEvents);
}
@Test
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java
index 0e7488ae191..3d418ef8c83 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java
@@ -41,4 +41,6 @@ public interface OMMXBean extends ServiceRuntimeInfo {
* @return the OM hostname for the datanode.
*/
String getHostname();
+
+ String getRatisEvents();
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
index dc67f3c00d5..7725f7835ac 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
@@ -1627,7 +1627,9 @@ public void addRatisEvent(String event) {
}
}
- @Metric("Ratis state machine events")
+ // Ratis state machine events are multi-line logs, which should not be
+ // published as time-series metrics to metrics systems like Prometheus.
+ // Instead, they are exposed via JMX / MXBean endpoints.
public String getRatisEvents() {
synchronized (ratisEvents) {
return String.join("\n", ratisEvents);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index db7717c81ea..1910d92e969 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -3309,6 +3309,11 @@ public String getHostname() {
return omHostName;
}
+ @Override
+ public String getRatisEvents() {
+ return metrics != null ? metrics.getRatisEvents() : "";
+ }
+
@VisibleForTesting
public OzoneManagerHttpServer getHttpServer() {
return httpServer;
diff --git
a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
index 313c79a3f08..2f501fc98d5 100644
---
a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
+++
b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
@@ -173,10 +173,10 @@
templateUrl: 'ratis-events.html',
controller: function ($http) {
var ctrl = this;
- $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics")
+
$http.get("jmx?qry=Hadoop:service=OzoneManager,name=OzoneManagerInfo,component=ServerRuntime")
.then(function (result) {
var metrics = result.data.beans[0];
- var rawEvents = metrics['tag.RatisEvents'] ?
metrics['tag.RatisEvents'].split('\n') : [];
+ var rawEvents = (metrics && metrics['RatisEvents']) ?
metrics['RatisEvents'].split('\n') : [];
ctrl.events = rawEvents.map(function(e) {
var parts = e.split('|');
return {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
