neils-dev commented on code in PR #4140:
URL: https://github.com/apache/ozone/pull/4140#discussion_r1107988022
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java:
##########
@@ -1826,6 +1828,20 @@ public void updatePeerList(List<String> newPeers) {
}
}
}
+ String leaderId = "";
+ if (isRatisEnabled) {
+ RaftPeer leader = null;
+ try {
+ leader = omRatisServer.getLeader();
+ } catch (IOException ex) {
+ LOG.error("IOException while getting the " +
+ "Ratis server leader.", ex);
+ }
+ if (Objects.nonNull(leader)) {
+ leaderId = leader.getId().toString();
+ }
+ }
+ omHAMetricsInit(leaderId);
Review Comment:
If `leaderId` is not set due to ratis not enabled or error getting leader a
blank "" string is passed to `omHAMetricsInit`. In `omHAMetrics` iin this case
what happens? We should handle that. (see also comment in
`omHAMetrics.getMetrics()`)
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.ha;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.Interns;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * Class to maintain metrics and info related to OM HA.
+ */
+@Metrics(about = "OzoneManager HA Metrics", context = OzoneConsts.OZONE)
+public final class OMHAMetrics implements MetricsSource {
+
+ /**
+ * Private nested class to hold the values
+ * of MetricsInfo for OMHAMetrics.
+ */
+ private static final class OMHAMetricsInfo {
+
+ private static final MetricsInfo OZONE_MANAGER_HA_LEADER_STATE =
+ Interns.info("OzoneManagerHALeaderState",
+ "Leader active state of OzoneManager node (1 leader, 0 follower)");
+
+ private static final MetricsInfo NODE_ID =
+ Interns.info("NodeId", "OM node Id");
+
+ private long ozoneManagerHALeaderState;
+ private String nodeId;
+
+ OMHAMetricsInfo() {
+ this.ozoneManagerHALeaderState = 0L;
+ this.nodeId = "";
+ }
+
+ public long getOzoneManagerHALeaderState() {
+ return ozoneManagerHALeaderState;
+ }
+
+ public void setOzoneManagerHALeaderState(long ozoneManagerHALeaderState) {
+ this.ozoneManagerHALeaderState = ozoneManagerHALeaderState;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public void setNodeId(String nodeId) {
+ this.nodeId = nodeId;
+ }
+ }
+
+ public static final String SOURCE_NAME =
+ OMHAMetrics.class.getSimpleName();
+ private final OMHAMetricsInfo omhaMetricsInfo = new OMHAMetricsInfo();
+ private MetricsRegistry metricsRegistry;
+
+ private String currNodeId;
+ private String leaderId;
+
+ private OMHAMetrics(String currNodeId, String leaderId) {
+ this.currNodeId = currNodeId;
+ this.leaderId = leaderId;
+ this.metricsRegistry = new MetricsRegistry(SOURCE_NAME);
+ }
+
+ /**
+ * Create and return OMHAMetrics instance.
+ * @return OMHAMetrics
+ */
+ public static OMHAMetrics create(
+ String nodeId, String leaderId) {
+ OMHAMetrics metrics = new OMHAMetrics(nodeId, leaderId);
+ return DefaultMetricsSystem.instance()
+ .register(SOURCE_NAME, "Metrics for OM HA", metrics);
+ }
+
+ /**
+ * Unregister the metrics instance.
+ */
+ public static void unRegister() {
+ DefaultMetricsSystem.instance().unregisterSource(SOURCE_NAME);
+ }
+
+ @Override
+ public synchronized void getMetrics(MetricsCollector collector, boolean all)
{
+
+ MetricsRecordBuilder recordBuilder = collector.addRecord(SOURCE_NAME);
+
+ // Check current node state (1 leader, 0 follower)
+ int state = currNodeId.equals(leaderId) ? 1 : 0;
Review Comment:
If `leaderId` is an empty string due to leader unavail at metrics
registration, we should probably check here and omit setting metrics. Just
return.
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java:
##########
@@ -104,6 +106,83 @@ public void testMultipartUpload() throws Exception {
testMultipartUploadWithOneOmNodeDown();
}
+ @Test
+ public void testOMHAMetrics() throws InterruptedException,
+ TimeoutException, IOException {
+ waitForLeaderToBeReady();
+
+ // Get leader OM
+ OzoneManager leaderOM = getCluster().getOMLeader();
+ // Store current leader's node ID,
+ // to use it after restarting the OM
+ String leaderNodeId = leaderOM.getOMNodeId();
+ // Get a list of all OMs
+ List<OzoneManager> omList = getCluster().getOzoneManagersList();
+
+ // Check metrics for all OMs
+ checkOMHAMetricsForAllOMs(omList, leaderOM);
+
+ // Restart current leader OM
+ leaderOM.stop();
+ leaderOM.restart();
+
+ waitForLeaderToBeReady();
+
+ // Get the new leader
+ OzoneManager newLeaderOM = getCluster().getOMLeader();
+ // Get a list of all OMs again
+ omList = getCluster().getOzoneManagersList();
+
+ // New state for the old leader
+ int newState = leaderNodeId.equals(newLeaderOM.getOMNodeId()) ? 1 : 0;
+
+ // Get old leader
+ OzoneManager oldLeader = getCluster().getOzoneManager(leaderNodeId);
+ // Get old leader's metrics
+ OMHAMetrics omhaMetrics = oldLeader.getOmhaMetrics();
+
+ Assertions.assertEquals(newState,
+ omhaMetrics.getOmhaInfoOzoneManagerHALeaderState());
+
+ // Check that metrics for all OMs have been updated
+ checkOMHAMetricsForAllOMs(omList, newLeaderOM);
+ }
+
+ private void checkOMHAMetricsForAllOMs(List<OzoneManager> omList,
+ OzoneManager leaderOM) {
Review Comment:
This check doesn't need the `leaderOM` `OzoneManager`. We can just pass the
`leaderNodeId` `string` and use it inside to check on line 160.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.ha;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.Interns;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * Class to maintain metrics and info related to OM HA.
+ */
+@Metrics(about = "OzoneManager HA Metrics", context = OzoneConsts.OZONE)
+public final class OMHAMetrics implements MetricsSource {
+
+ /**
+ * Private nested class to hold the values
+ * of MetricsInfo for OMHAMetrics.
+ */
+ private static final class OMHAMetricsInfo {
+
+ private static final MetricsInfo OZONE_MANAGER_HA_LEADER_STATE =
+ Interns.info("OzoneManagerHALeaderState",
+ "Leader active state of OzoneManager node (1 leader, 0 follower)");
+
+ private static final MetricsInfo NODE_ID =
+ Interns.info("NodeId", "OM node Id");
+
+ private long ozoneManagerHALeaderState;
+ private String nodeId;
+
+ OMHAMetricsInfo() {
+ this.ozoneManagerHALeaderState = 0L;
+ this.nodeId = "";
+ }
+
+ public long getOzoneManagerHALeaderState() {
+ return ozoneManagerHALeaderState;
+ }
+
+ public void setOzoneManagerHALeaderState(long ozoneManagerHALeaderState) {
+ this.ozoneManagerHALeaderState = ozoneManagerHALeaderState;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public void setNodeId(String nodeId) {
+ this.nodeId = nodeId;
+ }
+ }
+
+ public static final String SOURCE_NAME =
+ OMHAMetrics.class.getSimpleName();
+ private final OMHAMetricsInfo omhaMetricsInfo = new OMHAMetricsInfo();
+ private MetricsRegistry metricsRegistry;
+
+ private String currNodeId;
+ private String leaderId;
+
+ private OMHAMetrics(String currNodeId, String leaderId) {
+ this.currNodeId = currNodeId;
+ this.leaderId = leaderId;
+ this.metricsRegistry = new MetricsRegistry(SOURCE_NAME);
+ }
+
+ /**
+ * Create and return OMHAMetrics instance.
+ * @return OMHAMetrics
+ */
+ public static OMHAMetrics create(
+ String nodeId, String leaderId) {
+ OMHAMetrics metrics = new OMHAMetrics(nodeId, leaderId);
Review Comment:
Thanks for the updates @xBis7. We should check whether `leaderId` is blank
or not. If blank, log an error indicating no valid leader was given.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
