This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2560e64515 HDDS-8363. Make SCM Ratis roles available in /prom endpoint
(#4521)
2560e64515 is described below
commit 2560e645159ed0fdf1c0508459fc2cb5ef392d1b
Author: Mladjan Gadzic <[email protected]>
AuthorDate: Mon Apr 17 08:27:35 2023 +0200
HDDS-8363. Make SCM Ratis roles available in /prom endpoint (#4521)
---
.../apache/hadoop/hdds/scm/ha/SCMHAMetrics.java | 116 ++++++++++++++++++++
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 +
.../hdds/scm/server/StorageContainerManager.java | 22 ++++
.../hadoop/hdds/scm/ha/TestSCMHAMetrics.java | 69 ++++++++++++
.../apache/hadoop/hdds/scm/ha/package-info.java | 18 ++++
.../ozone/scm/TestStorageContainerManagerHA.java | 120 +++++++++++++--------
.../org/apache/hadoop/ozone/scm/package-info.java | 18 ++++
7 files changed, 323 insertions(+), 42 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAMetrics.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAMetrics.java
new file mode 100644
index 0000000000..3a8c5bcacf
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAMetrics.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.ha;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.Interns;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * SCM HA metrics.
+ */
+@Metrics(about = "SCM HA metrics", context = OzoneConsts.OZONE)
+public final class SCMHAMetrics implements MetricsSource {
+
+ /**
+ * Metrics value holder.
+ */
+ private static final class SCMHAMetricsInfo {
+
+ private static final MetricsInfo SCM_MANAGER_HA_LEADER_STATE =
+ Interns.info("SCMHALeaderState", "Leader active " +
+ "state of SCM node (1 leader, 0 follower");
+ private static final MetricsInfo NODE_ID = Interns.info("NodeId",
+ "SCM node Id");
+ private int scmHALeaderState;
+ private String nodeId;
+
+ public int getScmHALeaderState() {
+ return scmHALeaderState;
+ }
+
+ public void setScmHALeaderState(int scmHALeaderState) {
+ this.scmHALeaderState = scmHALeaderState;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public void setNodeId(String nodeId) {
+ this.nodeId = nodeId;
+ }
+ }
+
+ private static final String SOURCE_NAME = SCMHAMetrics.class.getSimpleName();
+ private final SCMHAMetricsInfo scmHAMetricsInfo = new SCMHAMetricsInfo();
+ private final String currNodeId;
+ private final String leaderId;
+
+ private SCMHAMetrics(String currNodeId, String leaderId) {
+ this.currNodeId = currNodeId;
+ this.leaderId = leaderId;
+ }
+
+ /**
+ * Creates and returns SCMHAMetrics instance.
+ * @return SCMHAMetrics
+ */
+ public static SCMHAMetrics create(String nodeId, String leaderId) {
+ SCMHAMetrics metrics = new SCMHAMetrics(nodeId, leaderId);
+ return DefaultMetricsSystem.instance()
+ .register(SOURCE_NAME, "SCM HA metrics", metrics);
+ }
+
+ /**
+ * Unregisters the metrics instance.
+ */
+ public static void unRegister() {
+ DefaultMetricsSystem.instance().unregisterSource(SOURCE_NAME);
+ }
+
+ @Override
+ public synchronized void getMetrics(MetricsCollector collector, boolean all)
{
+ // Check current node state (1 leader, 0 follower)
+ int state = currNodeId.equals(leaderId) ? 1 : 0;
+ scmHAMetricsInfo.setNodeId(currNodeId);
+ scmHAMetricsInfo.setScmHALeaderState(state);
+
+ MetricsRecordBuilder recordBuilder = collector.addRecord(SOURCE_NAME);
+ recordBuilder
+ .tag(SCMHAMetricsInfo.NODE_ID, currNodeId)
+ .addGauge(SCMHAMetricsInfo.SCM_MANAGER_HA_LEADER_STATE, state);
+ recordBuilder.endRecord();
+ }
+
+ @VisibleForTesting
+ public String getSCMHAMetricsInfoNodeId() {
+ return scmHAMetricsInfo.getNodeId();
+ }
+
+ @VisibleForTesting
+ public int getSCMHAMetricsInfoLeaderState() {
+ return scmHAMetricsInfo.getScmHALeaderState();
+ }
+
+}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
index 5e5298a967..c88331db98 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
@@ -283,6 +283,8 @@ public class SCMStateMachine extends BaseStateMachine {
deletedBlockLog instanceof DeletedBlockLogImpl);
((DeletedBlockLogImpl) deletedBlockLog).onBecomeLeader();
scm.getScmDecommissionManager().onBecomeLeader();
+
+ scm.scmHAMetricsUpdate(newLeaderId.toString());
}
@Override
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 28249d4210..e5faff4b69 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.hdds.scm.ha.HASecurityUtils;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SCMHAManagerImpl;
+import org.apache.hadoop.hdds.scm.ha.SCMHAMetrics;
import org.apache.hadoop.hdds.scm.ha.SCMHANodeDetails;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
@@ -217,6 +218,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
* SCM metrics.
*/
private static SCMMetrics metrics;
+ private SCMHAMetrics scmHAMetrics;
/*
* RPC Endpoints exposed by SCM.
@@ -1517,6 +1519,9 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
}
setStartTime();
+
+ // At this point leader is not known
+ scmHAMetricsUpdate(null);
}
/** Persist SCM certs to DB on bootstrap scm nodes.
@@ -1663,6 +1668,10 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
LOG.error("SCM HA Manager stop failed", ex);
}
+ if (scmHAMetrics != null) {
+ SCMHAMetrics.unRegister();
+ }
+
IOUtils.cleanupWithLogger(LOG, containerManager);
IOUtils.cleanupWithLogger(LOG, pipelineManager);
@@ -1996,6 +2005,11 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
return scmHANodeDetails.getLocalNodeDetails().getNodeId();
}
+ @VisibleForTesting
+ public SCMHAMetrics getScmHAMetrics() {
+ return scmHAMetrics;
+ }
+
private void startSecretManagerIfNecessary() {
boolean shouldRun = securityConfig.isSecurityEnabled()
&& securityConfig.isContainerTokenEnabled()
@@ -2155,4 +2169,12 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
return scmHAManager.removeSCM(request);
}
+
+ public void scmHAMetricsUpdate(String leaderId) {
+ // unregister, in case metrics already exist
+ // so that the metric tags will get updated.
+ SCMHAMetrics.unRegister();
+
+ scmHAMetrics = SCMHAMetrics.create(getScmId(), leaderId);
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAMetrics.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAMetrics.java
new file mode 100644
index 0000000000..3a504fc84d
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAMetrics.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.ha;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link SCMHAMetrics}.
+ */
+class TestSCMHAMetrics {
+
+ private static final MetricsCollectorImpl METRICS_COLLECTOR =
+ new MetricsCollectorImpl();
+ private static final String NODE_ID =
+ "scm" + RandomStringUtils.randomNumeric(5);
+ private String leaderId;
+ private SCMHAMetrics scmhaMetrics;
+
+ @AfterEach
+ public void cleanup() {
+ SCMHAMetrics.unRegister();
+ }
+
+ @Test
+ public void testGetMetricsWithLeader() {
+ // GIVEN
+ leaderId = NODE_ID;
+
+ // WHEN
+ scmhaMetrics = SCMHAMetrics.create(NODE_ID, leaderId);
+ scmhaMetrics.getMetrics(METRICS_COLLECTOR, true);
+
+ // THEN
+ Assertions.assertEquals(1, scmhaMetrics.getSCMHAMetricsInfoLeaderState());
+ }
+
+ @Test
+ public void testGetMetricsWithFollower() {
+ // GIVEN
+ leaderId = "scm" + RandomStringUtils.randomNumeric(5);
+
+ // WHEN
+ scmhaMetrics = SCMHAMetrics.create(NODE_ID, leaderId);
+ scmhaMetrics.getMetrics(METRICS_COLLECTOR, true);
+
+ // THEN
+ Assertions.assertEquals(0, scmhaMetrics.getSCMHAMetricsInfoLeaderState());
+ }
+
+}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/package-info.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/package-info.java
new file mode 100644
index 0000000000..12c241dd0f
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.ha;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
index f4cd64bd59..4424324ea4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ha.SCMHAMetrics;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -42,17 +43,13 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.ozone.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Rule;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.time.Instant;
@@ -61,14 +58,16 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.TimeoutException;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.client.ReplicationType.RATIS;
/**
- * Base class for Ozone Manager HA tests.
+ * Base class for SCM HA tests.
*/
+@Timeout(300)
public class TestStorageContainerManagerHA {
private MiniOzoneHAClusterImpl cluster = null;
@@ -80,12 +79,6 @@ public class TestStorageContainerManagerHA {
private String scmServiceId;
private static int numOfSCMs = 3;
- private static final Logger LOG = LoggerFactory
- .getLogger(TestStorageContainerManagerHA.class);
-
- @Rule
- public Timeout timeout = new Timeout(300_000);
-
/**
* Create a MiniDFSCluster for testing.
* <p>
@@ -127,28 +120,29 @@ public class TestStorageContainerManagerHA {
public void testAllSCMAreRunning() throws Exception {
int count = 0;
List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
- Assert.assertEquals(numOfSCMs, scms.size());
+ Assertions.assertEquals(numOfSCMs, scms.size());
int peerSize = cluster.getStorageContainerManager().getScmHAManager()
.getRatisServer().getDivision().getGroup().getPeers().size();
for (StorageContainerManager scm : scms) {
if (scm.checkLeader()) {
count++;
}
- Assert.assertTrue(peerSize == numOfSCMs);
+ Assertions.assertEquals(peerSize, numOfSCMs);
}
- Assert.assertEquals(1, count);
+ Assertions.assertEquals(1, count);
count = 0;
List<OzoneManager> oms = cluster.getOzoneManagersList();
- Assert.assertEquals(numOfOMs, oms.size());
+ Assertions.assertEquals(numOfOMs, oms.size());
for (OzoneManager om : oms) {
if (om.isLeaderReady()) {
count++;
}
}
- Assert.assertEquals(1, count);
+ Assertions.assertEquals(1, count);
testPutKey();
}
+ @Test
public void testPutKey() throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
@@ -169,13 +163,13 @@ public class TestStorageContainerManagerHA {
out.write(value.getBytes(UTF_8));
out.close();
OzoneKey key = bucket.getKey(keyName);
- Assert.assertEquals(keyName, key.getName());
+ Assertions.assertEquals(keyName, key.getName());
OzoneInputStream is = bucket.readKey(keyName);
byte[] fileContent = new byte[value.getBytes(UTF_8).length];
is.read(fileContent);
- Assert.assertEquals(value, new String(fileContent, UTF_8));
- Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
- Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ Assertions.assertEquals(value, new String(fileContent, UTF_8));
+ Assertions.assertFalse(key.getCreationTime().isBefore(testStartTime));
+
Assertions.assertFalse(key.getModificationTime().isBefore(testStartTime));
is.close();
final OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
@@ -194,7 +188,7 @@ public class TestStorageContainerManagerHA {
index = getLastAppliedIndex(scm);
}
}
- Assert.assertFalse(index == -1);
+ Assertions.assertNotEquals(-1, index);
long finalIndex = index;
// Ensure all follower scms have caught up with the leader
GenericTestUtils.waitFor(() -> areAllScmInSync(finalIndex), 100, 10000);
@@ -205,9 +199,9 @@ public class TestStorageContainerManagerHA {
// flush to DB on each SCM
((SCMRatisServerImpl) scm.getScmHAManager().getRatisServer())
.getStateMachine().takeSnapshot();
- Assert.assertTrue(scm.getContainerManager()
+ Assertions.assertTrue(scm.getContainerManager()
.containerExist(ContainerID.valueOf(containerID)));
- Assert.assertNotNull(scm.getScmMetadataStore().getContainerTable()
+ Assertions.assertNotNull(scm.getScmMetadataStore().getContainerTable()
.get(ContainerID.valueOf(containerID)));
}
}
@@ -237,12 +231,12 @@ public class TestStorageContainerManagerHA {
scm1.getSCMNodeId());
conf2.set(ScmConfigKeys.OZONE_SCM_PRIMORDIAL_NODE_ID_KEY,
scm1.getSCMNodeId());
- Assert.assertTrue(StorageContainerManager.scmBootstrap(conf1));
+ Assertions.assertTrue(StorageContainerManager.scmBootstrap(conf1));
scm1.getScmHAManager().stop();
- Assert.assertTrue(
+ Assertions.assertTrue(
StorageContainerManager.scmInit(conf1, scm1.getClusterId()));
- Assert.assertTrue(StorageContainerManager.scmBootstrap(conf2));
- Assert.assertTrue(
+ Assertions.assertTrue(StorageContainerManager.scmBootstrap(conf2));
+ Assertions.assertTrue(
StorageContainerManager.scmInit(conf2, scm2.getClusterId()));
}
@@ -252,7 +246,7 @@ public class TestStorageContainerManagerHA {
StorageContainerManager scm0 = cluster.getStorageContainerManager(0);
scm0.stop();
boolean isDeleted = scm0.getScmStorageConfig().getVersionFile().delete();
- Assert.assertTrue(isDeleted);
+ Assertions.assertTrue(isDeleted);
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, isRatisEnabled);
final SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
scmStorageConfig.setClusterId(UUID.randomUUID().toString());
@@ -260,8 +254,8 @@ public class TestStorageContainerManagerHA {
scmStorageConfig.setSCMHAFlag(isRatisEnabled);
DefaultConfigManager.clearDefaultConfigs();
scmStorageConfig.initialize();
- scm0.scmInit(conf, clusterId);
- Assert.assertEquals(DefaultConfigManager.getValue(
+ StorageContainerManager.scmInit(conf, clusterId);
+ Assertions.assertEquals(DefaultConfigManager.getValue(
ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, !isRatisEnabled),
isRatisEnabled);
}
@@ -272,7 +266,7 @@ public class TestStorageContainerManagerHA {
StorageContainerManager scm0 = cluster.getStorageContainerManager(0);
scm0.stop();
boolean isDeleted = scm0.getScmStorageConfig().getVersionFile().delete();
- Assert.assertTrue(isDeleted);
+ Assertions.assertTrue(isDeleted);
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, isRatisEnabled);
final SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
scmStorageConfig.setClusterId(UUID.randomUUID().toString());
@@ -284,14 +278,12 @@ public class TestStorageContainerManagerHA {
() -> StorageContainerManager.scmInit(conf, clusterId));
}
-
-
@Test
public void testBootStrapSCM() throws Exception {
StorageContainerManager scm2 =
cluster.getStorageContainerManagers().get(1);
OzoneConfiguration conf2 = scm2.getConfiguration();
boolean isDeleted = scm2.getScmStorageConfig().getVersionFile().delete();
- Assert.assertTrue(isDeleted);
+ Assertions.assertTrue(isDeleted);
final SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf2);
scmStorageConfig.setClusterId(UUID.randomUUID().toString());
scmStorageConfig.getCurrentDir().delete();
@@ -299,10 +291,10 @@ public class TestStorageContainerManagerHA {
scmStorageConfig.initialize();
conf2.setBoolean(ScmConfigKeys.OZONE_SCM_SKIP_BOOTSTRAP_VALIDATION_KEY,
false);
- Assert.assertFalse(StorageContainerManager.scmBootstrap(conf2));
+ Assertions.assertFalse(StorageContainerManager.scmBootstrap(conf2));
conf2.setBoolean(ScmConfigKeys.OZONE_SCM_SKIP_BOOTSTRAP_VALIDATION_KEY,
true);
- Assert.assertTrue(StorageContainerManager.scmBootstrap(conf2));
+ Assertions.assertTrue(StorageContainerManager.scmBootstrap(conf2));
}
@Test
@@ -312,8 +304,52 @@ public class TestStorageContainerManagerHA {
resultSet.addAll(scm.getScmHAManager().getRatisServer().getRatisRoles());
}
System.out.println(resultSet);
- Assert.assertEquals(3, resultSet.size());
- Assert.assertEquals(1,
+ Assertions.assertEquals(3, resultSet.size());
+ Assertions.assertEquals(1,
resultSet.stream().filter(x -> x.contains("LEADER")).count());
}
+
+ @Test
+ public void testSCMHAMetrics() throws InterruptedException, TimeoutException
{
+ waitForLeaderToBeReady();
+
+ StorageContainerManager leaderSCM = cluster.getActiveSCM();
+ String leaderSCMId = leaderSCM.getScmId();
+ List<StorageContainerManager> scms =
+ cluster.getStorageContainerManagersList();
+
+ checkSCMHAMetricsForAllSCMs(scms, leaderSCMId);
+ }
+
+ private void checkSCMHAMetricsForAllSCMs(List<StorageContainerManager> scms,
+ String leaderSCMId) {
+ for (StorageContainerManager scm : scms) {
+ String nodeId = scm.getScmId();
+
+ SCMHAMetrics scmHAMetrics = scm.getScmHAMetrics();
+ // If current SCM is leader, state should be 1
+ int expectedState = nodeId.equals(leaderSCMId) ? 1 : 0;
+
+ Assertions.assertEquals(expectedState,
+ scmHAMetrics.getSCMHAMetricsInfoLeaderState());
+ Assertions.assertEquals(nodeId,
scmHAMetrics.getSCMHAMetricsInfoNodeId());
+ }
+ }
+
+ /**
+ * Some tests are stopping or restarting SCMs.
+ * There are test cases where we might need to
+ * wait for a leader to be elected and ready.
+ */
+ private void waitForLeaderToBeReady()
+ throws InterruptedException, TimeoutException {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return cluster.getActiveSCM().checkLeader();
+ } catch (Exception e) {
+ return false;
+ }
+ }, 1000, (int) ScmConfigKeys
+ .OZONE_SCM_HA_RATIS_LEADER_READY_WAIT_TIMEOUT_DEFAULT);
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/package-info.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/package-info.java
new file mode 100644
index 0000000000..d5563a43bf
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.scm;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
