This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 850bc96c63 HDDS-9298. Speed up TestStorageContainerManagerHA (#5307)
850bc96c63 is described below
commit 850bc96c630c837d45958553ebaa12e45e68a4ca
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Thu Sep 21 17:45:05 2023 +0200
HDDS-9298. Speed up TestStorageContainerManagerHA (#5307)
---
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 106 ++++++++++++++-------
.../ozone/scm/TestStorageContainerManagerHA.java | 71 +++-----------
2 files changed, 84 insertions(+), 93 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
index 4c5522275f..31a83fbd3c 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
@@ -22,20 +22,23 @@ import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.ha.ConfUtils;
import org.junit.Assert;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.Mockito;
+import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.UUID;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
@@ -55,16 +58,21 @@ import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PORT_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
/**
* Test for SCM HA-related configuration.
*/
-public class TestSCMHAConfiguration {
+class TestSCMHAConfiguration {
private OzoneConfiguration conf;
@BeforeEach
- public void setup() {
+ void setup(@TempDir File tempDir) {
conf = new OzoneConfiguration();
+ conf.set(OZONE_METADATA_DIRS, tempDir.getAbsolutePath());
+ DefaultConfigManager.clearDefaultConfigs();
}
@Test
@@ -132,64 +140,64 @@ public class TestSCMHAConfiguration {
port = 9880;
// Validate configs.
- Assertions.assertEquals("localhost:" + port++,
+ assertEquals("localhost:" + port++,
conf.get(ConfUtils.addKeySuffixes(OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
scmServiceId, "scm1")));
- Assertions.assertEquals(port,
+ assertEquals(port,
conf.getInt(ConfUtils.addKeySuffixes(OZONE_SCM_BLOCK_CLIENT_PORT_KEY,
scmServiceId, "scm1"), 9999));
- Assertions.assertEquals("172.28.9.1",
+ assertEquals("172.28.9.1",
conf.get(ConfUtils.addKeySuffixes(OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY,
scmServiceId, "scm1")));
- Assertions.assertEquals("localhost:" + port++,
+ assertEquals("localhost:" + port++,
conf.get(ConfUtils.addKeySuffixes(
OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY, scmServiceId, "scm1")));
- Assertions.assertEquals(port, conf.getInt(ConfUtils.addKeySuffixes(
+ assertEquals(port, conf.getInt(ConfUtils.addKeySuffixes(
OZONE_SCM_SECURITY_SERVICE_PORT_KEY, scmServiceId, "scm1"), 9999));
- Assertions.assertEquals("172.28.9.1",
+ assertEquals("172.28.9.1",
conf.get(ConfUtils.addKeySuffixes(
OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY, scmServiceId, "scm1")));
- Assertions.assertEquals("localhost:" + port++,
+ assertEquals("localhost:" + port++,
conf.get(ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_ADDRESS_KEY,
scmServiceId, "scm1")));
- Assertions.assertEquals(port,
+ assertEquals(port,
conf.getInt(ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_PORT_KEY,
scmServiceId, "scm1"), 9999));
- Assertions.assertEquals("172.28.9.1", conf.get(
+ assertEquals("172.28.9.1", conf.get(
ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_BIND_HOST_KEY, scmServiceId,
"scm1")));
- Assertions.assertEquals("localhost:" + port++,
+ assertEquals("localhost:" + port++,
conf.get(ConfUtils.addKeySuffixes(OZONE_SCM_DATANODE_ADDRESS_KEY,
scmServiceId, "scm1")));
- Assertions.assertEquals(port,
+ assertEquals(port,
conf.getInt(ConfUtils.addKeySuffixes(OZONE_SCM_DATANODE_PORT_KEY,
scmServiceId, "scm1"), 9999));
- Assertions.assertEquals("172.28.9.1", conf.get(
+ assertEquals("172.28.9.1", conf.get(
ConfUtils.addKeySuffixes(OZONE_SCM_DATANODE_BIND_HOST_KEY,
scmServiceId,
"scm1")));
- Assertions.assertEquals("localhost:" + port++,
+ assertEquals("localhost:" + port++,
conf.get(ConfUtils.addKeySuffixes(OZONE_SCM_HTTP_ADDRESS_KEY,
scmServiceId, "scm1")));
- Assertions.assertEquals("172.28.9.1",
+ assertEquals("172.28.9.1",
conf.get(ConfUtils.addKeySuffixes(OZONE_SCM_HTTP_BIND_HOST_KEY,
scmServiceId, "scm1")));
- Assertions.assertEquals("localhost", conf.get(ConfUtils.addKeySuffixes(
+ assertEquals("localhost", conf.get(ConfUtils.addKeySuffixes(
OZONE_SCM_ADDRESS_KEY, scmServiceId,
"scm1")));
- Assertions.assertEquals("/var/scm-metadata1",
+ assertEquals("/var/scm-metadata1",
conf.get(ConfUtils.addKeySuffixes(OZONE_SCM_DB_DIRS, scmServiceId,
"scm1")));
- Assertions.assertEquals(port++,
+ assertEquals(port++,
conf.getInt(ConfUtils.addKeySuffixes(OZONE_SCM_RATIS_PORT_KEY,
scmServiceId, "scm1"), 9999));
@@ -226,8 +234,8 @@ public class TestSCMHAConfiguration {
SCMHANodeDetails scmhaNodeDetails =
SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig);
- Assertions.assertEquals("10000", conf.get(OZONE_SCM_RATIS_PORT_KEY));
- Assertions.assertEquals("10001", conf.get(OZONE_SCM_GRPC_PORT_KEY));
+ assertEquals("10000", conf.get(OZONE_SCM_RATIS_PORT_KEY));
+ assertEquals("10001", conf.get(OZONE_SCM_GRPC_PORT_KEY));
InetSocketAddress clientAddress =
@@ -237,37 +245,37 @@ public class TestSCMHAConfiguration {
NetUtils.createSocketAddr("0.0.0.0", 9896);
InetSocketAddress datanodeAddress =
NetUtils.createSocketAddr("0.0.0.0", 9898);
- Assertions.assertEquals(clientAddress,
+ assertEquals(clientAddress,
scmhaNodeDetails.getLocalNodeDetails()
.getClientProtocolServerAddress());
- Assertions.assertEquals(blockAddress,
scmhaNodeDetails.getLocalNodeDetails()
+ assertEquals(blockAddress, scmhaNodeDetails.getLocalNodeDetails()
.getBlockProtocolServerAddress());
- Assertions.assertEquals(datanodeAddress,
+ assertEquals(datanodeAddress,
scmhaNodeDetails.getLocalNodeDetails()
.getDatanodeProtocolServerAddress());
- Assertions.assertEquals(10000,
+ assertEquals(10000,
scmhaNodeDetails.getLocalNodeDetails().getRatisPort());
- Assertions.assertEquals(10001,
+ assertEquals(10001,
scmhaNodeDetails.getLocalNodeDetails().getGrpcPort());
for (SCMNodeDetails peer : scmhaNodeDetails.getPeerNodeDetails()) {
- Assertions.assertEquals(clientAddress,
+ assertEquals(clientAddress,
peer.getClientProtocolServerAddress());
- Assertions.assertEquals(blockAddress,
+ assertEquals(blockAddress,
peer.getBlockProtocolServerAddress());
- Assertions.assertEquals(datanodeAddress,
+ assertEquals(datanodeAddress,
peer.getDatanodeProtocolServerAddress());
- Assertions.assertEquals(10000, peer.getRatisPort());
- Assertions.assertEquals(10001,
+ assertEquals(10000, peer.getRatisPort());
+ assertEquals(10001,
peer.getGrpcPort());
}
// Security protocol address is not set in SCMHANode Details.
// Check conf is properly set with expected port.
- Assertions.assertEquals(
+ assertEquals(
NetUtils.createSocketAddr("0.0.0.0", 9899),
HddsServerUtil.getScmSecurityInetAddress(conf));
@@ -318,7 +326,37 @@ public class TestSCMHAConfiguration {
.thenReturn(Storage.StorageState.INITIALIZED);
Mockito.when(scmStorageConfig.isSCMHAEnabled()).thenReturn(isRatisEnabled);
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, !isRatisEnabled);
- Assertions.assertThrows(ConfigurationException.class,
+ assertThrows(ConfigurationException.class,
() -> SCMHANodeDetails.loadSCMHAConfig(conf, scmStorageConfig));
}
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testHAConfig(boolean ratisEnabled) throws IOException {
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, ratisEnabled);
+ SCMStorageConfig scmStorageConfig = newStorageConfig(ratisEnabled);
+ StorageContainerManager.scmInit(conf, scmStorageConfig.getClusterID());
+ assertEquals(ratisEnabled, DefaultConfigManager.getValue(
+ ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, !ratisEnabled));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testInvalidHAConfig(boolean ratisEnabled) throws IOException {
+ conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, ratisEnabled);
+ SCMStorageConfig scmStorageConfig = newStorageConfig(!ratisEnabled);
+ String clusterID = scmStorageConfig.getClusterID();
+ assertThrows(ConfigurationException.class,
+ () -> StorageContainerManager.scmInit(conf, clusterID));
+ }
+
+ private SCMStorageConfig newStorageConfig(
+ boolean ratisEnabled) throws IOException {
+ final SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
+ scmStorageConfig.setClusterId(UUID.randomUUID().toString());
+ scmStorageConfig.setSCMHAFlag(ratisEnabled);
+ scmStorageConfig.initialize();
+ return scmStorageConfig;
+ }
+
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
index ff2d9dae69..573b9fd68c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.ozone.scm;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
-import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.DefaultConfigManager;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -49,8 +48,6 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.time.Instant;
@@ -107,6 +104,7 @@ public class TestStorageContainerManagerHA {
.setNumOfOzoneManagers(numOfOMs)
.build();
cluster.waitForClusterToBeReady();
+ waitForLeaderToBeReady();
}
/**
@@ -121,7 +119,7 @@ public class TestStorageContainerManagerHA {
}
@Test
- public void testAllSCMAreRunning() throws Exception {
+ void testAllSCMAreRunning() throws Exception {
int count = 0;
List<StorageContainerManager> scms = cluster.getStorageContainerManagers();
Assertions.assertEquals(numOfSCMs, scms.size());
@@ -136,6 +134,13 @@ public class TestStorageContainerManagerHA {
Assertions.assertEquals(peerSize, numOfSCMs);
}
Assertions.assertEquals(1, count);
+ Assertions.assertNotNull(leaderScm);
+
+ assertRatisRoles();
+
+ String leaderSCMId = leaderScm.getScmId();
+ checkSCMHAMetricsForAllSCMs(scms, leaderSCMId);
+
count = 0;
List<OzoneManager> oms = cluster.getOzoneManagersList();
Assertions.assertEquals(numOfOMs, oms.size());
@@ -149,7 +154,7 @@ public class TestStorageContainerManagerHA {
// verify timer based transaction buffer flush is working
SnapshotInfo latestSnapshot = leaderScm.getScmHAManager()
.asSCMHADBTransactionBuffer().getLatestSnapshot();
- testPutKey();
+ doPutKey();
final StorageContainerManager leaderScmTmp = leaderScm;
GenericTestUtils.waitFor(() -> {
if (leaderScmTmp.getScmHAManager().asSCMHADBTransactionBuffer()
@@ -163,8 +168,7 @@ public class TestStorageContainerManagerHA {
}, 2000, 30000);
}
- @Test
- public void testPutKey() throws Exception {
+ private void doPutKey() throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
Instant testStartTime = Instant.now();
@@ -261,44 +265,6 @@ public class TestStorageContainerManagerHA {
StorageContainerManager.scmInit(conf2, scm2.getClusterId()));
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- public void testHAConfig(boolean isRatisEnabled) throws Exception {
- StorageContainerManager scm0 = cluster.getStorageContainerManager(0);
- scm0.stop();
- boolean isDeleted = scm0.getScmStorageConfig().getVersionFile().delete();
- Assertions.assertTrue(isDeleted);
- conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, isRatisEnabled);
- final SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
- scmStorageConfig.setClusterId(UUID.randomUUID().toString());
- scmStorageConfig.getCurrentDir().delete();
- scmStorageConfig.setSCMHAFlag(isRatisEnabled);
- DefaultConfigManager.clearDefaultConfigs();
- scmStorageConfig.initialize();
- StorageContainerManager.scmInit(conf, clusterId);
- Assertions.assertEquals(DefaultConfigManager.getValue(
- ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, !isRatisEnabled),
- isRatisEnabled);
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- public void testInvalidHAConfig(boolean isRatisEnabled) throws Exception {
- StorageContainerManager scm0 = cluster.getStorageContainerManager(0);
- scm0.stop();
- boolean isDeleted = scm0.getScmStorageConfig().getVersionFile().delete();
- Assertions.assertTrue(isDeleted);
- conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, isRatisEnabled);
- final SCMStorageConfig scmStorageConfig = new SCMStorageConfig(conf);
- scmStorageConfig.setClusterId(UUID.randomUUID().toString());
- scmStorageConfig.getCurrentDir().delete();
- scmStorageConfig.setSCMHAFlag(!isRatisEnabled);
- DefaultConfigManager.clearDefaultConfigs();
- scmStorageConfig.initialize();
- Assertions.assertThrows(ConfigurationException.class,
- () -> StorageContainerManager.scmInit(conf, clusterId));
- }
-
@Test
public void testBootStrapSCM() throws Exception {
StorageContainerManager scm2 =
cluster.getStorageContainerManagers().get(1);
@@ -318,8 +284,7 @@ public class TestStorageContainerManagerHA {
Assertions.assertTrue(StorageContainerManager.scmBootstrap(conf2));
}
- @Test
- public void testGetRatisRolesDetail() {
+ private void assertRatisRoles() {
Set<String> resultSet = new HashSet<>();
for (StorageContainerManager scm: cluster.getStorageContainerManagers()) {
resultSet.addAll(scm.getScmHAManager().getRatisServer().getRatisRoles());
@@ -330,18 +295,6 @@ public class TestStorageContainerManagerHA {
resultSet.stream().filter(x -> x.contains("LEADER")).count());
}
- @Test
- public void testSCMHAMetrics() throws InterruptedException, TimeoutException
{
- waitForLeaderToBeReady();
-
- StorageContainerManager leaderSCM = cluster.getActiveSCM();
- String leaderSCMId = leaderSCM.getScmId();
- List<StorageContainerManager> scms =
- cluster.getStorageContainerManagersList();
-
- checkSCMHAMetricsForAllSCMs(scms, leaderSCMId);
- }
-
private void checkSCMHAMetricsForAllSCMs(List<StorageContainerManager> scms,
String leaderSCMId) {
for (StorageContainerManager scm : scms) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
