This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 79da6b7 HDDS-5766. Speed up some OM tests by skipping SCM safemode
check (#2662)
79da6b7 is described below
commit 79da6b70172f8b09140cd150d1c9940ca7b7a03f
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Tue Sep 21 09:38:50 2021 +0100
HDDS-5766. Speed up some OM tests by skipping SCM safemode check (#2662)
---
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 31 +++++++++++++++++-----
.../ozone/om/TestOzoneManagerConfiguration.java | 5 +++-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 29 +++++++++++++++++---
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 17 ++++++++++++
.../ozone/om/TestOzoneManagerListVolumes.java | 7 ++++-
5 files changed, 78 insertions(+), 11 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
index ee7ce6b..880b403 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@ -31,6 +31,7 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils;
@@ -64,6 +65,8 @@ public class TestOmMetrics {
@Rule
public Timeout timeout = Timeout.seconds(300);
private MiniOzoneCluster cluster;
+ private MiniOzoneCluster.Builder clusterBuilder;
+ private OzoneConfiguration conf;
private OzoneManager ozoneManager;
/**
@@ -76,10 +79,17 @@ public class TestOmMetrics {
*/
@Before
public void setup() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
+ conf = new OzoneConfiguration();
conf.setTimeDuration(OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL,
1000, TimeUnit.MILLISECONDS);
- cluster = MiniOzoneCluster.newBuilder(conf).build();
+ // Most tests in this class do not use any features of SCM, so we can skip
+ // safemode which gets the cluster to come up much faster.
+ conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED, false);
+ clusterBuilder = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(0);
+ }
+
+ private void startCluster() throws Exception {
+ cluster = clusterBuilder.build();
cluster.waitForClusterToBeReady();
ozoneManager = cluster.getOzoneManager();
}
@@ -97,7 +107,8 @@ public class TestOmMetrics {
@Test
- public void testVolumeOps() throws IOException {
+ public void testVolumeOps() throws Exception {
+ startCluster();
VolumeManager volumeManager =
(VolumeManager) HddsWhiteboxTestUtils.getInternalState(
ozoneManager, "volumeManager");
@@ -174,7 +185,8 @@ public class TestOmMetrics {
}
@Test
- public void testBucketOps() throws IOException {
+ public void testBucketOps() throws Exception {
+ startCluster();
BucketManager bucketManager =
(BucketManager) HddsWhiteboxTestUtils.getInternalState(
ozoneManager, "bucketManager");
@@ -240,7 +252,8 @@ public class TestOmMetrics {
}
@Test
- public void testKeyOps() throws IOException {
+ public void testKeyOps() throws Exception {
+ startCluster();
KeyManager keyManager = (KeyManager) HddsWhiteboxTestUtils
.getInternalState(ozoneManager, "keyManager");
KeyManager mockKm = Mockito.spy(keyManager);
@@ -330,7 +343,8 @@ public class TestOmMetrics {
}
@Test
- public void testAclOperations() throws IOException {
+ public void testAclOperations() throws Exception {
+ startCluster();
try {
// Create a volume.
cluster.getClient().getObjectStore().createVolume("volumeacl");
@@ -367,6 +381,11 @@ public class TestOmMetrics {
@Test
public void testAclOperationsHA() throws Exception {
+ // This test needs a cluster with DNs and SCM to wait on safemode
+ clusterBuilder.setNumDatanodes(3);
+ conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED, true);
+ startCluster();
+
ObjectStore objectStore = cluster.getClient().getObjectStore();
// Create a volume.
objectStore.createVolume("volumeacl");
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java
index 6e57445..8181eea 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java
@@ -81,7 +81,9 @@ public class TestOzoneManagerConfiguration {
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
conf.setTimeDuration(OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
RATIS_RPC_TIMEOUT, TimeUnit.MILLISECONDS);
-
+ // These test do not use any features of SCM, so we can skip safemode
+ // which gets the cluster to come up much faster.
+ conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED, false);
OMStorage omStore = new OMStorage(conf);
omStore.setClusterId("testClusterId");
// writes the version file properties
@@ -100,6 +102,7 @@ public class TestOzoneManagerConfiguration {
.setClusterId(clusterId)
.setScmId(scmId)
.setOmId(omId)
+ .setNumDatanodes(0)
.build();
cluster.waitForClusterToBeReady();
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
index ffc1643..2383f0a 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
@@ -69,6 +69,7 @@ import static org.junit.Assert.fail;
public abstract class TestOzoneManagerHA {
private MiniOzoneOMHAClusterImpl cluster = null;
+ private MiniOzoneCluster.Builder clusterBuilder = null;
private ObjectStore objectStore;
private OzoneConfiguration conf;
private String clusterId;
@@ -101,6 +102,10 @@ public abstract class TestOzoneManagerHA {
return conf;
}
+ public MiniOzoneCluster.Builder getClusterBuilder() {
+ return clusterBuilder;
+ }
+
public String getOmServiceId() {
return omServiceId;
}
@@ -166,19 +171,37 @@ public abstract class TestOzoneManagerHA {
*/
conf.set(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, "10s");
conf.set(OZONE_KEY_DELETING_LIMIT_PER_TASK, "2");
- cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
+ additionalConfiguration();
+
+ clusterBuilder = MiniOzoneCluster.newOMHABuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId(omServiceId)
.setOmId(omId)
- .setNumOfOzoneManagers(numOfOMs)
- .build();
+ .setNumOfOzoneManagers(numOfOMs);
+ additionalClusterSettings();
+
+ cluster = (MiniOzoneOMHAClusterImpl)clusterBuilder.build();
cluster.waitForClusterToBeReady();
objectStore = OzoneClientFactory.getRpcClient(omServiceId, conf)
.getObjectStore();
}
/**
+ * Override this method in sub-classes to additional test specific
+ * configuration. Add settings to the conf instance variable.
+ */
+ protected void additionalConfiguration() {
+ }
+
+ /**
+ * Override this method in sub-classes to additional test specific
+ * mini-cluster settings. Add settings the clusterBuilder instance variable.
+ */
+ protected void additionalClusterSettings() {
+ }
+
+ /**
* Shutdown MiniDFSCluster.
*/
@After
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java
index 1be7717..462d2af 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java
@@ -17,8 +17,11 @@
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdfs.LogVerificationAppender;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneTestUtils;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -73,6 +76,20 @@ import static org.junit.Assert.fail;
*/
public class TestOzoneManagerHAMetadataOnly extends TestOzoneManagerHA {
+ @Override
+ protected void additionalConfiguration() {
+ OzoneConfiguration conf = getConf();
+ // These test do not use any features of SCM, so we can skip safemode
+ // which gets the cluster to come up much faster.
+ conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED, false);
+ }
+
+ @Override
+ protected void additionalClusterSettings() {
+ MiniOzoneCluster.Builder builder = getClusterBuilder();
+ builder.setNumDatanodes(0);
+ }
+
private OzoneVolume createAndCheckVolume(String volumeName)
throws Exception {
String userName = "user" + RandomStringUtils.randomNumeric(5);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java
index ef7765e..a2da99d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java
@@ -27,6 +27,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneAcl;
@@ -97,9 +98,13 @@ public class TestOzoneManagerListVolumes {
// Use native impl here, default impl doesn't do actual checks
conf.set(OZONE_ACL_AUTHORIZER_CLASS, OZONE_ACL_AUTHORIZER_CLASS_NATIVE);
+ // These test do not use any features of SCM, so we can skip safemode
+ // which gets the cluster to come up much faster.
+ conf.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_ENABLED, false);
cluster = MiniOzoneCluster.newBuilder(conf)
- .setClusterId(clusterId).setScmId(scmId).setOmId(omId).build();
+ .setClusterId(clusterId).setScmId(scmId).setOmId(omId).
+ setNumDatanodes(0).build();
cluster.waitForClusterToBeReady();
// Create volumes with non-default owners and ACLs
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
