This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 195737d HDDS-6114. Intermittent error due to Failed to init RocksDB
(#2947)
195737d is described below
commit 195737dc72f7814207c1ec478675a61e01ac1666
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Jan 12 20:28:08 2022 +0100
HDDS-6114. Intermittent error due to Failed to init RocksDB (#2947)
---
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 39 +--
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 4 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 2 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 16 -
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 368 +++++++++++++--------
.../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 116 -------
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 23 ++
.../hadoop/ozone/TestMiniOzoneOMHACluster.java | 6 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 6 +-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 6 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 8 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 2 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 8 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 6 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 18 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 4 +-
18 files changed, 307 insertions(+), 329 deletions(-)
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
index 728ded0..05094b0 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.ozone;
import java.io.IOException;
import java.time.Duration;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -102,15 +101,13 @@ public class MiniOzoneChaosCluster extends
MiniOzoneHAClusterImpl {
@SuppressWarnings("parameternumber")
public MiniOzoneChaosCluster(OzoneConfiguration conf,
- List<OzoneManager> ozoneManagers, List<StorageContainerManager> scms,
- List<HddsDatanodeService> hddsDatanodes, String omServiceID,
- String scmServiceId, String clusterPath,
+ OMHAService omService, SCMHAService scmService,
+ List<HddsDatanodeService> hddsDatanodes, String clusterPath,
Set<Class<? extends Failures>> clazzes) {
- super(conf, ozoneManagers, scms, hddsDatanodes, omServiceID, scmServiceId,
- clusterPath);
+ super(conf, omService, scmService, hddsDatanodes, clusterPath, null);
this.numDatanodes = getHddsDatanodes().size();
- this.numOzoneManagers = ozoneManagers.size();
- this.numStorageContainerManagers = scms.size();
+ this.numOzoneManagers = omService.getServices().size();
+ this.numStorageContainerManagers = scmService.getServices().size();
this.failedOmSet = new HashSet<>();
this.failedDnSet = new HashSet<>();
@@ -304,33 +301,21 @@ public class MiniOzoneChaosCluster extends
MiniOzoneHAClusterImpl {
initOMRatisConf();
}
- List<OzoneManager> omList;
- List<StorageContainerManager> scmList;
+ SCMHAService scmService;
+ OMHAService omService;
try {
- if (numOfSCMs > 1) {
- scmList = createSCMService();
- } else {
- StorageContainerManager scm = createSCM();
- scm.start();
- scmList = Arrays.asList(scm);
- }
- if (numOfOMs > 1) {
- omList = createOMService();
- } else {
- OzoneManager om = createOM();
- om.start();
- omList = Arrays.asList(om);
- }
+ scmService = createSCMService();
+ omService = createOMService();
} catch (AuthenticationException ex) {
throw new IOException("Unable to build MiniOzoneCluster. ", ex);
}
final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(
- scmList, null);
+ scmService.getActiveServices(), null);
MiniOzoneChaosCluster cluster =
- new MiniOzoneChaosCluster(conf, omList, scmList, hddsDatanodes,
- omServiceId, scmServiceId, path, clazzes);
+ new MiniOzoneChaosCluster(conf, omService, scmService, hddsDatanodes,
+ path, clazzes);
if (startDataNodes) {
cluster.startHddsDatanodes();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
index b9fbde6..974912a 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
@@ -167,7 +167,7 @@ public class TestOzoneFsHAURLs {
* @return the leader OM's RPC address in the MiniOzoneHACluster
*/
private String getLeaderOMNodeAddr() {
- MiniOzoneOMHAClusterImpl haCluster = (MiniOzoneOMHAClusterImpl) cluster;
+ MiniOzoneHAClusterImpl haCluster = (MiniOzoneHAClusterImpl) cluster;
OzoneManager omLeader = haCluster.getOMLeader();
Assert.assertNotNull("There should be a leader OM at this point.",
omLeader);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index dc20050..b6d4b31 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -63,7 +63,7 @@ public interface MiniOzoneCluster {
* @return MiniOzoneCluster builder
*/
static Builder newOMHABuilder(OzoneConfiguration conf) {
- return new MiniOzoneOMHAClusterImpl.Builder(conf);
+ return new MiniOzoneHAClusterImpl.Builder(conf);
}
static Builder newHABuilder(OzoneConfiguration conf) {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index d9b5b3c..b2a7107 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -153,22 +153,6 @@ public class MiniOzoneClusterImpl implements
MiniOzoneCluster {
}
/**
- * Creates a new MiniOzoneCluster without the OzoneManager. This is used by
- * {@link MiniOzoneOMHAClusterImpl} for starting multiple OzoneManagers.
- *
- * @param conf
- * @param scm
- * @param hddsDatanodes
- */
- MiniOzoneClusterImpl(OzoneConfiguration conf, StorageContainerManager scm,
- List<HddsDatanodeService> hddsDatanodes, ReconServer reconServer) {
- this.conf = conf;
- this.scm = scm;
- this.hddsDatanodes = hddsDatanodes;
- this.reconServer = reconServer;
- }
-
- /**
* Creates a new MiniOzoneCluster without the OzoneManager and
* StorageContainerManager. This is used by
* {@link MiniOzoneHAClusterImpl} for starting multiple
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
index 9bfae2a..e0d061f 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -23,6 +23,7 @@ import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.hdds.ExitManager;
+import org.apache.hadoop.hdds.conf.ConfigurationTarget;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
@@ -39,22 +40,30 @@ import
org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
import org.apache.hadoop.ozone.recon.ReconServer;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ratis.util.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.Closeable;
import java.io.IOException;
import java.net.BindException;
+import java.net.ServerSocket;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Random;
+import java.util.PrimitiveIterator;
+import java.util.Queue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Function;
-import java.util.stream.Collectors;
+import static java.util.Collections.singletonList;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static
org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION;
+import static org.apache.hadoop.ozone.OzoneTestUtils.reservePorts;
import static
org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
/**
@@ -75,51 +84,27 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
private int waitForClusterToBeReadyTimeout = 120000; // 2 min
- private static final Random RANDOM = new Random();
private static final int RATIS_RPC_TIMEOUT = 1000; // 1 second
- private static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
+ public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
/**
* Creates a new MiniOzoneCluster.
*
* @throws IOException if there is an I/O error
*/
- @SuppressWarnings("checkstyle:ParameterNumber")
public MiniOzoneHAClusterImpl(
OzoneConfiguration conf,
- List<OzoneManager> activeOMList,
- List<OzoneManager> inactiveOMList,
- List<StorageContainerManager> activeSCMList,
- List<StorageContainerManager> inactiveSCMList,
+ OMHAService omhaService,
+ SCMHAService scmhaService,
List<HddsDatanodeService> hddsDatanodes,
- String omServiceId,
- String scmServiceId,
String clusterPath,
ReconServer reconServer) {
super(conf, hddsDatanodes, reconServer);
- omhaService =
- new OMHAService(activeOMList, inactiveOMList, omServiceId);
- scmhaService =
- new SCMHAService(activeSCMList, inactiveSCMList, scmServiceId);
+ this.omhaService = omhaService;
+ this.scmhaService = scmhaService;
this.clusterMetaPath = clusterPath;
}
- /**
- * Creates a new MiniOzoneCluster with all OMs active.
- * This is used by MiniOzoneChaosCluster.
- */
- protected MiniOzoneHAClusterImpl(
- OzoneConfiguration conf,
- List<OzoneManager> omList,
- List<StorageContainerManager> scmList,
- List<HddsDatanodeService> hddsDatanodes,
- String omServiceId,
- String scmServiceId,
- String clusterPath) {
- this(conf, omList, null, scmList, null, hddsDatanodes,
- omServiceId, scmServiceId, clusterPath, null);
- }
-
@Override
public String getOMServiceId() {
return omhaService.getServiceId();
@@ -155,8 +140,8 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
return omhaService.isServiceActive(omNodeId);
}
- public boolean isSCMActive(String scmNodeId) {
- return scmhaService.isServiceActive(scmNodeId);
+ public Iterator<StorageContainerManager> getInactiveSCM() {
+ return scmhaService.inactiveServices();
}
public StorageContainerManager getSCM(String scmNodeId) {
@@ -310,6 +295,13 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
}
@Override
+ public void shutdown() {
+ super.shutdown();
+ omhaService.releasePorts();
+ scmhaService.releasePorts();
+ }
+
+ @Override
public void stop() {
for (OzoneManager ozoneManager : this.omhaService.getServices()) {
if (ozoneManager != null) {
@@ -343,6 +335,29 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
omhaService.deactivate(om);
}
+ private static void configureOMPorts(ConfigurationTarget conf,
+ String omServiceId, String omNodeId,
+ ReservedPorts omPorts, ReservedPorts omRpcPorts) {
+
+ String omAddrKey = ConfUtils.addKeySuffixes(
+ OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omNodeId);
+ String omHttpAddrKey = ConfUtils.addKeySuffixes(
+ OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, omServiceId, omNodeId);
+ String omHttpsAddrKey = ConfUtils.addKeySuffixes(
+ OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, omServiceId, omNodeId);
+ String omRatisPortKey = ConfUtils.addKeySuffixes(
+ OMConfigKeys.OZONE_OM_RATIS_PORT_KEY, omServiceId, omNodeId);
+
+ PrimitiveIterator.OfInt nodePorts = omPorts.assign(omNodeId);
+ PrimitiveIterator.OfInt rpcPorts = omRpcPorts.assign(omNodeId);
+ conf.set(omAddrKey, "127.0.0.1:" + rpcPorts.nextInt());
+ conf.set(omHttpAddrKey, "127.0.0.1:" + nodePorts.nextInt());
+ conf.set(omHttpsAddrKey, "127.0.0.1:" + nodePorts.nextInt());
+ conf.setInt(omRatisPortKey, nodePorts.nextInt());
+
+ omRpcPorts.release(omNodeId);
+ }
+
/**
* Builder for configuring the MiniOzoneCluster to run.
*/
@@ -356,6 +371,15 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
private List<StorageContainerManager> activeSCMs = new ArrayList<>();
private List<StorageContainerManager> inactiveSCMs = new ArrayList<>();
+ // These port reservations are for servers started when the component
+ // (OM or SCM) is started. These are Ratis, HTTP and HTTPS. We also have
+ // another set of ports for RPC endpoints, which are started as soon as
+ // the component is created (in methods called by OzoneManager and
+ // StorageContainerManager constructors respectively). So we need to
manage
+ // them separately, see initOMHAConfig() and initSCMHAConfig().
+ private final ReservedPorts omPorts = new ReservedPorts(3);
+ private final ReservedPorts scmPorts = new ReservedPorts(3);
+
/**
* Creates a new Builder.
*
@@ -365,14 +389,6 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
super(conf);
}
- public List<OzoneManager> getActiveOMs() {
- return activeOMs;
- }
-
- public List<OzoneManager> getInactiveOMs() {
- return inactiveOMs;
- }
-
@Override
public MiniOzoneCluster build() throws IOException {
if (numOfActiveOMs > numOfOMs) {
@@ -398,11 +414,12 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
DefaultMetricsSystem.setMiniClusterMode(true);
initializeConfiguration();
initOMRatisConf();
- StorageContainerManager scm;
+ SCMHAService scmService;
+ OMHAService omService;
ReconServer reconServer = null;
try {
- createSCMService();
- createOMService();
+ scmService = createSCMService();
+ omService = createOMService();
if (includeRecon) {
configureRecon();
reconServer = new ReconServer();
@@ -413,11 +430,10 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
}
final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(
- activeSCMs, reconServer);
+ scmService.getActiveServices(), reconServer);
MiniOzoneHAClusterImpl cluster = new MiniOzoneHAClusterImpl(conf,
- activeOMs, inactiveOMs, activeSCMs, inactiveSCMs,
- hddsDatanodes, omServiceId, scmServiceId, path, reconServer);
+ omService, scmService, hddsDatanodes, path, reconServer);
if (startDataNodes) {
cluster.startHddsDatanodes();
@@ -458,18 +474,21 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
/**
* Start OM service with multiple OMs.
*/
- protected List<OzoneManager> createOMService() throws IOException,
+ protected OMHAService createOMService() throws IOException,
AuthenticationException {
+ if (omServiceId == null) {
+ OzoneManager om = createOM();
+ om.start();
+ return new OMHAService(singletonList(om), null, null, null);
+ }
List<OzoneManager> omList = Lists.newArrayList();
int retryCount = 0;
- int basePort;
while (true) {
try {
- basePort = 10000 + RANDOM.nextInt(1000) * 4;
- initOMHAConfig(basePort);
+ initOMHAConfig();
for (int i = 1; i<= numOfOMs; i++) {
// Set nodeId
@@ -520,26 +539,30 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
omList.clear();
++retryCount;
LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
- retryCount);
+ retryCount, e);
}
}
- return omList;
+ return new OMHAService(activeOMs, inactiveOMs, omServiceId, omPorts);
}
/**
* Start OM service with multiple OMs.
*/
- protected List<StorageContainerManager> createSCMService()
+ protected SCMHAService createSCMService()
throws IOException, AuthenticationException {
+ if (scmServiceId == null) {
+ StorageContainerManager scm = createSCM();
+ scm.start();
+ return new SCMHAService(singletonList(scm), null, null, null);
+ }
+
List<StorageContainerManager> scmList = Lists.newArrayList();
int retryCount = 0;
- int basePort = 12000;
while (true) {
try {
- basePort = 12000 + RANDOM.nextInt(1000) * 4;
- initSCMHAConfig(basePort);
+ initSCMHAConfig();
for (int i = 1; i<= numOfSCMs; i++) {
// Set nodeId
@@ -575,11 +598,11 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
scm.start();
activeSCMs.add(scm);
LOG.info("Started SCM RPC server at {}",
- scm.getClientProtocolServer());
+ scm.getClientRpcAddress());
} else {
inactiveSCMs.add(scm);
LOG.info("Intialized SCM at {}. This SCM is currently "
- + "inactive (not running).", scm.getClientProtocolServer());
+ + "inactive (not running).", scm.getClientRpcAddress());
}
}
@@ -590,21 +613,22 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
scm.stop();
scm.join();
LOG.info("Stopping StorageContainerManager server at {}",
- scm.getClientProtocolServer());
+ scm.getClientRpcAddress());
}
scmList.clear();
++retryCount;
LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
- retryCount);
+ retryCount, e);
}
}
- return scmList;
+
+ return new SCMHAService(activeSCMs, inactiveSCMs, scmServiceId,
scmPorts);
}
/**
* Initialize HA related configurations.
*/
- private void initSCMHAConfig(int basePort) throws IOException {
+ private void initSCMHAConfig() {
// Set configurations required for starting OM HA service, because that
// is the serviceID being passed to start Ozone HA cluster.
// Here setting internal service and OZONE_OM_SERVICE_IDS_KEY, in this
@@ -616,11 +640,14 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
StringBuilder scmNodesKeyValue = new StringBuilder();
StringBuilder scmNames = new StringBuilder();
- int port = basePort;
+ scmPorts.reserve(numOfSCMs);
+ ReservedPorts scmRpcPorts = new ReservedPorts(4);
+ scmRpcPorts.reserve(numOfSCMs);
- for (int i = 1; i <= numOfSCMs; i++, port+=10) {
+ for (int i = 1; i <= numOfSCMs; i++) {
String scmNodeId = SCM_NODE_ID_PREFIX + i;
scmNodesKeyValue.append(",").append(scmNodeId);
+
String scmAddrKey = ConfUtils.addKeySuffixes(
ScmConfigKeys.OZONE_SCM_ADDRESS_KEY, scmServiceId, scmNodeId);
String scmHttpAddrKey = ConfUtils.addKeySuffixes(
@@ -641,18 +668,32 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
String scmGrpcPortKey = ConfUtils.addKeySuffixes(
ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY, scmServiceId, scmNodeId);
+ PrimitiveIterator.OfInt nodePorts = scmPorts.assign(scmNodeId);
+ PrimitiveIterator.OfInt rpcPorts = scmRpcPorts.assign(scmNodeId);
conf.set(scmAddrKey, "127.0.0.1");
- conf.set(scmHttpAddrKey, "127.0.0.1:" + (port + 2));
- conf.set(scmHttpsAddrKey, "127.0.0.1:" + (port + 3));
- conf.setInt(scmRatisPortKey, port + 4);
- //conf.setInt("ozone.scm.ha.ratis.bind.port", port + 4);
- conf.set(dnPortKey, "127.0.0.1:" + (port + 5));
- conf.set(blockClientKey, "127.0.0.1:" + (port + 6));
- conf.set(ssClientKey, "127.0.0.1:" + (port + 7));
- conf.setInt(scmGrpcPortKey, port + 8);
- scmNames.append(",").append("localhost:" + (port + 5));
- conf.set(ScmConfigKeys.
- OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, "127.0.0.1:" + (port + 6));
+ conf.set(scmHttpAddrKey, "127.0.0.1:" + nodePorts.nextInt());
+ conf.set(scmHttpsAddrKey, "127.0.0.1:" + nodePorts.nextInt());
+
+ int ratisPort = nodePorts.nextInt();
+ conf.setInt(scmRatisPortKey, ratisPort);
+ //conf.setInt("ozone.scm.ha.ratis.bind.port", ratisPort);
+
+ int dnPort = rpcPorts.nextInt();
+ conf.set(dnPortKey, "127.0.0.1:" + dnPort);
+ scmNames.append(",localhost:").append(dnPort);
+
+ conf.set(ssClientKey, "127.0.0.1:" + rpcPorts.nextInt());
+ conf.setInt(scmGrpcPortKey, rpcPorts.nextInt());
+
+ int blockPort = rpcPorts.nextInt();
+ conf.set(blockClientKey, "127.0.0.1:" + blockPort);
+ conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+ "127.0.0.1:" + blockPort);
+
+ if (i <= numOfActiveSCMs) {
+ scmPorts.release(scmNodeId);
+ }
+ scmRpcPorts.release(scmNodeId);
}
conf.set(scmNodesKey, scmNodesKeyValue.substring(1));
@@ -662,7 +703,7 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
/**
* Initialize HA related configurations.
*/
- private void initOMHAConfig(int basePort) throws IOException {
+ private void initOMHAConfig() {
// Set configurations required for starting OM HA service, because that
// is the serviceID being passed to start Ozone HA cluster.
// Here setting internal service and OZONE_OM_SERVICE_IDS_KEY, in this
@@ -673,25 +714,19 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
OMConfigKeys.OZONE_OM_NODES_KEY, omServiceId);
List<String> omNodeIds = new ArrayList<>();
- int port = basePort;
+ omPorts.reserve(numOfOMs);
+ ReservedPorts omRpcPorts = new ReservedPorts(1);
+ omRpcPorts.reserve(numOfOMs);
- for (int i = 1; i <= numOfOMs; i++, port+=6) {
+ for (int i = 1; i <= numOfOMs; i++) {
String omNodeId = OM_NODE_ID_PREFIX + i;
omNodeIds.add(omNodeId);
- String omAddrKey = ConfUtils.addKeySuffixes(
- OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omNodeId);
- String omHttpAddrKey = ConfUtils.addKeySuffixes(
- OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, omServiceId, omNodeId);
- String omHttpsAddrKey = ConfUtils.addKeySuffixes(
- OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, omServiceId, omNodeId);
- String omRatisPortKey = ConfUtils.addKeySuffixes(
- OMConfigKeys.OZONE_OM_RATIS_PORT_KEY, omServiceId, omNodeId);
-
- conf.set(omAddrKey, "127.0.0.1:" + port);
- conf.set(omHttpAddrKey, "127.0.0.1:" + (port + 2));
- conf.set(omHttpsAddrKey, "127.0.0.1:" + (port + 3));
- conf.setInt(omRatisPortKey, port + 4);
+ configureOMPorts(conf, omServiceId, omNodeId, omPorts, omRpcPorts);
+
+ if (i <= numOfActiveOMs) {
+ omPorts.release(omNodeId);
+ }
}
conf.set(omNodesKey, String.join(",", omNodeIds));
@@ -729,9 +764,8 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
while (true) {
try {
- List<Integer> portSet = getFreePortList(4);
OzoneConfiguration newConf = addNewOMToConfig(getOMServiceId(),
- omNodeId, portSet);
+ omNodeId);
if (updateConfigs) {
updateOMConfigs(newConf);
@@ -755,7 +789,7 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
e.getCause() instanceof BindException) {
++retryCount;
LOG.info("MiniOzoneHACluster port conflicts, retried {} times",
- retryCount);
+ retryCount, e);
} else {
throw e;
}
@@ -772,30 +806,19 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
* Set the configs for new OMs.
*/
private OzoneConfiguration addNewOMToConfig(String omServiceId,
- String omNodeId, List<Integer> portList) {
+ String omNodeId) {
+
+ ReservedPorts omPorts = omhaService.getPorts();
+ omPorts.reserve(1);
+ ReservedPorts omRpcPorts = new ReservedPorts(1);
+ omRpcPorts.reserve(1);
OzoneConfiguration newConf = new OzoneConfiguration(getConf());
+ configureOMPorts(newConf, omServiceId, omNodeId, omPorts, omRpcPorts);
+
String omNodesKey = ConfUtils.addKeySuffixes(
OMConfigKeys.OZONE_OM_NODES_KEY, omServiceId);
- StringBuilder omNodesKeyValue = new StringBuilder();
- omNodesKeyValue.append(newConf.get(omNodesKey))
- .append(",").append(omNodeId);
-
- String omAddrKey = ConfUtils.addKeySuffixes(
- OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omNodeId);
- String omHttpAddrKey = ConfUtils.addKeySuffixes(
- OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, omServiceId, omNodeId);
- String omHttpsAddrKey = ConfUtils.addKeySuffixes(
- OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, omServiceId, omNodeId);
- String omRatisPortKey = ConfUtils.addKeySuffixes(
- OMConfigKeys.OZONE_OM_RATIS_PORT_KEY, omServiceId, omNodeId);
-
- newConf.set(omAddrKey, "127.0.0.1:" + portList.get(0));
- newConf.set(omHttpAddrKey, "127.0.0.1:" + portList.get(1));
- newConf.set(omHttpsAddrKey, "127.0.0.1:" + portList.get(2));
- newConf.setInt(omRatisPortKey, portList.get(3));
-
- newConf.set(omNodesKey, omNodesKeyValue.toString());
+ newConf.set(omNodesKey, newConf.get(omNodesKey) + "," + omNodeId);
return newConf;
}
@@ -838,9 +861,7 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
ExitManagerForOM exitManager = new ExitManagerForOM(this, nodeId);
om.setExitManagerForTesting(exitManager);
omhaService.addInstance(om, false);
-
- om.start();
- omhaService.activate(om);
+ startInactiveOM(nodeId);
return om;
}
@@ -902,6 +923,7 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
private List<Type> services;
private String serviceId;
private String serviceName;
+ private final ReservedPorts ports;
// Active services s denote OM/SCM services which are up and running
private List<Type> activeServices;
@@ -911,9 +933,10 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
private Function<Type, String> serviceIdProvider;
MiniOzoneHAService(String name, List<Type> activeList,
- List<Type> inactiveList, String serviceId,
- Function<Type, String> idProvider) {
+ List<Type> inactiveList, String serviceId,
+ ReservedPorts ports, Function<Type, String> idProvider) {
this.serviceName = name;
+ this.ports = ports != null ? ports : new ReservedPorts(0);
this.serviceMap = Maps.newHashMap();
this.serviceIdProvider = idProvider;
if (activeList != null) {
@@ -945,6 +968,10 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
return services;
}
+ public void releasePorts() {
+ ports.releaseAll();
+ }
+
public List<Type> getActiveServices() {
return activeServices;
}
@@ -958,6 +985,8 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
serviceMap.put(serviceIdProvider.apply(t), t);
if (isActive) {
activeServices.add(t);
+ } else {
+ inactiveServices.add(t);
}
}
@@ -975,6 +1004,10 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
return activeServices.contains(serviceMap.get(id));
}
+ public Iterator<Type> inactiveServices() {
+ return new ArrayList<>(inactiveServices).iterator();
+ }
+
public Type getServiceByIndex(int index) {
return this.services.get(index);
}
@@ -989,17 +1022,22 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
if (!inactiveServices.contains(service)) {
throw new IOException(serviceName + " is already active.");
} else {
+ ports.release(id);
serviceStarter.execute(service);
activeServices.add(service);
inactiveServices.remove(service);
}
}
+
+ public ReservedPorts getPorts() {
+ return ports;
+ }
}
static class OMHAService extends MiniOzoneHAService<OzoneManager> {
OMHAService(List<OzoneManager> activeList, List<OzoneManager> inactiveList,
- String serviceId) {
- super("OM", activeList, inactiveList, serviceId,
+ String serviceId, ReservedPorts omPorts) {
+ super("OM", activeList, inactiveList, serviceId, omPorts,
OzoneManager::getOMNodeId);
}
}
@@ -1007,10 +1045,10 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
static class SCMHAService extends
MiniOzoneHAService<StorageContainerManager> {
SCMHAService(List<StorageContainerManager> activeList,
- List<StorageContainerManager> inactiveList,
- String serviceId) {
+ List<StorageContainerManager> inactiveList,
+ String serviceId, ReservedPorts scmPorts) {
super("SCM", activeList, inactiveList, serviceId,
- StorageContainerManager::getScmId);
+ scmPorts, StorageContainerManager::getSCMNodeId);
}
}
@@ -1022,13 +1060,6 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
return getStorageContainerManagers().get(0);
}
- private List<Integer> getFreePortList(int size) {
- return org.apache.ratis.util.NetUtils.createLocalServerAddress(size)
- .stream()
- .map(inetSocketAddress -> inetSocketAddress.getPort())
- .collect(Collectors.toList());
- }
-
private static final class ExitManagerForOM extends ExitManager {
private MiniOzoneHAClusterImpl cluster;
@@ -1055,4 +1086,77 @@ public class MiniOzoneHAClusterImpl extends
MiniOzoneClusterImpl {
throw new IOException(message);
}
}
+
+ /**
+ * Reserves a number of ports for services.
+ */
+ private static class ReservedPorts {
+
+ private final Queue<ServerSocket> allPorts = new LinkedList<>();
+ private final Map<String, List<ServerSocket>> assignedPorts =
+ new HashMap<>();
+ private final int portsPerNode;
+
+ ReservedPorts(int portsPerNode) {
+ this.portsPerNode = portsPerNode;
+ }
+
+ /**
+ * Reserve {@code portsPerNode * nodes} ports by binding server sockets
+ * to random free ports. The sockets are kept open until
+ * {@link #release(String)} or {@link #releaseAll} is called.
+ */
+ public void reserve(int nodes) {
+ Preconditions.checkState(allPorts.isEmpty());
+ allPorts.addAll(reservePorts(portsPerNode * nodes));
+ }
+
+ /**
+ * Assign {@code portsPerNode} ports to a service identified by {@code id}.
+ * This set of ports should be released right before starting the service
+ * by calling {@link #release(String)}.
+ *
+ * @return iterator of the ports assigned
+ */
+ public PrimitiveIterator.OfInt assign(String id) {
+ Preconditions.checkState(allPorts.size() >= portsPerNode);
+ List<ServerSocket> nodePorts = new LinkedList<>();
+ for (int i = 0; i < portsPerNode; i++) {
+ nodePorts.add(allPorts.remove());
+ }
+ assignedPorts.put(id, nodePorts);
+ LOG.debug("assign ports for {}: {}", id, nodePorts);
+
+ return
nodePorts.stream().mapToInt(ServerSocket::getLocalPort).iterator();
+ }
+
+ /**
+ * Release the ports assigned to the service identified by {@code id}.
+ *
+ * This closes the server sockets, making the same ports available for
+ * the service. Note: there is a race condition with other processes
+ * running on the host, but that's OK since this is for tests.
+ *
+ * If no ports are assigned to the service, this is a no-op.
+ */
+ public void release(String id) {
+ List<ServerSocket> ports = assignedPorts.remove(id);
+ LOG.debug("release ports for {}: {}", id, ports);
+ if (ports != null) {
+ IOUtils.cleanup(LOG, ports.toArray(new Closeable[0]));
+ }
+ }
+
+ /**
+ * Release all reserved ports, assigned or not.
+ */
+ public void releaseAll() {
+ IOUtils.cleanup(LOG, allPorts.toArray(new Closeable[0]));
+ allPorts.clear();
+
+ for (String id : new ArrayList<>(assignedPorts.keySet())) {
+ release(id);
+ }
+ }
+ }
}
\ No newline at end of file
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMHAClusterImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMHAClusterImpl.java
deleted file mode 100644
index 2f2b773..0000000
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMHAClusterImpl.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.recon.ReconServer;
-import
org.apache.hadoop.security.authentication.client.AuthenticationException;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Collections;
-
-/**
- * MiniOzoneOMHAClusterImpl creates a complete in-process Ozone cluster
- * with OM HA suitable for running tests. The cluster consists of a set of
- * OzoneManagers, StorageContainerManager and multiple DataNodes.
- */
-public final class MiniOzoneOMHAClusterImpl extends MiniOzoneHAClusterImpl {
- public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
-
- /**
- * Creates a new MiniOzoneOMHACluster.
- *
- * @throws IOException if there is an I/O error
- */
- @SuppressWarnings("checkstyle:ParameterNumber")
- private MiniOzoneOMHAClusterImpl(
- OzoneConfiguration conf,
- List<OzoneManager> activeOMList,
- List<OzoneManager> inactiveOMList,
- StorageContainerManager scm,
- List<HddsDatanodeService> hddsDatanodes,
- String omServiceId,
- String metaPath,
- ReconServer reconServer) {
- super(conf, activeOMList, inactiveOMList, Collections.singletonList(scm),
- null, hddsDatanodes, omServiceId, null, metaPath, reconServer);
- }
-
- /**
- * Builder for configuring the MiniOzoneCluster to run.
- */
- public static class Builder extends MiniOzoneHAClusterImpl.Builder {
-
- /**
- * Creates a new Builder.
- *
- * @param conf configuration
- */
- public Builder(OzoneConfiguration conf) {
- super(conf);
- }
-
- @Override
- public MiniOzoneCluster build() throws IOException {
- if (numOfActiveOMs > numOfOMs) {
- throw new IllegalArgumentException("Number of active OMs cannot be " +
- "more than the total number of OMs");
- }
-
- // If num of ActiveOMs is not set, set it to numOfOMs.
- if (numOfActiveOMs == ACTIVE_OMS_NOT_SET) {
- numOfActiveOMs = numOfOMs;
- }
-
- DefaultMetricsSystem.setMiniClusterMode(true);
- initializeConfiguration();
- initOMRatisConf();
- StorageContainerManager scm;
- ReconServer reconServer = null;
- try {
- scm = createSCM();
- scm.start();
- createOMService();
- if (includeRecon) {
- configureRecon();
- reconServer = new ReconServer();
- reconServer.execute(new String[] {});
- }
- } catch (AuthenticationException ex) {
- throw new IOException("Unable to build MiniOzoneCluster. ", ex);
- }
-
- final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(
- Collections.singletonList(scm), reconServer);
-
- MiniOzoneClusterImpl cluster = new MiniOzoneOMHAClusterImpl(conf,
- getActiveOMs(), getInactiveOMs(), scm, hddsDatanodes,
- omServiceId, path, reconServer);
-
- if (startDataNodes) {
- cluster.startHddsDatanodes();
- }
- return cluster;
- }
- }
-}
\ No newline at end of file
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
index 23ac5e1..e4cb1a1 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
@@ -17,7 +17,13 @@
*/
package org.apache.hadoop.ozone;
+import java.io.Closeable;
import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hdds.client.BlockID;
@@ -32,6 +38,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.ozone.test.LambdaTestUtils.VoidCallable;
+import org.apache.ratis.util.IOUtils;
import org.apache.ratis.util.function.CheckedConsumer;
import org.junit.Assert;
@@ -146,4 +153,20 @@ public final class OzoneTestUtils {
Assert.assertEquals(code, ex.getResult());
}
}
+
+ public static List<ServerSocket> reservePorts(int count) {
+ List<ServerSocket> sockets = new ArrayList<>(count);
+ try {
+ for (int i = 0; i < count; i++) {
+ ServerSocket s = new ServerSocket();
+ sockets.add(s);
+ s.setReuseAddress(true);
+ s.bind(new InetSocketAddress(InetAddress.getByName(null), 0), 1);
+ }
+ } catch (IOException e) {
+ IOUtils.cleanup(null, sockets.toArray(new Closeable[0]));
+ throw new UncheckedIOException(e);
+ }
+ return sockets;
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java
index 61225a9..44b635f 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java
@@ -39,11 +39,11 @@ import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDC
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
/**
- * This class tests MiniOzoneOMHAClusterImpl.
+ * This class tests MiniOzoneHAClusterImpl.
*/
public class TestMiniOzoneOMHACluster {
- private MiniOzoneOMHAClusterImpl cluster = null;
+ private MiniOzoneHAClusterImpl cluster = null;
private OzoneConfiguration conf;
private String clusterId;
private String scmId;
@@ -71,7 +71,7 @@ public class TestMiniOzoneOMHACluster {
conf.set(OzoneConfigKeys.OZONE_ADMINISTRATORS,
OZONE_ADMINISTRATORS_WILDCARD);
conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2);
- cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId(omServiceId)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
index cd3aee4..42bfd11 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
@@ -27,7 +27,7 @@ import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
@@ -58,7 +58,7 @@ import org.slf4j.event.Level;
@Timeout(500)
public class TestOMRatisSnapshots {
- private MiniOzoneOMHAClusterImpl cluster = null;
+ private MiniOzoneHAClusterImpl cluster = null;
private ObjectStore objectStore;
private OzoneConfiguration conf;
private String clusterId;
@@ -89,7 +89,7 @@ public class TestOMRatisSnapshots {
conf.setLong(
OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
SNAPSHOT_THRESHOLD);
- cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId("om-service-test1")
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
index b2c4662..5a109f7 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java
@@ -37,7 +37,7 @@ import java.util.concurrent.TimeoutException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
@@ -67,7 +67,7 @@ public class TestOMUpgradeFinalization {
*/
@Rule
public Timeout timeout = new Timeout(300000);
- private MiniOzoneOMHAClusterImpl cluster;
+ private MiniOzoneHAClusterImpl cluster;
private OzoneManager ozoneManager;
private ClientProtocol clientProtocol;
private int fromLayoutVersion;
@@ -100,7 +100,7 @@ public class TestOMUpgradeFinalization {
OzoneConfiguration conf = new OzoneConfiguration();
String omServiceId = UUID.randomUUID().toString();
- cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
.setClusterId(UUID.randomUUID().toString())
.setScmId(UUID.randomUUID().toString())
.setOMServiceId(omServiceId)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
index 6118a5e..b3af2f9 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -69,7 +69,7 @@ import static org.junit.Assert.fail;
*/
public abstract class TestOzoneManagerHA {
- private static MiniOzoneOMHAClusterImpl cluster = null;
+ private static MiniOzoneHAClusterImpl cluster = null;
private static MiniOzoneCluster.Builder clusterBuilder = null;
private static ObjectStore objectStore;
private static OzoneConfiguration conf;
@@ -91,7 +91,7 @@ public abstract class TestOzoneManagerHA {
@Rule
public Timeout timeout = Timeout.seconds(300);
- public MiniOzoneOMHAClusterImpl getCluster() {
+ public MiniOzoneHAClusterImpl getCluster() {
return cluster;
}
@@ -180,7 +180,7 @@ public abstract class TestOzoneManagerHA {
.setOmId(omId)
.setNumOfOzoneManagers(numOfOMs);
- cluster = (MiniOzoneOMHAClusterImpl) clusterBuilder.build();
+ cluster = (MiniOzoneHAClusterImpl) clusterBuilder.build();
cluster.waitForClusterToBeReady();
objectStore = OzoneClientFactory.getRpcClient(omServiceId, conf)
.getObjectStore();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java
index 8c085fb..fc4fa36 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java
@@ -62,7 +62,7 @@ import java.util.TreeSet;
import java.util.Iterator;
import java.util.UUID;
-import static
org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl.NODE_FAILURE_TIMEOUT;
+import static
org.apache.hadoop.ozone.MiniOzoneHAClusterImpl.NODE_FAILURE_TIMEOUT;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT;
import static
org.apache.ratis.metrics.RatisMetrics.RATIS_APPLICATION_NAME_METRICS;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java
index a0c014a..7dbacc4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java
@@ -43,7 +43,7 @@ import java.util.Map;
import java.util.UUID;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static
org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl.NODE_FAILURE_TIMEOUT;
+import static
org.apache.hadoop.ozone.MiniOzoneHAClusterImpl.NODE_FAILURE_TIMEOUT;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DIRECTORY_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_ALREADY_EXISTS;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java
index ecbb6e8..a31ed2e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java
@@ -21,7 +21,7 @@ import org.apache.hadoop.ozone.om.ha.OMFailoverProxyProvider;
import org.junit.Assert;
import org.junit.Test;
-import static
org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl.NODE_FAILURE_TIMEOUT;
+import static
org.apache.hadoop.ozone.MiniOzoneHAClusterImpl.NODE_FAILURE_TIMEOUT;
/**
* Test Ozone Manager operation in distributed handler scenario with failover.
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java
index c35b45b..4db0e8e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java
@@ -24,7 +24,7 @@ import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneVolume;
@@ -47,7 +47,7 @@ import org.junit.rules.Timeout;
*/
public class TestOzoneManagerSnapshotProvider {
- private MiniOzoneOMHAClusterImpl cluster = null;
+ private MiniOzoneHAClusterImpl cluster = null;
private ObjectStore objectStore;
private OzoneConfiguration conf;
private String clusterId;
@@ -69,7 +69,7 @@ public class TestOzoneManagerSnapshotProvider {
omServiceId = "om-service-test1";
conf.setBoolean(OMConfigKeys.OZONE_OM_HTTP_ENABLED_KEY, true);
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
- cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId(omServiceId)
@@ -140,4 +140,4 @@ public class TestOzoneManagerSnapshotProvider {
return trxnInfoFromCheckpoint.getTransactionIndex();
}
-}
\ No newline at end of file
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
index 1d34000..540f174 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.utils.db.RocksDBConfiguration;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
@@ -55,7 +55,7 @@ public class TestReconWithOzoneManagerHA {
@Rule
public Timeout timeout = Timeout.seconds(300);;
- private MiniOzoneOMHAClusterImpl cluster;
+ private MiniOzoneHAClusterImpl cluster;
private ObjectStore objectStore;
private static final String OM_SERVICE_ID = "omService1";
private static final String VOL_NAME = "testrecon";
@@ -70,7 +70,7 @@ public class TestReconWithOzoneManagerHA {
dbConf.setSyncOption(true);
conf.setFromObject(dbConf);
- cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf)
.setClusterId(UUID.randomUUID().toString())
.setScmId(UUID.randomUUID().toString())
.setOMServiceId(OM_SERVICE_ID)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
index e9a7cf3..1932b66 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java
@@ -20,6 +20,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.UUID;
@@ -120,7 +121,7 @@ public class TestSCMInstallSnapshotWithHA {
StorageContainerManager leaderSCM = getLeader(cluster);
Assert.assertNotNull(leaderSCM);
// Find the inactive SCM
- String followerId = getInactiveSCM(cluster).getScmId();
+ String followerId = getInactiveSCM(cluster).getSCMNodeId();
StorageContainerManager followerSCM = cluster.getSCM(followerId);
// Do some transactions so that the log index increases
@@ -159,7 +160,7 @@ public class TestSCMInstallSnapshotWithHA {
public void testInstallOldCheckpointFailure() throws Exception {
// Get the leader SCM
StorageContainerManager leaderSCM = getLeader(cluster);
- String followerId = getInactiveSCM(cluster).getScmId();
+ String followerId = getInactiveSCM(cluster).getSCMNodeId();
// Find the inactive SCM
StorageContainerManager followerSCM = cluster.getSCM(followerId);
@@ -213,7 +214,7 @@ public class TestSCMInstallSnapshotWithHA {
public void testInstallCorruptedCheckpointFailure() throws Exception {
StorageContainerManager leaderSCM = getLeader(cluster);
// Find the inactive SCM
- String followerId = getInactiveSCM(cluster).getScmId();
+ String followerId = getInactiveSCM(cluster).getSCMNodeId();
StorageContainerManager followerSCM = cluster.getSCM(followerId);
// Do some transactions so that the log index increases
writeToIncreaseLogIndex(leaderSCM, 100);
@@ -316,13 +317,10 @@ public class TestSCMInstallSnapshotWithHA {
return null;
}
- static StorageContainerManager getInactiveSCM(MiniOzoneHAClusterImpl impl) {
- for (StorageContainerManager scm : impl.getStorageContainerManagers()) {
- if (!impl.isSCMActive(scm.getScmId())) {
- return scm;
- }
- }
- return null;
+ private static StorageContainerManager getInactiveSCM(
+ MiniOzoneHAClusterImpl cluster) {
+ Iterator<StorageContainerManager> inactiveScms = cluster.getInactiveSCM();
+ return inactiveScms.hasNext() ? inactiveScms.next() : null;
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index d63b303..f525942 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.fs.ozone.OzoneFsShell;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.ha.ConfUtils;
import org.apache.hadoop.ozone.om.OMConfigKeys;
@@ -241,7 +241,7 @@ public class TestOzoneShellHA {
* @return the leader OM's Node ID in the MiniOzoneHACluster.
*/
private String getLeaderOMNodeId() {
- MiniOzoneOMHAClusterImpl haCluster = (MiniOzoneOMHAClusterImpl) cluster;
+ MiniOzoneHAClusterImpl haCluster = (MiniOzoneHAClusterImpl) cluster;
OzoneManager omLeader = haCluster.getOMLeader();
Assert.assertNotNull("There should be a leader OM at this point.",
omLeader);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
