This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e1acbb0 HDDS-5256. Fix fall back of config in SCM HA Cluster (#2272)
e1acbb0 is described below
commit e1acbb07002fc53f21c807418613d38c0dd9ff61
Author: Bharat Viswanadham <[email protected]>
AuthorDate: Fri May 21 09:29:11 2021 +0530
HDDS-5256. Fix fall back of config in SCM HA Cluster (#2272)
---
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 4 +-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 51 +++++++++++----
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 7 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 76 ++++++++++++++++++++++
4 files changed, 121 insertions(+), 17 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
index 8e8e73c..a8f4609 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java
@@ -69,6 +69,7 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT;
@@ -197,7 +198,8 @@ public final class HddsServerUtil {
return NetUtils.createSocketAddr(
host.orElse(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" +
- port.orElse(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
+ port.orElse(conf.getInt(OZONE_SCM_DATANODE_PORT_KEY,
+ ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)));
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
index cada48c..fa4bfcb 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
@@ -37,6 +37,7 @@ import java.util.OptionalInt;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_KEY;
@@ -95,12 +96,18 @@ public final class ScmUtils {
OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, localScmServiceId, nodeId);
final OptionalInt port = getPortNumberFromConfigKeys(conf, addressKey);
+ if (port.isPresent()) {
+ logWarn(OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+ OZONE_SCM_BLOCK_CLIENT_PORT_KEY);
+ }
return NetUtils.createSocketAddr(
host.orElse(
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT) + ":" +
- port.orElse(conf.getInt(ConfUtils.addKeySuffixes(
- OZONE_SCM_BLOCK_CLIENT_PORT_KEY, localScmServiceId, nodeId),
- OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT)));
+ OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT) + ":" +
+ port.orElse(conf.getInt(
+ ConfUtils.addKeySuffixes(OZONE_SCM_BLOCK_CLIENT_PORT_KEY,
+ localScmServiceId, nodeId),
+ conf.getInt(OZONE_SCM_BLOCK_CLIENT_PORT_KEY,
+ OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT))));
}
public static String getScmBlockProtocolServerAddressKey(
@@ -120,11 +127,18 @@ public final class ScmUtils {
String addressKey = ConfUtils.addKeySuffixes(
OZONE_SCM_CLIENT_ADDRESS_KEY, localScmServiceId, nodeId);
- final int port = getPortNumberFromConfigKeys(conf, addressKey)
- .orElse(conf.getInt(ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_PORT_KEY,
- localScmServiceId, nodeId), OZONE_SCM_CLIENT_PORT_DEFAULT));
+ OptionalInt port = getPortNumberFromConfigKeys(conf, addressKey);
- return NetUtils.createSocketAddr(host + ":" + port);
+ if (port.isPresent()) {
+ logWarn(OZONE_SCM_CLIENT_ADDRESS_KEY, OZONE_SCM_CLIENT_PORT_KEY);
+ }
+
+ return NetUtils.createSocketAddr(host + ":" +
+ port.orElse(
+ conf.getInt(ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_PORT_KEY,
+ localScmServiceId, nodeId),
+ conf.getInt(OZONE_SCM_CLIENT_PORT_KEY,
+ OZONE_SCM_CLIENT_PORT_DEFAULT))));
}
public static String getClientProtocolServerAddressKey(
@@ -136,20 +150,22 @@ public final class ScmUtils {
public static InetSocketAddress getScmDataNodeBindAddress(
ConfigurationSource conf, String localScmServiceId, String nodeId) {
String bindHostKey = ConfUtils.addKeySuffixes(
- OZONE_SCM_DATANODE_BIND_HOST_KEY,
- localScmServiceId, nodeId
- );
+ OZONE_SCM_DATANODE_BIND_HOST_KEY, localScmServiceId, nodeId);
final Optional<String> host = getHostNameFromConfigKeys(conf, bindHostKey);
String addressKey = ConfUtils.addKeySuffixes(
- OZONE_SCM_DATANODE_ADDRESS_KEY, localScmServiceId,
- nodeId);
+ OZONE_SCM_DATANODE_ADDRESS_KEY, localScmServiceId, nodeId);
final OptionalInt port = getPortNumberFromConfigKeys(conf, addressKey);
+ if (port.isPresent()) {
+ logWarn(OZONE_SCM_DATANODE_ADDRESS_KEY, OZONE_SCM_DATANODE_PORT_KEY);
+ }
+
return NetUtils.createSocketAddr(
host.orElse(OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" +
port.orElse(conf.getInt(ConfUtils.addKeySuffixes(
OZONE_SCM_DATANODE_PORT_KEY, localScmServiceId, nodeId),
- OZONE_SCM_DATANODE_PORT_DEFAULT)));
+ conf.getInt(OZONE_SCM_DATANODE_PORT_KEY,
+ OZONE_SCM_DATANODE_PORT_DEFAULT))));
}
public static String getScmDataNodeBindAddressKey(
@@ -158,4 +174,11 @@ public final class ScmUtils {
OZONE_SCM_DATANODE_ADDRESS_KEY,
serviceId, nodeId);
}
+
+ private static void logWarn(String confKey, String portKey) {
+ LOG.warn("ConfigKey {} is deprecated, For configuring different ports " +
+ "for each SCM use PortConfigKey {} appended with serviceId and " +
+ "nodeId. If want to configure same port configure {}", confKey,
+ portKey, portKey);
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
index 55f7d11..937a9d0 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java
@@ -200,12 +200,15 @@ public class SCMHANodeDetails {
String ratisPortKey =
ConfUtils.addKeySuffixes(OZONE_SCM_RATIS_PORT_KEY,
serviceId, nodeId);
- int ratisPort = conf.getInt(ratisPortKey,
OZONE_SCM_RATIS_PORT_DEFAULT);
+ int ratisPort = conf.getInt(ratisPortKey,
+ conf.getInt(OZONE_SCM_RATIS_PORT_KEY,
+ OZONE_SCM_RATIS_PORT_DEFAULT));
String grpcPortKey = ConfUtils
.addKeySuffixes(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY, serviceId,
nodeId);
- int grpcPort = conf.getInt(grpcPortKey, OZONE_SCM_GRPC_PORT_DEFAULT);
+ int grpcPort = conf.getInt(grpcPortKey,
+ conf.getInt(OZONE_SCM_GRPC_PORT_KEY, OZONE_SCM_GRPC_PORT_DEFAULT));
InetSocketAddress addr = null;
try {
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
index f913a70..596fbbe 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAConfiguration.java
@@ -19,11 +19,15 @@ package org.apache.hadoop.hdds.scm.ha;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.ha.ConfUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.net.InetSocketAddress;
+
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY;
@@ -35,6 +39,7 @@ import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRES
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_DIRS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTP_BIND_HOST_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PORT_KEY;
@@ -201,4 +206,75 @@ public class TestSCMHAConfiguration {
}
+
+
+ @Test
+ public void testHAWithSamePortConfig() throws Exception {
+ String scmServiceId = "scmserviceId";
+ conf.set(ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY, scmServiceId);
+
+ String[] nodes = new String[] {"scm1", "scm2", "scm3"};
+ conf.set(ScmConfigKeys.OZONE_SCM_NODES_KEY+"."+scmServiceId,
+ "scm1,scm2,scm3");
+ conf.set(ScmConfigKeys.OZONE_SCM_NODE_ID_KEY, "scm1");
+
+
+ for (String node : nodes) {
+ conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, scmServiceId,
+ node), "localhost");
+ }
+
+ conf.set(OZONE_SCM_RATIS_PORT_KEY, "10000");
+ conf.set(OZONE_SCM_GRPC_PORT_KEY, "10001");
+ conf.set(OZONE_SCM_BLOCK_CLIENT_PORT_KEY, "9896");
+ conf.set(OZONE_SCM_CLIENT_PORT_KEY, "9897");
+ conf.set(OZONE_SCM_DATANODE_PORT_KEY, "9898");
+ conf.set(OZONE_SCM_SECURITY_SERVICE_PORT_KEY, "9899");
+
+
+ SCMHANodeDetails scmhaNodeDetails = SCMHANodeDetails.loadSCMHAConfig(conf);
+
+ Assert.assertEquals("10000", conf.get(OZONE_SCM_RATIS_PORT_KEY));
+ Assert.assertEquals("10001", conf.get(OZONE_SCM_GRPC_PORT_KEY));
+
+
+ InetSocketAddress clientAddress =
+ NetUtils.createSocketAddr("0.0.0.0",
+ 9897);
+ InetSocketAddress blockAddress =
+ NetUtils.createSocketAddr("0.0.0.0", 9896);
+ InetSocketAddress datanodeAddress =
+ NetUtils.createSocketAddr("0.0.0.0", 9898);
+ Assert.assertEquals(clientAddress, scmhaNodeDetails.getLocalNodeDetails()
+ .getClientProtocolServerAddress());
+ Assert.assertEquals(blockAddress, scmhaNodeDetails.getLocalNodeDetails()
+ .getBlockProtocolServerAddress());
+ Assert.assertEquals(datanodeAddress, scmhaNodeDetails.getLocalNodeDetails()
+ .getDatanodeProtocolServerAddress());
+
+ Assert.assertEquals(10000,
+ scmhaNodeDetails.getLocalNodeDetails().getRatisPort());
+ Assert.assertEquals(10001,
+ scmhaNodeDetails.getLocalNodeDetails().getGrpcPort());
+
+ for (SCMNodeDetails peer : scmhaNodeDetails.getPeerNodeDetails()) {
+ Assert.assertEquals(clientAddress,
peer.getClientProtocolServerAddress());
+ Assert.assertEquals(blockAddress, peer.getBlockProtocolServerAddress());
+ Assert.assertEquals(datanodeAddress,
+ peer.getDatanodeProtocolServerAddress());
+
+ Assert.assertEquals(10000, peer.getRatisPort());
+ Assert.assertEquals(10001,
+ peer.getGrpcPort());
+ }
+
+
+ // Security protocol address is not set in SCMHANode Details.
+ // Check conf is properly set with expected port.
+ Assert.assertEquals(
+ NetUtils.createSocketAddr("0.0.0.0", 9899),
+ HddsServerUtil.getScmSecurityInetAddress(conf));
+
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
