This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 841d82bef5e HDDS-15514. Pass Ratis peer addresses as hostnames (#10485)
841d82bef5e is described below
commit 841d82bef5e2e2a55364a44d06ac7935cac4d89d
Author: Ritesh H Shukla <[email protected]>
AuthorDate: Fri Jun 12 22:29:43 2026 -0700
HDDS-15514. Pass Ratis peer addresses as hostnames (#10485)
---
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 13 +++--
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 12 ++++-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 63 +++++++++-------------
.../om/ratis/TestOzoneManagerRatisServer.java | 40 ++++++++++++++
4 files changed, 86 insertions(+), 42 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
index 01f1fee0d36..d78e4a8d526 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
@@ -118,9 +118,16 @@ public void start() throws IOException {
final boolean success = HAUtils.addSCM(ozoneConf,
new AddSCMRequest.Builder().setClusterId(scm.getClusterId())
.setScmId(scm.getScmId())
- .setRatisAddr(nodeDetails
- // TODO : Should we use IP instead of hostname??
- .getRatisHostPortStr()).build(), scm.getSCMNodeId());
+ // Pass the configured host:port string verbatim. Do NOT
+ // resolve it into an InetSocketAddress first -- that bakes
+ // a resolved IP into Ratis's peer address for the channel's
+ // lifetime. With the string passed through, gRPC's
+ // DnsNameResolver re-resolves hostname addresses on
+ // connection failure (peer pod restarts recover
+ // automatically), and IP-literal configs are still honored
+ // exactly as configured. See HDDS-15514.
+ .setRatisAddr(nodeDetails.getRatisHostPortStr())
+ .build(), scm.getSCMNodeId());
if (!success) {
throw new IOException("Adding SCM to existing HA group failed");
} else {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
index 81ebf1c2cc7..cfc5898bb95 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java
@@ -396,8 +396,18 @@ private static RaftGroup buildRaftGroup(SCMNodeDetails
details,
final RaftGroupId groupId = buildRaftGroupId(clusterId);
RaftPeerId selfPeerId = getSelfPeerId(scmId);
+ // Pass the configured host:port string through verbatim. The
+ // invariant is "do not pre-resolve" -- never construct an
+ // InetSocketAddress from the configured address and hand the
+ // resolved form to RaftPeer.setAddress, which would freeze the peer
+ // at one IP for the channel's lifetime. Ratis routes the string to
+ // gRPC's NettyChannelBuilder, whose default DnsNameResolver
+ // re-resolves hostname addresses on connection failure (peer pod
+ // restarts recover automatically in Kubernetes-style environments
+ // where DNS names are stable but IPs are not). IP-literal configs
+ // are still honored exactly as configured. See HDDS-15514
+ // (DNS-refresh-on-failure for all RPC paths).
RaftPeer localRaftPeer = RaftPeer.newBuilder().setId(selfPeerId)
- // TODO : Should we use IP instead of hostname??
.setAddress(details.getRatisHostPortStr()).build();
List<RaftPeer> raftPeers = new ArrayList<>();
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
index dab93e75900..18defcf808a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
@@ -227,10 +227,8 @@ public static OzoneManagerRatisServer newOMRatisServer(
// On regular startup, add all OMs to Ratis ring
raftPeers.add(localRaftPeer);
- for (Map.Entry<String, OMNodeDetails> peerInfo : peerNodes.entrySet()) {
- String peerNodeId = peerInfo.getKey();
- OMNodeDetails peerNode = peerInfo.getValue();
- RaftPeer raftPeer = OzoneManagerRatisServer.createRaftPeer(peerNode,
peerNodeId);
+ for (OMNodeDetails peerNode : peerNodes.values()) {
+ RaftPeer raftPeer = OzoneManagerRatisServer.createRaftPeer(peerNode);
// Add other OM nodes belonging to the same OM service to the Ratis
ring
raftPeers.add(raftPeer);
@@ -435,42 +433,31 @@ private void updateRatisConfiguration(List<RaftPeer>
followers, List<RaftPeer> l
}
}
- private static RaftPeer createRaftPeer(OMNodeDetails omNode) {
- String nodeId = omNode.getNodeId();
- RaftPeerId raftPeerId = RaftPeerId.valueOf(nodeId);
- InetSocketAddress ratisAddr = new InetSocketAddress(
- omNode.getHostAddress(), omNode.getRatisPort());
- RaftPeerRole startRole = omNode.isRatisListener() ?
- RaftPeerRole.LISTENER : RaftPeerRole.FOLLOWER;
-
- return RaftPeer.newBuilder()
- .setId(raftPeerId)
- .setAddress(ratisAddr)
- .setStartupRole(startRole)
- .build();
- }
-
/**
- * Helper method to create a RaftPeer from OMNodeDetails, handling
unresolved hosts.
- * @param omNode the OM node details
- * @param nodeId the node ID to use
- * @return the created RaftPeer
+ * Build a RaftPeer for the given OM node. The peer address is set from
+ * {@link OMNodeDetails#getRatisHostPortStr()} -- the configured host
+ * string (hostname or IP literal) paired with the Ratis port. The
+ * configured string is passed through verbatim; this method never
+ * resolves it into an {@link InetSocketAddress} (which would bake the
+ * resolved IP into the peer address).
+ * <p>
+ * Why this matters: Ratis hands the address string to gRPC's
+ * {@code NettyChannelBuilder.forTarget(...)}, whose default
+ * {@code DnsNameResolver} re-resolves hostnames on connection failure
+ * / refresh. If the address is a hostname, gRPC recovers automatically
+ * from peer pod restarts in environments like Kubernetes where DNS
+ * names are stable but IPs are not. If the operator configured an IP
+ * literal, gRPC of course uses that IP directly -- the invariant is
+ * "don't pre-resolve", not "must be a hostname". See HDDS-15514
+ * (DNS-refresh-on-failure for all RPC paths).
*/
- private static RaftPeer createRaftPeer(OMNodeDetails omNode, String nodeId) {
- RaftPeerId raftPeerId = RaftPeerId.valueOf(nodeId);
- RaftPeer.Builder builder = RaftPeer.newBuilder()
- .setId(raftPeerId)
- .setStartupRole(omNode.isRatisListener() ? RaftPeerRole.LISTENER :
RaftPeerRole.FOLLOWER);
-
- if (omNode.isHostUnresolved()) {
- builder.setAddress(omNode.getRatisHostPortStr());
- } else {
- InetSocketAddress ratisAddr = new InetSocketAddress(
- omNode.getInetAddress(), omNode.getRatisPort());
- builder.setAddress(ratisAddr);
- }
-
- return builder.build();
+ static RaftPeer createRaftPeer(OMNodeDetails omNode) {
+ return RaftPeer.newBuilder()
+ .setId(RaftPeerId.valueOf(omNode.getNodeId()))
+ .setAddress(omNode.getRatisHostPortStr())
+ .setStartupRole(omNode.isRatisListener()
+ ? RaftPeerRole.LISTENER : RaftPeerRole.FOLLOWER)
+ .build();
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisServer.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisServer.java
index f5e1b2dce81..6eda7a6ba1e 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisServer.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisServer.java
@@ -145,6 +145,46 @@ public void testStartOMRatisServer() throws Exception {
"Ratis Server should be in running state");
}
+ /**
+ * RaftPeer.address must preserve the configured host string
+ * verbatim -- not a Java-resolved {@code InetSocketAddress} form. When
+ * the operator configured a hostname, the hostname must survive into
+ * RaftPeer.address so gRPC's {@code DnsNameResolver} can re-resolve it
+ * on connection failure (Kubernetes pod restarts). When the operator
+ * configured an IP literal, that literal must survive too. The
+ * regression this test guards is "{@code createRaftPeer} pre-resolved
+ * a hostname into a numeric IP and handed the resolved form to
+ * RaftPeer," which would freeze the gRPC channel at that IP for the
+ * channel's lifetime.
+ */
+ @Test
+ public void testCreateRaftPeerUsesHostnameAddress() {
+ String hostname = "om-2.om.example.svc.cluster.local";
+ int rpcPort = 9862;
+ int ratisPort = 9872;
+ OMNodeDetails peer = new OMNodeDetails.Builder()
+ .setOMServiceId("test-service")
+ .setOMNodeId("om2")
+ .setHostAddress(hostname)
+ .setRpcPort(rpcPort)
+ .setRatisPort(ratisPort)
+ .build();
+
+ org.apache.ratis.protocol.RaftPeer raftPeer =
+ OzoneManagerRatisServer.createRaftPeer(peer);
+ String addr = raftPeer.getAddress();
+ assertEquals(hostname + ":" + ratisPort, addr,
+ "RaftPeer address must preserve the configured host string "
+ + "verbatim. The configured hostname must survive into "
+ + "RaftPeer so gRPC can re-resolve it -- pre-resolving into a "
+ + "numeric IP would freeze the channel at that IP for its "
+ + "lifetime.");
+ // Defensive: the configured hostname must not have been pre-resolved
+ // into a numeric IPv4 octet form before reaching RaftPeer.
+ String host = addr.substring(0, addr.lastIndexOf(':'));
+ assertThat(host).doesNotMatch("^\\d{1,3}(\\.\\d{1,3}){3}$");
+ }
+
@Test
public void testLoadSnapshotInfoOnStart() throws Exception {
// Stop the Ratis server and manually update the snapshotInfo.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
