This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8c6c9fa7229 HDDS-16139. Define and enforce OFS and O3FS IPv6 authority
contracts (#11108)
8c6c9fa7229 is described below
commit 8c6c9fa7229faa62af1b81e7190934e605e5071c
Author: Eric C. Ho <[email protected]>
AuthorDate: Wed Sep 23 01:11:42 2026 +0800
HDDS-16139. Define and enforce OFS and O3FS IPv6 authority contracts
(#11108)
Generated-by: Codex (GPT-5)
---
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 16 ++--
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 52 ++++++++++
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 17 ++++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 8 ++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 15 ++-
.../hadoop/fs/ozone/TestBasicOzoneFileSystems.java | 105 +++++++++++++++++++++
6 files changed, 205 insertions(+), 8 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index dc676300b42..617e1636440 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -151,8 +151,8 @@ public static String getOmRpcAddress(ConfigurationSource
conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_OM_ADDRESS_KEY);
- return host.orElse(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
- getOmRpcPort(conf);
+ return getHostPortString(host.orElse(OZONE_OM_BIND_HOST_DEFAULT),
+ getOmRpcPort(conf));
}
/**
@@ -167,8 +167,9 @@ public static String getOmRpcAddress(ConfigurationSource
conf,
final Optional<String> host = getHostNameFromConfigKeys(conf, confKey);
if (host.isPresent()) {
- return host.get() + ":" + getPortNumberFromConfigKeys(conf, confKey)
- .orElse(OZONE_OM_PORT_DEFAULT);
+ return getHostPortString(host.get(),
+ getPortNumberFromConfigKeys(conf, confKey)
+ .orElse(OZONE_OM_PORT_DEFAULT));
} else {
// The specified confKey is not set
return null;
@@ -626,7 +627,8 @@ public static String
getHttpAddressForOMPeerNode(ConfigurationSource conf,
String hostName = bindHost.orElse(addressHost.orElse(omNodeHostAddr));
- return hostName + ":" +
addressPort.orElse(OZONE_OM_HTTP_BIND_PORT_DEFAULT);
+ return getHostPortString(hostName,
+ addressPort.orElse(OZONE_OM_HTTP_BIND_PORT_DEFAULT));
}
/**
@@ -652,8 +654,8 @@ public static String
getHttpsAddressForOMPeerNode(ConfigurationSource conf,
String hostName = bindHost.orElse(addressHost.orElse(omNodeHostAddr));
- return hostName + ":" +
- addressPort.orElse(OZONE_OM_HTTPS_BIND_PORT_DEFAULT);
+ return getHostPortString(hostName,
+ addressPort.orElse(OZONE_OM_HTTPS_BIND_PORT_DEFAULT));
}
public static File createOMDir(String dirPath) {
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
index 1521cdd24ee..0657d3b4b24 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
@@ -38,11 +38,13 @@
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
import java.nio.file.Path;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.ha.ConfUtils;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -216,6 +218,56 @@ void getOmSocketAddressHostPort() {
assertEquals(100, addr.getPort());
}
+ @Test
+ void getOmSocketAddressIpv6HostPort() throws UnknownHostException {
+ final OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_OM_ADDRESS_KEY, "[2001:db8::10]:100");
+
+ assertEquals("[2001:db8::10]:100", OmUtils.getOmRpcAddress(conf));
+ InetSocketAddress addr = OmUtils.getOmAddress(conf);
+ assertEquals(InetAddress.getByName("2001:db8::10"), addr.getAddress());
+ assertEquals(100, addr.getPort());
+ }
+
+ @Test
+ void getOmSocketAddressIpv6Host() throws UnknownHostException {
+ final OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_OM_ADDRESS_KEY, "[2001:db8::10]");
+
+ int defaultPort = OMConfigKeys.OZONE_OM_PORT_DEFAULT;
+ assertEquals("[2001:db8::10]:" + defaultPort,
+ OmUtils.getOmRpcAddress(conf));
+ InetSocketAddress addr = OmUtils.getOmAddress(conf);
+ assertEquals(InetAddress.getByName("2001:db8::10"), addr.getAddress());
+ assertEquals(defaultPort, addr.getPort());
+ }
+
+ @Test
+ void getOmRpcAddressIpv6ForConfigKey() {
+ final OzoneConfiguration conf = new OzoneConfiguration();
+ String addressKey = OZONE_OM_ADDRESS_KEY + ".service.node";
+ conf.set(addressKey, "[2001:db8::10]:100");
+
+ assertEquals("[2001:db8::10]:100",
+ OmUtils.getOmRpcAddress(conf, addressKey));
+ }
+
+ @Test
+ void getOmPeerHttpAddressesIpv6() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ String serviceId = "service";
+ String nodeId = "node";
+ conf.set(ConfUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY,
serviceId, nodeId),
+ "[2001:db8::10]:100");
+ conf.set(ConfUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY,
serviceId, nodeId),
+ "[2001:db8::10]:200");
+
+ assertEquals("[2001:db8::10]:100",
+ OmUtils.getHttpAddressForOMPeerNode(conf, serviceId, nodeId,
"fallback"));
+ assertEquals("[2001:db8::10]:200",
+ OmUtils.getHttpsAddressForOMPeerNode(conf, serviceId, nodeId,
"fallback"));
+ }
+
@Test
void getOmSocketAddressEmpty() {
final OzoneConfiguration conf = new OzoneConfiguration();
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java
index 4fdb3e1cb1d..4a91b44ee34 100644
---
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java
@@ -177,6 +177,23 @@ public void testExcludesListenerNodes() throws Exception {
}
}
+ @Test
+ public void testConfiguredIpv6Address() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_OM_ADDRESS_KEY, "[2001:db8::10]:9862");
+
+ try (HadoopRpcOMFailoverProxyProvider<OzoneManagerProtocolPB> ipv6Provider
=
+ new HadoopRpcOMFailoverProxyProvider<>(conf,
+ UserGroupInformation.getCurrentUser(), null,
+ OzoneManagerProtocolPB.class)) {
+ List<OMProxyInfo<OzoneManagerProtocolPB>> proxies =
+ ipv6Provider.getOMProxies();
+ assertEquals(1, proxies.size());
+ assertEquals("[2001:db8::10]:9862",
+ proxies.get(0).getAddressString());
+ }
+ }
+
@Test
public void testOMProxyMap() {
final String serviceID = "service0";
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index 7ade8339232..28cdb10dddb 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -33,6 +33,7 @@
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
+import com.google.common.net.InetAddresses;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -129,6 +130,9 @@ public class BasicOzoneFileSystem extends FileSystem {
"o3fs://bucket.volume.om-host.example.com:5678/key OR " +
"o3fs://bucket.volume.omServiceId/key";
+ private static final String IPV6_EXCEPTION_TEXT =
+ "Raw IPv6 literals are not supported in O3FS authorities; use an OM
service ID or DNS name instead";
+
private static final int PATH_DEPTH_TO_BUCKET = 0;
@Override
@@ -162,6 +166,10 @@ public void initialize(URI name, Configuration conf)
throws IOException {
String volumeStr = matcher.group(2);
String remaining = matcher.groupCount() == 3 ? matcher.group(3) : null;
+ if (!isEmpty(remaining) && remaining.contains(":") &&
InetAddresses.isInetAddress(remaining)) {
+ throw new IllegalArgumentException(IPV6_EXCEPTION_TEXT);
+ }
+
String omHost = null;
int omPort = -1;
if (!isEmpty(remaining)) {
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 089f0c453e6..5eb54a21335 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -38,6 +38,7 @@
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.net.HostAndPort;
+import com.google.common.net.InetAddresses;
import io.opentelemetry.api.trace.Span;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -132,7 +133,12 @@ public class BasicRootedOzoneFileSystem extends FileSystem
{
"URL should be one of the following formats: " +
"ofs://om-service-id/path/to/key OR " +
"ofs://om-host.example.com/path/to/key OR " +
- "ofs://om-host.example.com:5678/path/to/key";
+ "ofs://om-host.example.com:5678/path/to/key OR " +
+ "ofs://[2001:db8::10]:9862/path/to/key";
+
+ private static final String UNBRACKETED_IPV6_EXCEPTION_TEXT =
+ "IPv6 literals in OFS authorities must be enclosed in brackets, for
example " +
+ "ofs://[2001:db8::10]/ or ofs://[2001:db8::10]:9862/";
private static final int PATH_DEPTH_TO_BUCKET = 2;
private OzoneConfiguration ozoneConfiguration;
@@ -159,6 +165,13 @@ public void initialize(URI name, Configuration conf)
throws IOException {
throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
}
+ if (authority.indexOf(':') != authority.lastIndexOf(':') &&
!authority.startsWith("[")) {
+ if (InetAddresses.isInetAddress(authority)) {
+ throw new IllegalArgumentException(UNBRACKETED_IPV6_EXCEPTION_TEXT);
+ }
+ throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
+ }
+
// Parse hostname and port. HostAndPort is bracket-aware, so IPv6 literal
// authorities (for example [::1]:9862) are split correctly instead of on
// every colon.
diff --git
a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicOzoneFileSystems.java
b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicOzoneFileSystems.java
index 6cab32c9ea0..f9e9ebfc6da 100644
---
a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicOzoneFileSystems.java
+++
b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicOzoneFileSystems.java
@@ -29,13 +29,18 @@
import static org.mockito.Mockito.anyInt;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.isNull;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.nullable;
+import static org.mockito.Mockito.same;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
@@ -178,6 +183,106 @@ public void testRootedAuthorityParsing(String uri, String
expectedHost,
assertEquals(new URI(uri).getAuthority(), ofs.getUri().getAuthority());
}
+ @Test
+ public void testRootedIpv6UriConstructionAndRoundTrip() throws Exception {
+ URI uri = new URI("ofs", null, "2001:db8::10", 9862, "/", null, null);
+ assertEquals("ofs://[2001:db8::10]:9862/", uri.toString());
+
+ BasicRootedOzoneFileSystem ofs = spy(new BasicRootedOzoneFileSystem());
+ BasicRootedOzoneClientAdapterImpl adapter =
mock(BasicRootedOzoneClientAdapterImpl.class);
+ doReturn(adapter).when(ofs).createAdapter(any(), anyString(), anyInt());
+ ofs.initialize(uri, new OzoneConfiguration());
+
+ Path qualified = ofs.makeQualified(new Path("/volume/bucket/key"));
+ assertEquals("ofs://[2001:db8::10]:9862/volume/bucket/key",
qualified.toString());
+ assertEquals(qualified, new Path(qualified.toUri()));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "ofs://2001:db8::10/",
+ "ofs://2001:db8::10:9862/"
+ })
+ public void testRootedAuthorityRejectsUnbracketedIpv6(String uri) throws
Exception {
+ BasicRootedOzoneFileSystem ofs = new BasicRootedOzoneFileSystem();
+ IllegalArgumentException exception =
assertThrows(IllegalArgumentException.class,
+ () -> ofs.initialize(new URI(uri), new OzoneConfiguration()));
+ assertTrue(exception.getMessage().contains("must be enclosed in
brackets"));
+ }
+
+ @Test
+ public void testRootedMalformedAuthorityUsesGeneralError() throws Exception {
+ BasicRootedOzoneFileSystem ofs = new BasicRootedOzoneFileSystem();
+ IllegalArgumentException exception =
assertThrows(IllegalArgumentException.class,
+ () -> ofs.initialize(new URI("ofs://host:123:456/"), new
OzoneConfiguration()));
+ assertTrue(exception.getMessage().contains("URL should be one of the
following formats"));
+ }
+
+ @ParameterizedTest
+ @CsvSource(value = {
+ "o3fs://bucket.volume/, NULL, -1",
+ "o3fs://bucket.volume.omservice1/, omservice1, -1",
+ "o3fs://bucket.volume.om.example.com:9862/, om.example.com, 9862",
+ "o3fs://bucket.volume.192.0.2.1:9862/, 192.0.2.1, 9862"
+ }, nullValues = "NULL")
+ public void testO3fsAuthorityParsing(String uri, String expectedHost, int
expectedPort) throws Exception {
+ BasicOzoneFileSystem o3fs = spy(new BasicOzoneFileSystem());
+ BasicOzoneClientAdapterImpl adapter =
mock(BasicOzoneClientAdapterImpl.class);
+ doReturn(adapter).when(o3fs).createAdapter(any(), anyString(),
anyString(), nullable(String.class), anyInt());
+
+ o3fs.initialize(new URI(uri), new OzoneConfiguration());
+
+ ArgumentCaptor<String> hostCaptor = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<Integer> portCaptor =
ArgumentCaptor.forClass(Integer.class);
+ verify(o3fs).createAdapter(any(), anyString(), anyString(),
hostCaptor.capture(), portCaptor.capture());
+ assertEquals(expectedHost, hostCaptor.getValue());
+ assertEquals(expectedPort, portCaptor.getValue().intValue());
+ assertEquals(new URI(uri).getAuthority(), o3fs.getUri().getAuthority());
+ }
+
+ @Test
+ public void testO3fsNullEndpoint() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ BasicOzoneFileSystem o3fs = spy(new BasicOzoneFileSystem());
+ BasicOzoneClientAdapterImpl adapter =
mock(BasicOzoneClientAdapterImpl.class);
+ doReturn(adapter).when(o3fs).createAdapter(any(), anyString(),
anyString(), nullable(String.class), anyInt());
+
+ o3fs.initialize(new URI("o3fs://bucket.volume/"), conf);
+
+ verify(o3fs).createAdapter(same(conf), eq("bucket"), eq("volume"),
isNull(), eq(-1));
+ }
+
+ @Test
+ public void testO3fsDottedIpv6UriCannotBeConstructed() {
+ assertThrows(URISyntaxException.class,
+ () -> new URI("o3fs://bucket.volume.[2001:db8::10]:9862/"));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "o3fs://bucket.volume.2001:db8::10/",
+ "o3fs://bucket.volume.2001:db8::10:9862/"
+ })
+ public void testO3fsInitializationRejectsIpv6LiteralForms(String uri) throws
Exception {
+ BasicOzoneFileSystem o3fs = new BasicOzoneFileSystem();
+ IllegalArgumentException exception =
assertThrows(IllegalArgumentException.class,
+ () -> o3fs.initialize(new URI(uri), new OzoneConfiguration()));
+ assertTrue(exception.getMessage().contains("use an OM service ID or DNS
name instead"));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "o3fs://[2001:db8::10]/",
+ "o3fs://[2001:db8::10]:9862/",
+ "o3fs://bucket.volume.host:123:456/"
+ })
+ public void testO3fsMalformedAuthoritiesUseGeneralError(String uri) throws
Exception {
+ BasicOzoneFileSystem o3fs = new BasicOzoneFileSystem();
+ IllegalArgumentException exception =
assertThrows(IllegalArgumentException.class,
+ () -> o3fs.initialize(new URI(uri), new OzoneConfiguration()));
+ assertTrue(exception.getMessage().contains("Ozone file system URL should
be one of the following formats"));
+ }
+
@Test
public void testRootedAuthorityRejectsOutOfRangePort() {
BasicRootedOzoneFileSystem ofs = spy(new BasicRootedOzoneFileSystem());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
