This is an automated email from the ASF dual-hosted git repository.
ivandika3 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 74a7ab4b48f HDDS-16286. Use StaticMapping in MiniOzoneCluster topology
setup (#11158)
74a7ab4b48f is described below
commit 74a7ab4b48f4917f27ea99393b7d105a42ed6561
Author: Shuo Huang <[email protected]>
AuthorDate: Mon Aug 31 14:01:11 2026 +0800
HDDS-16286. Use StaticMapping in MiniOzoneCluster topology setup (#11158)
---
.../hadoop/hdds/scm/TestRackAwarePlacement.java | 4 +
.../hdds/scm/TestStorageContainerManager.java | 24 +---
hadoop-ozone/mini-cluster/pom.xml | 5 +
.../org/apache/hadoop/ozone/FixedHostMapping.java | 153 ---------------------
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 19 ++-
5 files changed, 30 insertions(+), 175 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRackAwarePlacement.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRackAwarePlacement.java
index 6df252fdc79..d915e10380f 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRackAwarePlacement.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRackAwarePlacement.java
@@ -43,6 +43,7 @@
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.StaticMapping;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -130,6 +131,9 @@ void testContainerPlacementWithPolicy(
assertPipelinesSpanMultipleRacks(cluster);
assertContainerReplicationIsRackAware(cluster);
}
+
+ assertTrue(new StaticMapping().getSwitchMap().isEmpty(),
+ "Static mapping should be cleared after cluster shutdown");
}
@Nested
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
index 8aa364b19df..da93072e40c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.hdds.scm;
-import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
@@ -47,7 +46,6 @@
import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
-import java.net.UnknownHostException;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
@@ -112,9 +110,7 @@
import
org.apache.hadoop.hdds.server.events.FixedThreadPoolWithAffinityExecutor;
import org.apache.hadoop.hdds.utils.HddsVersionInfo;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.net.DNSToSwitchMapping;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.net.StaticMapping;
import org.apache.hadoop.ozone.DataTestUtil;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -160,7 +156,6 @@
*/
public class TestStorageContainerManager {
private static final int KEY_COUNT = 5;
- private static final String LOCALHOST_IP = "127.0.0.1";
private static final Logger LOG = LoggerFactory.getLogger(
TestStorageContainerManager.class);
@@ -169,12 +164,16 @@ public class TestStorageContainerManager {
void test(@TempDir Path tempDir) throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setBoolean(OZONE_TEST_AUTHORIZATION_ENABLED, true);
- configureTopology(conf);
configureBlockDeletion(conf);
Path scmPath = tempDir.resolve("scm-meta");
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ String host = NetUtils.normalizeHostName(HddsUtils.getHostName(conf));
- try (MiniOzoneCluster cluster =
MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build()) {
+ try (MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(1)
+ .setHosts(new String[] {host})
+ .setRacks(new String[] {"/rack1"})
+ .build()) {
cluster.waitForClusterToBeReady();
// non-destructive test cases
@@ -643,17 +642,6 @@ private void
testScmProcessDatanodeHeartbeat(MiniOzoneCluster cluster) {
}
}
- private static void configureTopology(OzoneConfiguration conf) throws
UnknownHostException {
- String rackName = "/rack1";
- conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
- StaticMapping.class, DNSToSwitchMapping.class);
-
StaticMapping.addNodeToRack(NetUtils.normalizeHostName(HddsUtils.getHostName(conf)),
- rackName);
- // In case of JDK17, the IP address is resolved to localhost mapped to
127.0.0.1 which is not in sync with JDK8
- // and hence need to make following entry under HDDS-10132
- StaticMapping.addNodeToRack(LOCALHOST_IP, rackName);
- }
-
@Test
@SuppressWarnings("unchecked")
public void testCloseContainerCommandOnRestart() throws Exception {
diff --git a/hadoop-ozone/mini-cluster/pom.xml
b/hadoop-ozone/mini-cluster/pom.xml
index a6a250be449..c95b80eeb43 100644
--- a/hadoop-ozone/mini-cluster/pom.xml
+++ b/hadoop-ozone/mini-cluster/pom.xml
@@ -50,6 +50,11 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-common</artifactId>
diff --git
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/FixedHostMapping.java
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/FixedHostMapping.java
deleted file mode 100644
index 801af9fb1cf..00000000000
---
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/FixedHostMapping.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-import org.apache.hadoop.net.CachedDNSToSwitchMapping;
-import org.apache.hadoop.net.DNSToSwitchMapping;
-import org.apache.hadoop.net.NetworkTopology;
-
-/**
- * A {@link CachedDNSToSwitchMapping} implementation that resolves hostnames
- * to rack locations using a statically configured map, bypassing DNS lookups.
- *
- * <p>This is intended for use in test environments (e.g. {@code
MiniOzoneCluster})
- * where DataNode hostnames may be synthetic or unresolvable via DNS. The
standard
- * {@link CachedDNSToSwitchMapping} performs DNS normalization before rack
resolution,
- * which can cause synthetic hostnames to be incorrectly resolved to a real IP
address,
- * leading to rack mapping failures. This class avoids that by resolving
directly
- * against the registered hostname.
- *
- * <p>The mapping is stored in a JVM-wide static map. Callers must invoke
- * {@link #addNode(String, String)} before cluster startup to register
hostname-to-rack
- * entries, and should call {@link #clear()} after each test to avoid
cross-test pollution.
- *
- * <p>Usage:
- * <pre>{@code
- * FixedHostMapping.addNode("dn-0.test", "/rack1");
- * FixedHostMapping.addNode("dn-1.test", "/rack1");
- * FixedHostMapping.addNode("dn-2.test", "/rack2");
- *
- * conf.setClass(
- * CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
- * FixedHostMapping.class,
- * DNSToSwitchMapping.class);
- * }</pre>
- */
-public class FixedHostMapping extends CachedDNSToSwitchMapping {
-
- private static final Map<String, String> RACK_MAP = new
ConcurrentHashMap<>();
-
- /**
- * Constructs a {@code FixedHostMapping} with a no-op raw mapping.
- * The raw mapping is unused since {@link #resolve(List)} is fully
overridden.
- */
- public FixedHostMapping() {
- super(new NoOpMapping());
- }
-
- /**
- * Constructs a {@code FixedHostMapping} with the given raw mapping.
- * The raw mapping is unused since {@link #resolve(List)} is fully
overridden,
- * but is accepted to satisfy {@link CachedDNSToSwitchMapping} constructor
requirements.
- *
- * @param rawMapping the raw DNS mapping (not used for resolution)
- */
- public FixedHostMapping(DNSToSwitchMapping rawMapping) {
- super(rawMapping);
- }
-
- /**
- * Registers a hostname-to-rack mapping entry.
- * Must be called before cluster startup for the mapping to take effect
during
- * DataNode registration.
- *
- * @param host the DataNode hostname as it will appear in {@link
#resolve(List)}
- * @param rack the rack path (e.g. {@code "/rack1"})
- */
- public static void addNode(String host, String rack) {
- RACK_MAP.put(host, rack);
- }
-
- /**
- * Clears all registered hostname-to-rack mappings.
- * Should be called in test teardown (e.g. {@code @AfterEach}) to prevent
- * cross-test pollution of the JVM-wide static map.
- */
- public static void clear() {
- RACK_MAP.clear();
- }
-
- /**
- * Resolves a list of hostnames to their rack locations using the static map.
- * Hostnames not present in the map are assigned {@link
NetworkTopology#DEFAULT_RACK}.
- * Unlike the parent class, this method does not perform DNS normalization.
- *
- * @param names the list of hostnames to resolve
- * @return a list of rack paths in the same order as the input
- */
- @Override
- public List<String> resolve(List<String> names) {
- return names.stream()
- .map(name -> RACK_MAP.getOrDefault(name, NetworkTopology.DEFAULT_RACK))
- .collect(Collectors.toList());
- }
-
- /**
- * No-op: this implementation does not maintain a cache.
- */
- @Override
- public void reloadCachedMappings() {
- }
-
- /**
- * No-op: this implementation does not maintain a cache.
- *
- * @param names the hostnames whose cached mappings should be reloaded
(ignored)
- */
- @Override
- public void reloadCachedMappings(List<String> names) {
- }
-
- /**
- * A no-op {@link DNSToSwitchMapping} used as a placeholder raw mapping.
- * All hostnames are mapped to {@link NetworkTopology#DEFAULT_RACK}.
- * This is never invoked during normal resolution since {@link
FixedHostMapping#resolve(List)}
- * is fully overridden.
- */
- private static class NoOpMapping implements DNSToSwitchMapping {
-
- @Override
- public List<String> resolve(List<String> names) {
- return names.stream()
- .map(n -> NetworkTopology.DEFAULT_RACK)
- .collect(Collectors.toList());
- }
-
- @Override
- public void reloadCachedMappings() {
- }
-
- @Override
- public void reloadCachedMappings(List<String> names) {
- }
- }
-}
diff --git
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index e13719dde6e..cd675ddc309 100644
---
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -74,6 +74,7 @@
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectMetrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.DNSToSwitchMapping;
+import org.apache.hadoop.net.StaticMapping;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.common.Storage.StorageState;
@@ -394,6 +395,16 @@ public void shutdown() {
ManagedRocksObjectMetrics.INSTANCE.assertNoLeaks();
} catch (Exception e) {
LOG.error("Exception while shutting down the cluster.", e);
+ } finally {
+ resetStaticMappingIfConfigured(conf);
+ }
+ }
+
+ private static void resetStaticMappingIfConfigured(
+ OzoneConfiguration conf) {
+ if (StaticMapping.class.getName().equals(
+
conf.get(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY)))
{
+ StaticMapping.resetMap();
}
}
@@ -536,7 +547,7 @@ public MiniOzoneCluster build() throws IOException {
// Recreate the Ratis pipeline to prevent imbalanced node placement
across racks
// caused by asynchronous DN registration.
- if (racks != null && startDataNodes) {
+ if (racks != null && startDataNodes && numOfDatanodes >=
HddsProtos.ReplicationFactor.THREE.getNumber()) {
resetPipelinesForRackAwareness(cluster);
}
@@ -622,11 +633,11 @@ protected void initializeConfiguration() throws
IOException {
}
private void configureHostAndRackTopology() throws IOException {
- FixedHostMapping.clear();
if (racks == null && hosts == null) {
return;
}
+ StaticMapping.resetMap();
conf.setBoolean(HddsConfigKeys.HDDS_DATANODE_USE_DN_HOSTNAME, true);
if (hosts == null) {
@@ -645,10 +656,10 @@ private void configureHostAndRackTopology() throws
IOException {
}
conf.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
- FixedHostMapping.class, DNSToSwitchMapping.class);
+ StaticMapping.class, DNSToSwitchMapping.class);
for (int i = 0; i < racks.length; i++) {
- FixedHostMapping.addNode(hosts[i], racks[i]);
+ StaticMapping.addNodeToRack(hosts[i], racks[i]);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
