This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 984edb2ed81 HDDS-16206. Start MiniOzoneCluster datanodes finalized
(#11111)
984edb2ed81 is described below
commit 984edb2ed81f8d6eacec49faedc2dcc3e39f65a5
Author: rjgoyln <[email protected]>
AuthorDate: Fri Sep 25 02:57:05 2026 +0800
HDDS-16206. Start MiniOzoneCluster datanodes finalized (#11111)
Generated-by: Claude Code (Claude Opus 5)
---
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 24 ++++++++++++++++++++++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 24 ++++++++++++++++++----
2 files changed, 44 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index 6f90a54835f..ef1034b7cd7 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -19,6 +19,7 @@
import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
import static
org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
+import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_IPC_RANDOM_PORT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -47,6 +48,7 @@
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import
org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
@@ -57,6 +59,7 @@
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
/**
* Test cases for mini ozone cluster.
@@ -111,6 +114,27 @@ public void testStartMultipleDatanodes() throws Exception {
}
}
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void testDatanodesStartWithLatestLayoutVersion(boolean withCustomHostnames)
+ throws Exception {
+ MiniOzoneCluster.Builder builder =
+ MiniOzoneCluster.newBuilder(new OzoneConfiguration(conf))
+ .setNumDatanodes(1)
+ .setStartDataNodes(false);
+ if (withCustomHostnames) {
+ builder.setHosts(new String[] {"host0.test"});
+ }
+ cluster = builder.build();
+
+ OzoneConfiguration dnConf = cluster.getHddsDatanodes().get(0).getConf();
+ assertEquals(maxLayoutVersion(),
+ new DatanodeLayoutStorage(dnConf).getLayoutVersion(),
+ "Datanode must start at the latest metadata layout version. A "
+ + "datanode.id file written before the datanode starts makes it "
+ + "come up pre-finalized, as if upgraded from an older install.");
+ }
+
@Test
void testClientDefaultsWithIndividualFlags() {
OzoneConfiguration config = new OzoneConfiguration();
diff --git
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index cd675ddc309..b38f500d592 100644
---
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -20,6 +20,7 @@
import static java.util.Collections.singletonList;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
import static org.apache.hadoop.hdds.server.http.BaseHttpServer.SERVER_DIR;
+import static
org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR;
import static org.apache.ozone.test.GenericTestUtils.PortAllocator.getFreePort;
import static
org.apache.ozone.test.GenericTestUtils.PortAllocator.localhostWithFreePort;
@@ -78,6 +79,7 @@
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.common.Storage.StorageState;
+import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
import org.apache.hadoop.ozone.container.common.utils.DatanodeStoreCache;
@@ -785,10 +787,9 @@ protected List<HddsDatanodeService> createHddsDatanodes()
OzoneConfiguration dnConf = dnFactory.apply(conf);
if (hosts != null) {
dnConf.set(HddsConfigKeys.HDDS_DATANODE_HOST_NAME_KEY, hosts[i]);
+ initializeDatanodeIdentity(dnConf, hosts[i]);
}
- // Bypass InetAddress.getName() resolution for custom hostnames by
starting DN via YAML.
- confDatanodeViaYaml(dnConf);
HddsDatanodeService datanode = new HddsDatanodeService(NO_ARGS);
dnConf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES,
conf.getStrings(ScmConfigKeys.OZONE_SCM_NAMES));
datanode.setConfiguration(dnConf);
@@ -798,13 +799,28 @@ protected List<HddsDatanodeService> createHddsDatanodes()
return hddsDatanodes;
}
- private void confDatanodeViaYaml(OzoneConfiguration dnConf) throws
IOException {
+ /**
+ * Seeds the datanode.id file for a Datanode with a synthetic hostname:
such a hostname
+ * cannot be resolved, so the Datanode cannot discover its own IP address,
which is a
+ * required field of the registration request. The layout version is
stamped as well,
+ * since a datanode.id file without a VERSION file marks the Datanode as
an installation
+ * predating the upgrade framework, which would make it start
pre-finalized.
+ */
+ private void initializeDatanodeIdentity(OzoneConfiguration dnConf, String
hostName)
+ throws IOException {
DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder()
.setID(DatanodeID.randomID())
- .setHostName(dnConf.get(HddsConfigKeys.HDDS_DATANODE_HOST_NAME_KEY))
+ .setHostName(hostName)
.setIpAddress("127.0.0.1")
.build();
datanodeDetails.setNetworkName(datanodeDetails.getUuidString());
+
+ DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(dnConf,
+ datanodeDetails.getUuidString(), maxLayoutVersion());
+ if (layoutStorage.getState() != StorageState.INITIALIZED) {
+ layoutStorage.initialize();
+ }
+
String dnFilePath = HddsServerUtil.getDatanodeIdFilePath(dnConf);
ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, new
File(dnFilePath), dnConf);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
