This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4a2c8db444e HDDS-14764. Allow Datanode to dynamically reconfigure SCM
node addresses (#9863)
4a2c8db444e is described below
commit 4a2c8db444e3c799c847aff723597711dc8ae12f
Author: Ivan Andika <[email protected]>
AuthorDate: Thu Mar 5 13:18:37 2026 +0800
HDDS-14764. Allow Datanode to dynamically reconfigure SCM node addresses
(#9863)
---
.../apache/hadoop/ozone/HddsDatanodeService.java | 8 +++++--
.../hadoop/hdds/conf/ReconfigurationHandler.java | 28 ++++++++++++++++++++--
.../scm/TestDatanodeSCMNodesReconfiguration.java | 8 +++++--
3 files changed, 38 insertions(+), 6 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index b0fbfe30b93..190c542809f 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -19,6 +19,7 @@
import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.HTTP;
import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.HTTPS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NODES_KEY;
import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser;
import static
org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClientWithMaxRetry;
@@ -93,6 +94,7 @@
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import
org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerProtocolServer;
+import org.apache.hadoop.ozone.ha.ConfUtils;
import org.apache.hadoop.ozone.util.OzoneNetUtils;
import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.apache.hadoop.security.SecurityUtil;
@@ -313,9 +315,11 @@ public String getNamespace() {
this::reconfigReplicationStreamsLimit);
scmServiceId = HddsUtils.getScmServiceId(conf);
+
if (scmServiceId != null) {
- reconfigurationHandler.register(OZONE_SCM_NODES_KEY + "." +
scmServiceId,
- this::reconfigScmNodes);
+ reconfigurationHandler
+ .registerPrefix(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY,
scmServiceId))
+ .register(OZONE_SCM_NODES_KEY + "." + scmServiceId,
this::reconfigScmNodes);
}
reconfigurationHandler.setReconfigurationCompleteCallback(reconfigurationHandler.defaultLoggingCallback());
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/ReconfigurationHandler.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/ReconfigurationHandler.java
index 979525f7a1a..32eb67c0797 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/ReconfigurationHandler.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/ReconfigurationHandler.java
@@ -23,11 +23,13 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListSet;
import java.util.function.BiConsumer;
import java.util.function.UnaryOperator;
import org.apache.hadoop.conf.Configuration;
@@ -51,6 +53,7 @@ public class ReconfigurationHandler extends ReconfigurableBase
private final CheckedConsumer<String, IOException> requireAdminPrivilege;
private final Map<String, UnaryOperator<String>> properties =
new ConcurrentHashMap<>();
+ private final Set<String> prefixProperties = new ConcurrentSkipListSet<>();
private final List<ReconfigurationChangeCallback> completeCallbacks = new
ArrayList<>();
private BiConsumer<ReconfigurationTaskStatus, Configuration>
reconfigurationStatusListener;
@@ -122,6 +125,13 @@ public ReconfigurationHandler
register(ReconfigurableConfig config) {
return this;
}
+ public ReconfigurationHandler registerPrefix(String prefixProperty) {
+ prefixProperties.add(
+ prefixProperty.endsWith(".") ? prefixProperty :
prefixProperty.concat(".")
+ );
+ return this;
+ }
+
@Override
protected Configuration getNewConf() {
return new OzoneConfiguration();
@@ -129,7 +139,19 @@ protected Configuration getNewConf() {
@Override
public Set<String> getReconfigurableProperties() {
- return unmodifiableSet(properties.keySet());
+ Set<String> reconfigureProperties = new HashSet<>(properties.keySet());
+ reconfigureProperties.addAll(prefixProperties);
+ return unmodifiableSet(reconfigureProperties);
+ }
+
+ @Override
+ public boolean isPropertyReconfigurable(String property) {
+ for (String prefixProperty : prefixProperties) {
+ if (property.startsWith(prefixProperty)) {
+ return true;
+ }
+ }
+ return properties.containsKey(property);
}
@Override
@@ -164,7 +186,9 @@ public ReconfigurationTaskStatus getReconfigureStatus()
throws IOException {
@Override
public List<String> listReconfigureProperties() throws IOException {
requireAdminPrivilege.accept("listReconfigurableProperties");
- return new ArrayList<>(new TreeSet<>(getReconfigurableProperties()));
+ Set<String> reconfigureProperties = new
TreeSet<>(getReconfigurableProperties());
+ reconfigureProperties.addAll(prefixProperties);
+ return new ArrayList<>(reconfigureProperties);
}
@Override
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java
index d2c90c21d65..6f38d89fb2b 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java
@@ -128,7 +128,9 @@ void testSCMMigration() throws Exception {
for (StorageContainerManager scm :
cluster.getStorageContainerManagers()) {
String scmAddrKey = ConfUtils.addKeySuffixes(
ScmConfigKeys.OZONE_SCM_ADDRESS_KEY, scmServiceId,
scm.getSCMNodeId());
- datanode.getConf().set(scmAddrKey, cluster.getConf().get(scmAddrKey));
+
assertTrue(datanode.getReconfigurationHandler().isPropertyReconfigurable(scmAddrKey));
+ datanode.getReconfigurationHandler().reconfigureProperty(scmAddrKey,
+ cluster.getConf().get(scmAddrKey));
String dnPortKey = ConfUtils.addKeySuffixes(
ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, scmServiceId,
scm.getSCMNodeId());
datanode.getConf().set(dnPortKey, cluster.getConf().get(dnPortKey));
@@ -273,7 +275,9 @@ void testAddAndRemoveOneSCM() throws Exception {
for (StorageContainerManager scm :
cluster.getStorageContainerManagers()) {
String scmAddrKey = ConfUtils.addKeySuffixes(
ScmConfigKeys.OZONE_SCM_ADDRESS_KEY, scmServiceId,
scm.getSCMNodeId());
- datanode.getConf().set(scmAddrKey, cluster.getConf().get(scmAddrKey));
+
assertTrue(datanode.getReconfigurationHandler().isPropertyReconfigurable(scmAddrKey));
+ datanode.getReconfigurationHandler().reconfigureProperty(scmAddrKey,
+ cluster.getConf().get(scmAddrKey));
String dnPortKey = ConfUtils.addKeySuffixes(
ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, scmServiceId,
scm.getSCMNodeId());
datanode.getConf().set(dnPortKey, cluster.getConf().get(dnPortKey));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
