This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new cd31fd57ef HDDS-9038. Enable dynamic reconfiguration of EC container
provider properties (#5088)
cd31fd57ef is described below
commit cd31fd57efa4f86d45b45aa8ad9e7e2eac1ddf3b
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Thu Jul 20 13:43:27 2023 +0200
HDDS-9038. Enable dynamic reconfiguration of EC container provider
properties (#5088)
---
.../apache/hadoop/hdds/scm/pipeline/WritableContainerFactory.java | 6 +++++-
.../hadoop/hdds/scm/pipeline/WritableECContainerProvider.java | 6 +++++-
.../org/apache/hadoop/hdds/scm/server/StorageContainerManager.java | 1 -
.../org/apache/hadoop/ozone/reconfig/TestScmReconfiguration.java | 3 +++
4 files changed, 13 insertions(+), 3 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableContainerFactory.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableContainerFactory.java
index 000f2d296b..81189538b5 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableContainerFactory.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableContainerFactory.java
@@ -50,13 +50,17 @@ public class WritableContainerFactory {
scm.getContainerManager(), scm.getPipelineChoosePolicy());
this.standaloneProvider = ratisProvider;
+ WritableECContainerProviderConfig ecProviderConfig =
+ conf.getObject(WritableECContainerProviderConfig.class);
this.ecProvider = new WritableECContainerProvider(
- conf.getObject(WritableECContainerProviderConfig.class),
+ ecProviderConfig,
getConfiguredContainerSize(conf),
scm.getScmNodeManager(),
scm.getPipelineManager(),
scm.getContainerManager(),
scm.getEcPipelineChoosePolicy());
+
+ scm.getReconfigurationHandler().register(ecProviderConfig);
}
public ContainerInfo getContainer(final long size,
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
index 6e2950a286..8cd2c933ca 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.ConfigType;
import org.apache.hadoop.hdds.conf.PostConstruct;
+import org.apache.hadoop.hdds.conf.ReconfigurableConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.PipelineChoosePolicy;
import org.apache.hadoop.hdds.scm.PipelineRequestInformation;
@@ -235,12 +236,14 @@ public class WritableECContainerProvider
* Class to hold configuration for WriteableECContainerProvider.
*/
@ConfigGroup(prefix = WritableECContainerProviderConfig.PREFIX)
- public static class WritableECContainerProviderConfig {
+ public static class WritableECContainerProviderConfig
+ extends ReconfigurableConfig {
private static final String PREFIX = "ozone.scm.ec";
@Config(key = "pipeline.minimum",
defaultValue = "5",
+ reconfigurable = true,
type = ConfigType.INT,
description = "The minimum number of pipelines to have open for each "
+
"Erasure Coding configuration",
@@ -265,6 +268,7 @@ public class WritableECContainerProvider
@Config(key = PIPELINE_PER_VOLUME_FACTOR_KEY,
type = ConfigType.DOUBLE,
defaultValue = PIPELINE_PER_VOLUME_FACTOR_DEFAULT_VALUE,
+ reconfigurable = true,
tags = {SCM},
description = "TODO"
)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 9a0ab7e936..08636f1f0f 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -2180,7 +2180,6 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
scmHAMetrics = SCMHAMetrics.create(getScmId(), leaderId);
}
- @VisibleForTesting
public ReconfigurationHandler getReconfigurationHandler() {
return reconfigurationHandler;
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestScmReconfiguration.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestScmReconfiguration.java
index 2b13986276..27d9a9736d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestScmReconfiguration.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestScmReconfiguration.java
@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.conf.ReconfigurationException;
import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
import
org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration;
+import
org.apache.hadoop.hdds.scm.pipeline.WritableECContainerProvider.WritableECContainerProviderConfig;
import org.junit.jupiter.api.Test;
import java.time.Duration;
@@ -50,6 +51,8 @@ class TestScmReconfiguration extends ReconfigurationTestBase {
.add(OZONE_READONLY_ADMINISTRATORS)
.addAll(new ReplicationManagerConfiguration()
.reconfigurableProperties())
+ .addAll(new WritableECContainerProviderConfig()
+ .reconfigurableProperties())
.build();
assertProperties(getSubject(), expected);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
