This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new a64945b HDDS-5325. EC: Add configuration to set an EC container
placement policy (#2483)
a64945b is described below
commit a64945bd7c897267b8095c011d679c1f83d73a43
Author: Symious <[email protected]>
AuthorDate: Tue Aug 3 00:15:12 2021 +0800
HDDS-5325. EC: Add configuration to set an EC container placement policy
(#2483)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 ++
.../common/src/main/resources/ozone-default.xml | 14 ++++++++++++
.../ContainerPlacementPolicyFactory.java | 25 +++++++++++++++++++++-
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 6 +++---
.../algorithms/TestContainerPlacementFactory.java | 7 ++++++
5 files changed, 50 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 8d0aab3..a3f0585 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -345,6 +345,8 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY =
"ozone.scm.container.placement.impl";
+ public static final String OZONE_SCM_CONTAINER_PLACEMENT_EC_IMPL_KEY =
+ "ozone.scm.container.placement.ec.impl";
public static final String OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT =
"ozone.scm.pipeline.owner.container.count";
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7c75b63..8468800 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -821,6 +821,20 @@
</description>
</property>
<property>
+ <name>ozone.scm.container.placement.ec.impl</name>
+ <value>
+
org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom
+ </value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ The full name of class which implements
+ org.apache.hadoop.hdds.scm.PlacementPolicy.
+ The class decides which datanode will be used to host the container
replica in EC mode. If not set,
+
org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom
will be used as default
+ value.
+ </description>
+ </property>
+ <property>
<name>ozone.scm.pipeline.owner.container.count</name>
<value>3</value>
<tag>OZONE, SCM, PIPELINE</tag>
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
index c799b02..983d935 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
@@ -30,7 +30,8 @@ import org.slf4j.LoggerFactory;
/**
* A factory to create container placement instance based on configuration
- * property {@link ScmConfigKeys#OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY}.
+ * property {@link ScmConfigKeys#OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY} and
+ * property {@link ScmConfigKeys#OZONE_SCM_CONTAINER_PLACEMENT_EC_IMPL_KEY}.
*/
public final class ContainerPlacementPolicyFactory {
private static final Logger LOG =
@@ -52,6 +53,28 @@ public final class ContainerPlacementPolicyFactory {
.getClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT,
PlacementPolicy.class);
+ return getPolicyInternal(placementClass, conf, nodeManager, clusterMap,
+ fallback, metrics);
+ }
+
+ public static PlacementPolicy getECPolicy(
+ ConfigurationSource conf, final NodeManager nodeManager,
+ NetworkTopology clusterMap, final boolean fallback,
+ SCMContainerPlacementMetrics metrics) throws SCMException{
+ // TODO: Change default placement policy for EC
+ final Class<? extends PlacementPolicy> placementClass = conf
+ .getClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_EC_IMPL_KEY,
+ OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT,
+ PlacementPolicy.class);
+ return getPolicyInternal(placementClass, conf, nodeManager, clusterMap,
+ fallback, metrics);
+ }
+
+ public static PlacementPolicy getPolicyInternal(
+ Class<? extends PlacementPolicy> placementClass,
+ ConfigurationSource conf, final NodeManager nodeManager,
+ NetworkTopology clusterMap, final boolean fallback,
+ SCMContainerPlacementMetrics metrics) throws SCMException{
Constructor<? extends PlacementPolicy> constructor;
try {
constructor = placementClass.getDeclaredConstructor(NodeManager.class,
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java
index aec48cd..e879ebb 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java
@@ -54,9 +54,9 @@ public class PipelineFactory {
new RatisPipelineProvider(nodeManager,
stateManager, conf,
eventPublisher, scmContext));
- PlacementPolicy placementPolicy;
+ PlacementPolicy ecPlacementPolicy;
try {
- placementPolicy = ContainerPlacementPolicyFactory.getPolicy(conf,
+ ecPlacementPolicy = ContainerPlacementPolicyFactory.getECPolicy(conf,
nodeManager, nodeManager.getClusterNetworkTopologyMap(), true,
SCMContainerPlacementMetrics.create());
} catch (SCMException e) {
@@ -65,7 +65,7 @@ public class PipelineFactory {
}
providers.put(ReplicationType.EC,
new ECPipelineProvider(nodeManager, stateManager, conf,
- placementPolicy));
+ ecPlacementPolicy));
}
protected PipelineFactory() {
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
index 1f0853d..b6499a6 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
@@ -165,6 +165,13 @@ public class TestContainerPlacementFactory {
Assert.assertSame(SCMContainerPlacementRandom.class, policy.getClass());
}
+ @Test
+ public void testECPolicy() throws IOException {
+ PlacementPolicy policy = ContainerPlacementPolicyFactory
+ .getECPolicy(conf, null, null, true, null);
+ Assert.assertSame(SCMContainerPlacementRandom.class, policy.getClass());
+ }
+
/**
* A dummy container placement implementation for test.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
