This is an automated email from the ASF dual-hosted git repository.
siddhant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4181b2d783 HDDS-9033. Increase EC pipeline limit for final allocation
attempt (#5084)
4181b2d783 is described below
commit 4181b2d7838537702d347c0679dead595b04283a
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Jul 26 06:04:43 2023 +0200
HDDS-9033. Increase EC pipeline limit for final allocation attempt (#5084)
---
.../scm/pipeline/WritableECContainerProvider.java | 20 ++++++++++----
.../hadoop/hdds/scm/container/MockNodeManager.java | 4 +++
.../pipeline/TestWritableECContainerProvider.java | 31 +++++++++++++++++++---
3 files changed, 47 insertions(+), 8 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
index 8cd2c933ca..98cfc995d9 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
@@ -45,6 +45,7 @@ import java.util.List;
import java.util.NavigableSet;
import static org.apache.hadoop.hdds.conf.ConfigTag.SCM;
+import static org.apache.hadoop.hdds.scm.node.NodeStatus.inServiceHealthy;
/**
* Writable Container provider to obtain a writable container for EC pipelines.
@@ -95,7 +96,7 @@ public class WritableECContainerProvider
ECReplicationConfig repConfig, String owner, ExcludeList excludeList)
throws IOException {
int maximumPipelines = getMaximumPipelines(repConfig);
- int openPipelineCount = 0;
+ int openPipelineCount;
synchronized (this) {
openPipelineCount = pipelineManager.getPipelineCount(repConfig,
Pipeline.PipelineState.OPEN);
@@ -118,6 +119,7 @@ public class WritableECContainerProvider
repConfig, Pipeline.PipelineState.OPEN,
excludeList.getDatanodes(), excludeList.getPipelineIds());
final int pipelineCount = existingPipelines.size();
+ LOG.debug("Checking existing pipelines: {}", existingPipelines);
PipelineRequestInformation pri =
PipelineRequestInformation.Builder.getBuilder()
@@ -160,13 +162,21 @@ public class WritableECContainerProvider
// If we get here, all the pipelines we tried were no good. So try to
// allocate a new one.
try {
- synchronized (this) {
- if (openPipelineCount < maximumPipelines) {
+ if (openPipelineCount >= maximumPipelines) {
+ final int nodeCount = nodeManager.getNodeCount(inServiceHealthy());
+ if (nodeCount > maximumPipelines) {
+ LOG.debug("Increasing pipeline limit {} -> {} for final attempt",
+ maximumPipelines, nodeCount);
+ maximumPipelines = nodeCount;
+ }
+ }
+ if (openPipelineCount < maximumPipelines) {
+ synchronized (this) {
return allocateContainer(repConfig, size, owner, excludeList);
}
- throw new IOException("Pipeline limit (" + maximumPipelines
- + ") reached (" + openPipelineCount + "), none closed");
}
+ throw new IOException("Pipeline limit (" + maximumPipelines
+ + ") reached (" + openPipelineCount + "), none closed");
} catch (IOException e) {
LOG.warn("Unable to allocate a container after trying {} existing ones; "
+ "requested size={}, replication={}, owner={}, {}",
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index ae37161f36..6ce5caa21e 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -309,6 +309,10 @@ public class MockNodeManager implements NodeManager {
return deadNodes;
}
+ if (nodestate == null) {
+ return new ArrayList<>(nodeMetricMap.keySet());
+ }
+
return null;
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
index 9d2626591f..87c7f151b6 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
@@ -67,6 +67,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.verify;
@@ -237,6 +238,8 @@ public class TestWritableECContainerProvider {
@MethodSource("policies")
public void testNewPipelineNotCreatedIfAllPipelinesExcluded(
PipelineChoosePolicy policy) throws IOException {
+ final int nodeCount = nodeManager.getNodeCount(null, null);
+ providerConf.setMinimumPipelines(nodeCount);
provider = createSubject(policy);
Set<ContainerInfo> allocatedContainers = new HashSet<>();
for (int i = 0; i < providerConf.getMinimumPipelines(); i++) {
@@ -244,8 +247,7 @@ public class TestWritableECContainerProvider {
1, repConfig, OWNER, new ExcludeList());
allocatedContainers.add(container);
}
- // We have the min limit of pipelines, but then exclude all the associated
- // containers.
+ // We have the min limit of pipelines, but then exclude them all
ExcludeList exclude = new ExcludeList();
for (ContainerInfo c : allocatedContainers) {
exclude.addPipeline(c.getPipelineID());
@@ -254,10 +256,32 @@ public class TestWritableECContainerProvider {
1, repConfig, OWNER, exclude));
}
+ @ParameterizedTest
+ @MethodSource("policies")
+ void newPipelineCreatedIfSoftLimitReached(PipelineChoosePolicy policy)
+ throws IOException {
+
+ providerConf.setMinimumPipelines(1);
+ provider = createSubject(policy);
+ ContainerInfo container = provider.getContainer(
+ 1, repConfig, OWNER, new ExcludeList());
+
+ ExcludeList exclude = new ExcludeList();
+ exclude.addPipeline(container.getPipelineID());
+ exclude.addDatanode(
+ pipelineManager.getPipeline(container.getPipelineID()).getFirstNode());
+
+ ContainerInfo newContainer = provider.getContainer(
+ 1, repConfig, OWNER, exclude);
+ assertNotSame(container, newContainer);
+ }
+
@ParameterizedTest
@MethodSource("policies")
public void testNewPipelineNotCreatedIfAllContainersExcluded(
PipelineChoosePolicy policy) throws IOException {
+ final int nodeCount = nodeManager.getNodeCount(null, null);
+ providerConf.setMinimumPipelines(nodeCount);
provider = createSubject(policy);
Set<ContainerInfo> allocatedContainers = new HashSet<>();
for (int i = 0; i < providerConf.getMinimumPipelines(); i++) {
@@ -265,7 +289,8 @@ public class TestWritableECContainerProvider {
1, repConfig, OWNER, new ExcludeList());
allocatedContainers.add(container);
}
- // We have the min limit of pipelines, but then exclude them all
+ // We have the min limit of pipelines, but then exclude all the associated
+ // containers.
ExcludeList exclude = new ExcludeList();
for (ContainerInfo c : allocatedContainers) {
exclude.addConatinerId(c.containerID());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
