This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 37309490a5 HDDS-8736. Add usedNodes to limit check and log messages in
SCMContainerPlacementRackScatter (#4804)
37309490a5 is described below
commit 37309490a50de0134af533966ab27e236e9107ed
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Thu Jun 1 08:02:10 2023 +0100
HDDS-8736. Add usedNodes to limit check and log messages in
SCMContainerPlacementRackScatter (#4804)
---
.../SCMContainerPlacementRackScatter.java | 23 ++++++++++++++--------
.../TestSCMContainerPlacementRackScatter.java | 16 +++++++++++++++
2 files changed, 31 insertions(+), 8 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java
index 495f97e0c3..2d5ade3993 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java
@@ -210,6 +210,7 @@ public final class SCMContainerPlacementRackScatter
* @throws SCMException SCMException
*/
@Override
+ @SuppressWarnings("checkstyle:methodlength")
protected List<DatanodeDetails> chooseDatanodesInternal(
List<DatanodeDetails> usedNodes,
final List<DatanodeDetails> excludedNodes,
@@ -226,18 +227,23 @@ public final class SCMContainerPlacementRackScatter
}
int nodesRequired = nodesRequiredToChoose;
int excludedNodesCount = excludedNodes == null ? 0 : excludedNodes.size();
+ int usedNodesCount = usedNodes == null ? 0 : usedNodes.size();
List<Node> availableNodes = networkTopology.getNodes(
networkTopology.getMaxLevel());
int totalNodesCount = availableNodes.size();
if (excludedNodes != null) {
availableNodes.removeAll(excludedNodes);
}
+ if (usedNodes != null) {
+ availableNodes.removeAll(usedNodes);
+ }
if (availableNodes.size() < nodesRequired) {
throw new SCMException("No enough datanodes to choose. " +
- "TotalNode = " + totalNodesCount +
- " AvailableNode = " + availableNodes.size() +
- " RequiredNode = " + nodesRequired +
- " ExcludedNode = " + excludedNodesCount,
+ "TotalNodes = " + totalNodesCount +
+ " AvailableNodes = " + availableNodes.size() +
+ " RequiredNodes = " + nodesRequired +
+ " ExcludedNodes = " + excludedNodesCount +
+ " UsedNodes = " + usedNodesCount,
FAILED_TO_FIND_SUITABLE_NODE);
}
List<DatanodeDetails> mutableFavoredNodes = new ArrayList<>();
@@ -316,8 +322,8 @@ public final class SCMContainerPlacementRackScatter
+ additionalRacksRequired + " do not match.";
LOG.warn("Placement policy could not choose the enough nodes from " +
"available racks. {} Available racks count: {},"
- + " Excluded nodes count: {}",
- reason, racks.size(), excludedNodesCount);
+ + " Excluded nodes count: {}, UsedNodes count: {}",
+ reason, racks.size(), excludedNodesCount, usedNodesCount);
throw new SCMException(reason,
SCMException.ResultCodes.FAILED_TO_FIND_HEALTHY_NODES);
}
@@ -337,8 +343,9 @@ public final class SCMContainerPlacementRackScatter
.size() + ", but required nodes to choose: "
+ nodesRequiredToChoose + " do not match.";
LOG.warn("Placement policy could not choose the enough nodes."
- + " {} Available nodes count: {}, Excluded nodes count: {}",
- reason, totalNodesCount, excludedNodesCount);
+ + " {} Available nodes count: {}, Excluded nodes count: {}, "
+ + " Used nodes count: {}",
+ reason, totalNodesCount, excludedNodesCount, usedNodesCount);
throw new SCMException(reason,
SCMException.ResultCodes.FAILED_TO_FIND_HEALTHY_NODES);
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java
index 815c70ffb3..dbdc8ae7d2 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java
@@ -63,6 +63,8 @@ import static
org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.FAI
import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.matchesPattern;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -621,6 +623,20 @@ public class TestSCMContainerPlacementRackScatter {
exception.getResult());
}
+ @Test
+ public void testChooseNodesWithInsufficientNodesAvailable() {
+ setup(5, 2);
+ List<DatanodeDetails> usedDns = getDatanodes(Lists.newArrayList(0, 1));
+ List<DatanodeDetails> excludedDns = getDatanodes(Lists.newArrayList(2));
+ SCMException exception = Assertions.assertThrows(SCMException.class, () ->
+ policy.chooseDatanodes(usedDns, excludedDns,
+ null, 3, 0, 5));
+ assertThat(exception.getMessage(),
+ matchesPattern("^No enough datanodes to choose.*"));
+ assertEquals(SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE,
+ exception.getResult());
+ }
+
@Test
public void testInValidChooseNodesWithUsedNodesWithInsufficientRacks() {
setup(6, 2);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
