This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4dd44a4127 HDDS-8950. NPE in
SCMCommonPlacementPolicy#validateContainerPlacement (#4999)
4dd44a4127 is described below
commit 4dd44a41277f6944e8e896d70c3574ef07abc19e
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Jun 28 22:29:53 2023 +0200
HDDS-8950. NPE in SCMCommonPlacementPolicy#validateContainerPlacement
(#4999)
---
.../apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 14 +++++++++-----
.../hdds/scm/pipeline/TestPipelinePlacementPolicy.java | 6 ++++++
2 files changed, 15 insertions(+), 5 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
index c6833fffa8..151e72227a 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
@@ -48,6 +48,7 @@ import java.util.PriorityQueue;
import java.util.Queue;
import java.util.Random;
import java.util.Set;
+import java.util.function.Function;
import java.util.stream.Collectors;
/**
@@ -435,9 +436,13 @@ public abstract class SCMCommonPlacementPolicy implements
return invalidPlacement;
}
}
- Map<Node, Long> currentRackCount = dns.stream()
- .collect(Collectors.groupingBy(this::getPlacementGroup,
- Collectors.counting()));
+ List<Integer> currentRackCount = new ArrayList<>(dns.stream()
+ .map(this::getPlacementGroup)
+ .filter(Objects::nonNull)
+ .collect(Collectors.groupingBy(
+ Function.identity(),
+ Collectors.reducing(0, e -> 1, Integer::sum)))
+ .values());
final int maxLevel = topology.getMaxLevel();
// The leaf nodes are all at max level, so the number of nodes at
// leafLevel - 1 is the rack count
@@ -449,8 +454,7 @@ public abstract class SCMCommonPlacementPolicy implements
Math.min(requiredRacks, numRacks));
return new ContainerPlacementStatusDefault(
currentRackCount.size(), requiredRacks, numRacks, maxReplicasPerRack,
- currentRackCount.values().stream().map(Long::intValue)
- .collect(Collectors.toList()));
+ currentRackCount);
}
/**
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
index 82a2ab3246..ceb5d24f58 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -517,6 +517,12 @@ public class TestPipelinePlacementPolicy {
subSet.add(dns.get(0));
status = placementPolicy.validateContainerPlacement(subSet, 1);
Assertions.assertTrue(status.isPolicySatisfied());
+
+ // three nodes, one dead, one rack
+ cluster.remove(dns.get(2));
+ status = placementPolicy.validateContainerPlacement(dns, 3);
+ Assertions.assertFalse(status.isPolicySatisfied());
+ Assertions.assertEquals(1, status.misReplicationCount());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
