This is an automated email from the ASF dual-hosted git repository.
sarvekshayr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1dff0050e07 HDDS-16199. Log skipped unhealthy containers at INFO
(#11041)
1dff0050e07 is described below
commit 1dff0050e07fb9b3d87b369ccfde8e34b198b4d0
Author: Shuo Huang <[email protected]>
AuthorDate: Mon Sep 21 15:39:59 2026 +0800
HDDS-16199. Log skipped unhealthy containers at INFO (#11041)
Co-authored-by: Doroszlai, Attila <[email protected]>
---
.../ContainerBalancerSelectionCriteria.java | 22 +++++++-
.../container/balancer/ContainerBalancerTask.java | 1 +
.../TestContainerBalancerSelectionCriteria.java | 64 ++++++++++++++++++++++
3 files changed, 86 insertions(+), 1 deletion(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java
index c6828e635bd..0e70b3950d6 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java
@@ -21,10 +21,13 @@
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Set;
import java.util.TreeSet;
+import java.util.stream.Collectors;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
@@ -47,6 +50,7 @@
public class ContainerBalancerSelectionCriteria {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerBalancerSelectionCriteria.class);
+ private static final int MAX_EXCLUDED_CONTAINERS_TO_LOG = 50;
private ContainerBalancerConfiguration balancerConfiguration;
private NodeManager nodeManager;
@@ -55,11 +59,11 @@ public class ContainerBalancerSelectionCriteria {
private Map<ContainerID, DatanodeDetails> containerToSourceMap;
private Set<ContainerID> excludeContainers;
private Set<ContainerID> includeContainers;
-
// excludeContainersDueToFailure: Containers excluded for current iteration
only.
// excludeContainersNotFound: Container exclusions persist across iterations.
private final Set<ContainerID> excludeContainersDueToFailure;
private final Set<ContainerID> excludeContainersNotFound;
+ private final Map<ContainerID, ContainerHealthResult.HealthState>
excludeContainersDueToHealth;
private FindSourceStrategy findSourceStrategy;
private Map<DatanodeDetails, NavigableSet<ContainerID>> setMap;
@@ -92,6 +96,7 @@ public ContainerBalancerSelectionCriteria(
this.containerToSourceMap = containerToSourceMap;
this.excludeContainersDueToFailure = new HashSet<>();
this.excludeContainersNotFound = excludeContainersNotFound;
+ this.excludeContainersDueToHealth = new LinkedHashMap<>();
excludeContainers = balancerConfiguration.getExcludeContainers();
includeContainers = balancerConfiguration.getIncludeContainers();
this.findSourceStrategy = findSourceStrategy;
@@ -270,6 +275,7 @@ private boolean isContainerHealthyForMove(ContainerInfo
container, Set<Container
ContainerHealthResult.HealthState state =
replicationManager.getContainerReplicationHealth(container,
replicas).getHealthState();
if (state != ContainerHealthResult.HealthState.HEALTHY) {
+ excludeContainersDueToHealth.put(container.containerID(), state);
LOG.debug("Excluding container {} with replicas {} as its health is
{}.", container, replicas, state);
return false;
}
@@ -362,10 +368,24 @@ private boolean
isContainerHealthyForMoveRelaxed(ContainerInfo container, Set<Co
return true;
}
+ excludeContainersDueToHealth.put(container.containerID(), state);
LOG.debug("Excluding container {} with replicas {} as its health is {}.",
container, replicas, state);
return false;
}
+ void logExcludedContainersDueToHealth() {
+ if (excludeContainersDueToHealth.isEmpty()) {
+ return;
+ }
+
+ List<Map.Entry<ContainerID, ContainerHealthResult.HealthState>>
containersToLog =
+ excludeContainersDueToHealth.entrySet().stream()
+ .limit(MAX_EXCLUDED_CONTAINERS_TO_LOG)
+ .collect(Collectors.toList());
+ LOG.info("Excluded {} containers because of their health state. Logging
the first {}: {}.",
+ excludeContainersDueToHealth.size(), containersToLog.size(),
containersToLog);
+ }
+
private boolean breaksMaxSizeToMoveLimit(ContainerID containerID,
long usedBytes,
long sizeMovedAlready) {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
index 4dff0265445..8dbc4b8d9e2 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
@@ -291,6 +291,7 @@ private void balance() {
}
IterationResult currentIterationResult = doIteration();
+ selectionCriteria.logExcludedContainersDueToHealth();
ContainerBalancerTaskIterationStatusInfo iterationStatistic =
getIterationStatistic(i + 1, currentIterationResult,
getCurrentIterationDuration());
iterationsStatistic.offer(iterationStatistic);
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerSelectionCriteria.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerSelectionCriteria.java
index 5b87a13369d..10d98e19191 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerSelectionCriteria.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerSelectionCriteria.java
@@ -21,6 +21,7 @@
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -31,6 +32,7 @@
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static org.slf4j.event.Level.INFO;
import java.util.HashMap;
import java.util.HashSet;
@@ -49,6 +51,8 @@
import org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.GenericTestUtils.LogCapturer;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -107,6 +111,66 @@ public void shouldExcludeUnderReplicatedContainer() {
assertTrue(criteria.shouldBeExcluded(containerID, source, 0L));
}
+ @Test
+ public void shouldLogUnhealthyContainersAtInfo() throws Exception {
+ when(replicationManager.getContainerReplicationHealth(eq(containerInfo),
anySet())).thenReturn(
+ new ContainerHealthResult.UnderReplicatedHealthResult(containerInfo, 1,
+ false, false, false));
+
+ GenericTestUtils.setLogLevel(ContainerBalancerSelectionCriteria.class,
INFO);
+ LogCapturer logCapturer =
LogCapturer.captureLogs(ContainerBalancerSelectionCriteria.class);
+ try {
+ assertTrue(criteria.shouldBeExcluded(containerID, source, 0L));
+ assertThat(logCapturer.getOutput()).isEmpty();
+
+ ContainerID secondContainerID = mockUnhealthyContainer(2L);
+ balancerConfiguration.setIncludeNonStandardContainers(true);
+ assertTrue(criteria.shouldBeExcluded(secondContainerID, source, 0L));
+ assertTrue(criteria.shouldBeExcluded(containerID, source, 0L));
+ criteria.logExcludedContainersDueToHealth();
+
+ assertThat(logCapturer.getOutput())
+ .contains("Excluded 2 containers because of their health state",
+ "#1=UNDER_REPLICATED", "#2=UNDER_REPLICATED");
+ } finally {
+ logCapturer.stopCapturing();
+ }
+ }
+
+ @Test
+ public void shouldLimitUnhealthyContainersLoggedAtInfo() throws Exception {
+ GenericTestUtils.setLogLevel(ContainerBalancerSelectionCriteria.class,
INFO);
+ LogCapturer logCapturer =
LogCapturer.captureLogs(ContainerBalancerSelectionCriteria.class);
+ try {
+ for (long id = 1; id <= 51; id++) {
+ assertTrue(criteria.shouldBeExcluded(mockUnhealthyContainer(id),
source, 0L));
+ }
+
+ criteria.logExcludedContainersDueToHealth();
+
+ assertThat(logCapturer.getOutput())
+ .containsOnlyOnce("Excluded 51 containers because of their health
state")
+ .contains("Logging the first 50", "#50=UNDER_REPLICATED")
+ .doesNotContain("#51=UNDER_REPLICATED");
+ } finally {
+ logCapturer.stopCapturing();
+ }
+ }
+
+ private ContainerID mockUnhealthyContainer(long id) throws Exception {
+ ContainerInfo unhealthyContainer = ReplicationTestUtil.createContainerInfo(
+ RatisReplicationConfig.getInstance(THREE), id,
HddsProtos.LifeCycleState.CLOSED, 1L, OzoneConsts.GB);
+ ContainerID unhealthyContainerID = unhealthyContainer.containerID();
+ Set<ContainerReplica> replicas = new HashSet<>();
+
replicas.add(ReplicationTestUtil.createContainerReplica(unhealthyContainerID,
0, IN_SERVICE, CLOSED,
+ 1L, OzoneConsts.GB, source, source.getID()));
+
when(containerManager.getContainer(unhealthyContainerID)).thenReturn(unhealthyContainer);
+
when(containerManager.getContainerReplicas(unhealthyContainerID)).thenReturn(replicas);
+
when(replicationManager.getContainerReplicationHealth(eq(unhealthyContainer),
anySet())).thenReturn(
+ new
ContainerHealthResult.UnderReplicatedHealthResult(unhealthyContainer, 1, false,
false, false));
+ return unhealthyContainerID;
+ }
+
@Test
public void shouldExcludeOverReplicatedContainer() {
when(replicationManager.getContainerReplicationHealth(eq(containerInfo),
anySet())).thenReturn(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
