This is an automated email from the ASF dual-hosted git repository.
Gargi-jais11 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1975e74d096 HDDS-15688. Retain in-memory PAUSED state when
diskBalancer.info write fails on DN state change (#10818).
1975e74d096 is described below
commit 1975e74d096120f5ed9c08ffb487dcbfc09256b0
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Tue Jul 21 17:23:41 2026 +0530
HDDS-15688. Retain in-memory PAUSED state when diskBalancer.info write
fails on DN state change (#10818).
---
.../diskbalancer/DiskBalancerService.java | 17 ++++++---
.../diskbalancer/TestDiskBalancerService.java | 43 ++++++++++++++++++++++
2 files changed, 54 insertions(+), 6 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index fdf90afb61b..7ab0243db06 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -851,16 +851,21 @@ public synchronized void
nodeStateUpdated(HddsProtos.NodeOperationalState state)
}
if (stateChanged) {
+ DiskBalancerRunningStatus newOperationalState = this.operationalState;
LOG.info("DiskBalancer operational state changed from {} to {} due to
Datanode state update . Persisting.",
- originalServiceState, this.operationalState);
+ originalServiceState, newOperationalState);
try {
writeDiskBalancerInfoTo(getDiskBalancerInfo(), diskBalancerInfoFile);
} catch (IOException e) {
- LOG.error("Failed to persist DiskBalancerInfo after state change in
nodeStateUpdated. " +
- "Reverting operational state to {} to maintain consistency.",
originalServiceState, e);
- // Revert state on persistence error to keep in-memory state
consistent with last known persisted state.
- this.operationalState = originalServiceState;
- LOG.warn("DiskBalancer operational state reverted to {} due to
persistence failure.", this.operationalState);
+ if (newOperationalState == DiskBalancerRunningStatus.PAUSED) {
+ LOG.error("Failed to persist DiskBalancerInfo after pausing
DiskBalancer due to " +
+ "Datanode state update. Retaining in-memory PAUSED state for
safety.", e);
+ } else {
+ LOG.error("Failed to persist DiskBalancerInfo after state change in
nodeStateUpdated. " +
+ "Reverting operational state to {} to maintain consistency.",
originalServiceState, e);
+ // Revert state on persistence error to keep in-memory state
consistent with last known persisted state.
+ this.operationalState = originalServiceState;
+ }
}
}
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
index 1352caadede..ad336f12dc9 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
@@ -48,6 +48,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.DiskBalancerRunningStatus;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
@@ -436,6 +437,48 @@ public void
testDiskBalancerInfoWriteReportsDirectoryCreationFailure(
.hasMessageStartingWith("Unable to create DiskBalancerInfo
directories: ");
}
+ @ContainerTestVersionInfo.ContainerTest
+ public void testNodeStateUpdatedRetainsPausedWhenPersistFails(
+ ContainerTestVersionInfo versionInfo) throws Exception {
+ setLayoutAndSchemaForTest(versionInfo);
+ File infoDir =
tmpDir.resolve("diskBalancer-pause-persist-failure").toFile();
+ DiskBalancerServiceTestImpl svc =
+ getDiskBalancerService(confWithDiskBalancerInfoDir(infoDir));
+ svc.refresh(new DiskBalancerInfo(DiskBalancerRunningStatus.RUNNING, 10.0d,
100L, 5, true));
+ breakDiskBalancerInfoPersistence(infoDir);
+
+ svc.nodeStateUpdated(NodeOperationalState.DECOMMISSIONING);
+
+ assertEquals(DiskBalancerRunningStatus.PAUSED,
+ svc.getDiskBalancerInfo().getOperationalState());
+ assertTrue(svc.getTasks().isEmpty());
+ svc.shutdown();
+ }
+
+ @ContainerTestVersionInfo.ContainerTest
+ public void testNodeStateUpdatedRevertsToPausedWhenResumePersistFails(
+ ContainerTestVersionInfo versionInfo) throws Exception {
+ setLayoutAndSchemaForTest(versionInfo);
+ File infoDir =
tmpDir.resolve("diskBalancer-resume-persist-failure").toFile();
+ DiskBalancerServiceTestImpl svc =
+ getDiskBalancerService(confWithDiskBalancerInfoDir(infoDir));
+ svc.refresh(new DiskBalancerInfo(DiskBalancerRunningStatus.PAUSED, 10.0d,
100L, 5, true));
+ breakDiskBalancerInfoPersistence(infoDir);
+
+ svc.nodeStateUpdated(NodeOperationalState.IN_SERVICE);
+
+ assertEquals(DiskBalancerRunningStatus.PAUSED,
+ svc.getDiskBalancerInfo().getOperationalState());
+ assertTrue(svc.getTasks().isEmpty());
+ svc.shutdown();
+ }
+
+ private void breakDiskBalancerInfoPersistence(File infoDir) throws
IOException {
+ File infoFile = getDiskBalancerInfoFile(infoDir);
+ FileUtils.deleteQuietly(infoFile);
+ assertTrue(infoFile.mkdirs(), "Failed to replace diskBalancer.info with a
directory");
+ }
+
@ContainerTestVersionInfo.ContainerTest
public void testDiskBalancerInfoWriteReportsFileWriteFailure(
ContainerTestVersionInfo versionInfo) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
