(ozone) branch HDDS-5713 updated: HDDS-12431. [DiskBalancer] Use committedBytes to reserve the space pre-allocated for container (#8297)

[sammichen]

This is an automated email from the ASF dual-hosted git repository.

sammichen pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git


The following commit(s) were added to refs/heads/HDDS-5713 by this push:
     new 73a813b5bf HDDS-12431. [DiskBalancer] Use committedBytes to reserve 
the space pre-allocated for container (#8297)
73a813b5bf is described below

commit 73a813b5bfaa3625fe1da4be2de7efce9144717e
Author: Gargi Jaiswal <134698352+gargi-jai...@users.noreply.github.com>
AuthorDate: Fri Apr 25 13:58:14 2025 +0530

    HDDS-12431. [DiskBalancer] Use committedBytes to reserve the space 
pre-allocated for container (#8297)
---
 .../container/diskbalancer/DiskBalancerService.java   | 19 +++++++++++--------
 .../policy/DefaultVolumeChoosingPolicy.java           | 10 +++++++---
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git 
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
 
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index 5bd0992e52..627623ba08 100644
--- 
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++ 
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -92,10 +92,15 @@ public class DiskBalancerService extends BackgroundService {
   private Map<DiskBalancerTask, Integer> inProgressTasks;
   private Set<Long> inProgressContainers;
 
-  // Every time a container is decided to be moved from Vol A to Vol B,
-  // the size will be deducted from Vol A and added to Vol B.
-  // This map is used to help calculate the expected storage size after
-  // the container balancing finished.
+  /**
+   * A map that tracks the total bytes which will be freed from each source 
volume
+   * during container moves in the current disk balancing cycle.
+   *
+   * Unlike committedBytes, which is used for pre-allocating space on
+   * destination volumes, deltaSizes helps track how many space will be
+   * freed on the source volumes without modifying their
+   * committedBytes (which could otherwise go negative).
+   */
   private Map<HddsVolume, Long> deltaSizes;
   private MutableVolumeSet volumeSet;
 
@@ -346,8 +351,7 @@ public BackgroundTaskQueue getTasks() {
         inProgressContainers.add(toBalanceContainer.getContainerID());
         deltaSizes.put(sourceVolume, deltaSizes.getOrDefault(sourceVolume, 0L)
             - toBalanceContainer.getBytesUsed());
-        deltaSizes.put(destVolume, deltaSizes.getOrDefault(destVolume, 0L)
-            + toBalanceContainer.getBytesUsed());
+        destVolume.incCommittedBytes(toBalanceContainer.getBytesUsed());
       }
     }
 
@@ -506,8 +510,7 @@ private void postCall() {
       inProgressContainers.remove(containerData.getContainerID());
       deltaSizes.put(sourceVolume, deltaSizes.get(sourceVolume) +
           containerData.getBytesUsed());
-      deltaSizes.put(destVolume, deltaSizes.get(destVolume)
-          - containerData.getBytesUsed());
+      destVolume.incCommittedBytes(-containerData.getBytesUsed());
     }
   }
 
diff --git 
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
 
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
index 393b3e15d5..a0166f9e2a 100644
--- 
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
+++ 
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
@@ -29,6 +29,10 @@
 
 /**
  * Choose a random volume for balancing.
+ *
+ * Source volumes use deltaMap to simulate space that will be freed 
(pre-deleted).
+ * Destination volumes use committedBytes to account for space already 
reserved.
+ * Both deltaMap and committedBytes are considered to calculate usage.
  */
 public class DefaultVolumeChoosingPolicy implements VolumeChoosingPolicy {
 
@@ -46,15 +50,15 @@ public Pair<HddsVolume, HddsVolume> 
chooseVolume(MutableVolumeSet volumeSet,
         .filter(volume ->
             Math.abs(
                 ((double)((volume.getCurrentUsage().getCapacity() - 
volume.getCurrentUsage().getAvailable())
-                    + deltaMap.getOrDefault(volume, 0L)))
+                    + deltaMap.getOrDefault(volume, 0L) + 
volume.getCommittedBytes()))
                     / volume.getCurrentUsage().getCapacity() - idealUsage) >= 
threshold)
         .sorted((v1, v2) ->
             Double.compare(
                 (double) ((v2.getCurrentUsage().getCapacity() - 
v2.getCurrentUsage().getAvailable())
-                    + deltaMap.getOrDefault(v2, 0L)) /
+                    + deltaMap.getOrDefault(v2, 0L) + v2.getCommittedBytes()) /
                     v2.getCurrentUsage().getCapacity(),
                 (double) ((v1.getCurrentUsage().getCapacity() - 
v1.getCurrentUsage().getAvailable())
-                    + deltaMap.getOrDefault(v1, 0L)) /
+                    + deltaMap.getOrDefault(v1, 0L) + v1.getCommittedBytes()) /
                     v1.getCurrentUsage().getCapacity()))
         .collect(Collectors.toList());
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@ozone.apache.org
For additional commands, e-mail: commits-h...@ozone.apache.org