This is an automated email from the ASF dual-hosted git repository.
ashishkumar50 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c689b148989 HDDS-14925. Enhance DN Disk space management with soft and
hard minfreespace limits. (#10054)
c689b148989 is described below
commit c689b1489897df0d5a83194cf9c0048df71c7531
Author: Ashish Kumar <[email protected]>
AuthorDate: Wed May 13 13:47:13 2026 +0530
HDDS-14925. Enhance DN Disk space management with soft and hard
minfreespace limits. (#10054)
---
.../container/common/helpers/ContainerUtils.java | 20 +-
.../common/statemachine/DatanodeConfiguration.java | 81 +++++++-
.../common/volume/AvailableSpaceFilter.java | 19 +-
.../ozone/container/common/volume/HddsVolume.java | 16 +-
.../container/common/volume/VolumeInfoMetrics.java | 44 ++++
.../common/helpers/TestContainerUtils.java | 222 +++++++++++++++++++++
.../container/common/impl/TestHddsDispatcher.java | 109 ++++++++++
.../statemachine/TestDatanodeConfiguration.java | 46 ++++-
.../common/volume/TestAvailableSpaceFilter.java | 177 ++++++++++++++++
.../volume/TestCapacityVolumeChoosingPolicy.java | 2 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 2 +-
.../replication/TestReplicationSupervisor.java | 2 +-
.../content/design/dn-min-space-configuration.md | 120 +++++++++++
.../dn/TestDatanodeMinFreeSpaceIntegration.java | 121 +++++++++++
14 files changed, 961 insertions(+), 20 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
index 7d16546fb69..f2817e37b51 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
@@ -58,6 +58,7 @@
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.VolumeInfoMetrics;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -367,12 +368,25 @@ public static long getPendingDeletionBlocks(ContainerData
containerData) {
public static void assertSpaceAvailability(long containerId, HddsVolume
volume, int sizeRequested)
throws StorageContainerException {
final SpaceUsageSource currentUsage = volume.getCurrentUsage();
- final long spared = volume.getFreeSpaceToSpare(currentUsage.getCapacity());
+ final long capacity = currentUsage.getCapacity();
+ final long available = currentUsage.getAvailable();
+ final long hardSpare = volume.getFreeSpaceToSpare(capacity);
- if (currentUsage.getAvailable() - spared < sizeRequested) {
+ if (available - hardSpare < sizeRequested) {
+ VolumeInfoMetrics stats = volume.getVolumeInfoStats();
+ if (stats != null) {
+ stats.incNumWriteRequestsRejectedHardMinFreeSpace();
+ }
throw new StorageContainerException("Failed to write " + sizeRequested +
" bytes to container "
+ containerId + " due to volume " + volume + " out of space "
- + currentUsage + ", minimum free space spared=" + spared,
DISK_OUT_OF_SPACE);
+ + currentUsage + ", minimum free space spared=" + hardSpare,
DISK_OUT_OF_SPACE);
+ }
+ final long reportedSpare = volume.getReportedFreeSpaceToSpare(capacity);
+ if (available - reportedSpare < sizeRequested) {
+ VolumeInfoMetrics stats = volume.getVolumeInfoStats();
+ if (stats != null) {
+ stats.incNumWriteRequestsInSoftBandMinFreeSpace();
+ }
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
index e47fdbb0c6b..1fd094b55f1 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
@@ -25,6 +25,7 @@
import static org.apache.hadoop.hdds.conf.ConfigTag.STORAGE;
import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.CONFIG_PREFIX;
+import com.google.common.annotations.VisibleForTesting;
import java.time.Duration;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.apache.hadoop.hdds.conf.Config;
@@ -81,6 +82,10 @@ public class DatanodeConfiguration extends
ReconfigurableConfig {
public static final String HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT =
"hdds.datanode.volume.min.free.space.percent";
public static final float
HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT_DEFAULT = 0.02f;
+ public static final String
HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT =
+ "hdds.datanode.volume.min.free.space.hard.limit.percent";
+ public static final float
HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT_DEFAULT =
+ 0.015f;
public static final String WAIT_ON_ALL_FOLLOWERS =
"hdds.datanode.wait.on.all.followers";
public static final String CONTAINER_SCHEMA_V3_ENABLED =
"hdds.datanode.container.schema.v3.enabled";
@@ -318,11 +323,10 @@ public class DatanodeConfiguration extends
ReconfigurableConfig {
defaultValue = "-1",
type = ConfigType.SIZE,
tags = { OZONE, CONTAINER, STORAGE, MANAGEMENT },
- description = "This determines the free space to be used for closing
containers" +
- " When the difference between volume capacity and used reaches this
number," +
- " containers that reside on this volume will be closed and no new
containers" +
- " would be allocated on this volume." +
- " Max of min.free.space and min.free.space.percent will be used as
final value."
+ description = "Minimum free space (bytes) applied together with
min.free.space.percent "
+ + "(reported to SCM in heartbeat as freeSpaceToSpare) and "
+ + "min.free.space.hard.limit.percent (local write enforcement). "
+ + "The effective value for each tier is max(this bytes, capacity *
ratio)."
)
private long minFreeSpace = getDefaultFreeSpace();
@@ -330,13 +334,25 @@ public class DatanodeConfiguration extends
ReconfigurableConfig {
defaultValue = "0.02", // match
HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT_DEFAULT
type = ConfigType.FLOAT,
tags = { OZONE, CONTAINER, STORAGE, MANAGEMENT },
- description = "This determines the free space percent to be used for
closing containers" +
- " When the difference between volume capacity and used reaches
(free.space.percent of volume capacity)," +
- " containers that reside on this volume will be closed and no new
containers" +
- " would be allocated on this volume." +
- " Max of min.free.space or min.free.space.percent will be used as
final value."
+ description = "Minimum fraction of volume capacity reported to SCM as
freeSpaceToSpare "
+ + "(heartbeat / storage reports). Local write rejection uses "
+ + "hdds.datanode.volume.min.free.space.hard.limit.percent instead. "
+ + "The soft band is the gap between these two (e.g. 2000GB disk: 2%
= 40GB reported vs "
+ + "1.5% = 30GB hard → 10GB band) where the DN may send
close-container actions while "
+ + "writes still succeed."
)
private float minFreeSpaceRatio =
HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT_DEFAULT;
+ @Config(key = "hdds.datanode.volume.min.free.space.hard.limit.percent",
+ defaultValue = "0.015",
+ type = ConfigType.FLOAT,
+ tags = { OZONE, CONTAINER, STORAGE, MANAGEMENT },
+ description = "Minimum fraction of volume capacity reserved for local
enforcement: "
+ + "writes fail when available space would drop below max(this ratio
* capacity, "
+ + "hdds.datanode.volume.min.free.space). Should be <=
min.free.space.percent "
+ + "so SCM can plan for a larger headroom than the DN enforces
locally."
+ )
+ private float minFreeSpaceHardLimitRatio =
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT_DEFAULT;
@Config(key = "hdds.datanode.periodic.disk.check.interval.minutes",
defaultValue = "60",
@@ -876,6 +892,23 @@ private void validateMinFreeSpace() {
minFreeSpaceRatio);
minFreeSpaceRatio = HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT_DEFAULT;
}
+ if (minFreeSpaceHardLimitRatio > 1 || minFreeSpaceHardLimitRatio < 0) {
+ LOG.warn("{} = {} is invalid, should be between 0 and 1; resetting to
default {}",
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT,
+ minFreeSpaceHardLimitRatio,
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT_DEFAULT);
+ minFreeSpaceHardLimitRatio =
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT_DEFAULT;
+ }
+ if (minFreeSpaceHardLimitRatio > minFreeSpaceRatio) {
+ LOG.warn("{} = {} must not exceed {} = {}, setting hard limit to soft
limit. "
+ + "Set hard.limit.percent <= min.free.space.percent to enable
the soft band.",
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT,
+ minFreeSpaceHardLimitRatio,
+ HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
+ minFreeSpaceRatio);
+ minFreeSpaceHardLimitRatio = minFreeSpaceRatio;
+ }
if (minFreeSpace < 0) {
minFreeSpace = getDefaultFreeSpace();
@@ -942,10 +975,33 @@ public void setContainerCloseThreads(int
containerCloseThreads) {
this.containerCloseThreads = containerCloseThreads;
}
+ /**
+ * Minimum free space reported to SCM (freeSpaceToSpare in storage reports).
+ */
public long getMinFreeSpace(long capacity) {
return Math.max((long) (capacity * minFreeSpaceRatio), minFreeSpace);
}
+ /**
+ * Minimum free space enforced locally for writes (disk full / out-of-space)
+ * and for choosing a volume for a new container (same threshold as writes).
+ */
+ public long getHardLimitMinFreeSpace(long capacity) {
+ return Math.max((long) (capacity * minFreeSpaceHardLimitRatio),
minFreeSpace);
+ }
+
+ /**
+ * Width of the soft band: reported spare minus hard spare. For example,
with 2000GB capacity,
+ * 2% reported (40GB) and 1.5% hard (30GB), this is 10GB — the gap where the
DN may send
+ * close-container actions while writes still succeed.
+ */
+ @VisibleForTesting
+ public long getSoftBandMinFreeSpaceWidth(long capacity) {
+ long reported = getMinFreeSpace(capacity);
+ long hard = getHardLimitMinFreeSpace(capacity);
+ return Math.max(0L, reported - hard);
+ }
+
public long getMinFreeSpace() {
return minFreeSpace;
}
@@ -954,6 +1010,11 @@ public float getMinFreeSpaceRatio() {
return minFreeSpaceRatio;
}
+ @VisibleForTesting
+ public float getMinFreeSpaceHardLimitRatio() {
+ return minFreeSpaceHardLimitRatio;
+ }
+
public long getPeriodicDiskCheckIntervalMinutes() {
return periodicDiskCheckIntervalMinutes;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
index bbd2bc97517..5ddc1efa68c 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
@@ -24,6 +24,9 @@
/**
* Filter for selecting volumes with enough space for a new container.
+ * Uses the <em>hard</em> min-free spare (same as write checks), not the
SCM-reported spare in
+ * {@link StorageLocationReport#getFreeSpaceToSpare()}. The gap between
reported and hard is the
+ * soft band (e.g. 40GB − 30GB on a 2000GB disk with 2% vs 1.5%).
* Keeps track of ineligible volumes for logging/debug purposes.
*/
public class AvailableSpaceFilter implements Predicate<HddsVolume> {
@@ -39,9 +42,23 @@ public AvailableSpaceFilter(long requiredSpace) {
@Override
public boolean test(HddsVolume vol) {
StorageLocationReport report = vol.getReport();
- long available = report.getUsableSpace();
+ long capacity = report.getCapacity();
+ long spareAtHardLimit = vol.getFreeSpaceToSpare(capacity);
+ long available =
+ report.getRemaining() - report.getCommitted() - spareAtHardLimit;
+ long availableAtReportedSpare = report.getUsableSpace();
+
boolean hasEnoughSpace = available > requiredSpace;
+ VolumeInfoMetrics stats = vol.getVolumeInfoStats();
+ if (stats != null) {
+ if (!hasEnoughSpace) {
+ stats.incNumContainerCreateRequestsRejectedHardMinFreeSpace();
+ } else if (availableAtReportedSpare <= requiredSpace) {
+ stats.incNumContainerCreateRequestsInSoftBandMinFreeSpace();
+ }
+ }
+
mostAvailableSpace = Math.max(available, mostAvailableSpace);
if (!hasEnoughSpace) {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index 8ac48fae748..f1deedc8d33 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -192,7 +192,7 @@ protected StorageLocationReport.Builder reportBuilder() {
StorageLocationReport.Builder builder = super.reportBuilder();
if (!builder.isFailed()) {
builder.setCommitted(getCommittedBytes())
- .setFreeSpaceToSpare(getFreeSpaceToSpare(builder.getCapacity()));
+
.setFreeSpaceToSpare(getReportedFreeSpaceToSpare(builder.getCapacity()));
}
return builder;
}
@@ -388,10 +388,22 @@ public long getCommittedBytes() {
return committedBytes.get();
}
- public long getFreeSpaceToSpare(long volumeCapacity) {
+ /**
+ * Minimum free space reported to SCM (heartbeat), from
+ * {@code hdds.datanode.volume.min.free.space.percent}.
+ */
+ public long getReportedFreeSpaceToSpare(long volumeCapacity) {
return getDatanodeConfig().getMinFreeSpace(volumeCapacity);
}
+ /**
+ * Minimum free space enforced locally for writes (see
+ * {@code hdds.datanode.volume.min.free.space.hard.limit.percent}).
+ */
+ public long getFreeSpaceToSpare(long volumeCapacity) {
+ return getDatanodeConfig().getHardLimitMinFreeSpace(volumeCapacity);
+ }
+
@Override
public void setGatherContainerUsages(Function<HddsVolume, Long>
gatherContainerUsages) {
this.gatherContainerUsages = gatherContainerUsages;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
index 8340c1c4f7f..0cb0c9d56a9 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
@@ -76,6 +76,18 @@ public class VolumeInfoMetrics implements MetricsSource {
@Metric("Number of scans skipped for the volume")
private MutableCounterLong numScansSkipped;
+ @Metric("Write requests allowed while usable space is between the reported
(soft) and hard min-free-space thresholds")
+ private MutableCounterLong numWriteRequestsInSoftBandMinFreeSpace;
+
+ @Metric("Write requests rejected because the hard min-free-space limit would
be violated")
+ private MutableCounterLong numWriteRequestsRejectedHardMinFreeSpace;
+ @Metric("Container create allowed while usable space is between the reported
(soft) " +
+ "and hard min-free-space thresholds")
+ private MutableCounterLong numContainerCreateRequestsInSoftBandMinFreeSpace;
+
+ @Metric("Container create requests rejected because the hard min-free-space
limit would be violated")
+ private MutableCounterLong
numContainerCreateRequestsRejectedHardMinFreeSpace;
+
/**
* @param identifier Typically, path to volume root. E.g. /data/hdds
*/
@@ -185,6 +197,38 @@ public void incNumScansSkipped() {
numScansSkipped.incr();
}
+ public long getNumWriteRequestsInSoftBandMinFreeSpace() {
+ return numWriteRequestsInSoftBandMinFreeSpace.value();
+ }
+
+ public void incNumWriteRequestsInSoftBandMinFreeSpace() {
+ numWriteRequestsInSoftBandMinFreeSpace.incr();
+ }
+
+ public long getNumWriteRequestsRejectedHardMinFreeSpace() {
+ return numWriteRequestsRejectedHardMinFreeSpace.value();
+ }
+
+ public void incNumWriteRequestsRejectedHardMinFreeSpace() {
+ numWriteRequestsRejectedHardMinFreeSpace.incr();
+ }
+
+ public long getNumContainerCreateRequestsInSoftBandMinFreeSpace() {
+ return numContainerCreateRequestsInSoftBandMinFreeSpace.value();
+ }
+
+ public void incNumContainerCreateRequestsInSoftBandMinFreeSpace() {
+ numContainerCreateRequestsInSoftBandMinFreeSpace.incr();
+ }
+
+ public long getNumContainerCreateRequestsRejectedHardMinFreeSpace() {
+ return numContainerCreateRequestsRejectedHardMinFreeSpace.value();
+ }
+
+ public void incNumContainerCreateRequestsRejectedHardMinFreeSpace() {
+ numContainerCreateRequestsRejectedHardMinFreeSpace.incr();
+ }
+
@Override
public void getMetrics(MetricsCollector collector, boolean all) {
MetricsRecordBuilder builder = collector.addRecord(metricsSourceName);
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestContainerUtils.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestContainerUtils.java
index e262e795aa6..a2ef8f53798 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestContainerUtils.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestContainerUtils.java
@@ -24,11 +24,14 @@
import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type.ReadChunk;
import static
org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.getReadChunkResponse;
import static
org.apache.hadoop.ozone.container.ContainerTestHelper.getDummyCommandRequestProto;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
@@ -42,13 +45,18 @@
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ByteStringConversion;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.common.ChunkBuffer;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.VolumeInfoMetrics;
import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -227,4 +235,218 @@ private static void assertDetailsEquals(DatanodeDetails
expected,
assertEquals(expected.getInitialVersion(), actual.getInitialVersion());
assertEquals(expected.getIpAddress(), actual.getIpAddress());
}
+
+ @Test
+ public void
assertSpaceAvailabilityIncrementsSoftBandWhenBetweenReportedAndHard()
+ throws Exception {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 100L,
900L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getReportedFreeSpaceToSpare(1000L)).thenReturn(100L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ ContainerUtils.assertSpaceAvailability(1L, volume, 50);
+
+ verify(metrics).incNumWriteRequestsInSoftBandMinFreeSpace();
+ verify(metrics, never()).incNumWriteRequestsRejectedHardMinFreeSpace();
+ }
+
+ @Test
+ public void
assertSpaceAvailabilityIncrementsHardRejectWhenHardLimitViolated() {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 100L,
900L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getReportedFreeSpaceToSpare(1000L)).thenReturn(100L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ StorageContainerException ex =
assertThrows(StorageContainerException.class,
+ () -> ContainerUtils.assertSpaceAvailability(1L, volume, 80));
+ assertEquals(Result.DISK_OUT_OF_SPACE, ex.getResult());
+
+ verify(metrics).incNumWriteRequestsRejectedHardMinFreeSpace();
+ verify(metrics, never()).incNumWriteRequestsInSoftBandMinFreeSpace();
+ }
+
+ /**
+ * available(100) - hardSpare(30) = 70 == sizeRequested(70).
+ * The check is strict less-than, so the write passes at the exact boundary.
+ * The volume is still inside the soft band (available - softSpare = 0 < 70),
+ * so the soft-band metric fires.
+ */
+ @Test
+ public void assertSpaceAvailabilityPassesAtExactHardBoundary() throws
Exception {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 100L,
900L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getReportedFreeSpaceToSpare(1000L)).thenReturn(100L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ // available(100) - hardSpare(30) = 70 == sizeRequested → passes (< not <=)
+ assertDoesNotThrow(() -> ContainerUtils.assertSpaceAvailability(1L,
volume, 70));
+
+ verify(metrics).incNumWriteRequestsInSoftBandMinFreeSpace();
+ verify(metrics, never()).incNumWriteRequestsRejectedHardMinFreeSpace();
+ }
+
+ /**
+ * available(100) - hardSpare(30) = 70 < sizeRequested(71).
+ * One byte past the hard boundary; write must be rejected.
+ */
+ @Test
+ public void assertSpaceAvailabilityRejectsOneByteOverHardBoundary() {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 100L,
900L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getReportedFreeSpaceToSpare(1000L)).thenReturn(100L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ // available(100) - hardSpare(30) = 70 < 71 → rejected
+ StorageContainerException ex =
assertThrows(StorageContainerException.class,
+ () -> ContainerUtils.assertSpaceAvailability(1L, volume, 71));
+ assertEquals(Result.DISK_OUT_OF_SPACE, ex.getResult());
+
+ verify(metrics).incNumWriteRequestsRejectedHardMinFreeSpace();
+ verify(metrics, never()).incNumWriteRequestsInSoftBandMinFreeSpace();
+ }
+
+ /**
+ * available(200) is well above both soft(100) and hard(30) spares.
+ * available - hardSpare = 170 > sizeRequested(50): write passes.
+ * available - softSpare = 100 > sizeRequested(50): not in soft band.
+ * Neither metric should fire.
+ */
+ @Test
+ public void assertSpaceAvailabilityFiresNoMetricWhenWellAboveBothLimits()
throws Exception {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 200L,
800L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getReportedFreeSpaceToSpare(1000L)).thenReturn(100L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ // available(200) - hardSpare(30) = 170 > 50, and 200 - softSpare(100) =
100 > 50
+ ContainerUtils.assertSpaceAvailability(1L, volume, 50);
+
+
+ verify(metrics, never()).incNumWriteRequestsInSoftBandMinFreeSpace();
+ verify(metrics, never()).incNumWriteRequestsRejectedHardMinFreeSpace();
+ }
+
+ /**
+ * When VolumeInfoMetrics is null (volume not yet initialised or metrics
disabled),
+ * assertSpaceAvailability must not throw NullPointerException on the
soft-band path.
+ */
+ @Test
+ public void assertSpaceAvailabilityHandlesNullMetrics() {
+ HddsVolume volume = mock(HddsVolume.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 100L,
900L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getReportedFreeSpaceToSpare(1000L)).thenReturn(100L);
+ when(volume.getVolumeInfoStats()).thenReturn(null);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ assertDoesNotThrow(() -> ContainerUtils.assertSpaceAvailability(1L,
volume, 50));
+ }
+
+ /**
+ * When VolumeInfoMetrics is null and the hard limit is violated,
+ * assertSpaceAvailability must throw DISK_OUT_OF_SPACE without
NullPointerException
+ * when trying to increment the hard-reject metric.
+ */
+ @Test
+ public void assertSpaceAvailabilityHandlesNullMetricsOnHardReject() {
+ HddsVolume volume = mock(HddsVolume.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 100L,
900L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getVolumeInfoStats()).thenReturn(null);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ // available(100) - hardSpare(30) = 70 < 80 → hard reject; null metrics
must not NPE
+ StorageContainerException ex =
assertThrows(StorageContainerException.class,
+ () -> ContainerUtils.assertSpaceAvailability(1L, volume, 80));
+ assertEquals(Result.DISK_OUT_OF_SPACE, ex.getResult());
+ }
+
+ /**
+ * Exact soft boundary: available(150) - softSpare(100) == sizeRequested(50).
+ * The soft-band check is strict {@code <}, so at equality it must NOT fire.
+ * The hard check: 150 - 30 = 120 > 50, so the write passes.
+ */
+ @Test
+ public void assertSpaceAvailabilityNoSoftBandMetricAtExactSoftBoundary()
throws Exception {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 150L,
850L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getReportedFreeSpaceToSpare(1000L)).thenReturn(100L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ // available(150) - softSpare(100) = 50 == sizeRequested(50): boundary is
exclusive, no soft metric
+ ContainerUtils.assertSpaceAvailability(1L, volume, 50);
+
+ verify(metrics, never()).incNumWriteRequestsInSoftBandMinFreeSpace();
+ verify(metrics, never()).incNumWriteRequestsRejectedHardMinFreeSpace();
+ }
+
+ /**
+ * Zero-byte write: sizeRequested == 0 should always pass regardless of
available space,
+ * and must not fire any metric.
+ */
+ @Test
+ public void assertSpaceAvailabilityNoMetricsForZeroSizeRequest() throws
Exception {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 100L,
900L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getReportedFreeSpaceToSpare(1000L)).thenReturn(100L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ ContainerUtils.assertSpaceAvailability(1L, volume, 0);
+
+ verify(metrics, never()).incNumWriteRequestsInSoftBandMinFreeSpace();
+ verify(metrics, never()).incNumWriteRequestsRejectedHardMinFreeSpace();
+ }
+
+ /**
+ * When hard spare == soft spare the soft band is effectively disabled.
+ * A write that passes the hard check must not trigger the soft-band metric
+ * even though the usable space is tight.
+ * available(100) - spare(30) = 70 > 50: passes; soft band width = 0.
+ */
+ @Test
+ public void assertSpaceAvailabilityNoSoftBandWhenHardAndSoftSpareAreEqual()
throws Exception {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ SpaceUsageSource.Fixed usage = new SpaceUsageSource.Fixed(1000L, 100L,
900L);
+ when(volume.getCurrentUsage()).thenReturn(usage);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getReportedFreeSpaceToSpare(1000L)).thenReturn(30L); // same
as hard → no soft band
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+ when(volume.toString()).thenReturn("mockVolume");
+
+ ContainerUtils.assertSpaceAvailability(1L, volume, 50);
+
+ verify(metrics, never()).incNumWriteRequestsInSoftBandMinFreeSpace();
+ verify(metrics, never()).incNumWriteRequestsRejectedHardMinFreeSpace();
+ }
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index da84d23d3e5..b5d4ea10647 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -856,6 +856,115 @@ public void verify(Token<?> token,
}
}
+ /**
+ * Verifies the soft/hard min-free-space split on the write path:
+ *
+ * <p>Setup (capacity=500 bytes):
+ * <pre>
+ * minFreeSpace bytes floor = 1 (ratio always dominates)
+ * softRatio = 10% → softSpare = 50 bytes (reported to SCM)
+ * hardRatio = 6% → hardSpare = 30 bytes (local write enforcement)
+ * softBand = 20 bytes
+ * writeChunk size ≈ 36 bytes (UUID string)
+ * </pre>
+ *
+ * <p>Three scenarios exercised in sequence using the same volume by calling
+ * {@code hddsVolume.incrementUsedSpace(delta)} to update the
CachingSpaceUsageSource cache:
+ * <ol>
+ * <li>Well above both limits (usedSpace=400, available=100): write
passes, no metric fires.</li>
+ * <li>Inside the soft band (usedSpace=425, available=75): write passes
(75-30=45 > 36),
+ * {@code numWriteRequestsInSoftBandMinFreeSpace} incremented
(75-50=25 < 36).</li>
+ * <li>Below hard limit (usedSpace=465, available=35): write rejected with
DISK_OUT_OF_SPACE
+ * (35-30=5 < 36), {@code numWriteRequestsRejectedHardMinFreeSpace}
incremented.</li>
+ * </ol>
+ */
+ @ContainerLayoutTestInfo.ContainerTest
+ public void testWriteChunkEnforcesSoftHardMinFreeSpace(
+ ContainerLayoutVersion layoutVersion) throws Exception {
+ String testDirPath = testDir.getPath();
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // 1-byte floor so the percentage ratios always dominate
+
conf.setStorageSize(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE,
+ 1.0, StorageUnit.BYTES);
+ // soft spare = 10% of 500 = 50 bytes; hard spare = 6% of 500 = 30 bytes;
band = 20 bytes
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
0.1f);
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT,
0.06f);
+ conf.set(HDDS_DATANODE_DIR_KEY, testDirPath);
+ conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDirPath);
+ DatanodeDetails dd = randomDatanodeDetails();
+ UUID scmId = UUID.randomUUID();
+ AtomicLong usedSpace = new AtomicLong(400); // available = 100, well above
both limits
+ SpaceUsageSource spaceUsage = MockSpaceUsageSource.of(500, usedSpace);
+ SpaceUsageCheckFactory factory = MockSpaceUsageCheckFactory.of(
+ spaceUsage, Duration.ZERO, inMemory(new AtomicLong(0)));
+ HddsVolume.Builder volumeBuilder =
+ new HddsVolume.Builder(testDirPath).datanodeUuid(dd.getUuidString())
+
.conf(conf).usageCheckFactory(MockSpaceUsageCheckFactory.NONE).clusterID("test");
+ volumeBuilder.usageCheckFactory(factory);
+ MutableVolumeSet volumeSet = mock(MutableVolumeSet.class);
+ when(volumeSet.getVolumesList())
+ .thenReturn(Collections.singletonList(volumeBuilder.build()));
+
volumeSet.getVolumesList().get(0).setState(StorageVolume.VolumeState.NORMAL);
+ volumeSet.getVolumesList().get(0).start();
+ HddsVolume hddsVolume = StorageVolumeUtil
+ .getHddsVolumesList(volumeSet.getVolumesList()).get(0);
+ try {
+ KeyValueContainerData containerData = new KeyValueContainerData(1L,
+ layoutVersion, 50, UUID.randomUUID().toString(), dd.getUuidString());
+ Container container = new KeyValueContainer(containerData, conf);
+ StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList())
+ .forEach(v -> v.setDbParentDir(tempDir.toFile()));
+ container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(),
scmId.toString());
+ ContainerSet containerSet = newContainerSet();
+ containerSet.addContainer(container);
+ StateContext context = ContainerTestUtils.getMockContext(dd, conf);
+ ContainerMetrics metrics = ContainerMetrics.create(conf);
+ Map<ContainerType, Handler> handlers = Maps.newHashMap();
+ for (ContainerType containerType : ContainerType.values()) {
+ handlers.put(containerType,
+ Handler.getHandlerForContainerType(containerType, conf,
+ dd.getUuidString(), containerSet, volumeSet,
volumeChoosingPolicy,
+ metrics, NO_OP_ICR_SENDER, new
ContainerChecksumTreeManager(conf)));
+ }
+ HddsDispatcher hddsDispatcher = new HddsDispatcher(
+ conf, containerSet, volumeSet, handlers, context, metrics, null);
+ hddsDispatcher.setClusterId(scmId.toString());
+ // --- Scenario 1: well above both limits (available=100) ---
+ // available(100) - hardSpare(30) = 70 > writeSize(~36): passes
+ // available(100) - softSpare(50) = 50 > writeSize(~36): not in soft band
+ ContainerCommandResponseProto response =
+ hddsDispatcher.dispatch(getWriteChunkRequest(dd.getUuidString(), 1L,
1L), null);
+ assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
+ assertEquals(0,
+
hddsVolume.getVolumeInfoStats().getNumWriteRequestsInSoftBandMinFreeSpace());
+ assertEquals(0,
+
hddsVolume.getVolumeInfoStats().getNumWriteRequestsRejectedHardMinFreeSpace());
+ // --- Scenario 2: inside the soft band (usedSpace → 425, available=75)
---
+ // available(75) - hardSpare(30) = 45 > writeSize(~36): passes hard check
+ // available(75) - softSpare(50) = 25 < writeSize(~36): soft-band metric
fires
+ // Use incrementUsedSpace so the CachingSpaceUsageSource internal cache
is updated;
+ hddsVolume.incrementUsedSpace(25); // 400 → 425
+ response =
hddsDispatcher.dispatch(getWriteChunkRequest(dd.getUuidString(), 1L, 2L), null);
+ assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
+ assertEquals(1,
+
hddsVolume.getVolumeInfoStats().getNumWriteRequestsInSoftBandMinFreeSpace());
+ assertEquals(0,
+
hddsVolume.getVolumeInfoStats().getNumWriteRequestsRejectedHardMinFreeSpace());
+ // --- Scenario 3: below hard limit (usedSpace → 465, available=35) ---
+ // available(35) - hardSpare(30) = 5 < writeSize(~36): DISK_OUT_OF_SPACE
+ hddsVolume.incrementUsedSpace(40); // 425 → 465
+ response =
hddsDispatcher.dispatch(getWriteChunkRequest(dd.getUuidString(), 1L, 3L), null);
+ assertEquals(ContainerProtos.Result.DISK_OUT_OF_SPACE,
response.getResult());
+ assertEquals(1,
+
hddsVolume.getVolumeInfoStats().getNumWriteRequestsInSoftBandMinFreeSpace());
+ assertEquals(1,
+
hddsVolume.getVolumeInfoStats().getNumWriteRequestsRejectedHardMinFreeSpace());
+ } finally {
+ volumeSet.shutdown();
+ ContainerMetrics.remove();
+ }
+ }
+
static DispatcherContext newContext(Op op) {
return newContext(op, WriteChunkStage.COMBINED);
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java
index b12c6575960..59e90ba30b3 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/TestDatanodeConfiguration.java
@@ -31,6 +31,7 @@
import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.FAILED_VOLUMES_TOLERATED_DEFAULT;
import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.GRPC_SO_BACKLOG_DEFAULT;
import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.GRPC_SO_BACKLOG_KEY;
+import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT_DEFAULT;
import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT_DEFAULT;
import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.PERIODIC_DISK_CHECK_INTERVAL_MINUTES_DEFAULT;
import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.PERIODIC_DISK_CHECK_INTERVAL_MINUTES_KEY;
@@ -186,12 +187,22 @@ public void isCreatedWitDefaultValues() {
subject.getBlockDeleteCommandWorkerInterval());
assertEquals(DatanodeConfiguration.getDefaultFreeSpace(),
subject.getMinFreeSpace());
assertEquals(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT_DEFAULT,
subject.getMinFreeSpaceRatio());
+
assertEquals(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT_DEFAULT,
+ subject.getMinFreeSpaceHardLimitRatio());
final long oneGB = 1024 * 1024 * 1024;
// capacity is less, consider default min_free_space
assertEquals(DatanodeConfiguration.getDefaultFreeSpace(),
subject.getMinFreeSpace(oneGB));
+ assertEquals(DatanodeConfiguration.getDefaultFreeSpace(),
subject.getHardLimitMinFreeSpace(oneGB));
+ assertEquals(0L, subject.getSoftBandMinFreeSpaceWidth(oneGB));
// capacity is large, consider min_free_space_percent, max(min_free_space,
min_free_space_percent * capacity)ß
assertEquals(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT_DEFAULT * oneGB *
oneGB,
subject.getMinFreeSpace(oneGB * oneGB));
+
assertEquals(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT_DEFAULT *
oneGB * oneGB,
+ subject.getHardLimitMinFreeSpace(oneGB * oneGB));
+ // e.g. 2000GB: 40GB reported − 30GB hard = 10GB soft bandwidth (derived,
not configured)
+ assertEquals(
+ subject.getMinFreeSpace(oneGB * oneGB) -
subject.getHardLimitMinFreeSpace(oneGB * oneGB),
+ subject.getSoftBandMinFreeSpaceWidth(oneGB * oneGB));
// Verify that no warnings were logged when using default values
String logOutput = logCapturer.getOutput();
@@ -226,6 +237,34 @@ void useMaxIfBothMinFreeSpacePropertiesSet() {
}
}
+ /**
+ * If hard limit percent is greater than soft (reported) percent, {@link
DatanodeConfiguration}
+ * uses the hard threshold for SCM-reported spare as well, so there is no
negative "soft band".
+ */
+ @Test
+ void whenHardRatioExceedsSoftRatioReportedSpareMatchesHardOnly() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.unset(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE);
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
0.01f);
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT,
0.02f);
+
+ DatanodeConfiguration subject =
conf.getObject(DatanodeConfiguration.class);
+ long capacityBytes = 1000L * 1024 * 1024 * 1024;
+
+ assertEquals(subject.getHardLimitMinFreeSpace(capacityBytes),
+ subject.getMinFreeSpace(capacityBytes));
+ assertEquals(0L, subject.getSoftBandMinFreeSpaceWidth(capacityBytes));
+ }
+
+ @Test
+ void rejectsInvalidMinFreeSpaceHardLimitRatio() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT,
1.5f);
+ DatanodeConfiguration subject =
conf.getObject(DatanodeConfiguration.class);
+
assertEquals(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT_DEFAULT,
+ subject.getMinFreeSpaceHardLimitRatio());
+ }
+
@ParameterizedTest
@ValueSource(longs = {1_000, 10_000, 100_000})
void usesFixedMinFreeSpace(long bytes) {
@@ -233,6 +272,8 @@ void usesFixedMinFreeSpace(long bytes) {
conf.setLong(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE,
bytes);
// keeping %cent low so that min free space is picked up
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
0.00001f);
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT,
+ 0.00001f);
DatanodeConfiguration subject =
conf.getObject(DatanodeConfiguration.class);
@@ -249,7 +290,10 @@ void calculatesMinFreeSpaceRatio(int percent) {
OzoneConfiguration conf = new OzoneConfiguration();
// keeping min free space low so that %cent is picked up after calculation
conf.set(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE,
"1000"); // set in ozone-site.xml
-
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
percent / 100.0f);
+ float softRatio = percent / 100.0f;
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
softRatio);
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT,
+ Math.min(softRatio, 0.01f));
DatanodeConfiguration subject =
conf.getObject(DatanodeConfiguration.class);
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestAvailableSpaceFilter.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestAvailableSpaceFilter.java
new file mode 100644
index 00000000000..dade7f4b570
--- /dev/null
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestAvailableSpaceFilter.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.volume;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link AvailableSpaceFilter}.
+ */
+public class TestAvailableSpaceFilter {
+
+ @Test
+ public void testIncrementsSoftBandWhenBetweenReportedAndHard() {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ StorageLocationReport report = mock(StorageLocationReport.class);
+ when(volume.getReport()).thenReturn(report);
+ when(report.getCapacity()).thenReturn(1000L);
+ when(report.getRemaining()).thenReturn(100L);
+ when(report.getCommitted()).thenReturn(0L);
+ when(report.getUsableSpace()).thenReturn(0L);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+
+ AvailableSpaceFilter filter = new AvailableSpaceFilter(50L);
+ assertTrue(filter.test(volume));
+
+ verify(metrics).incNumContainerCreateRequestsInSoftBandMinFreeSpace();
+ verify(metrics,
never()).incNumContainerCreateRequestsRejectedHardMinFreeSpace();
+ }
+
+ @Test
+ public void testIncrementsHardRejectWhenHardLimitViolated() {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ StorageLocationReport report = mock(StorageLocationReport.class);
+ when(volume.getReport()).thenReturn(report);
+ when(report.getCapacity()).thenReturn(1000L);
+ when(report.getRemaining()).thenReturn(100L);
+ when(report.getCommitted()).thenReturn(0L);
+ when(report.getUsableSpace()).thenReturn(0L);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+
+ AvailableSpaceFilter filter = new AvailableSpaceFilter(80L);
+ assertFalse(filter.test(volume));
+
+ verify(metrics).incNumContainerCreateRequestsRejectedHardMinFreeSpace();
+ verify(metrics,
never()).incNumContainerCreateRequestsInSoftBandMinFreeSpace();
+ }
+
+ @Test
+ public void testNoMetricIncrementWhenWellAboveSoftBand() {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ StorageLocationReport report = mock(StorageLocationReport.class);
+ when(volume.getReport()).thenReturn(report);
+ when(report.getCapacity()).thenReturn(1000L);
+ when(report.getRemaining()).thenReturn(1000L);
+ when(report.getCommitted()).thenReturn(0L);
+ when(report.getUsableSpace()).thenReturn(900L);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+
+ AvailableSpaceFilter filter = new AvailableSpaceFilter(50L);
+ assertTrue(filter.test(volume));
+
+ verify(metrics,
never()).incNumContainerCreateRequestsInSoftBandMinFreeSpace();
+ verify(metrics,
never()).incNumContainerCreateRequestsRejectedHardMinFreeSpace();
+ }
+
+ /**
+ * Without committed bytes: remaining(200) - hardSpare(30) = 170 >
requiredSpace(60),
+ * and 200 - softSpare(100) = 100 > 60 → well above both limits, no metric.
+ * With committed(80): 200 - 80 - hardSpare(30) = 90 > 60 → still passes
hard,
+ * but 200 - 80 - softSpare(100) = 20 ≤ 60 → now inside the soft band.
+ * Committed bytes representing in-flight pipeline allocations push the
volume into
+ * the soft band even though raw remaining space looks healthy.
+ */
+ @Test
+ public void testCommittedBytesCanPushVolumeIntoSoftBand() {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ StorageLocationReport report = mock(StorageLocationReport.class);
+ when(volume.getReport()).thenReturn(report);
+ when(report.getCapacity()).thenReturn(1000L);
+ when(report.getRemaining()).thenReturn(200L);
+ when(report.getCommitted()).thenReturn(80L);
+ when(report.getUsableSpace()).thenReturn(20L);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+
+ // available = 200 - 80 - 30 = 90 > requiredSpace(60) → passes hard check
+ // getUsableSpace = 200 - 80 - spareOnReport(100) = 20 <= 60 → inside soft
band
+ AvailableSpaceFilter filter = new AvailableSpaceFilter(60L);
+ assertTrue(filter.test(volume));
+
+ verify(metrics).incNumContainerCreateRequestsInSoftBandMinFreeSpace();
+ verify(metrics,
never()).incNumContainerCreateRequestsRejectedHardMinFreeSpace();
+ }
+
+ /**
+ * Committed bytes representing in-flight pipeline allocations cause a hard
reject
+ * that would not occur if committed were zero.
+ * remaining(130) - committed(80) - hardSpare(30) = 20 < requiredSpace(50) →
rejected.
+ * Without committed: 130 - 0 - 30 = 100 > 50 → would have passed.
+ */
+ @Test
+ public void testCommittedBytesCanCauseHardReject() {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ StorageLocationReport report = mock(StorageLocationReport.class);
+ when(volume.getReport()).thenReturn(report);
+ when(report.getCapacity()).thenReturn(1000L);
+ when(report.getRemaining()).thenReturn(130L);
+ when(report.getCommitted()).thenReturn(80L);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+
+ // available = 130 - 80 - 30 = 20 < requiredSpace(50) → hard rejected
+ AvailableSpaceFilter filter = new AvailableSpaceFilter(50L);
+ assertFalse(filter.test(volume));
+
+ verify(metrics).incNumContainerCreateRequestsRejectedHardMinFreeSpace();
+ verify(metrics,
never()).incNumContainerCreateRequestsInSoftBandMinFreeSpace();
+ }
+
+ /**
+ * Even with non-zero committed bytes, if remaining space is large enough,
+ * the volume remains well above both limits and no metric fires.
+ * remaining(300) - committed(50) - hardSpare(30) = 220 > requiredSpace(50):
passes hard.
+ * 300 - 50 - softSpare(100) = 150 > 50: not in soft band.
+ */
+ @Test
+ public void testCommittedBytesDoNotAffectMetricsWhenVolumeStillHealthy() {
+ HddsVolume volume = mock(HddsVolume.class);
+ VolumeInfoMetrics metrics = mock(VolumeInfoMetrics.class);
+ StorageLocationReport report = mock(StorageLocationReport.class);
+ when(volume.getReport()).thenReturn(report);
+ when(report.getCapacity()).thenReturn(1000L);
+ when(report.getRemaining()).thenReturn(300L);
+ when(report.getCommitted()).thenReturn(50L);
+ when(report.getUsableSpace()).thenReturn(150L);
+ when(volume.getFreeSpaceToSpare(1000L)).thenReturn(30L);
+ when(volume.getVolumeInfoStats()).thenReturn(metrics);
+
+ // available = 300 - 50 - 30 = 220 > 50; getUsableSpace = 300 - 50 - 100 =
150 > 50
+ AvailableSpaceFilter filter = new AvailableSpaceFilter(50L);
+ assertTrue(filter.test(volume));
+
+ verify(metrics,
never()).incNumContainerCreateRequestsInSoftBandMinFreeSpace();
+ verify(metrics,
never()).incNumContainerCreateRequestsRejectedHardMinFreeSpace();
+ }
+}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
index deae4f83951..7917ebf80bd 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
@@ -129,7 +129,7 @@ public void
throwsDiskOutOfSpaceIfRequestMoreThanAvailable() {
String msg = e.getMessage();
assertThat(msg)
.contains("No volumes have enough space for a new container. " +
- "Most available space: 240 bytes");
+ "Most available space: 243 bytes");
}
@Test
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
index 36fabff1fe8..91bc9caa2c2 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
@@ -114,7 +114,7 @@ public void
throwsDiskOutOfSpaceIfRequestMoreThanAvailable() {
String msg = e.getMessage();
assertThat(msg).contains("No volumes have enough space for a new
container. " +
- "Most available space: 140 bytes");
+ "Most available space: 143 bytes");
}
@Test
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
index 9ceb0a99e9f..abfef6fbffd 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@ -419,7 +419,7 @@ public void
testReplicationImportReserveSpace(ContainerLayoutVersion layout)
assertEquals(0, usedSpace);
// Increase committed bytes so that volume has only remaining 3 times
container size space
long minFreeSpace =
-
conf.getObject(DatanodeConfiguration.class).getMinFreeSpace(vol1.getCurrentUsage().getCapacity());
+
conf.getObject(DatanodeConfiguration.class).getHardLimitMinFreeSpace(vol1.getCurrentUsage().getCapacity());
long initialCommittedBytes = vol1.getCurrentUsage().getCapacity() -
containerMaxSize * 3 - minFreeSpace;
vol1.incCommittedBytes(initialCommittedBytes);
ContainerReplicator replicator =
diff --git a/hadoop-hdds/docs/content/design/dn-min-space-configuration.md
b/hadoop-hdds/docs/content/design/dn-min-space-configuration.md
index ab62e51428d..037ad6da4cf 100644
--- a/hadoop-hdds/docs/content/design/dn-min-space-configuration.md
+++ b/hadoop-hdds/docs/content/design/dn-min-space-configuration.md
@@ -105,4 +105,124 @@ This case is more useful for test environment where disk
space is less and no ne
- So Approach 1 is selected considering advantage where higher free space can
be configured by default.
2. Min Space will be 20GB as default
+---
+
+# Soft and Hard Min-Free-Space Limits
+
+## Overview
+
+The min-free-space value chosen above is split into two distinct thresholds
that serve different purposes:
+a **soft** (reported) limit and a **hard** (locally-enforced) limit. The gap
between them is the
+*soft band* — a warning zone where writes are still accepted but the Datanode
is already signalling
+to SCM that it is running low.
+
+## Configuration
+
+| Key | Default | Purpose |
+|-----|---------|---------|
+| `hdds.datanode.volume.min.free.space` | `20GB` | Absolute floor shared by
both tiers. Effective spare = `max(this, capacity × ratio)`. |
+| `hdds.datanode.volume.min.free.space.percent` | `2%` | **Soft limit ratio**
— reported to SCM via `freeSpaceToSpare` in storage heartbeats. |
+| `hdds.datanode.volume.min.free.space.hard.limit.percent` | `1.5%` | **Hard
limit ratio** — enforced locally for write rejection and new container
placement. |
+
+Rules:
+- `hard.limit.percent` must be ≤ `min.free.space.percent`. If it is set
higher, it is silently
+ clamped down to the soft ratio, making the soft band width zero (effectively
disabling it).
+- Setting both ratios to the same value disables the soft band entirely —
behaviour is then identical
+ to the pre-split single-threshold code.
+
+### Effective spare values for a 2 TB volume
+
+| Threshold | Calculation | Result |
+|-----------|-------------|--------|
+| Soft (reported) | `max(20 GB, 2 TB × 2%)` | **40 GB** |
+| Hard (local) | `max(20 GB, 2 TB × 1.5%)` | **30 GB** |
+| Soft band width | `40 GB − 30 GB` | **10 GB** |
+
+## How Each Limit Is Used
+
+### Hard limit — local write enforcement and new container placement
+
+`getFreeSpaceToSpare(capacity)` returns the hard spare. It is checked in two
places:
+
+1. **Write rejection** (`ContainerUtils.assertSpaceAvailability`): a write is
rejected with
+ `DISK_OUT_OF_SPACE` when `available − hardSpare < requestedBytes`. This is
the definitive gate
+ that prevents disk exhaustion.
+
+2. **New container placement** (`AvailableSpaceFilter`,
`PendingContainerTracker`): SCM only
+ schedules a new container on a volume when `available − committed −
hardSpare > maxContainerSize`.
+ This ensures that containers are placed only where writes will actually
succeed.
+
+### Soft limit — SCM visibility and proactive close-container actions
+
+`getReportedFreeSpaceToSpare(capacity)` returns the soft spare. It appears in
two places:
+
+1. **Storage heartbeat** (`HddsVolume.reportBuilder`): each storage report
sent from the Datanode to
+ SCM carries `freeSpaceToSpare = softSpare`. SCM uses this value to gauge
how much usable space
+ remains on the volume when making placement decisions for new pipelines and
replication.
+
+2. **Soft-band metric** (`ContainerUtils.assertSpaceAvailability`,
`AvailableSpaceFilter`):
+ when a write (or container placement attempt) succeeds the hard check but
`available − softSpare`
+ is below the required size, the Datanode increments
+ `numWriteRequestsInSoftBandMinFreeSpace` (for writes) or
+ `numContainerCreateRequestsInSoftBandMinFreeSpace` (for container
placement). These metrics
+ alert operators that the volume is inside the warning zone.
+
+## The Soft Band: Improving Write Continuity Near Capacity Limits
+
+Without a soft band, a single threshold would have to serve two conflicting
goals:
+- **High enough** to give SCM enough lead time to stop routing work to a
nearly-full node.
+- **Low enough** to maximise usable disk space and avoid premature write
rejection.
+
+Splitting the threshold resolves the conflict:
+
+```
+ Disk capacity
+ ─────────────────────────────────────────────────────────────────
+ usable data space (writes accepted here)
+ ─────────────────────────────────────────── ← hard spare (30 GB)
+ soft band (10 GB warning zone):
+ writes still accepted, but DN metrics fire and SCM starts
+ steering new work away from this volume
+ ─────────────────────────────────────────── ← soft spare (40 GB)
+ reserved buffer (SCM sees this as "full")
+ ─────────────────────────────────────────────────────────────────
+```
+
+**What happens as a volume fills up:**
+
+1. **Above soft spare (> 40 GB free):** Normal operation.
+
+2. **Inside the soft band (30 – 40 GB free):** Writes to existing open
containers still succeed.
+ The `InSoftBand` metrics increment, and because SCM already sees
`freeSpaceToSpare = 40 GB` via
+ the heartbeat, it stops preferring this volume for new pipelines. The
Datanode may send
+ close-container actions (see [Full Volume
Handling](full-volume-handling.md)) to speed up
+ migration.
+
+3. **Below hard spare (< 30 GB free):** Writes are rejected. The
`RejectedHardMinFreeSpace` metrics
+ increment. No new containers are placed on this volume by SCM.
+
+**Why this improves write continuity:** without the band, SCM would only learn
a volume is nearly
+full when write rejections start happening. With the band, SCM gets advance
warning via a smaller
+reported `freeSpaceToSpare` and can route new containers to other volumes
*before* the hard limit is
+hit, reducing client-visible write failures.
+
+## Turning Off the Soft Band
+
+To disable the soft band (equivalent to single-threshold behaviour), set the
two ratios to the same
+value:
+
+```xml
+<property>
+ <name>hdds.datanode.volume.min.free.space.percent</name>
+ <value>0.02</value>
+</property>
+<property>
+ <name>hdds.datanode.volume.min.free.space.hard.limit.percent</name>
+ <value>0.02</value>
+</property>
+```
+
+When both ratios are equal, `softSpare == hardSpare`, the soft band width is
zero, and the
+`InSoftBand` metrics never fire.
+
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeMinFreeSpaceIntegration.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeMinFreeSpaceIntegration.java
new file mode 100644
index 00000000000..ca0e231a207
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeMinFreeSpaceIntegration.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.dn;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.function.BooleanSupplier;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.HddsDatanodeService;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+/**
+ * Integration tests: For min free space as hard and soft limit.
+ */
+@Timeout(300)
+public class TestDatanodeMinFreeSpaceIntegration {
+
+ @Test
+ public void storageReportsAtScmMatchSoftMinFreeSpaceFromConfig() throws
Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.unset(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE);
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
0.03f);
+
conf.setFloat(DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_HARD_LIMIT_PERCENT,
0.015f);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 2, SECONDS);
+
+ DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class);
+
+ try (MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(1)
+ .build()) {
+ cluster.waitForClusterToBeReady();
+ cluster.waitTobeOutOfSafeMode();
+
+ HddsDatanodeService dnService = cluster.getHddsDatanodes().get(0);
+ DatanodeDetails dn = dnService.getDatanodeDetails();
+
+ StorageContainerManager scm = cluster.getStorageContainerManager();
+ NodeManager nm = scm.getScmNodeManager();
+
+ BooleanSupplier softSpareVisibleAtScm =
+ () -> storageReportsMatchSoftMinFree(nm, dn, dnConf);
+ GenericTestUtils.waitFor(softSpareVisibleAtScm, 500, 120_000);
+
+ DatanodeInfo info = nm.getDatanodeInfo(dn);
+ assertNotNull(info);
+ assertFalse(info.getStorageReports().isEmpty());
+
+ for (StorageReportProto report : info.getStorageReports()) {
+ if (report.getFailed()) {
+ continue;
+ }
+ long capacity = report.getCapacity();
+ assertTrue(capacity > 0, "data volume should have positive capacity");
+
+ long expectedSoft = dnConf.getMinFreeSpace(capacity);
+ long expectedHard = dnConf.getHardLimitMinFreeSpace(capacity);
+ long expectedBand = dnConf.getSoftBandMinFreeSpaceWidth(capacity);
+
+ assertEquals(expectedSoft, report.getFreeSpaceToSpare(),
+ "freeSpaceToSpare in SCM storage report should match soft min-free
for capacity");
+ assertThat(expectedSoft).isGreaterThanOrEqualTo(expectedHard);
+ assertThat(expectedBand).isGreaterThan(0L);
+ assertEquals(expectedBand, expectedSoft - expectedHard);
+ }
+ }
+ }
+
+ /**
+ * SCM has caught up with DN heartbeats: every non-failed data report's
{@code freeSpaceToSpare}
+ * equals the configured soft min-free for that volume capacity.
+ */
+ private static boolean storageReportsMatchSoftMinFree(
+ NodeManager nm, DatanodeDetails dn, DatanodeConfiguration dnConf) {
+ DatanodeInfo info = nm.getDatanodeInfo(dn);
+ if (info == null || info.getStorageReports().isEmpty()) {
+ return false;
+ }
+ boolean anyDataVolume = false;
+ for (StorageReportProto r : info.getStorageReports()) {
+ if (r.getFailed() || r.getCapacity() <= 0) {
+ continue;
+ }
+ anyDataVolume = true;
+ long expectedSoft = dnConf.getMinFreeSpace(r.getCapacity());
+ if (expectedSoft != r.getFreeSpaceToSpare()) {
+ return false;
+ }
+ }
+ return anyDataVolume;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
