This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8f16a3b94b HDDS-12163. Reduce number of individual
getCapacity/getAvailable/getUsedSpace calls (#7790)
8f16a3b94b is described below
commit 8f16a3b94ba138b46ed2ea514a1653c1f6fe5208
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Sun Feb 2 13:05:17 2025 +0100
HDDS-12163. Reduce number of individual
getCapacity/getAvailable/getUsedSpace calls (#7790)
---
.../container/common/impl/HddsDispatcher.java | 7 +++---
.../common/volume/AvailableSpaceFilter.java | 7 ++++--
.../volume/CapacityVolumeChoosingPolicy.java | 4 ++--
.../container/common/volume/MutableVolumeSet.java | 8 ++++---
.../container/common/volume/StorageVolume.java | 14 -----------
.../ozone/container/common/volume/VolumeInfo.java | 22 -----------------
.../ozone/container/common/volume/VolumeUsage.java | 24 ++++---------------
.../volume/TestCapacityVolumeChoosingPolicy.java | 6 ++---
.../container/common/volume/TestHddsVolume.java | 28 +++++++++++++---------
.../common/volume/TestReservedVolumeSpace.java | 4 ++--
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 4 ++--
.../container/common/volume/TestVolumeSet.java | 2 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
13 files changed, 46 insertions(+), 86 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index d1ea73fbfd..c4552dee01 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -622,12 +622,11 @@ private boolean isVolumeFull(Container container) {
.orElse(Boolean.FALSE);
if (isOpen) {
HddsVolume volume = container.getContainerData().getVolume();
- SpaceUsageSource precomputedVolumeSpace =
- volume.getCurrentUsage();
- long volumeCapacity = precomputedVolumeSpace.getCapacity();
+ SpaceUsageSource usage = volume.getCurrentUsage();
+ long volumeCapacity = usage.getCapacity();
long volumeFreeSpaceToSpare =
freeSpaceCalculator.get(volumeCapacity);
- long volumeFree = precomputedVolumeSpace.getAvailable();
+ long volumeFree = usage.getAvailable();
long volumeCommitted = volume.getCommittedBytes();
long volumeAvailable = volumeFree - volumeCommitted;
return (volumeAvailable <= volumeFreeSpaceToSpare);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
index 0d82a38d1c..bb74a051af 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.ozone.container.common.volume;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
+
import java.util.HashMap;
import java.util.Map;
import java.util.function.Predicate;
@@ -38,8 +40,9 @@ public class AvailableSpaceFilter implements
Predicate<HddsVolume> {
@Override
public boolean test(HddsVolume vol) {
- long volumeCapacity = vol.getCapacity();
- long free = vol.getAvailable();
+ SpaceUsageSource usage = vol.getCurrentUsage();
+ long volumeCapacity = usage.getCapacity();
+ long free = usage.getAvailable();
long committed = vol.getCommittedBytes();
long available = free - committed;
long volumeFreeSpaceToSpare =
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java
index 14afce020b..f7204ad606 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/CapacityVolumeChoosingPolicy.java
@@ -91,9 +91,9 @@ public HddsVolume chooseVolume(List<HddsVolume> volumes,
HddsVolume firstVolume = volumesWithEnoughSpace.get(firstIndex);
HddsVolume secondVolume = volumesWithEnoughSpace.get(secondIndex);
- long firstAvailable = firstVolume.getAvailable()
+ long firstAvailable = firstVolume.getCurrentUsage().getAvailable()
- firstVolume.getCommittedBytes();
- long secondAvailable = secondVolume.getAvailable()
+ long secondAvailable = secondVolume.getCurrentUsage().getAvailable()
- secondVolume.getCommittedBytes();
return firstAvailable < secondAvailable ? secondVolume : firstVolume;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
index 9afea8e6b0..4a85880105 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
@@ -33,6 +33,7 @@
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
@@ -479,9 +480,10 @@ public StorageLocationReport[] getStorageReport() {
if (volumeInfo.isPresent()) {
try {
rootDir = volumeInfo.get().getRootDir();
- scmUsed = volumeInfo.get().getScmUsed();
- remaining = volumeInfo.get().getAvailable();
- capacity = volumeInfo.get().getCapacity();
+ SpaceUsageSource usage = volumeInfo.get().getCurrentUsage();
+ scmUsed = usage.getUsedSpace();
+ remaining = usage.getAvailable();
+ capacity = usage.getCapacity();
committed = (volume instanceof HddsVolume) ?
((HddsVolume) volume).getCommittedBytes() : 0;
failed = false;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
index b85ac15c54..db241da6f6 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
@@ -447,25 +447,11 @@ public String getVolumeRootDir() {
return volumeInfo.map(VolumeInfo::getRootDir).orElse(null);
}
- public long getCapacity() {
- return volumeInfo.map(VolumeInfo::getCapacity).orElse(0L);
- }
-
- public long getAvailable() {
- return volumeInfo.map(VolumeInfo::getAvailable).orElse(0L);
-
- }
-
public SpaceUsageSource getCurrentUsage() {
return volumeInfo.map(VolumeInfo::getCurrentUsage)
.orElse(SpaceUsageSource.UNKNOWN);
}
- public long getUsedSpace() {
- return volumeInfo.map(VolumeInfo::getScmUsed).orElse(0L);
-
- }
-
public File getStorageDir() {
return this.storageDir;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
index 3d1be9791e..fb99d35b62 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
@@ -152,24 +152,6 @@ private VolumeInfo(Builder b) throws IOException {
usage = new VolumeUsage(checkParams, b.conf);
}
- public long getCapacity() {
- return usage.getCapacity();
- }
-
- /**
- * <pre>
- * {@code
- * Calculate available space use method A.
- * |----used----| (avail) |++++++++reserved++++++++|
- * |<- capacity ->|
- * A) avail = capacity - used
- * }
- * </pre>
- */
- public long getAvailable() {
- return usage.getAvailable();
- }
-
public SpaceUsageSource getCurrentUsage() {
return usage.getCurrentUsage();
}
@@ -186,10 +168,6 @@ public void refreshNow() {
usage.refreshNow();
}
- public long getScmUsed() {
- return usage.getUsedSpace();
- }
-
void shutdownUsageThread() {
usage.shutdown();
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
index 34ba66c91b..838395d6ea 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
@@ -65,18 +65,6 @@ SpaceUsageSource realUsage() {
return source.snapshot();
}
- public long getCapacity() {
- return getCurrentUsage().getCapacity();
- }
-
- public long getAvailable() {
- return getCurrentUsage().getAvailable();
- }
-
- public long getUsedSpace() {
- return getCurrentUsage().getUsedSpace();
- }
-
/**
* <pre>
* {@code
@@ -114,15 +102,13 @@ public void decrementUsedSpace(long reclaimedSpace) {
* so there could be that DU value > totalUsed when there are deletes.
* @return other used space
*/
- private static long getOtherUsed(SpaceUsageSource precomputedVolumeSpace) {
- long totalUsed = precomputedVolumeSpace.getCapacity() -
- precomputedVolumeSpace.getAvailable();
- return Math.max(totalUsed - precomputedVolumeSpace.getUsedSpace(), 0L);
+ private static long getOtherUsed(SpaceUsageSource usage) {
+ long totalUsed = usage.getCapacity() - usage.getAvailable();
+ return Math.max(totalUsed - usage.getUsedSpace(), 0L);
}
- private long getRemainingReserved(
- SpaceUsageSource precomputedVolumeSpace) {
- return Math.max(reservedInBytes - getOtherUsed(precomputedVolumeSpace),
0L);
+ private long getRemainingReserved(SpaceUsageSource usage) {
+ return Math.max(reservedInBytes - getOtherUsed(usage), 0L);
}
public synchronized void start() {
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
index 1eba25c357..1a53291e0e 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestCapacityVolumeChoosingPolicy.java
@@ -104,9 +104,9 @@ public void testCapacityVolumeChoosingPolicy() throws
Exception {
HddsVolume hddsVolume2 = volumes.get(1);
HddsVolume hddsVolume3 = volumes.get(2);
- assertEquals(100L, hddsVolume1.getAvailable());
- assertEquals(200L, hddsVolume2.getAvailable());
- assertEquals(300L, hddsVolume3.getAvailable());
+ assertEquals(100L, hddsVolume1.getCurrentUsage().getAvailable());
+ assertEquals(200L, hddsVolume2.getCurrentUsage().getAvailable());
+ assertEquals(300L, hddsVolume3.getCurrentUsage().getAvailable());
Map<HddsVolume, Integer> chooseCount = new HashMap<>();
chooseCount.put(hddsVolume1, 0);
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
index 99056dd79b..ea27510948 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
@@ -248,7 +248,7 @@ public void testShutdown() throws Exception {
HddsVolume volume = volumeBuilder.build();
assertEquals(initialUsedSpace, savedUsedSpace.get());
- assertEquals(expectedUsedSpace, volume.getUsedSpace());
+ assertEquals(expectedUsedSpace, volume.getCurrentUsage().getUsedSpace());
// Shutdown the volume.
volume.shutdown();
@@ -264,10 +264,12 @@ public void testShutdown() throws Exception {
StorageSize size = StorageSize.parse(RESERVED_SPACE);
long reservedSpaceInBytes = (long) size.getUnit().toBytes(size.getValue());
+ SpaceUsageSource reportedUsage = volume.getCurrentUsage();
+
assertEquals(spaceUsage.getCapacity(),
- volume.getCapacity() + reservedSpaceInBytes);
+ reportedUsage.getCapacity() + reservedSpaceInBytes);
assertEquals(spaceUsage.getAvailable(),
- volume.getAvailable() + reservedSpaceInBytes);
+ reportedUsage.getAvailable() + reservedSpaceInBytes);
}
/**
@@ -300,8 +302,9 @@ public void testReportUsedBiggerThanActualUsed() throws
IOException {
HddsVolume volume = volumeBuilder.build();
- assertEquals(400, volume.getCapacity());
- assertEquals(100, volume.getAvailable());
+ SpaceUsageSource usage = volume.getCurrentUsage();
+ assertEquals(400, usage.getCapacity());
+ assertEquals(100, usage.getAvailable());
// Shutdown the volume.
volume.shutdown();
@@ -327,8 +330,9 @@ public void testReportUsedSmallerThanActualUsed() throws
IOException {
HddsVolume volume = volumeBuilder.build();
- assertEquals(400, volume.getCapacity());
- assertEquals(190, volume.getAvailable());
+ SpaceUsageSource usage = volume.getCurrentUsage();
+ assertEquals(400, usage.getCapacity());
+ assertEquals(190, usage.getAvailable());
// Shutdown the volume.
volume.shutdown();
@@ -353,8 +357,9 @@ public void testOverUsedReservedSpace() throws IOException {
HddsVolume volume = volumeBuilder.build();
- assertEquals(400, volume.getCapacity());
- assertEquals(300, volume.getAvailable());
+ SpaceUsageSource usage = volume.getCurrentUsage();
+ assertEquals(400, usage.getCapacity());
+ assertEquals(300, usage.getAvailable());
// Shutdown the volume.
volume.shutdown();
@@ -379,8 +384,9 @@ public void testOverUsedHddsSpace() throws IOException {
HddsVolume volume = volumeBuilder.build();
- assertEquals(400, volume.getCapacity());
- assertEquals(0, volume.getAvailable());
+ SpaceUsageSource usage = volume.getCurrentUsage();
+ assertEquals(400, usage.getCapacity());
+ assertEquals(0, usage.getAvailable());
// Shutdown the volume.
volume.shutdown();
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java
index 5e0a31944f..393b387015 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java
@@ -70,7 +70,7 @@ public void testDefaultConfig() throws Exception {
// Gets the total capacity reported by Ozone, which may be limited to less
than the volume's real capacity by the
// DU reserved configurations.
- long volumeCapacity = hddsVolume.getCapacity();
+ long volumeCapacity = hddsVolume.getCurrentUsage().getCapacity();
VolumeUsage usage = hddsVolume.getVolumeInfo().get().getUsageForTesting();
// Gets the actual total capacity without accounting for DU reserved space
configurations.
@@ -94,7 +94,7 @@ public void testVolumeCapacityAfterReserve() throws Exception
{
float percentage = conf.getFloat(HDDS_DATANODE_DIR_DU_RESERVED_PERCENT,
HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT);
- long volumeCapacity = hddsVolume.getCapacity();
+ long volumeCapacity = hddsVolume.getCurrentUsage().getCapacity();
VolumeUsage usage = hddsVolume.getVolumeInfo().get().getUsageForTesting();
//Gets the actual total capacity
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
index 1df2636553..a95cb40579 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java
@@ -93,8 +93,8 @@ public void testRRVolumeChoosingPolicy() throws Exception {
HddsVolume hddsVolume1 = volumes.get(0);
HddsVolume hddsVolume2 = volumes.get(1);
- assertEquals(100L, hddsVolume1.getAvailable());
- assertEquals(200L, hddsVolume2.getAvailable());
+ assertEquals(100L, hddsVolume1.getCurrentUsage().getAvailable());
+ assertEquals(200L, hddsVolume2.getCurrentUsage().getAvailable());
// Test two rounds of round-robin choosing
assertEquals(hddsVolume1, policy.chooseVolume(volumes, 0));
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
index 68e687fefa..054e7a9fb2 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
@@ -220,7 +220,7 @@ public void testShutdown() throws Exception {
for (StorageVolume volume : volumesList) {
assertNotNull(volume.getVolumeInfo().get()
.getUsageForTesting());
- volume.getAvailable();
+ volume.getCurrentUsage();
}
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 2f2cbc81e9..c2e8393a8d 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -234,7 +234,7 @@ public void
testContainerCreateDiskFull(ContainerTestVersionInfo versionInfo)
volume.format(clusterId);
// eat up all available space except size of 1 container
- volume.incCommittedBytes(volume.getAvailable() - containerSize);
+ volume.incCommittedBytes(volume.getCurrentUsage().getAvailable() -
containerSize);
// eat up 10 bytes more, now available space is less than 1 container
volume.incCommittedBytes(10);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
