This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 62bce67d7c HDDS-9380. Reduce the number of system calls for space
usage when DN writes a key (#5388)
62bce67d7c is described below
commit 62bce67d7c7bf082ce05c0e404c6257312341381
Author: XiChen <[email protected]>
AuthorDate: Thu Oct 5 20:36:06 2023 +0800
HDDS-9380. Reduce the number of system calls for space usage when DN writes
a key (#5388)
---
.../container/common/impl/HddsDispatcher.java | 7 +++-
.../container/common/volume/StorageVolume.java | 12 ++++++
.../ozone/container/common/volume/VolumeInfo.java | 10 +++++
.../ozone/container/common/volume/VolumeUsage.java | 45 ++++++++++++++++++++++
4 files changed, 72 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 300c15bb72..71b8486600 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -78,6 +78,7 @@ import java.util.TreeMap;
import static
org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.malformedRequest;
import static
org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
import static
org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult;
+import static
org.apache.hadoop.ozone.container.common.volume.VolumeUsage.PrecomputedVolumeSpace;
/**
* Ozone Container dispatcher takes a call from the netty server and routes it
@@ -580,10 +581,12 @@ public class HddsDispatcher implements
ContainerDispatcher, Auditor {
.orElse(Boolean.FALSE);
if (isOpen) {
HddsVolume volume = container.getContainerData().getVolume();
- long volumeCapacity = volume.getCapacity();
+ PrecomputedVolumeSpace precomputedVolumeSpace =
+ volume.getPrecomputedVolumeSpace();
+ long volumeCapacity = precomputedVolumeSpace.getCapacity();
long volumeFreeSpaceToSpare =
VolumeUsage.getMinVolumeFreeSpace(conf, volumeCapacity);
- long volumeFree = volume.getAvailable();
+ long volumeFree = volume.getAvailable(precomputedVolumeSpace);
long volumeCommitted = volume.getCommittedBytes();
long volumeAvailable = volumeFree - volumeCommitted;
return (volumeAvailable <= volumeFreeSpaceToSpare);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
index dd48f0bb17..fc329ebeb0 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
@@ -50,6 +50,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import static
org.apache.hadoop.ozone.container.common.HDDSVolumeLayoutVersion.getLatestVersion;
+import static
org.apache.hadoop.ozone.container.common.volume.VolumeUsage.PrecomputedVolumeSpace;
+
/**
* StorageVolume represents a generic Volume in datanode, could be
@@ -454,6 +456,16 @@ public abstract class StorageVolume
}
+ public long getAvailable(PrecomputedVolumeSpace precomputedVolumeSpace) {
+ return volumeInfo.map(info -> info.getAvailable(precomputedVolumeSpace))
+ .orElse(0L);
+ }
+
+ public PrecomputedVolumeSpace getPrecomputedVolumeSpace() {
+ return volumeInfo.map(VolumeInfo::getPrecomputedVolumeSpace)
+ .orElse(new PrecomputedVolumeSpace(0L, 0L));
+ }
+
public long getUsedSpace() {
return volumeInfo.map(VolumeInfo::getScmUsed).orElse(0L);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
index 310f3ca1ae..61c0b422c7 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
@@ -32,6 +32,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static
org.apache.hadoop.ozone.container.common.volume.VolumeUsage.PrecomputedVolumeSpace;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT;
@@ -239,6 +240,15 @@ public final class VolumeInfo {
return Math.max(Math.min(avail, usage.getAvailable()), 0);
}
+ public long getAvailable(PrecomputedVolumeSpace precomputedValues) {
+ long avail = precomputedValues.getCapacity() - usage.getUsedSpace();
+ return Math.max(Math.min(avail, usage.getAvailable(precomputedValues)), 0);
+ }
+
+ public PrecomputedVolumeSpace getPrecomputedVolumeSpace() {
+ return usage.getPrecomputedVolumeSpace();
+ }
+
public void incrementUsedSpace(long usedSpace) {
usage.incrementUsedSpace(usedSpace);
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
index 87e50d680a..e7a06abc9e 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
@@ -65,6 +65,11 @@ public class VolumeUsage implements SpaceUsageSource {
return source.getAvailable() - getRemainingReserved();
}
+ public long getAvailable(PrecomputedVolumeSpace precomputedVolumeSpace) {
+ long available = precomputedVolumeSpace.getAvailable();
+ return available - getRemainingReserved(precomputedVolumeSpace);
+ }
+
@Override
public long getUsedSpace() {
return source.getUsedSpace();
@@ -89,10 +94,21 @@ public class VolumeUsage implements SpaceUsageSource {
return Math.max(totalUsed - source.getUsedSpace(), 0L);
}
+ private long getOtherUsed(PrecomputedVolumeSpace precomputedVolumeSpace) {
+ long totalUsed = precomputedVolumeSpace.getCapacity() -
+ precomputedVolumeSpace.getAvailable();
+ return Math.max(totalUsed - source.getUsedSpace(), 0L);
+ }
+
private long getRemainingReserved() {
return Math.max(reservedInBytes - getOtherUsed(), 0L);
}
+ private long getRemainingReserved(
+ PrecomputedVolumeSpace precomputedVolumeSpace) {
+ return Math.max(reservedInBytes - getOtherUsed(precomputedVolumeSpace),
0L);
+ }
+
public synchronized void start() {
source.start();
}
@@ -145,4 +161,33 @@ public class VolumeUsage implements SpaceUsageSource {
HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_DEFAULT, StorageUnit.BYTES);
}
+
+ /**
+ * Class representing precomputed space values of a volume.
+ * This class is intended to store precomputed values, such as capacity
+ * and available space of a volume, to avoid recalculating these
+ * values multiple times and to make method signatures simpler.
+ */
+ public static class PrecomputedVolumeSpace {
+ private final long capacity;
+ private final long available;
+
+ public PrecomputedVolumeSpace(long capacity, long available) {
+ this.capacity = capacity;
+ this.available = available;
+ }
+
+ public long getCapacity() {
+ return capacity;
+ }
+
+ public long getAvailable() {
+ return available;
+ }
+ }
+
+ public PrecomputedVolumeSpace getPrecomputedVolumeSpace() {
+ return new PrecomputedVolumeSpace(source.getCapacity(),
+ source.getAvailable());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
