This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6d0a8306ab HDDS-12721. Log more details about volumes (#8181)
6d0a8306ab is described below
commit 6d0a8306aba01483edfeab04e8b527985fa471b0
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Apr 7 06:36:38 2025 +0200
HDDS-12721. Log more details about volumes (#8181)
---
.../container/common/impl/HddsDispatcher.java | 8 +++++--
.../common/impl/StorageLocationReport.java | 26 +++++++++++++++++++++
.../common/volume/AvailableSpaceFilter.java | 27 +++++-----------------
.../ozone/container/common/volume/DbVolume.java | 7 +++---
.../ozone/container/common/volume/HddsVolume.java | 7 ++----
.../container/common/volume/StorageVolume.java | 5 +++-
.../replication/DownloadAndImportReplicator.java | 8 +++----
.../replication/SendContainerRequestHandler.java | 8 +++----
8 files changed, 55 insertions(+), 41 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index a8a73c7838..26c6bcf4ee 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -70,7 +70,6 @@
import
org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.VolumeUsage;
import
org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
import org.apache.hadoop.util.Time;
import org.apache.ratis.statemachine.StateMachine;
@@ -615,7 +614,12 @@ private boolean isVolumeFull(Container container) {
.orElse(Boolean.FALSE);
if (isOpen) {
HddsVolume volume = container.getContainerData().getVolume();
- return VolumeUsage.getUsableSpace(volume.getReport()) <= 0;
+ StorageLocationReport volumeReport = volume.getReport();
+ boolean full = volumeReport.getUsableSpace() <= 0;
+ if (full) {
+ LOG.info("Container {} volume is full: {}",
container.getContainerData().getContainerID(), volumeReport);
+ }
+ return full;
}
return false;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
index 9fd094c6a6..5f539e121b 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
@@ -24,6 +24,7 @@
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
import
org.apache.hadoop.ozone.container.common.interfaces.StorageLocationReportMXBean;
+import org.apache.hadoop.ozone.container.common.volume.VolumeUsage;
/**
* Storage location stats of datanodes that provide back store for containers.
@@ -54,6 +55,10 @@ private StorageLocationReport(Builder builder) {
this.storageLocation = builder.storageLocation;
}
+ public long getUsableSpace() {
+ return VolumeUsage.getUsableSpace(this);
+ }
+
@Override
public String getId() {
return id;
@@ -227,6 +232,27 @@ public static StorageLocationReport
getFromProtobuf(StorageReportProto report)
return builder.build();
}
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(128)
+ .append("{")
+ .append(" id=").append(id)
+ .append(" dir=").append(storageLocation)
+ .append(" type=").append(storageType);
+
+ if (failed) {
+ sb.append(" failed");
+ } else {
+ sb.append(" capacity=").append(capacity)
+ .append(" used=").append(scmUsed)
+ .append(" available=").append(remaining)
+ .append(" minFree=").append(freeSpaceToSpare)
+ .append(" committed=").append(committed);
+ }
+
+ return sb.append(" }").toString();
+ }
+
/**
* Returns StorageLocation.Builder instance.
*
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
index ef50e73cb3..6769a8eeae 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
@@ -17,8 +17,8 @@
package org.apache.hadoop.ozone.container.common.volume;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.LinkedList;
+import java.util.List;
import java.util.function.Predicate;
import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
@@ -29,24 +29,23 @@
public class AvailableSpaceFilter implements Predicate<HddsVolume> {
private final long requiredSpace;
- private final Map<HddsVolume, AvailableSpace> fullVolumes =
- new HashMap<>();
+ private final List<StorageLocationReport> fullVolumes = new LinkedList<>();
private long mostAvailableSpace = Long.MIN_VALUE;
- public AvailableSpaceFilter(long requiredSpace) {
+ AvailableSpaceFilter(long requiredSpace) {
this.requiredSpace = requiredSpace;
}
@Override
public boolean test(HddsVolume vol) {
StorageLocationReport report = vol.getReport();
- long available = VolumeUsage.getUsableSpace(report);
+ long available = report.getUsableSpace();
boolean hasEnoughSpace = available > requiredSpace;
mostAvailableSpace = Math.max(available, mostAvailableSpace);
if (!hasEnoughSpace) {
- fullVolumes.put(vol, new AvailableSpace(report));
+ fullVolumes.add(report);
}
return hasEnoughSpace;
@@ -65,18 +64,4 @@ public String toString() {
return "required space: " + requiredSpace +
", volumes: " + fullVolumes;
}
-
- private static class AvailableSpace {
- private final StorageLocationReport report;
-
- AvailableSpace(StorageLocationReport report) {
- this.report = report;
- }
-
- @Override
- public String toString() {
- return "free: " + report.getRemaining() +
- ", committed: " + report.getCommitted();
- }
- }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java
index 0f188e53b1..a1f13998b8 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java
@@ -55,13 +55,12 @@ protected DbVolume(Builder b) throws IOException {
super(b);
this.hddsDbStorePathMap = new HashMap<>();
- if (!b.getFailedVolume()) {
- LOG.info("Creating DbVolume: {} of storage type: {}, {}",
- getStorageDir(), b.getStorageType(),
- getCurrentUsage());
+ if (!b.getFailedVolume()) {
initialize();
}
+
+ LOG.info("DbVolume: {}", getReport());
}
@Override
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index a81bde59b4..55bd550210 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -129,11 +129,6 @@ private HddsVolume(Builder b) throws IOException {
this.volumeInfoMetrics =
new VolumeInfoMetrics(b.getVolumeRootStr(), this);
- LOG.info("Creating HddsVolume: {} of storage type: {}, {}",
- getStorageDir(),
- b.getStorageType(),
- getCurrentUsage());
-
initialize();
} else {
// Builder is called with failedVolume set, so create a failed volume
@@ -142,6 +137,8 @@ private HddsVolume(Builder b) throws IOException {
volumeIOStats = null;
volumeInfoMetrics = new VolumeInfoMetrics(b.getVolumeRootStr(), this);
}
+
+ LOG.info("HddsVolume: {}", getReport());
}
@Override
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
index 9269619da9..fe0fa89692 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java
@@ -123,6 +123,8 @@ public enum VolumeState {
private final StorageType storageType;
private final String volumeRoot;
private final File storageDir;
+ /** This is the raw storage dir location, saved for logging, to avoid
repeated filesystem lookup. */
+ private final String storageDirStr;
private String workingDirName;
private File tmpDir;
private File diskCheckDir;
@@ -173,6 +175,7 @@ protected StorageVolume(Builder<?> b) throws IOException {
this.conf = null;
this.dnConf = null;
}
+ this.storageDirStr = storageDir.getAbsolutePath();
}
public void format(String cid) throws IOException {
@@ -459,7 +462,7 @@ protected StorageLocationReport.Builder reportBuilder() {
StorageLocationReport.Builder builder = StorageLocationReport.newBuilder()
.setFailed(isFailed())
.setId(getStorageID())
- .setStorageLocation(volumeRoot)
+ .setStorageLocation(storageDirStr)
.setStorageType(storageType);
if (!builder.isFailed()) {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
index 8c44d0d078..dc53ea8c73 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
@@ -26,7 +26,7 @@
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.volume.AvailableSpaceFilter;
+import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import
org.apache.hadoop.ozone.container.replication.AbstractReplicationTask.Status;
import org.slf4j.Logger;
@@ -83,10 +83,10 @@ public void replicate(ReplicationTask task) {
targetVolume = containerImporter.chooseNextVolume();
// Increment committed bytes and verify if it doesn't cross the space
left.
targetVolume.incCommittedBytes(containerSize * 2);
+ StorageLocationReport volumeReport = targetVolume.getReport();
// Already committed bytes increased above, so required space is not
required here in AvailableSpaceFilter
- AvailableSpaceFilter filter = new AvailableSpaceFilter(0);
- if (!filter.test(targetVolume)) {
- LOG.warn("Container {} replication was unsuccessful, due to no space
left", containerID);
+ if (volumeReport.getUsableSpace() <= 0) {
+ LOG.warn("Container {} replication was unsuccessful, no space left on
volume {}", containerID, volumeReport);
task.setStatus(Status.FAILED);
return;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java
index e76a44e680..5224498e72 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java
@@ -29,7 +29,7 @@
import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.volume.AvailableSpaceFilter;
+import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.util.DiskChecker;
import org.apache.ratis.grpc.util.ZeroCopyMessageMarshaller;
@@ -89,11 +89,11 @@ public void onNext(SendContainerRequest req) {
volume = importer.chooseNextVolume();
// Increment committed bytes and verify if it doesn't cross the space
left.
volume.incCommittedBytes(importer.getDefaultContainerSize() * 2);
+ StorageLocationReport volumeReport = volume.getReport();
// Already committed bytes increased above, so required space is not
required here in AvailableSpaceFilter
- AvailableSpaceFilter filter = new AvailableSpaceFilter(0);
- if (!filter.test(volume)) {
+ if (volumeReport.getUsableSpace() <= 0) {
volume.incCommittedBytes(-importer.getDefaultContainerSize() * 2);
- LOG.warn("Container {} import was unsuccessful, due to no space
left", containerId);
+ LOG.warn("Container {} import was unsuccessful, no space left on
volume {}", containerId, volumeReport);
volume = null;
throw new DiskChecker.DiskOutOfSpaceException("No more available
volumes");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
