This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7b829183a0e HDDS-15147. Improve Recon Datanode Insight export:
filename, size unit (#10177)
7b829183a0e is described below
commit 7b829183a0eddde0d0de542b19fef757eff5d7d4
Author: Navink <[email protected]>
AuthorDate: Sun May 17 16:15:51 2026 +0530
HDDS-15147. Improve Recon Datanode Insight export: filename, size unit
(#10177)
---
.../recon/api/StorageDistributionEndpoint.java | 50 ++++++++++++++++------
.../recon/api/TestStorageDistributionEndpoint.java | 34 +++++++++------
2 files changed, 56 insertions(+), 28 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java
index de72041e2dd..4b189ddd3d3 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java
@@ -21,6 +21,9 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -34,11 +37,13 @@
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.ozone.recon.ReconContext;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
import org.apache.hadoop.ozone.recon.api.types.DataNodeMetricsServiceResponse;
@@ -79,18 +84,23 @@ public class StorageDistributionEndpoint {
private final ReconGlobalStatsManager reconGlobalStatsManager;
private final ReconGlobalMetricsService reconGlobalMetricsService;
private final DataNodeMetricsService dataNodeMetricsService;
+ private final ReconContext reconContext;
+ private static final DateTimeFormatter TIMESTAMP_FORMATTER =
+ DateTimeFormatter.ofPattern("yyyyMMdd_HHmm'Z'");
@Inject
public StorageDistributionEndpoint(OzoneStorageContainerManager reconSCM,
NSSummaryEndpoint nsSummaryEndpoint,
ReconGlobalStatsManager
reconGlobalStatsManager,
ReconGlobalMetricsService
reconGlobalMetricsService,
- DataNodeMetricsService
dataNodeMetricsService) {
+ DataNodeMetricsService
dataNodeMetricsService,
+ ReconContext reconContext) {
this.nodeManager = (ReconNodeManager) reconSCM.getScmNodeManager();
this.nsSummaryEndpoint = nsSummaryEndpoint;
this.reconGlobalStatsManager = reconGlobalStatsManager;
this.reconGlobalMetricsService = reconGlobalMetricsService;
this.dataNodeMetricsService = dataNodeMetricsService;
+ this.reconContext = reconContext;
}
@GET
@@ -140,7 +150,12 @@ public Response getStorageDistribution() {
* The CSV includes the following headers: HostName, Datanode UUID,
Filesystem Capacity,
* Filesystem Used Space, Filesystem Remaining Space, Ozone Capacity, Ozone
Used Space,
* Ozone Remaining Space, PreAllocated Container Space, Reserved Space,
Minimum Free
- * Space, and Pending Block Size.
+ * Space, and Pending Block Size. The values for all size-related headers
are represented
+ * in bytes.
+ *
+ * The downloaded csv file is dynamically named using the cluster ID and a
UTC timestamp,
+ * to ensure clear timezone-independent record keeping.
+ * Example: Datanode_Insights_<clusterId>_yyyyMMdd_HHmmZ.csv
*
* @return A Response object. Depending on the state of metrics collection,
this can be:
* - An HTTP 202 (Accepted) response with a status and metrics data
if the
@@ -190,16 +205,16 @@ public Response downloadDataNodeStorageDistribution() {
List<String> headers = Arrays.asList(
"HostName",
"Datanode UUID",
- "Filesystem Capacity",
- "Filesystem Used Space",
- "Filesystem Remaining Space",
- "Ozone Capacity",
- "Ozone Used Space",
- "Ozone Remaining Space",
- "PreAllocated Container Space",
- "Reserved Space",
- "Minimum Free Space",
- "Pending Block Size"
+ "Filesystem Capacity (Bytes)",
+ "Filesystem Used Space (Bytes)",
+ "Filesystem Remaining Space (Bytes)",
+ "Ozone Capacity (Bytes)",
+ "Ozone Used Space (Bytes)",
+ "Ozone Remaining Space (Bytes)",
+ "PreAllocated Container Space (Bytes)",
+ "Reserved Space (Bytes)",
+ "Minimum Free Space (Bytes)",
+ "Pending Block Size (Bytes)"
);
List<Function<DataNodeStoragePendingDeletionView, Object>> columns =
@@ -215,10 +230,17 @@ public Response downloadDataNodeStorageDistribution() {
v -> v.getReport() != null ? v.getReport().getCommitted() : -1,
v -> v.getReport() != null ? v.getReport().getReserved() : -1,
v -> v.getReport() != null ? v.getReport().getMinimumFreeSpace() :
-1,
- v -> v.getReport() != null ? v.getMetric().getPendingBlockSize() :
-1
+ v -> v.getMetric() != null ? v.getMetric().getPendingBlockSize() :
-1
);
- return
ReconUtils.downloadCsv("datanode_storage_and_pending_deletion_stats.csv",
headers, data, columns);
+ String timestamp =
LocalDateTime.now(ZoneOffset.UTC).format(TIMESTAMP_FORMATTER);
+ String clusterId = "UnknownCluster";
+ if (StringUtils.isNotBlank(reconContext.getClusterId())) {
+ clusterId = reconContext.getClusterId();
+ }
+ String fileName = String.format("Datanode_Insights_%s_%s.csv", clusterId,
timestamp);
+
+ return ReconUtils.downloadCsv(fileName, headers, data, columns);
}
/**
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java
index d62aae15913..fc96c60677f 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestStorageDistributionEndpoint.java
@@ -42,6 +42,7 @@
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.ozone.recon.ReconContext;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
import org.apache.hadoop.ozone.recon.api.types.DataNodeMetricsServiceResponse;
import org.apache.hadoop.ozone.recon.api.types.DatanodePendingDeletionMetrics;
@@ -82,8 +83,6 @@ public class TestStorageDistributionEndpoint {
private static final String TEXT_PLAIN = "text/plain";
private static final String TEXT_CSV = "text/csv";
private static final String CONTENT_DISPOSITION = "Content-Disposition";
- private static final String DOWNLOAD_CONTENT_DISPOSITION =
- "attachment;
filename=\"datanode_storage_and_pending_deletion_stats.csv\"";
private static final String METRICS_MISSING_ERROR =
"Metrics data is missing despite FINISHED status.";
private static final String ROOT_PATH = "/";
@@ -91,6 +90,7 @@ public class TestStorageDistributionEndpoint {
private static final String PENDING_DIRECTORY_SIZE_KEY =
"pendingDirectorySize";
private static final String PENDING_KEY_SIZE_KEY = "pendingKeySize";
private static final String TOTAL_REPLICATED_DATA_SIZE_KEY =
"totalReplicatedDataSize";
+ private static final String CLUSTER_ID = "TestClusterID";
private DataNodeMetricsService dataNodeMetricsService;
private StorageDistributionEndpoint storageDistributionEndpoint;
@@ -108,11 +108,14 @@ public void setup() {
OzoneStorageContainerManager reconSCM =
mock(OzoneStorageContainerManager.class);
when(reconSCM.getScmNodeManager()).thenReturn(nodeManager);
reconGlobalStatsManager = mock(ReconGlobalStatsManager.class);
+ ReconContext reconContext = mock(ReconContext.class);
+ when(reconContext.getClusterId()).thenReturn(CLUSTER_ID);
storageDistributionEndpoint = new StorageDistributionEndpoint(reconSCM,
nssummaryEndpoint,
reconGlobalStatsManager,
reconGlobalMetricsService,
- dataNodeMetricsService);
+ dataNodeMetricsService,
+ reconContext);
}
@Test
@@ -189,7 +192,10 @@ public void testDownloadReturnsCsvWithMetrics() throws
Exception {
// then
assertEquals(Response.Status.OK.getStatusCode(), response.getStatus());
assertEquals(TEXT_CSV, response.getMediaType().toString());
- assertEquals(DOWNLOAD_CONTENT_DISPOSITION,
response.getHeaderString(CONTENT_DISPOSITION));
+ String contentDisposition = response.getHeaderString(CONTENT_DISPOSITION);
+ assertNotNull(contentDisposition);
+ assertTrue(contentDisposition.startsWith("attachment;
filename=\"Datanode_Insights_" + CLUSTER_ID + "_"));
+ assertTrue(contentDisposition.endsWith("Z.csv\""));
String csv = readCsv(response);
for (String row : csvRows) {
assertTrue(csv.contains(row));
@@ -201,16 +207,16 @@ private List<String> mockStorageDistributionData(int
numNodes) throws Exception
List<String> headers = Arrays.asList(
"HostName",
"Datanode UUID",
- "Filesystem Capacity",
- "Filesystem Used Space",
- "Filesystem Remaining Space",
- "Ozone Capacity",
- "Ozone Used Space",
- "Ozone Remaining Space",
- "PreAllocated Container Space",
- "Reserved Space",
- "Minimum Free Space",
- "Pending Block Size");
+ "Filesystem Capacity (Bytes)",
+ "Filesystem Used Space (Bytes)",
+ "Filesystem Remaining Space (Bytes)",
+ "Ozone Capacity (Bytes)",
+ "Ozone Used Space (Bytes)",
+ "Ozone Remaining Space (Bytes)",
+ "PreAllocated Container Space (Bytes)",
+ "Reserved Space (Bytes)",
+ "Minimum Free Space (Bytes)",
+ "Pending Block Size (Bytes)");
csvRows.add(String.join(",", headers));
List<DatanodePendingDeletionMetrics> pendingDeletionMetrics = new
ArrayList<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
