This is an automated email from the ASF dual-hosted git repository.
ivandika pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4e603aa93c HDDS-11462. Enhancing DataNode I/O Monitoring Capabilities.
(#7206)
4e603aa93c is described below
commit 4e603aa93c8479349776c1597ef54504453cb512
Author: slfan1989 <[email protected]>
AuthorDate: Fri Nov 8 13:54:23 2024 +0800
HDDS-11462. Enhancing DataNode I/O Monitoring Capabilities. (#7206)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +
.../common/src/main/resources/ozone-default.xml | 12 +++
.../java/org/apache/hadoop/ozone/DNMXBean.java | 28 ++++++
.../java/org/apache/hadoop/ozone/DNMXBeanImpl.java | 49 +++++++++-
.../apache/hadoop/ozone/HddsDatanodeService.java | 18 ++--
.../ozone/container/common/impl/ContainerSet.java | 15 ++++
.../ozone/container/common/volume/HddsVolume.java | 20 ++++-
.../container/common/volume/VolumeIOStats.java | 57 +++++++++---
.../container/common/volume/VolumeInfoMetrics.java | 8 ++
.../ozoneimpl/BackgroundContainerDataScanner.java | 1 +
.../container/ozoneimpl/ContainerController.java | 10 +++
.../ozoneimpl/ContainerDataScannerMetrics.java | 11 +++
.../ozone/container/ozoneimpl/OzoneContainer.java | 14 +++
.../webapps/hddsDatanode/dn-overview.html | 28 +++++-
.../resources/webapps/hddsDatanode/dn-scanner.html | 47 ++++++++++
.../src/main/resources/webapps/hddsDatanode/dn.js | 100 +++++++++++++++++++--
.../main/resources/webapps/hddsDatanode/index.html | 9 +-
.../resources/webapps/hddsDatanode/iostatus.html | 76 ++++++++++++++++
.../TestVolumeIOStatsWithPrometheusSink.java | 9 +-
.../src/main/resources/webapps/static/ozone.css | 21 ++++-
.../src/main/resources/webapps/static/ozone.js | 14 ++-
.../resources/webapps/static/templates/jvm.html | 13 ++-
.../resources/webapps/static/templates/menu.html | 2 +
.../webapps/static/templates/overview.html | 2 +-
24 files changed, 526 insertions(+), 41 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 87707f75dc..4d630243e5 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -401,4 +401,7 @@ public final class HddsConfigKeys {
"hdds.datanode.slow.op.warning.threshold";
public static final String HDDS_DATANODE_SLOW_OP_WARNING_THRESHOLD_DEFAULT =
"500ms";
+
+ public static final String
OZONE_DATANODE_IO_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY =
+ "ozone.volume.io.percentiles.intervals.seconds";
}
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index bc90a87b11..f3e45c47ee 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -4544,4 +4544,16 @@
maximum number of buckets across all volumes.
</description>
</property>
+
+ <property>
+ <name>ozone.volume.io.percentiles.intervals.seconds</name>
+ <value>60</value>
+ <tag>OZONE, DATANODE</tag>
+ <description>
+ This setting specifies the interval (in seconds) for monitoring
percentile performance metrics.
+ It helps in tracking the read and write performance of DataNodes in
real-time,
+ allowing for better identification and analysis of performance issues.
+ </description>
+ </property>
+
</configuration>
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBean.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBean.java
index d36fcdb6fc..9c077a8e27 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBean.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBean.java
@@ -26,4 +26,32 @@ import org.apache.hadoop.hdds.server.ServiceRuntimeInfo;
*/
@InterfaceAudience.Private
public interface DNMXBean extends ServiceRuntimeInfo {
+
+ /**
+ * Gets the datanode hostname.
+ *
+ * @return the datanode hostname for the datanode.
+ */
+ String getHostname();
+
+ /**
+ * Gets the client rpc port.
+ *
+ * @return the client rpc port
+ */
+ String getClientRpcPort();
+
+ /**
+ * Gets the http port.
+ *
+ * @return the http port
+ */
+ String getHttpPort();
+
+ /**
+ * Gets the https port.
+ *
+ * @return the http port
+ */
+ String getHttpsPort();
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBeanImpl.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBeanImpl.java
index f7b484c6bb..5a0a455663 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBeanImpl.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/DNMXBeanImpl.java
@@ -25,8 +25,53 @@ import org.apache.hadoop.hdds.utils.VersionInfo;
* This is the JMX management class for DN information.
*/
public class DNMXBeanImpl extends ServiceRuntimeInfoImpl implements DNMXBean {
- public DNMXBeanImpl(
- VersionInfo versionInfo) {
+
+ private String hostName;
+ private String clientRpcPort;
+ private String httpPort;
+ private String httpsPort;
+
+ public DNMXBeanImpl(VersionInfo versionInfo) {
super(versionInfo);
}
+
+ @Override
+ public String getHostname() {
+ return hostName;
+ }
+
+ @Override
+ public String getClientRpcPort() {
+ return clientRpcPort;
+ }
+
+ @Override
+ public String getHttpPort() {
+ return httpPort;
+ }
+
+ @Override
+ public String getHttpsPort() {
+ return httpsPort;
+ }
+
+ public void setHttpPort(String httpPort) {
+ this.httpPort = httpPort;
+ }
+
+ public void setHostName(String hostName) {
+ this.hostName = hostName;
+ }
+
+ public void setClientRpcPort(String rpcPort) {
+ this.clientRpcPort = rpcPort;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
+
+ public void setHttpsPort(String httpsPort) {
+ this.httpsPort = httpsPort;
+ }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 55aeb466e7..de21e37503 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -228,6 +228,7 @@ public class HddsDatanodeService extends GenericCli
implements ServicePlugin {
String ip = InetAddress.getByName(hostname).getHostAddress();
datanodeDetails = initializeDatanodeDetails();
datanodeDetails.setHostName(hostname);
+ serviceRuntimeInfo.setHostName(hostname);
datanodeDetails.setIpAddress(ip);
datanodeDetails.setVersion(
HddsVersionInfo.HDDS_VERSION_INFO.getVersion());
@@ -300,23 +301,30 @@ public class HddsDatanodeService extends GenericCli
implements ServicePlugin {
httpServer = new HddsDatanodeHttpServer(conf);
httpServer.start();
HttpConfig.Policy policy = HttpConfig.getHttpPolicy(conf);
+
if (policy.isHttpEnabled()) {
- datanodeDetails.setPort(DatanodeDetails.newPort(HTTP,
- httpServer.getHttpAddress().getPort()));
+ int httpPort = httpServer.getHttpAddress().getPort();
+ datanodeDetails.setPort(DatanodeDetails.newPort(HTTP, httpPort));
+ serviceRuntimeInfo.setHttpPort(String.valueOf(httpPort));
}
+
if (policy.isHttpsEnabled()) {
- datanodeDetails.setPort(DatanodeDetails.newPort(HTTPS,
- httpServer.getHttpsAddress().getPort()));
+ int httpsPort = httpServer.getHttpAddress().getPort();
+ datanodeDetails.setPort(DatanodeDetails.newPort(HTTPS, httpsPort));
+ serviceRuntimeInfo.setHttpsPort(String.valueOf(httpsPort));
}
+
} catch (Exception ex) {
LOG.error("HttpServer failed to start.", ex);
}
-
clientProtocolServer = new HddsDatanodeClientProtocolServer(
datanodeDetails, conf, HddsVersionInfo.HDDS_VERSION_INFO,
reconfigurationHandler);
+ int clientRpcport = clientProtocolServer.getClientRpcAddress().getPort();
+ serviceRuntimeInfo.setClientRpcPort(String.valueOf(clientRpcport));
+
// Get admin list
String starterUser =
UserGroupInformation.getCurrentUser().getShortUserName();
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
index 15cc6245dd..5335021da9 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
@@ -251,6 +251,21 @@ public class ContainerSet implements
Iterable<Container<?>> {
.iterator();
}
+ /**
+ * Get the number of containers based on the given volume.
+ *
+ * @param volume hdds volume.
+ * @return number of containers
+ */
+ public long containerCount(HddsVolume volume) {
+ Preconditions.checkNotNull(volume);
+ Preconditions.checkNotNull(volume.getStorageID());
+ String volumeUuid = volume.getStorageID();
+ return containerMap.values().stream()
+ .filter(x -> volumeUuid.equals(x.getContainerData().getVolume()
+ .getStorageID())).count();
+ }
+
/**
* Return an containerMap iterator over {@link ContainerSet#containerMap}.
* @return containerMap Iterator
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
index c58aab2e5b..5fced0e39b 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java
@@ -29,6 +29,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
@@ -36,6 +37,7 @@ import
org.apache.hadoop.ozone.container.common.utils.DatanodeStoreCache;
import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil;
import org.apache.hadoop.ozone.container.common.utils.RawDB;
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
+import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import
org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures.SchemaV3;
import org.apache.hadoop.util.Time;
@@ -44,6 +46,7 @@ import org.slf4j.LoggerFactory;
import jakarta.annotation.Nullable;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.OZONE_DATANODE_IO_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_NAME;
import static
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore;
@@ -80,6 +83,8 @@ public class HddsVolume extends StorageVolume {
private final VolumeIOStats volumeIOStats;
private final VolumeInfoMetrics volumeInfoMetrics;
+ private ContainerController controller;
+
private final AtomicLong committedBytes = new AtomicLong(); // till Open
containers become full
// Mentions the type of volume
@@ -119,8 +124,10 @@ public class HddsVolume extends StorageVolume {
if (!b.getFailedVolume() && getVolumeInfo().isPresent()) {
this.setState(VolumeState.NOT_INITIALIZED);
+ ConfigurationSource conf = getConf();
+ int[] intervals =
conf.getInts(OZONE_DATANODE_IO_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY);
this.volumeIOStats = new VolumeIOStats(b.getVolumeRootStr(),
- this.getStorageDir().toString());
+ this.getStorageDir().toString(), intervals);
this.volumeInfoMetrics =
new VolumeInfoMetrics(b.getVolumeRootStr(), this);
@@ -382,6 +389,17 @@ public class HddsVolume extends StorageVolume {
getStorageID());
}
+ public void setController(ContainerController controller) {
+ this.controller = controller;
+ }
+
+ public long getContainers() {
+ if (controller != null) {
+ return controller.getContainerCount(this);
+ }
+ return 0;
+ }
+
/**
* Pick a DbVolume for HddsVolume and init db instance.
* Use the HddsVolume directly if no DbVolume found.
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeIOStats.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeIOStats.java
index e22addd354..2ce19c3bf1 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeIOStats.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeIOStats.java
@@ -21,7 +21,10 @@ package org.apache.hadoop.ozone.container.common.volume;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+import org.apache.hadoop.metrics2.lib.MutableRate;
/**
* This class is used to track Volume IO stats for each HDDS Volume.
@@ -29,12 +32,23 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong;
public class VolumeIOStats {
private String metricsSourceName = VolumeIOStats.class.getSimpleName();
private String storageDirectory;
- private @Metric MutableCounterLong readBytes;
- private @Metric MutableCounterLong readOpCount;
- private @Metric MutableCounterLong writeBytes;
- private @Metric MutableCounterLong writeOpCount;
- private @Metric MutableCounterLong readTime;
- private @Metric MutableCounterLong writeTime;
+ private final MetricsRegistry registry = new
MetricsRegistry("VolumeIOStats");
+ @Metric
+ private MutableCounterLong readBytes;
+ @Metric
+ private MutableCounterLong readOpCount;
+ @Metric
+ private MutableCounterLong writeBytes;
+ @Metric
+ private MutableCounterLong writeOpCount;
+ @Metric
+ private MutableRate readTime;
+ @Metric
+ private MutableQuantiles[] readLatencyQuantiles;
+ @Metric
+ private MutableRate writeTime;
+ @Metric
+ private MutableQuantiles[] writeLatencyQuantiles;
@Deprecated
public VolumeIOStats() {
@@ -44,9 +58,24 @@ public class VolumeIOStats {
/**
* @param identifier Typically, path to volume root. e.g. /data/hdds
*/
- public VolumeIOStats(String identifier, String storageDirectory) {
+ public VolumeIOStats(String identifier, String storageDirectory, int[]
intervals) {
this.metricsSourceName += '-' + identifier;
this.storageDirectory = storageDirectory;
+
+ // Try initializing `readLatencyQuantiles` and `writeLatencyQuantiles`
+ if (intervals != null && intervals.length > 0) {
+ final int length = intervals.length;
+ readLatencyQuantiles = new MutableQuantiles[intervals.length];
+ writeLatencyQuantiles = new MutableQuantiles[intervals.length];
+ for (int i = 0; i < length; i++) {
+ readLatencyQuantiles[i] = registry.newQuantiles(
+ "readLatency" + intervals[i] + "s",
+ "Read Data File Io Latency in ms", "ops", "latency", intervals[i]);
+ writeLatencyQuantiles[i] = registry.newQuantiles(
+ "writeLatency" + intervals[i] + "s",
+ "Write Data File Io Latency in ms", "ops", "latency",
intervals[i]);
+ }
+ }
init();
}
@@ -99,7 +128,10 @@ public class VolumeIOStats {
* @param time
*/
public void incReadTime(long time) {
- readTime.incr(time);
+ readTime.add(time);
+ for (MutableQuantiles q : readLatencyQuantiles) {
+ q.add(time);
+ }
}
/**
@@ -107,7 +139,10 @@ public class VolumeIOStats {
* @param time
*/
public void incWriteTime(long time) {
- writeTime.incr(time);
+ writeTime.add(time);
+ for (MutableQuantiles q : writeLatencyQuantiles) {
+ q.add(time);
+ }
}
/**
@@ -147,7 +182,7 @@ public class VolumeIOStats {
* @return long
*/
public long getReadTime() {
- return readTime.value();
+ return (long) readTime.lastStat().total();
}
/**
@@ -155,7 +190,7 @@ public class VolumeIOStats {
* @return long
*/
public long getWriteTime() {
- return writeTime.value();
+ return (long) writeTime.lastStat().total();
}
@Metric
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
index 68140600db..cd31b8063d 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
@@ -37,6 +37,7 @@ public class VolumeInfoMetrics {
private final HddsVolume volume;
@Metric("Returns the RocksDB compact times of the Volume")
private MutableRate dbCompactLatency;
+ private long containers;
/**
* @param identifier Typically, path to volume root. E.g. /data/hdds
@@ -153,4 +154,11 @@ public class VolumeInfoMetrics {
dbCompactLatency.add(time);
}
+ /**
+ * Return the Container Count of the Volume.
+ */
+ @Metric("Returns the Container Count of the Volume")
+ public long getContainers() {
+ return volume.getContainers();
+ }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java
index 327f019224..8ff2e30876 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerDataScanner.java
@@ -63,6 +63,7 @@ public class BackgroundContainerDataScanner extends
throttler = new HddsDataTransferThrottler(conf.getBandwidthPerVolume());
canceler = new Canceler();
this.metrics = ContainerDataScannerMetrics.create(volume.toString());
+ this.metrics.setStorageDirectory(volume.toString());
this.minScanGap = conf.getContainerScanMinGap();
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
index 0db98a01d8..567741a98d 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java
@@ -235,6 +235,16 @@ public class ContainerController {
return containerSet.getContainerIterator(volume);
}
+ /**
+ * Get the number of containers based on the given volume.
+ *
+ * @param volume hdds volume.
+ * @return number of containers.
+ */
+ public long getContainerCount(HddsVolume volume) {
+ return containerSet.containerCount(volume);
+ }
+
void updateDataScanTimestamp(long containerId, Instant timestamp)
throws IOException {
Container container = containerSet.getContainer(containerId);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScannerMetrics.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScannerMetrics.java
index a3f71d34ba..76e71312ae 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScannerMetrics.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScannerMetrics.java
@@ -37,6 +37,8 @@ public final class ContainerDataScannerMetrics
@Metric("disk bandwidth used by the container data scanner per volume")
private MutableRate numBytesScanned;
+ private String storageDirectory;
+
public double getNumBytesScannedMean() {
return numBytesScanned.lastStat().mean();
}
@@ -66,4 +68,13 @@ public final class ContainerDataScannerMetrics
return ms.register(name, null, new ContainerDataScannerMetrics(name, ms));
}
+
+ @Metric("Returns the Directory name for the volume")
+ public String getStorageDirectory() {
+ return storageDirectory;
+ }
+
+ public void setStorageDirectory(final String volumeName) {
+ this.storageDirectory = volumeName;
+ }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 62196cdd87..56c4233836 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -388,6 +388,18 @@ public class OzoneContainer {
}
}
+ /**
+ * We need to inject the containerController into the hddsVolume.
+ * because we need to obtain the container count
+ * for each disk based on the container controller.
+ */
+ private void initHddsVolumeContainer() {
+ for (StorageVolume v : volumeSet.getVolumesList()) {
+ HddsVolume hddsVolume = (HddsVolume) v;
+ hddsVolume.setController(controller);
+ }
+ }
+
private void initMetadataScanner(ContainerScannerConfiguration c) {
if (this.metadataScanner == null) {
this.metadataScanner =
@@ -486,6 +498,8 @@ public class OzoneContainer {
blockDeletingService.start();
recoveringContainerScrubbingService.start();
+ initHddsVolumeContainer();
+
// mark OzoneContainer as INITIALIZED.
initializingStatus.set(InitializingStatus.INITIALIZED);
}
diff --git
a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html
index fd3d7407d2..4f51b423e8 100644
---
a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html
+++
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html
@@ -22,8 +22,32 @@
</tbody>
</table>
+<h2>HeartBeat Information</h2>
+<table class="table" class="col-md-6">
+ <thead>
+ <tr>
+ <th>Address</th>
+ <th>Last Successful HeartBeat</th>
+ <th>Missed Count</th>
+ <th>State</th>
+ <th>Type</th>
+ <th>Version Number</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="scm in $ctrl.heartbeatmetrics[0].SCMServers">
+ <td>{{scm.addressString}}</td>
+ <td>{{scm.lastSuccessfulHeartbeat}}</td>
+ <td>{{scm.missedCount}}</td>
+ <td>{{scm.state}}</td>
+ <td>{{scm.type}}</td>
+ <td>{{scm.versionNumber}}</td>
+ </tr>
+ </tbody>
+</table>
+
<h2>Volume Information</h2>
-<table class="table table-bordered table-striped" class="col-md-6">
+<table class="table" class="col-md-6">
<thead>
<tr>
<th>Directory</th>
@@ -33,6 +57,7 @@
<th>Available Space</th>
<th>Reserved</th>
<th>Total Capacity</th>
+ <th>Containers</th>
<th>State</th>
</tr>
</thead>
@@ -45,6 +70,7 @@
<td>{{volumeInfo.Available}}</td>
<td>{{volumeInfo.Reserved}}</td>
<td>{{volumeInfo.TotalCapacity}}</td>
+ <td>{{volumeInfo.Containers}}</td>
<td>{{volumeInfo["tag.VolumeState"]}}</td>
</tr>
</tbody>
diff --git
a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-scanner.html
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-scanner.html
new file mode 100644
index 0000000000..5c54a2aa0a
--- /dev/null
+++
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-scanner.html
@@ -0,0 +1,47 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta charset="UTF-8">
+ <title>DataNode Scanner Status</title>
+</head>
+<body>
+ <h2>DataNode Scanner Information</h2>
+ <table class="table" class="col-md-6">
+ <thead>
+ <tr>
+ <th>Directory</th>
+ <th>NumBytesScannedNumOps</th>
+ <th>NumBytesScannedAvgTime</th>
+ <th>NumContainersScanned</th>
+ <th>NumScanIterations</th>
+ <th>NumUnHealthyContainers</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="scanner in scannerStatusCtrl.dnscanner">
+ <td>{{scanner["tag.StorageDirectory"]}}</td>
+ <td>{{scanner.NumBytesScannedNumOps}}</td>
+ <td>{{scanner.NumBytesScannedAvgTime | millisecondsToMinutes}}</td>
+ <td>{{scanner.NumContainersScanned}}</td>
+ <td>{{scanner.NumScanIterations}}</td>
+ <td>{{scanner.NumUnHealthyContainers}}</td>
+ </tr>
+ </tbody>
+ </table>
+</body>
+</html>
\ No newline at end of file
diff --git
a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn.js
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn.js
index adc507acce..547e566ef8 100644
---
a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn.js
+++
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn.js
@@ -36,20 +36,104 @@
volume.TotalCapacity = transform(volume.TotalCapacity);
})
});
+
+
$http.get("jmx?qry=Hadoop:service=HddsDatanode,name=SCMConnectionManager")
+ .then(function (result) {
+ ctrl.heartbeatmetrics = result.data.beans;
+ ctrl.heartbeatmetrics.forEach(scm => {
+ var scmServers = scm.SCMServers;
+ scmServers.forEach(scmServer => {
+ scmServer.lastSuccessfulHeartbeat =
convertTimestampToDate(scmServer.lastSuccessfulHeartbeat)
+ })
+ })
+ });
}
});
- function transform(v) {
- var UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'ZB'];
- var prev = 0, i = 0;
- while (Math.floor(v) > 0 && i < UNITS.length) {
+
+ // Register ioStatus Controller
+ angular.module('ozone').config(function ($routeProvider) {
+ $routeProvider.when('/iostatus', {
+ templateUrl: 'iostatus.html',
+ controller: 'IOStatusController as ioStatusCtrl',
+ });
+ });
+
+ angular.module('ozone')
+ .controller('IOStatusController', function ($http) {
+ var ctrl = this;
+
$http.get("jmx?qry=Hadoop:service=HddsDatanode,name=VolumeIOStats*")
+ .then(function (result) {
+ ctrl.dniostatus = result.data.beans;
+ });
+ });
+
+ // Register Scanner Controller
+ angular.module('ozone').config(function ($routeProvider) {
+ $routeProvider.when('/dn-scanner', {
+ templateUrl: 'dn-scanner.html',
+ controller: 'DNScannerController as scannerStatusCtrl',
+ });
+ });
+
+ angular.module('ozone')
+ .controller('DNScannerController', function ($http) {
+ var ctrl = this;
+
$http.get("jmx?qry=Hadoop:service=HddsDatanode,name=ContainerDataScannerMetrics*")
+ .then(function (result) {
+ ctrl.dnscanner = result.data.beans;
+ });
+ });
+
+ angular.module('ozone')
+ .filter('millisecondsToMinutes', function() {
+ return function(milliseconds) {
+ if (isNaN(milliseconds)) {
+ return 'Invalid input';
+ }
+ var minutes = Math.floor(milliseconds / 60000); // 1 minute =
60000 milliseconds
+ var seconds = Math.floor((milliseconds % 60000) / 1000);
+ return minutes + ' mins ' + seconds + ' secs';
+ };
+ });
+
+ angular.module('ozone')
+ .filter('twoDecimalPlaces', function() {
+ return function(input) {
+ if (isNaN(input)) {
+ return 'Invalid input';
+ }
+ return parseFloat(input).toFixed(2);
+ };
+ });
+
+ function transform(v) {
+ var UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'ZB'];
+ var prev = 0, i = 0;
+ while (Math.floor(v) > 0 && i < UNITS.length) {
prev = v;
v /= 1024;
i += 1;
- }
- if (i > 0 && i < UNITS.length) {
+ }
+ if (i > 0 && i < UNITS.length) {
v = prev;
i -= 1;
- }
- return Math.round(v * 100) / 100 + ' ' + UNITS[i];
}
+ return Math.round(v * 100) / 100 + ' ' + UNITS[i];
+ }
+
+ function convertTimestampToDate(timestamp) {
+ if (!timestamp) return '';
+ var milliseconds = timestamp * 1000;
+
+ var date = new Date(milliseconds);
+
+ var year = date.getFullYear();
+ var month = date.getMonth() + 1;
+ var day = date.getDate();
+ var hours = date.getHours();
+ var minutes = date.getMinutes();
+ var seconds = date.getSeconds();
+
+ return `${year}-${month.toString().padStart(2,
'0')}-${day.toString().padStart(2, '0')} ${hours.toString().padStart(2,
'0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2,
'0')}`;
+ }
})();
diff --git
a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/index.html
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/index.html
index 1c32fe64e0..0e1cbf21a0 100644
---
a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/index.html
+++
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/index.html
@@ -49,11 +49,10 @@
<a class="navbar-brand" href="#">HDDS Datanode Service</a>
</div>
-
- <navmenu
- metrics="{ 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
-
-
+ <navmenu metrics="{ 'Rpc metrics' : '#!/metrics/rpc'}"
+ iostatus="true" io-link-href="#!/iostatus"
+ scanner="true" scanner-link-href="#!/dn-scanner"
+ />
</div>
</header>
diff --git
a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/iostatus.html
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/iostatus.html
new file mode 100644
index 0000000000..94916821bd
--- /dev/null
+++
b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/iostatus.html
@@ -0,0 +1,76 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta charset="UTF-8">
+ <title>DataNode IO Status</title>
+</head>
+<body>
+
+ <h2>Read Performance</h2>
+ <table class="table" class="col-md-6">
+ <thead>
+ <tr>
+ <th>Directory</th>
+ <th>ReadBytes</th>
+ <th>ReadOpCount</th>
+ <th>ReadAvgTime</th>
+ <th>ReadLatency60s(P90)</th>
+ <th>ReadLatency60s(P95)</th>
+ <th>ReadLatency60s(P99)</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="volumeInfo in ioStatusCtrl.dniostatus">
+ <td>{{volumeInfo["tag.StorageDirectory"]}}</td>
+ <td>{{volumeInfo.ReadBytes}}</td>
+ <td>{{volumeInfo.ReadOpCount}}</td>
+ <td>{{volumeInfo.ReadTimeAvgTime | twoDecimalPlaces}} ms</td>
+ <td>{{volumeInfo.ReadLatency60s90thPercentileLatency |
twoDecimalPlaces}} ms</td>
+ <td>{{volumeInfo.ReadLatency60s95thPercentileLatency |
twoDecimalPlaces}} ms</td>
+ <td>{{volumeInfo.ReadLatency60s99thPercentileLatency |
twoDecimalPlaces}} ms</td>
+ </tr>
+ </tbody>
+ </table>
+
+ <h2>Write Performance</h2>
+ <table class="table" class="col-md-6">
+ <thead>
+ <tr>
+ <th>Directory</th>
+ <th>WriteBytes</th>
+ <th>WriteOpCount</th>
+ <th>WriteAvgTime</th>
+ <th>WriteLatency60s(P90)</th>
+ <th>WriteLatency60s(P95)</th>
+ <th>WriteLatency60s(P99)</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="volumeInfo in ioStatusCtrl.dniostatus">
+ <td>{{volumeInfo["tag.StorageDirectory"]}}</td>
+ <td>{{volumeInfo.WriteBytes}}</td>
+ <td>{{volumeInfo.WriteOpCount}}</td>
+ <td>{{volumeInfo.WriteTimeAvgTime | twoDecimalPlaces}} ms</td>
+ <td>{{volumeInfo.WriteLatency60s90thPercentileLatency |
twoDecimalPlaces}} ms</td>
+ <td>{{volumeInfo.WriteLatency60s95thPercentileLatency |
twoDecimalPlaces}} ms</td>
+ <td>{{volumeInfo.WriteLatency60s99thPercentileLatency |
twoDecimalPlaces}} ms</td>
+ </tr>
+ </tbody>
+ </table>
+</body>
+</html>
\ No newline at end of file
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeIOStatsWithPrometheusSink.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeIOStatsWithPrometheusSink.java
index c8934bab41..1df886098a 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeIOStatsWithPrometheusSink.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeIOStatsWithPrometheusSink.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.container.common.volume;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.server.http.PrometheusMetricsSink;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -30,6 +31,7 @@ import java.io.OutputStreamWriter;
import static org.assertj.core.api.Assertions.assertThat;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.OZONE_DATANODE_IO_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY;
/**
* Test PrometheusMetricSink regarding VolumeIOStats.
@@ -54,11 +56,14 @@ public class TestVolumeIOStatsWithPrometheusSink {
@Test
public void testMultipleVolumeIOMetricsExist() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ int[] intervals =
conf.getInts(OZONE_DATANODE_IO_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY);
+
//GIVEN
VolumeIOStats volumeIOStats1 = new VolumeIOStats("VolumeIOStat1",
- "vol1/dir");
+ "vol1/dir", intervals);
VolumeIOStats volumeIOStat2 = new VolumeIOStats("VolumeIOStat2",
- "vol2/dir");
+ "vol2/dir", intervals);
//WHEN
String writtenMetrics = publishMetricsAndGetOutput();
diff --git a/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.css
b/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.css
index 389d9d78f2..4988cc8eeb 100644
--- a/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.css
+++ b/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.css
@@ -94,4 +94,23 @@ body {
.scm-roles-background {
background-color: #dcfbcd!important;
-}
\ No newline at end of file
+}
+.toggle-btn {
+ background: transparent; /* No background color */
+ color: #007bff; /* Button text color */
+ border: none; /* No border */
+ font-size: 12px; /* Font size for better readability */
+ cursor: pointer; /* Pointer cursor on hover */
+ padding: 5px 10px; /* Padding around the text */
+ margin-bottom: 5px; /* Space below the button */
+ transition: color 0.3s, transform 0.3s; /* Smooth transition for color and
transform */
+}
+
+.toggle-btn:hover {
+ color: #0056b3; /* Darker color on hover */
+ transform: scale(1.1); /* Slightly scale up the button on hover */
+}
+
+.toggle-btn:focus {
+ outline: none; /* Remove default focus outline */
+}
diff --git a/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js
b/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js
index a31078cfd7..7bb9310628 100644
--- a/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js
+++ b/hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js
@@ -48,8 +48,14 @@
});
angular.module('ozone').component('jvmParameters', {
templateUrl: 'static/templates/jvm.html',
- controller: function($http) {
+ controller: function($http, $scope) {
var ctrl = this;
+
+ $scope.contentVisible = false;
+ $scope.toggleContent = function() {
+ $scope.contentVisible = !$scope.contentVisible;
+ };
+
$http.get("jmx?qry=java.lang:type=Runtime")
.then(function(result) {
ctrl.jmx = result.data.beans[0];
@@ -245,7 +251,11 @@
angular.module('ozone').component('navmenu', {
bindings: {
- metrics: '<'
+ metrics: '<',
+ iostatus: '<',
+ ioLinkHref: '@',
+ scanner: '<',
+ scannerLinkHref: '@',
},
templateUrl: 'static/templates/menu.html',
controller: function($http) {
diff --git
a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/jvm.html
b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/jvm.html
index 9706ebdf6b..c562ae7d9a 100644
--- a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/jvm.html
+++ b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/jvm.html
@@ -20,7 +20,16 @@
<td>{{$ctrl.jmx.SystemProperties.java_vm_name}}
{{$ctrl.jmx.SystemProperties.java_vm_version}}</td>
</tr>
<tr>
- <th>Input arguments:</th>
- <td><pre>{{$ctrl.jmx.InputArguments.join('\n')}}</pre></td>
+ <th>
+ Input arguments:
+ <button class="toggle-btn" ng-click="toggleContent()">
+ {{ contentVisible ? 'Collapse(-)' : 'Expand(+)' }}
+ </button>
+ </th>
+ <td>
+ <div class="content" ng-show="contentVisible">
+ <pre>{{$ctrl.jmx.InputArguments.join('\n')}}</pre>
+ </div>
+ </td>
</tr>
</table>
diff --git
a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html
b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html
index 95f1b4842f..9a14f356d7 100644
---
a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html
+++
b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html
@@ -56,5 +56,7 @@
aria-hidden="true"></a></li>
</ul>
</li>
+ <li ng-show="$ctrl.iostatus"><a ng-href="{{$ctrl.ioLinkHref}}">IO
Status</a></li>
+ <li ng-show="$ctrl.scanner"><a
ng-href="{{$ctrl.scannerLinkHref}}">Data Scanner</a></li>
</ul>
</div><!--/.nav-collapse -->
diff --git
a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/overview.html
b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/overview.html
index 7ff118b330..2811e8c36a 100644
---
a/hadoop-hdds/framework/src/main/resources/webapps/static/templates/overview.html
+++
b/hadoop-hdds/framework/src/main/resources/webapps/static/templates/overview.html
@@ -14,7 +14,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<h1>Overview</h1>
+<h1>Overview <small
ng-if="$ctrl.jmx.Hostname">({{$ctrl.jmx.Hostname}})</small> </h1>
<table class="table table-bordered table-striped">
<tbody>
<tr ng-if="$ctrl.jmx.Namespace && $ctrl.jmx.Namespace !== ''">
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
