This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 786bb49f88 HDDS-11642. MutableQuantiles should be stopped (#7392)
786bb49f88 is described below
commit 786bb49f880c72dd49bb6ed68f305a00014ff6a7
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Nov 5 09:37:18 2024 +0100
HDDS-11642. MutableQuantiles should be stopped (#7392)
---
.../hadoop/hdds/scm/ContainerClientMetrics.java | 13 +++++
.../hadoop/hdds/scm/XceiverClientMetrics.java | 2 +
.../hadoop/ozone/grpc/metrics/GrpcMetrics.java | 3 +
.../org/apache/hadoop/ozone/util/MetricUtil.java | 16 ++++++
.../hadoop/ozone/util/PerformanceMetrics.java | 15 +++--
.../ozone/util/PerformanceMetricsInitializer.java | 9 ++-
.../container/common/helpers/ContainerMetrics.java | 9 ++-
.../ozone/container/ozoneimpl/OzoneContainer.java | 2 +
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 16 +++++-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/OzoneClientCreator.java | 65 ++++++++++++++++++++++
11 files changed, 144 insertions(+), 9 deletions(-)
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java
index 0333f4758c..f6367b5a53 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableQuantiles;
import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.util.MetricUtil;
import java.util.Map;
import java.util.UUID;
@@ -98,6 +99,7 @@ public final class ContainerClientMetrics {
}
referenceCount--;
if (referenceCount == 0) {
+ instance.stop();
DefaultMetricsSystem.instance().unregisterSource(
SOURCE_NAME + instanceCount);
instance = null;
@@ -156,6 +158,17 @@ public final class ContainerClientMetrics {
}
}
+ public void stop() {
+ MetricUtil.stop(listBlockLatency);
+ MetricUtil.stop(getBlockLatency);
+ MetricUtil.stop(getCommittedBlockLengthLatency);
+ MetricUtil.stop(readChunkLatency);
+ MetricUtil.stop(getSmallFileLatency);
+ MetricUtil.stop(hsyncLatencyNs);
+ MetricUtil.stop(omHsyncLatencyNs);
+ MetricUtil.stop(datanodeHsyncLatencyNs);
+ }
+
public void recordWriteChunk(Pipeline pipeline, long chunkSizeBytes) {
writeChunkCallsByPipeline.computeIfAbsent(pipeline.getId(),
pipelineID -> registry.newCounter(
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java
index 704e886659..50e010e85a 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
@@ -129,6 +130,7 @@ public class XceiverClientMetrics implements MetricsSource {
}
public void unRegister() {
+ IOUtils.closeQuietly(containerOpsLatency.values());
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java
index 6bd83b44a9..6e0dde6698 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.util.MetricUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,6 +101,8 @@ public class GrpcMetrics implements MetricsSource {
*/
public void unRegister() {
DefaultMetricsSystem.instance().unregisterSource(SOURCE_NAME);
+ MetricUtil.stop(grpcProcessingTimeMillisQuantiles);
+ MetricUtil.stop(grpcQueueTimeMillisQuantiles);
}
@Override
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/MetricUtil.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/MetricUtil.java
index 23ff3c0f29..9d903b900a 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/MetricUtil.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/MetricUtil.java
@@ -110,4 +110,20 @@ public final class MetricUtil {
sampleName, valueName, interval);
}).collect(Collectors.toList());
}
+
+ public static void stop(MutableQuantiles... quantiles) {
+ if (quantiles != null) {
+ stop(Arrays.asList(quantiles));
+ }
+ }
+
+ public static void stop(Iterable<MutableQuantiles> quantiles) {
+ if (quantiles != null) {
+ for (MutableQuantiles q : quantiles) {
+ if (q != null) {
+ q.stop();
+ }
+ }
+ }
+ }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetrics.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetrics.java
index 3f5150bd62..39e887eaa4 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetrics.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetrics.java
@@ -22,7 +22,9 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableQuantiles;
import org.apache.hadoop.metrics2.lib.MutableStat;
+import java.io.Closeable;
import java.util.List;
+import java.util.Map;
/**
* The {@code PerformanceMetrics} class encapsulates a collection of related
@@ -30,7 +32,7 @@ import java.util.List;
* This class provides methods to update these metrics and to
* snapshot their values for reporting.
*/
-public class PerformanceMetrics {
+public class PerformanceMetrics implements Closeable {
private final MutableStat stat;
private final List<MutableQuantiles> quantiles;
private final MutableMinMax minMax;
@@ -43,12 +45,13 @@ public class PerformanceMetrics {
* @param intervals the intervals for quantiles computation. Note, each
* interval in 'intervals' increases memory usage, as it corresponds
* to a separate quantile calculator.
+ * @return {@link PerformanceMetrics} instances created, mapped by field name
*/
- public static synchronized <T> void initializeMetrics(T source,
+ public static synchronized <T> Map<String, PerformanceMetrics>
initializeMetrics(T source,
MetricsRegistry registry, String sampleName, String valueName,
int[] intervals) {
try {
- PerformanceMetricsInitializer.initialize(
+ return PerformanceMetricsInitializer.initialize(
source, registry, sampleName, valueName, intervals);
} catch (IllegalAccessException e) {
throw new RuntimeException("Failed to initialize PerformanceMetrics", e);
@@ -73,6 +76,11 @@ public class PerformanceMetrics {
minMax = new MutableMinMax(registry, name, description, valueName);
}
+ @Override
+ public void close() {
+ MetricUtil.stop(quantiles);
+ }
+
/**
* Adds a value to all the aggregated metrics.
*
@@ -95,6 +103,5 @@ public class PerformanceMetrics {
this.quantiles.forEach(quantile -> quantile.snapshot(recordBuilder, all));
this.minMax.snapshot(recordBuilder, all);
}
-
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetricsInitializer.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetricsInitializer.java
index b2e83bb780..cb6f77e9f5 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetricsInitializer.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetricsInitializer.java
@@ -21,6 +21,8 @@ import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
/**
* Utility class for initializing PerformanceMetrics in a MetricsSource.
@@ -36,11 +38,13 @@ public final class PerformanceMetricsInitializer {
* @param sampleName sample name
* @param valueName value name
* @param intervals intervals for quantiles
+ * @return {@link PerformanceMetrics} instances created, mapped by field name
* @throws IllegalAccessException if unable to access the field
*/
- public static <T> void initialize(T source, MetricsRegistry registry,
+ public static <T> Map<String, PerformanceMetrics> initialize(T source,
MetricsRegistry registry,
String sampleName, String valueName, int[] intervals)
throws IllegalAccessException {
+ Map<String, PerformanceMetrics> instances = new HashMap<>();
Field[] fields = source.getClass().getDeclaredFields();
for (Field field : fields) {
@@ -54,8 +58,11 @@ public final class PerformanceMetricsInitializer {
sampleName, valueName, intervals);
field.setAccessible(true);
field.set(source, performanceMetrics);
+ instances.put(name, performanceMetrics);
}
}
}
+
+ return instances;
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
index 883f6cd851..46887441f0 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
@@ -30,7 +30,9 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableQuantiles;
import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.ozone.util.MetricUtil;
+import java.io.Closeable;
import java.util.EnumMap;
/**
@@ -47,7 +49,7 @@ import java.util.EnumMap;
*/
@InterfaceAudience.Private
@Metrics(about = "Storage Container DataNode Metrics", context = "dfs")
-public class ContainerMetrics {
+public class ContainerMetrics implements Closeable {
public static final String STORAGE_CONTAINER_METRICS =
"StorageContainerMetrics";
@Metric private MutableCounterLong numOps;
@@ -105,6 +107,11 @@ public class ContainerMetrics {
ms.unregisterSource(STORAGE_CONTAINER_METRICS);
}
+ @Override
+ public void close() {
+ opsLatQuantiles.values().forEach(MetricUtil::stop);
+ }
+
public void incContainerOpsMetrics(ContainerProtos.Type type) {
numOps.incr();
numOpsArray.get(type).incr();
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index e40fa635c1..62196cdd87 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -34,6 +34,7 @@ import
org.apache.hadoop.hdds.security.symmetric.SecretKeyVerifierClient;
import org.apache.hadoop.hdds.security.token.TokenVerifier;
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService;
@@ -512,6 +513,7 @@ public class OzoneContainer {
}
blockDeletingService.shutdown();
recoveringContainerScrubbingService.shutdown();
+ IOUtils.closeQuietly(metrics);
ContainerMetrics.remove();
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
index dd84d01917..ea34be638e 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.s3.metrics;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
@@ -33,12 +34,15 @@ import org.apache.hadoop.ozone.s3.S3GatewayConfigKeys;
import org.apache.hadoop.ozone.util.PerformanceMetrics;
import org.apache.hadoop.util.Time;
+import java.io.Closeable;
+import java.util.Map;
+
/**
* This class maintains S3 Gateway related metrics.
*/
@InterfaceAudience.Private
@Metrics(about = "S3 Gateway Metrics", context = OzoneConsts.OZONE)
-public final class S3GatewayMetrics implements MetricsSource {
+public final class S3GatewayMetrics implements Closeable, MetricsSource {
public static final String SOURCE_NAME =
S3GatewayMetrics.class.getSimpleName();
@@ -242,6 +246,8 @@ public final class S3GatewayMetrics implements
MetricsSource {
@Metric(about = "Latency for copy metadata of an key in nanoseconds")
private PerformanceMetrics copyKeyMetadataLatencyNs;
+ private final Map<String, PerformanceMetrics> performanceMetrics;
+
/**
* Private constructor.
*/
@@ -249,10 +255,15 @@ public final class S3GatewayMetrics implements
MetricsSource {
this.registry = new MetricsRegistry(SOURCE_NAME);
int[] intervals = conf.getInts(S3GatewayConfigKeys
.OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY);
- PerformanceMetrics.initializeMetrics(
+ performanceMetrics = PerformanceMetrics.initializeMetrics(
this, registry, "Ops", "Time", intervals);
}
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(performanceMetrics.values());
+ }
+
/**
* Create and returns S3 Gateway Metrics instance.
*
@@ -272,6 +283,7 @@ public final class S3GatewayMetrics implements
MetricsSource {
* Unregister the metrics instance.
*/
public static void unRegister() {
+ IOUtils.closeQuietly(instance);
instance = null;
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
index d0c9a33b33..877e25ac7c 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
@@ -75,7 +75,8 @@ import static
org.apache.hadoop.hdds.server.http.HttpServer2.setHttpBaseDir;
DatanodeSimulator.class,
OmMetadataGenerator.class,
DNRPCLoadGenerator.class,
- HsyncGenerator.class
+ HsyncGenerator.class,
+ OzoneClientCreator.class,
},
versionProvider = HddsVersionProvider.class,
mixinStandardHelpOptions = true)
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientCreator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientCreator.java
new file mode 100644
index 0000000000..2fc4cb48ea
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientCreator.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Creates and closes Ozone clients.
+ */
[email protected](name = "occ",
+ aliases = "ozone-client-creator",
+ description = "Create and close Ozone clients without doing anything
useful",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class OzoneClientCreator extends BaseFreonGenerator implements
Callable<Void> {
+
+ @CommandLine.Option(names = "--om-service-id",
+ description = "OM Service ID"
+ )
+ private String omServiceID;
+
+ private Timer timer;
+ private OzoneConfiguration conf;
+
+ @Override
+ public Void call() {
+ init();
+ conf = createOzoneConfiguration();
+ timer = getMetrics().timer("client-create");
+ runTests(this::createClient);
+ return null;
+ }
+
+ private void createClient(long step) {
+ timer.time(this::createClientSafely);
+ }
+
+ private void createClientSafely() {
+ try {
+ createOzoneClient(omServiceID, conf).close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
