This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3e2ae1da00e HDFS-16949 Introduce inverse quantiles for metrics where
higher numer… (#5495)
3e2ae1da00e is described below
commit 3e2ae1da00e055211914c90cca89d62432096530
Author: rdingankar
AuthorDate: Mon Apr 10 08:56:00 2023 -0700
HDFS-16949 Introduce inverse quantiles for metrics where higher numer…
(#5495)
---
.../hadoop/metrics2/lib/MetricsRegistry.java | 25 +-
.../metrics2/lib/MutableInverseQuantiles.java | 89
.../hadoop/metrics2/lib/MutableQuantiles.java | 95 +-
.../hadoop/metrics2/util/TestSampleQuantiles.java | 68 +---
.../org/apache/hadoop/test/MetricsAsserts.java | 25 +-
.../server/datanode/metrics/DataNodeMetrics.java | 2 +-
.../hdfs/server/datanode/TestDataNodeMetrics.java | 3 +-
7 files changed, 271 insertions(+), 36 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
index b71f7f8cc5e..31031b808ea 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
@@ -227,6 +227,29 @@ public class MetricsRegistry {
return ret;
}
+ /**
+ * Create a mutable inverse metric that estimates inverse quantiles of a
stream of values
+ * @param name of the metric
+ * @param desc metric description
+ * @param sampleName of the metric (e.g., "Ops")
+ * @param valueName of the metric (e.g., "Rate")
+ * @param interval rollover interval of estimator in seconds
+ * @return a new inverse quantile estimator object
+ * @throws MetricsException if interval is not a positive integer
+ */
+ public synchronized MutableQuantiles newInverseQuantiles(String name, String
desc,
+ String sampleName, String valueName, int interval) {
+checkMetricName(name);
+if (interval <= 0) {
+ throw new MetricsException("Interval should be positive. Value passed" +
+ " is: " + interval);
+}
+MutableQuantiles ret =
+new MutableInverseQuantiles(name, desc, sampleName, valueName,
interval);
+metricsMap.put(name, ret);
+return ret;
+ }
+
/**
* Create a mutable metric with stats
* @param name of the metric
@@ -278,7 +301,7 @@ public class MetricsRegistry {
}
/**
- * Create a mutable rate metric (for throughput measurement)
+ * Create a mutable rate metric (for throughput measurement).
* @param name of the metric
* @param desc description
* @param extended produce extended stat (stdev/min/max etc.) if true
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableInverseQuantiles.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableInverseQuantiles.java
new file mode 100644
index 000..a3d579cb9e7
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableInverseQuantiles.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics2.lib;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.metrics2.util.Quantile;
+import org.apache.hadoop.metrics2.util.SampleQuantiles;
+import java.text.DecimalFormat;
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+/**
+ * Watches a stream of long values, maintaining online estimates of specific
+ * quantiles with provably low error bounds. Inverse quantiles are meant for
+ * highly accurate low-percentile (e.g. 1st, 5th) metrics.
+ * InverseQuantiles are used for metrics where higher th
