This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 9084285 [INLONG-2451][TubeMQ] Add Histogram implementation classes
(#2452)
9084285 is described below
commit 9084285aa8890a3e2e791cbdd32d093317faf8c3
Author: gosonzhang <[email protected]>
AuthorDate: Fri Feb 11 11:42:38 2022 +0800
[INLONG-2451][TubeMQ] Add Histogram implementation classes (#2452)
---
.../tubemq/corebase/metric/impl/ESTHistogram.java | 187 +++++++++++++++++++++
.../corebase/metric/impl/SimpleHistogram.java | 80 +++++++++
.../tubemq/corebase/metric/HistogramTest.java | 130 ++++++++++++++
3 files changed, 397 insertions(+)
diff --git
a/inlong-tubemq/tubemq-core/src/main/java/org/apache/inlong/tubemq/corebase/metric/impl/ESTHistogram.java
b/inlong-tubemq/tubemq-core/src/main/java/org/apache/inlong/tubemq/corebase/metric/impl/ESTHistogram.java
new file mode 100644
index 0000000..5bbdbe9
--- /dev/null
+++
b/inlong-tubemq/tubemq-core/src/main/java/org/apache/inlong/tubemq/corebase/metric/impl/ESTHistogram.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.tubemq.corebase.metric.impl;
+
+import java.util.Map;
+import org.apache.inlong.tubemq.corebase.TBaseConstants;
+import org.apache.inlong.tubemq.corebase.metric.Histogram;
+
+/**
+ * ESTHistogram, Exponential Statistics for Time
+ *
+ * This class performs the index statistics of the time period with the
exponential power of 2.
+ * Here we count up to the 17th power of 2, a total of 18 buckets, according
to the input data and
+ * the corresponding time index, the corresponding cell is selected for
statistics.
+ * When outputting data, the full statistical output or
+ * only the output is not 0 according to different requirements
+ */
+public class ESTHistogram extends BaseMetric implements Histogram {
+ // Total number of exponential statistic blocks
+ private static final int NUM_BUCKETS = 18;
+ // The upper boundary index of exponential statistic blocks
+ private static final int MAX_BUCKET_INDEX = NUM_BUCKETS - 1;
+ // The lower boundary value of the last statistical block
+ private static final long POWER_2_17 = 131072L;
+ // Natural logarithm of 2
+ private static final double LOG2_VALUE = Math.log(2);
+ // Simple statistic items
+ private final LongStatsCounter count;
+ private final LongMinGauge min;
+ private final LongMaxGauge max;
+ // Exponential Statistic cells for Time
+ private final LongStatsCounter[] buckets = new
LongStatsCounter[NUM_BUCKETS];
+
+ public ESTHistogram(String metricName, String prefix) {
+ super(metricName, prefix);
+ this.count = new LongStatsCounter("count", getFullName());
+ this.min = new LongMinGauge("min", getFullName());
+ this.max = new LongMaxGauge("max", getFullName());
+ StringBuilder strBuff =
+ new StringBuilder(TBaseConstants.BUILDER_DEFAULT_SIZE);
+ // Initialize the name and store room of each cell
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ strBuff.append("cell_");
+ if (i == 0) {
+ strBuff.append(0).append("t").append((int) Math.pow(2, i + 1)
- 1);
+ } else if (i == MAX_BUCKET_INDEX) {
+ strBuff.append((int) Math.pow(2, i)).append("tMax");
+ } else {
+ strBuff.append((int) Math.pow(2, i))
+ .append("t").append((int) Math.pow(2, i + 1) - 1);
+ }
+ this.buckets[i] = new LongStatsCounter(strBuff.toString(),
getFullName());
+ strBuff.delete(0, strBuff.length());
+ }
+ }
+
+ @Override
+ public void update(long newValue) {
+ this.count.incValue();
+ this.min.update(newValue);
+ this.max.update(newValue);
+ int index = (newValue <= 0L) ? 0 : ((newValue >= POWER_2_17)
+ ? MAX_BUCKET_INDEX : ((int) (Math.log(newValue) /
LOG2_VALUE)));
+ this.buckets[index].incValue();
+ }
+
+ @Override
+ public void getValue(Map<String, Long> keyValMap, boolean includeZero) {
+ keyValMap.put(this.count.getFullName(), this.count.getValue());
+ keyValMap.put(this.min.getFullName(), this.min.getValue());
+ keyValMap.put(this.max.getFullName(), this.max.getValue());
+ if (includeZero) {
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ keyValMap.put(this.buckets[i].getFullName(),
this.buckets[i].getValue());
+ }
+ } else {
+ long tmpValue;
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ tmpValue = this.buckets[i].getValue();
+ if (tmpValue > 0) {
+ keyValMap.put(this.buckets[i].getFullName(), tmpValue);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void getValue(StringBuilder strBuff, boolean includeZero) {
+ strBuff.append("\"").append(getFullName()).append("\":")
+ .append("{\"").append(this.count.getShortName()).append("\":")
+ .append(this.count.getValue()).append(",\"")
+ .append(this.min.getShortName()).append("\":")
+ .append(this.min.getValue()).append(",\"")
+ .append(this.max.getShortName()).append("\":")
+ .append(this.max.getValue()).append(",\"cells\":{");
+ int count = 0;
+ if (includeZero) {
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ if (count++ > 0) {
+ strBuff.append(",");
+ }
+
strBuff.append("\"").append(this.buckets[i].getShortName()).append("\":")
+ .append(this.buckets[i].getValue());
+ }
+ } else {
+ long tmpValue;
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ tmpValue = this.buckets[i].getValue();
+ if (tmpValue > 0) {
+ if (count++ > 0) {
+ strBuff.append(",");
+ }
+
strBuff.append("\"").append(this.buckets[i].getShortName()).append("\":").append(tmpValue);
+ }
+ }
+ }
+ strBuff.append("}}");
+ }
+
+ @Override
+ public void snapShort(Map<String, Long> keyValMap, boolean includeZero) {
+ keyValMap.put(this.count.getFullName(), this.count.getAndResetValue());
+ keyValMap.put(this.min.getFullName(), this.min.getAndResetValue());
+ keyValMap.put(this.max.getFullName(), this.max.getAndResetValue());
+ if (includeZero) {
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ keyValMap.put(this.buckets[i].getFullName(),
this.buckets[i].getAndResetValue());
+ }
+ } else {
+ long tmpValue;
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ tmpValue = this.buckets[i].getAndResetValue();
+ if (tmpValue > 0) {
+ keyValMap.put(this.buckets[i].getFullName(), tmpValue);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void snapShort(StringBuilder strBuff, boolean includeZero) {
+ strBuff.append("\"").append(getFullName()).append("\":")
+ .append("{\"").append(this.count.getShortName()).append("\":")
+ .append(this.count.getAndResetValue()).append(",\"")
+ .append(this.min.getShortName()).append("\":")
+ .append(this.min.getAndResetValue()).append(",\"")
+ .append(this.max.getShortName()).append("\":")
+ .append(this.max.getAndResetValue()).append(",\"cells\":{");
+ int count = 0;
+ if (includeZero) {
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ if (count++ > 0) {
+ strBuff.append(",");
+ }
+
strBuff.append("\"").append(this.buckets[i].getShortName()).append("\":")
+ .append(this.buckets[i].getAndResetValue());
+ }
+ } else {
+ long tmpValue;
+ for (int i = 0; i < NUM_BUCKETS; i++) {
+ tmpValue = this.buckets[i].getAndResetValue();
+ if (tmpValue > 0) {
+ if (count++ > 0) {
+ strBuff.append(",");
+ }
+
strBuff.append("\"").append(this.buckets[i].getShortName()).append("\":").append(tmpValue);
+ }
+ }
+ }
+ strBuff.append("}}");
+ }
+}
diff --git
a/inlong-tubemq/tubemq-core/src/main/java/org/apache/inlong/tubemq/corebase/metric/impl/SimpleHistogram.java
b/inlong-tubemq/tubemq-core/src/main/java/org/apache/inlong/tubemq/corebase/metric/impl/SimpleHistogram.java
new file mode 100644
index 0000000..fe0695a
--- /dev/null
+++
b/inlong-tubemq/tubemq-core/src/main/java/org/apache/inlong/tubemq/corebase/metric/impl/SimpleHistogram.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.tubemq.corebase.metric.impl;
+
+import java.util.Map;
+import org.apache.inlong.tubemq.corebase.metric.Histogram;
+
+/**
+ * Simple Histogram Statistics, only include count, min, max value information.
+ */
+public class SimpleHistogram extends BaseMetric implements Histogram {
+ private final LongStatsCounter count;
+ private final LongMinGauge min;
+ private final LongMaxGauge max;
+
+ public SimpleHistogram(String metricName, String prefix) {
+ super(metricName, prefix);
+ this.count = new LongStatsCounter("count", getFullName());
+ this.min = new LongMinGauge("min", getFullName());
+ this.max = new LongMaxGauge("max", getFullName());
+ }
+
+ @Override
+ public void update(long newValue) {
+ this.count.incValue();
+ this.min.update(newValue);
+ this.max.update(newValue);
+ }
+
+ @Override
+ public void getValue(Map<String, Long> keyValMap, boolean includeZero) {
+ keyValMap.put(this.count.getFullName(), this.count.getValue());
+ keyValMap.put(this.min.getFullName(), this.min.getValue());
+ keyValMap.put(this.max.getFullName(), this.max.getValue());
+ }
+
+ @Override
+ public void getValue(StringBuilder strBuff, boolean includeZero) {
+ strBuff.append("\"").append(getFullName()).append("\":{\"")
+ .append(this.count.getShortName()).append("\":")
+ .append(this.count.getValue()).append(",\"")
+ .append(this.min.getShortName()).append("\":")
+ .append(this.min.getValue()).append(",\"")
+ .append(this.max.getShortName()).append("\":")
+ .append(this.max.getValue()).append("}");
+ }
+
+ @Override
+ public void snapShort(Map<String, Long> keyValMap, boolean includeZero) {
+ keyValMap.put(this.count.getFullName(), this.count.getAndResetValue());
+ keyValMap.put(this.min.getFullName(), this.min.getAndResetValue());
+ keyValMap.put(this.max.getFullName(), this.max.getAndResetValue());
+ }
+
+ @Override
+ public void snapShort(StringBuilder strBuff, boolean includeZero) {
+ strBuff.append("\"").append(getFullName()).append("\":{\"")
+ .append(this.count.getShortName()).append("\":")
+ .append(this.count.getAndResetValue()).append(",\"")
+ .append(this.min.getShortName()).append("\":")
+ .append(this.min.getAndResetValue()).append(",\"")
+ .append(this.max.getShortName()).append("\":")
+ .append(this.max.getAndResetValue()).append("}");
+ }
+}
diff --git
a/inlong-tubemq/tubemq-core/src/test/java/org/apache/inlong/tubemq/corebase/metric/HistogramTest.java
b/inlong-tubemq/tubemq-core/src/test/java/org/apache/inlong/tubemq/corebase/metric/HistogramTest.java
new file mode 100644
index 0000000..a97a221
--- /dev/null
+++
b/inlong-tubemq/tubemq-core/src/test/java/org/apache/inlong/tubemq/corebase/metric/HistogramTest.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.tubemq.corebase.metric;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.apache.inlong.tubemq.corebase.metric.impl.ESTHistogram;
+import org.apache.inlong.tubemq.corebase.metric.impl.SimpleHistogram;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class HistogramTest {
+
+ @Test
+ public void testSimpleHistogram() {
+ SimpleHistogram histogram =
+ new SimpleHistogram("stats", "api");
+ // test getValue by StringBuilder
+ histogram.update(10L);
+ histogram.update(100000L);
+ histogram.update(3000L);
+ histogram.update(-5L);
+ StringBuilder strBuff = new StringBuilder(512);
+ histogram.getValue(strBuff, false);
+ String result1 =
"\"api_stats\":{\"count\":4,\"min\":-5,\"max\":100000}";
+ Assert.assertEquals(result1, strBuff.toString());
+ strBuff.delete(0, strBuff.length());
+ // test getValue by Map
+ histogram.update(3L);
+ histogram.update(10000L);
+ histogram.update(900000L);
+ histogram.update(10L);
+ Map<String, Long> tmpMap = new LinkedHashMap();
+ histogram.getValue(tmpMap, false);
+ Assert.assertEquals(tmpMap.get("api_stats_count").longValue(), 8L);
+ Assert.assertEquals(tmpMap.get("api_stats_max").longValue(), 900000L);
+ Assert.assertEquals(tmpMap.get("api_stats_min").longValue(), -5L);
+ tmpMap.clear();
+ // test snapShort
+ histogram.update(2L);
+ histogram.update(10000000L);
+ histogram.update(500L);
+ histogram.update(1L);
+ histogram.snapShort(tmpMap, false);
+ Assert.assertEquals(tmpMap.get("api_stats_count").longValue(), 12L);
+ Assert.assertEquals(tmpMap.get("api_stats_max").longValue(),
10000000L);
+ Assert.assertEquals(tmpMap.get("api_stats_min").longValue(), -5L);
+ tmpMap.clear();
+ // test getValue by string buffer
+ histogram.update(3L);
+ histogram.update(10000L);
+ histogram.update(900000L);
+ histogram.update(10L);
+ histogram.snapShort(strBuff, false);
+ String result2 =
"\"api_stats\":{\"count\":4,\"min\":3,\"max\":900000}";
+ Assert.assertEquals(result2, strBuff.toString());
+ strBuff.delete(0, strBuff.length());
+ histogram.update(1L);
+ histogram.getValue(strBuff, false);
+ String result3 = "\"api_stats\":{\"count\":1,\"min\":1,\"max\":1}";
+ Assert.assertEquals(result3, strBuff.toString());
+ strBuff.delete(0, strBuff.length());
+ }
+
+ @Test
+ public void testESTHistogram() {
+ ESTHistogram estHistogram =
+ new ESTHistogram("dlt", "disk");
+ estHistogram.update(30L);
+ estHistogram.update(1000L);
+ estHistogram.update(-5L);
+ estHistogram.update(131070L);
+ estHistogram.update(131071L);
+ estHistogram.update(131072L);
+ estHistogram.update(131100L);
+ // test get value by strBuff
+ StringBuilder strBuff = new StringBuilder(512);
+ estHistogram.getValue(strBuff, false);
+ String result1 = "\"disk_dlt\":{\"count\":7,\"min\":-5,\"max\":131100,"
+ +
"\"cells\":{\"cell_0t1\":1,\"cell_16t31\":1,\"cell_512t1023\":1"
+ + ",\"cell_65536t131071\":2,\"cell_131072tMax\":2}}";
+ Assert.assertEquals(result1, strBuff.toString());
+ strBuff.delete(0, strBuff.length());
+ // test for map
+ Map<String, Long> tmpMap = new LinkedHashMap();
+ estHistogram.getValue(tmpMap, false);
+ Assert.assertEquals(tmpMap.get("disk_dlt_count").longValue(), 7L);
+ Assert.assertEquals(tmpMap.get("disk_dlt_max").longValue(), 131100L);
+ Assert.assertEquals(tmpMap.get("disk_dlt_min").longValue(), -5L);
+ Assert.assertEquals(tmpMap.get("disk_dlt_cell_0t1").longValue(), 1);
+ Assert.assertEquals(tmpMap.get("disk_dlt_cell_16t31").longValue(), 1);
+ Assert.assertEquals(tmpMap.get("disk_dlt_cell_512t1023").longValue(),
1);
+
Assert.assertEquals(tmpMap.get("disk_dlt_cell_65536t131071").longValue(), 2);
+
Assert.assertEquals(tmpMap.get("disk_dlt_cell_131072tMax").longValue(), 2);
+ tmpMap.clear();
+ // test snapShort
+ estHistogram.snapShort(tmpMap, false);
+ tmpMap.clear();
+ estHistogram.update(1L);
+ estHistogram.update(100L);
+ estHistogram.getValue(tmpMap, false);
+ Assert.assertEquals(tmpMap.get("disk_dlt_count").longValue(), 2L);
+ Assert.assertEquals(tmpMap.get("disk_dlt_max").longValue(), 100L);
+ Assert.assertEquals(tmpMap.get("disk_dlt_min").longValue(), 1L);
+ Assert.assertEquals(tmpMap.get("disk_dlt_cell_0t1").longValue(), 1);
+ Assert.assertEquals(tmpMap.get("disk_dlt_cell_64t127").longValue(), 1);
+ tmpMap.clear();
+ // test get value by strBuff
+ estHistogram.getValue(strBuff, false);
+ String result2 =
+
"\"disk_dlt\":{\"count\":2,\"min\":1,\"max\":100,\"cells\":{\"cell_0t1\":1,\"cell_64t127\":1}}";
+ Assert.assertEquals(result2, strBuff.toString());
+ strBuff.delete(0, strBuff.length());
+ }
+}
