This is an automated email from the ASF dual-hosted git repository. chaow pushed a commit to branch feature/metric2021 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e4c562aa53df6584dcb704edb4422de2de1873c9 Author: chaow <[email protected]> AuthorDate: Thu Feb 25 15:19:44 2021 +0800 add implements --- metrics/dropwizard-metrics/pom.xml | 27 ++ .../dropwizard/DropwizardMetricManager.java | 362 ++++++++++++++++++++ .../dropwizard/DropwizardMetricReporter.java | 93 ++++++ .../iotdb/metrics/dropwizard/MetricName.java | 116 +++++++ .../dropwizard/type/DropwizardCounter.java} | 15 +- .../metrics/dropwizard/type/DropwizardGauge.java | 62 ++++ .../dropwizard/type/DropwizardHistogram.java} | 30 +- .../type/DropwizardHistogramSnapshot.java | 73 ++++ .../metrics/dropwizard/type/DropwizardRate.java | 95 ++++++ .../metrics/dropwizard/type/DropwizardTimer.java} | 29 +- .../org.apache.iotdb.metrics.MetricManager | 18 + .../org.apache.iotdb.metrics.MetricReporter | 18 + .../org/apache/iotdb/metrics/MetricManager.java | 48 ++- .../org/apache/iotdb/metrics/MetricReporter.java | 2 +- .../org/apache/iotdb/metrics/MetricService.java | 55 +-- .../apache/iotdb/metrics/config/MetricConfig.java | 125 +++++++ .../metrics/config/MetricConfigDescriptor.java | 132 ++++++++ .../MetricConstant.java} | 10 +- .../iotdb/metrics/impl/DoNothingMetricManager.java | 29 +- .../iotdb/metrics/type/HistogramSnapshot.java | 2 - .../java/org/apache/iotdb/metrics/type/Timer.java | 7 +- .../ReporterType.java} | 37 ++- metrics/micrometer-metrics/pom.xml | 10 + .../iotdb/metrics/micrometer/MeterIdUtils.java} | 22 +- .../micrometer/MicrometerMetricFactory.java | 98 ------ .../micrometer/MicrometerMetricManager.java | 367 +++++++++++++++++++-- .../micrometer/MicrometerMetricReporter.java | 63 +++- .../metrics/micrometer/type/MicrometerCounter.java | 3 +- .../metrics/micrometer/type/MicrometerGauge.java} | 35 +- ...ometerCounter.java => MicrometerHistogram.java} | 28 +- .../type/MicrometerHistogramSnapshot.java | 91 +++++ .../metrics/micrometer/type/MicrometerRate.java | 81 +++++ ...MicrometerCounter.java => MicrometerTimer.java} | 32 +- ...otdb.metrics.micrometer.MicrometerMetricFactory | 1 - ...tdb.metrics.micrometer.MicrometerMetricReporter | 1 - .../org.apache.iotdb.metrics.MetricManager | 18 + .../org.apache.iotdb.metrics.MetricReporter | 18 + metrics/pom.xml | 2 +- server/pom.xml | 10 + .../resources/conf/iotdb-metric.properties | 30 ++ .../engine/storagegroup/StorageGroupProcessor.java | 36 ++ .../apache/iotdb/db/qp/executor/PlanExecutor.java | 25 ++ .../java/org/apache/iotdb/db/service/IoTDB.java | 5 +- .../org/apache/iotdb/db/service/TSServiceImpl.java | 41 ++- 44 files changed, 2092 insertions(+), 310 deletions(-) diff --git a/metrics/dropwizard-metrics/pom.xml b/metrics/dropwizard-metrics/pom.xml new file mode 100644 index 0000000..5779fae --- /dev/null +++ b/metrics/dropwizard-metrics/pom.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>iotdb-metrics</artifactId> + <groupId>org.apache.iotdb</groupId> + <version>0.12.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>dropwizard-metric</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>metrics-interface</artifactId> + <version>0.12.0-SNAPSHOT</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> + <version>4.1.2</version> + </dependency> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-jvm</artifactId> + </dependency> + </dependencies> +</project> diff --git a/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/DropwizardMetricManager.java b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/DropwizardMetricManager.java new file mode 100644 index 0000000..1f34053 --- /dev/null +++ b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/DropwizardMetricManager.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.dropwizard; + +import org.apache.iotdb.metrics.KnownMetric; +import org.apache.iotdb.metrics.MetricManager; +import org.apache.iotdb.metrics.config.MetricConfig; +import org.apache.iotdb.metrics.config.MetricConfigDescriptor; +import org.apache.iotdb.metrics.dropwizard.type.DropwizardCounter; +import org.apache.iotdb.metrics.dropwizard.type.DropwizardGauge; +import org.apache.iotdb.metrics.dropwizard.type.DropwizardHistogram; +import org.apache.iotdb.metrics.dropwizard.type.DropwizardRate; +import org.apache.iotdb.metrics.dropwizard.type.DropwizardTimer; +import org.apache.iotdb.metrics.type.Counter; +import org.apache.iotdb.metrics.type.Gauge; +import org.apache.iotdb.metrics.type.Histogram; +import org.apache.iotdb.metrics.type.IMetric; +import org.apache.iotdb.metrics.type.Rate; +import org.apache.iotdb.metrics.type.Timer; + +import com.codahale.metrics.JvmAttributeGaugeSet; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.jvm.BufferPoolMetricSet; +import com.codahale.metrics.jvm.CachedThreadStatesGaugeSet; +import com.codahale.metrics.jvm.ClassLoadingGaugeSet; +import com.codahale.metrics.jvm.GarbageCollectorMetricSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +public class DropwizardMetricManager implements MetricManager { + + private static final Logger logger = LoggerFactory.getLogger(DropwizardMetricManager.class); + + Map<MetricName, IMetric> currentMeters; + boolean isEnable; + + com.codahale.metrics.MetricRegistry metricRegistry; + MetricConfig metricConfig = MetricConfigDescriptor.getInstance().getMetricConfig(); + + public DropwizardMetricManager() { + metricRegistry = new MetricRegistry(); + isEnable = metricConfig.isEnabled(); + currentMeters = new ConcurrentHashMap<>(); + } + + @Override + public Counter counter(String metric, String... tags) { + if (!isEnable) { + return null; + } + MetricName name = new MetricName(metric, tags); + return (Counter) + currentMeters.computeIfAbsent( + name, key -> new DropwizardCounter(metricRegistry.counter(name.toFlatString()))); + } + + @Override + public Gauge gauge(String metric, String... tags) { + if (!isEnable) { + return null; + } + MetricName name = new MetricName(metric, tags); + return (Gauge) + currentMeters.computeIfAbsent( + name, + key -> { + DropwizardGauge dropwizardGauge = new DropwizardGauge(); + metricRegistry.register( + name.toFlatString(), dropwizardGauge.getDropwizardCachedGauge()); + return dropwizardGauge; + }); + } + + @Override + public Histogram histogram(String metric, String... tags) { + if (!isEnable) { + return null; + } + MetricName name = new MetricName(metric, tags); + return (Histogram) + currentMeters.computeIfAbsent( + name, key -> new DropwizardHistogram(metricRegistry.histogram(name.toFlatString()))); + } + + @Override + public Rate rate(String metric, String... tags) { + if (!isEnable) { + return null; + } + MetricName name = new MetricName(metric, tags); + return (Rate) + currentMeters.computeIfAbsent( + name, key -> new DropwizardRate(metricRegistry.meter(name.toFlatString()))); + } + + @Override + public Timer timer(String metric, String... tags) { + if (!isEnable) { + return null; + } + MetricName name = new MetricName(metric, tags); + return (Timer) + currentMeters.computeIfAbsent( + name, key -> new DropwizardTimer(metricRegistry.timer(name.toFlatString()))); + } + + @Override + public void count(int delta, String metric, String... tags) { + if (!isEnable) { + return; + } + MetricName name = new MetricName(metric, tags); + ((Counter) + currentMeters.computeIfAbsent( + name, key -> new DropwizardCounter(metricRegistry.counter(name.toFlatString())))) + .inc(delta); + } + + @Override + public void count(long delta, String metric, String... tags) { + if (!isEnable) { + return; + } + MetricName name = new MetricName(metric, tags); + ((Counter) + currentMeters.computeIfAbsent( + name, key -> new DropwizardCounter(metricRegistry.counter(name.toFlatString())))) + .inc(delta); + } + + @Override + public void histogram(int value, String metric, String... tags) { + if (!isEnable) { + return; + } + MetricName name = new MetricName(metric, tags); + ((Histogram) + currentMeters.computeIfAbsent( + name, + key -> new DropwizardHistogram(metricRegistry.histogram(name.toFlatString())))) + .update(value); + } + + @Override + public void histogram(long value, String metric, String... tags) { + if (!isEnable) { + return; + } + MetricName name = new MetricName(metric, tags); + ((Histogram) + currentMeters.computeIfAbsent( + name, + key -> new DropwizardHistogram(metricRegistry.histogram(name.toFlatString())))) + .update(value); + } + + @Override + public void gauge(int value, String metric, String... tags) { + if (!isEnable) { + return; + } + MetricName name = new MetricName(metric, tags); + ((Gauge) + currentMeters.computeIfAbsent( + name, + key -> { + DropwizardGauge dropwizardGauge = new DropwizardGauge(); + metricRegistry.register( + name.toFlatString(), dropwizardGauge.getDropwizardCachedGauge()); + return dropwizardGauge; + })) + .set(value); + } + + @Override + public void gauge(long value, String metric, String... tags) { + if (!isEnable) { + return; + } + MetricName name = new MetricName(metric, tags); + ((Gauge) + currentMeters.computeIfAbsent( + name, + key -> { + DropwizardGauge dropwizardGauge = new DropwizardGauge(); + metricRegistry.register( + name.toFlatString(), dropwizardGauge.getDropwizardCachedGauge()); + return dropwizardGauge; + })) + .set(value); + } + + @Override + public void rate(int value, String metric, String... tags) { + if (!isEnable) { + return; + } + MetricName name = new MetricName(metric, tags); + ((Rate) + currentMeters.computeIfAbsent( + name, key -> new DropwizardRate(metricRegistry.meter(name.toFlatString())))) + .mark(value); + } + + @Override + public void rate(long value, String metric, String... tags) { + if (!isEnable) { + return; + } + MetricName name = new MetricName(metric, tags); + ((Rate) + currentMeters.computeIfAbsent( + name, key -> new DropwizardRate(metricRegistry.meter(name.toFlatString())))) + .mark(value); + } + + @Override + public void timer(long delta, TimeUnit timeUnit, String metric, String... tags) { + if (!isEnable) { + return; + } + MetricName name = new MetricName(metric, tags); + ((Timer) + currentMeters.computeIfAbsent( + name, key -> new DropwizardTimer(metricRegistry.timer(name.toFlatString())))) + .update(delta, timeUnit); + } + + @Override + public List<String[]> getAllMetricKeys() { + if (!isEnable) { + return Collections.emptyList(); + } + List<String[]> keys = new ArrayList<>(currentMeters.size()); + currentMeters.keySet().forEach(k -> keys.add(k.toStringArray())); + return keys; + } + + @Override + public Map<String[], Counter> getAllCounters() { + Map<String[], Counter> counterMap = new HashMap<>(); + for (Map.Entry<MetricName, IMetric> entry : currentMeters.entrySet()) { + if (entry.getValue() instanceof Counter) { + counterMap.put(entry.getKey().toStringArray(), (Counter) entry.getValue()); + } + } + return counterMap; + } + + @Override + public Map<String[], Gauge> getAllGauges() { + Map<String[], Gauge> gaugeMap = new HashMap<>(); + for (Map.Entry<MetricName, IMetric> entry : currentMeters.entrySet()) { + if (entry.getValue() instanceof Gauge) { + gaugeMap.put(entry.getKey().toStringArray(), (Gauge) entry.getValue()); + } + } + return gaugeMap; + } + + @Override + public Map<String[], Rate> getAllRates() { + Map<String[], Rate> rateMap = new HashMap<>(); + for (Map.Entry<MetricName, IMetric> entry : currentMeters.entrySet()) { + if (entry.getValue() instanceof Rate) { + rateMap.put(entry.getKey().toStringArray(), (Rate) entry.getValue()); + } + } + return rateMap; + } + + @Override + public Map<String[], Histogram> getAllHistograms() { + Map<String[], Histogram> histogramMap = new HashMap<>(); + for (Map.Entry<MetricName, IMetric> entry : currentMeters.entrySet()) { + if (entry.getValue() instanceof Histogram) { + histogramMap.put(entry.getKey().toStringArray(), (Histogram) entry.getValue()); + } + } + return histogramMap; + } + + @Override + public Map<String[], Timer> getAllTimers() { + Map<String[], Timer> timerMap = new HashMap<>(); + for (Map.Entry<MetricName, IMetric> entry : currentMeters.entrySet()) { + if (entry.getValue() instanceof Timer) { + timerMap.put(entry.getKey().toStringArray(), (Timer) entry.getValue()); + } + } + return timerMap; + } + + @Override + public boolean isEnable() { + return isEnable; + } + + @Override + public void enableKnownMetric(KnownMetric metric) { + if (!isEnable) { + return; + } + switch (metric) { + case JVM: + enableJVMMetrics(); + break; + case SYSTEM: + break; + case THREAD: + break; + default: + // ignore; + } + } + + public MetricRegistry getMetricRegistry() { + return metricRegistry; + } + + private void enableJVMMetrics() { + if (!isEnable) { + return; + } + metricRegistry.registerAll(new JvmAttributeGaugeSet()); + metricRegistry.registerAll(new GarbageCollectorMetricSet()); + metricRegistry.registerAll(new ClassLoadingGaugeSet()); + metricRegistry.registerAll(new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer())); + metricRegistry.registerAll(new CachedThreadStatesGaugeSet(5, TimeUnit.MILLISECONDS)); + } + + @Override + public boolean init() { + return true; + } +} diff --git a/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/DropwizardMetricReporter.java b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/DropwizardMetricReporter.java new file mode 100644 index 0000000..235f9ad --- /dev/null +++ b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/DropwizardMetricReporter.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.dropwizard; + +import org.apache.iotdb.metrics.MetricManager; +import org.apache.iotdb.metrics.MetricReporter; +import org.apache.iotdb.metrics.config.MetricConfig; +import org.apache.iotdb.metrics.config.MetricConfigDescriptor; +import org.apache.iotdb.metrics.utils.ReporterType; + +import com.codahale.metrics.JmxReporter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class DropwizardMetricReporter implements MetricReporter { + private static Logger logger = LoggerFactory.getLogger(DropwizardMetricReporter.class); + private MetricManager dropwizardMetricManager; + private MetricConfig metricConfig = MetricConfigDescriptor.getInstance().getMetricConfig(); + + private JmxReporter jmxReporter; + + @Override + public boolean start() { + List<String> reporters = metricConfig.getReporterList(); + for (String reporter : reporters) { + switch (ReporterType.get(reporter)) { + case JMX: + { + jmxReporter = + JmxReporter.forRegistry( + ((DropwizardMetricManager) dropwizardMetricManager).getMetricRegistry()) + .build(); + startJmxReporter(jmxReporter); + } + break; + case IOTDB: + break; + case PROMETHEUS: + break; + default: + logger.warn("Dropwizard don't support reporter type {}", reporter); + } + } + return false; + } + + private void startJmxReporter(JmxReporter jmxReporter) { + jmxReporter.start(); + } + + @Override + public void setMetricManager(MetricManager metricManager) { + dropwizardMetricManager = metricManager; + } + + @Override + public boolean stop() { + List<String> reporters = metricConfig.getReporterList(); + for (String reporter : reporters) { + switch (ReporterType.get(reporter)) { + case JMX: + jmxReporter.stop(); + break; + case IOTDB: + break; + case PROMETHEUS: + break; + default: + logger.warn("Dropwizard don't support reporter type {}", reporter); + } + } + return true; + } +} diff --git a/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/MetricName.java b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/MetricName.java new file mode 100644 index 0000000..d214463 --- /dev/null +++ b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/MetricName.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.dropwizard; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class MetricName { + public static final String SEPARATOR = "."; + public static final Map<String, String> EMPTY_TAGS = Collections.emptyMap(); + + public String name; + public Map<String, String> tags; + + public MetricName(String name, String... tags) { + this.name = name; + this.tags = new HashMap<>(); + for (int i = 0; i < tags.length; i++) { + this.tags.put(tags[i], tags[i + 1]); + i += 2; + } + } + + public MetricName(String name, Map<String, String> tags) { + this.name = name; + this.tags = tags; + } + + public String toFlatString() { + StringBuilder stringBuilder = new StringBuilder(name); + tags.forEach((k, v) -> stringBuilder.append(k).append(SEPARATOR).append(v)); + return stringBuilder.toString().replace(" ", "_"); + } + + public String[] toStringArray() { + List<String> allNames = new ArrayList<>(); + allNames.add(name); + tags.forEach( + (k, v) -> { + allNames.add(k); + allNames.add(v); + }); + return allNames.toArray(new String[0]); + } + + @Override + public String toString() { + return "MetricName{" + "name='" + name + '\'' + ", tags=" + tags + '}'; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map<String, String> getTags() { + return tags; + } + + public void setTags(Map<String, String> tags) { + this.tags = tags; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof MetricName)) { + return false; + } + MetricName that = (MetricName) obj; + if (this.name != that.name) { + return false; + } + if (that.getTags().size() != this.tags.size()) { + return false; + } + Map<String, String> thatTags = that.getTags(); + for (Map.Entry<String, String> entry : this.tags.entrySet()) { + if (!thatTags.containsKey(entry.getKey())) { + return false; + } + if (!thatTags.get(entry.getKey()).equals(entry.getValue())) { + return false; + } + } + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, tags); + } +} diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardCounter.java similarity index 76% copy from metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java copy to metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardCounter.java index 9fbfa6f..3261ed2 100644 --- a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java +++ b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardCounter.java @@ -17,30 +17,29 @@ * under the License. */ -package org.apache.iotdb.metrics.micrometer.type; +package org.apache.iotdb.metrics.dropwizard.type; import org.apache.iotdb.metrics.type.Counter; -public class MicrometerCounter implements Counter { +public class DropwizardCounter implements Counter { + com.codahale.metrics.Counter counter; - public MicrometerCounter(io.micrometer.core.instrument.Counter counter) { + public DropwizardCounter(com.codahale.metrics.Counter counter) { this.counter = counter; } - io.micrometer.core.instrument.Counter counter; - @Override public void inc() { - counter.increment(); + counter.inc(); } @Override public void inc(long n) { - counter.increment(n); + counter.inc(n); } @Override public long count() { - return (long) counter.count(); + return counter.getCount(); } } diff --git a/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardGauge.java b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardGauge.java new file mode 100644 index 0000000..c7175f5 --- /dev/null +++ b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardGauge.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.dropwizard.type; + +import org.apache.iotdb.metrics.type.Gauge; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +public class DropwizardGauge implements Gauge { + + AtomicLong atomicLong; + DropwizardCachedGauge dropwizardCachedGauge; + + public DropwizardGauge() { + atomicLong = new AtomicLong(0); + dropwizardCachedGauge = new DropwizardCachedGauge(5, TimeUnit.MILLISECONDS); + } + + public class DropwizardCachedGauge extends com.codahale.metrics.CachedGauge<Long> { + + protected DropwizardCachedGauge(long timeout, TimeUnit timeoutUnit) { + super(timeout, timeoutUnit); + } + + @Override + protected Long loadValue() { + return atomicLong.get(); + } + } + + @Override + public long value() { + return dropwizardCachedGauge.getValue(); + } + + @Override + public void set(long value) { + atomicLong.set(value); + } + + public com.codahale.metrics.Gauge getDropwizardCachedGauge() { + return dropwizardCachedGauge; + } +} diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardHistogram.java similarity index 58% copy from metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java copy to metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardHistogram.java index 9fbfa6f..edf5f2b 100644 --- a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java +++ b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardHistogram.java @@ -17,30 +17,36 @@ * under the License. */ -package org.apache.iotdb.metrics.micrometer.type; +package org.apache.iotdb.metrics.dropwizard.type; -import org.apache.iotdb.metrics.type.Counter; +import org.apache.iotdb.metrics.type.Histogram; +import org.apache.iotdb.metrics.type.HistogramSnapshot; -public class MicrometerCounter implements Counter { +public class DropwizardHistogram implements Histogram { - public MicrometerCounter(io.micrometer.core.instrument.Counter counter) { - this.counter = counter; - } + com.codahale.metrics.Histogram histogram; - io.micrometer.core.instrument.Counter counter; + public DropwizardHistogram(com.codahale.metrics.Histogram histogram) { + this.histogram = histogram; + } @Override - public void inc() { - counter.increment(); + public void update(int value) { + histogram.update(value); } @Override - public void inc(long n) { - counter.increment(n); + public void update(long value) { + histogram.update(value); } @Override public long count() { - return (long) counter.count(); + return histogram.getCount(); + } + + @Override + public HistogramSnapshot takeSnapshot() { + return new DropwizardHistogramSnapshot(histogram.getSnapshot()); } } diff --git a/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardHistogramSnapshot.java b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardHistogramSnapshot.java new file mode 100644 index 0000000..0fe3a9c --- /dev/null +++ b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardHistogramSnapshot.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.dropwizard.type; + +import org.apache.iotdb.metrics.type.HistogramSnapshot; + +import java.io.OutputStream; + +public class DropwizardHistogramSnapshot implements HistogramSnapshot { + + com.codahale.metrics.Snapshot snapshot; + + public DropwizardHistogramSnapshot(com.codahale.metrics.Snapshot snapshot) { + this.snapshot = snapshot; + } + + @Override + public double getValue(double quantile) { + return snapshot.getValue(quantile); + } + + @Override + public long[] getValues() { + return snapshot.getValues(); + } + + @Override + public int size() { + return snapshot.size(); + } + + @Override + public double getMedian() { + return snapshot.getMedian(); + } + + @Override + public long getMax() { + return snapshot.getMax(); + } + + @Override + public double getMean() { + return snapshot.getMean(); + } + + @Override + public long getMin() { + return snapshot.getMin(); + } + + @Override + public void dump(OutputStream output) { + snapshot.dump(output); + } +} diff --git a/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardRate.java b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardRate.java new file mode 100644 index 0000000..8be1309 --- /dev/null +++ b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardRate.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.dropwizard.type; + +import org.apache.iotdb.metrics.type.Rate; + +import com.codahale.metrics.Meter; + +public class DropwizardRate implements Rate { + + Meter meter; + // read-only meter + com.codahale.metrics.Timer timer; + + public DropwizardRate(Meter meter) { + this.meter = meter; + this.timer = null; + } + + public DropwizardRate(com.codahale.metrics.Timer timer) { + this.timer = timer; + this.meter = null; + } + + @Override + public long getCount() { + if (meter != null) { + return meter.getCount(); + } + return timer.getCount(); + } + + @Override + public double getOneMinuteRate() { + if (meter != null) { + return meter.getOneMinuteRate(); + } + return timer.getOneMinuteRate(); + } + + @Override + public double getMeanRate() { + if (meter != null) { + return meter.getMeanRate(); + } + return timer.getMeanRate(); + } + + @Override + public double getFiveMinuteRate() { + if (meter != null) { + return meter.getFiveMinuteRate(); + } + return timer.getFiveMinuteRate(); + } + + @Override + public double getFifteenMinuteRate() { + if (meter != null) { + return meter.getFifteenMinuteRate(); + } + return timer.getFifteenMinuteRate(); + } + + @Override + public void mark() { + if (meter != null) { + meter.mark(); + } + } + + @Override + public void mark(long n) { + if (meter != null) { + meter.mark(n); + } + } +} diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardTimer.java similarity index 56% copy from metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java copy to metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardTimer.java index 9fbfa6f..24a0849 100644 --- a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java +++ b/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardTimer.java @@ -17,30 +17,33 @@ * under the License. */ -package org.apache.iotdb.metrics.micrometer.type; +package org.apache.iotdb.metrics.dropwizard.type; -import org.apache.iotdb.metrics.type.Counter; +import org.apache.iotdb.metrics.type.HistogramSnapshot; +import org.apache.iotdb.metrics.type.Rate; +import org.apache.iotdb.metrics.type.Timer; -public class MicrometerCounter implements Counter { +import java.util.concurrent.TimeUnit; - public MicrometerCounter(io.micrometer.core.instrument.Counter counter) { - this.counter = counter; - } +public class DropwizardTimer implements Timer { + com.codahale.metrics.Timer timer; - io.micrometer.core.instrument.Counter counter; + public DropwizardTimer(com.codahale.metrics.Timer timer) { + this.timer = timer; + } @Override - public void inc() { - counter.increment(); + public void update(long duration, TimeUnit unit) { + timer.update(duration, unit); } @Override - public void inc(long n) { - counter.increment(n); + public HistogramSnapshot takeSnapshot() { + return new DropwizardHistogramSnapshot(timer.getSnapshot()); } @Override - public long count() { - return (long) counter.count(); + public Rate getImmutableRate() { + return new DropwizardRate(timer); } } diff --git a/metrics/dropwizard-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricManager b/metrics/dropwizard-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricManager new file mode 100644 index 0000000..b606911 --- /dev/null +++ b/metrics/dropwizard-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricManager @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.iotdb.metrics.dropwizard.DropwizardMetricManager \ No newline at end of file diff --git a/metrics/dropwizard-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricReporter b/metrics/dropwizard-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricReporter new file mode 100644 index 0000000..1ac1d1e --- /dev/null +++ b/metrics/dropwizard-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricReporter @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.iotdb.metrics.dropwizard.DropwizardMetricReporter \ No newline at end of file diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricManager.java index d61bdb7..8add6ed 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricManager.java +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricManager.java @@ -24,11 +24,17 @@ import org.apache.iotdb.metrics.type.Histogram; import org.apache.iotdb.metrics.type.Rate; import org.apache.iotdb.metrics.type.Timer; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; public interface MetricManager { - + /* + * The following functions will create or get a exist Metric + * @param metric: the metric name + * @param tags: string appear in pairs, like sg="ln",user="user1" will be "sg", "ln", "user", "user1" + * @return Metric Instance + */ Counter counter(String metric, String... tags); Gauge gauge(String metric, String... tags); @@ -39,7 +45,13 @@ public interface MetricManager { Timer timer(String metric, String... tags); - // metric.counter(5, "insertRecords","interface","insertRecords","sg","sg1"); + /* + * The following functions just update the current record value + * @param the delta value will be recorded + * @param metric the metric name + * @param tags string appear in pairs, like sg="ln",user="user1" will be "sg", "ln", "user", "user1" + */ + void count(int delta, String metric, String... tags); void count(long delta, String metric, String... tags); @@ -52,26 +64,40 @@ public interface MetricManager { void gauge(long value, String metric, String... tags); - void meter(int value, String metric, String... tags); + void rate(int value, String metric, String... tags); - void meter(long value, String metric, String... tags); + void rate(long value, String metric, String... tags); void timer(long delta, TimeUnit timeUnit, String metric, String... tags); - void timerStart(String metric, String... tags); - - void timerEnd(String metric, String... tags); + /** @return all MetricKeys, key is metric name, value is tags, which is a string array */ + List<String[]> getAllMetricKeys(); - Map<String, String[]> getAllMetricKeys(); - - // key is name + tags + // key is name + tags, value Map<String[], Counter> getAllCounters(); Map<String[], Gauge> getAllGauges(); - Map<String[], Rate> getAllMeters(); + Map<String[], Rate> getAllRates(); Map<String[], Histogram> getAllHistograms(); Map<String[], Timer> getAllTimers(); + + /** @return whether enable metricService */ + boolean isEnable(); + + /** + * enable pre-defined metric set + * + * @param metric which metric set we want to collect + */ + void enableKnownMetric(KnownMetric metric); + + /** + * init something + * + * @return + */ + boolean init(); } diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricReporter.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricReporter.java index a1af228..023f0fe 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricReporter.java +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricReporter.java @@ -21,7 +21,7 @@ package org.apache.iotdb.metrics; public interface MetricReporter { boolean start(); - void setMetricFactory(MetricFactory metricFactory); + void setMetricManager(MetricManager metricManager); boolean stop(); } diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricService.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricService.java index 50b0d7d..8e1af33 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricService.java +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricService.java @@ -18,55 +18,65 @@ */ package org.apache.iotdb.metrics; -import org.apache.iotdb.metrics.impl.DoNothingFactory; +import org.apache.iotdb.metrics.impl.DoNothingMetricManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.ServiceLoader; -/** MetricService is the entr */ +/** MetricService is the entry to manage all Metric system */ public class MetricService { private static final Logger logger = LoggerFactory.getLogger(MetricService.class); private static final List<MetricReporter> reporters = new ArrayList<>(); - private static MetricFactory factory; - static { init(); } - private static void init() { + private static final MetricService INSTANCE = new MetricService(); + + private static MetricManager metricManager; + + public static MetricService getINSTANCE() { + return INSTANCE; + } - ServiceLoader<MetricFactory> metricFactories = ServiceLoader.load(MetricFactory.class); + private MetricService() {} + + private static void init() { + logger.debug("init metric service"); + ServiceLoader<MetricManager> metricManagers = ServiceLoader.load(MetricManager.class); int size = 0; - MetricFactory nothingFactory = null; + MetricManager nothingManager = new DoNothingMetricManager(); - for (MetricFactory mf : metricFactories) { - if (mf instanceof DoNothingFactory) { - nothingFactory = mf; + for (MetricManager mf : metricManagers) { + if (mf instanceof DoNothingMetricManager) { + nothingManager = mf; continue; } size++; - factory = mf; + metricManager = mf; } - // if no more implementation, we use nothingFactory. + // if no more implementations, we use nothingFactory. if (size == 0) { - factory = nothingFactory; + metricManager = nothingManager; } else if (size > 1) { - logger.warn("detect more than one MetricFactory, will use {}", factory.getClass().getName()); + logger.warn( + "detect more than one MetricManager, will use {}", metricManager.getClass().getName()); } + // do some init work + metricManager.init(); ServiceLoader<MetricReporter> reporter = ServiceLoader.load(MetricReporter.class); for (MetricReporter r : reporter) { reporters.add(r); - r.setMetricFactory(factory); + r.setMetricManager(metricManager); r.start(); logger.info("detect MetricReporter {}", r.getClass().getName()); } @@ -74,23 +84,20 @@ public class MetricService { public static void stop() { for (MetricReporter r : reporters) { + logger.info("detect MetricReporter {}", r.getClass().getName()); r.stop(); } } - public static MetricManager getMetric(String namespace) { - return factory.getMetric(namespace); + public static MetricManager getMetricManager() { + return metricManager; } public static void enableKnownMetric(KnownMetric metric) { - factory.enableKnownMetric(metric); - } - - public static Map<String, MetricManager> getAllMetrics() { - return factory.getAllMetrics(); + metricManager.enableKnownMetric(metric); } public static boolean isEnable() { - return factory.isEnable(); + return metricManager.isEnable(); } } diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java new file mode 100644 index 0000000..fd40973 --- /dev/null +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; + +public class MetricConfig { + private static final Logger logger = LoggerFactory.getLogger(MetricConfig.class); + static final String CONFIG_NAME = "iotdb-metric.properties"; + + /** The period of data pushed by the reporter to the remote monitoring system */ + private int pushPeriodInSecond = 5; + + /** enable publishing data */ + private boolean isEnabled = true; + + /** provide or push metric data to remote system, could be jmx, prometheus, iotdb, etc. */ + private List<String> reporterList = + Arrays.asList("jmx", "prometheus"); // Collections.singletonList("jmx"); + + // the following is prometheus related config + /** the http server's port for prometheus exporter to get metric data */ + private String prometheusExporterPort = "8090"; + + // the following is iotdb related config + + private String iotdbSg = "monitor"; + private String iotdbUser = "root"; + private String iotdbPasswd = "root"; + private String iotdbIp = "127.0.0.1"; + private String iotdbPort = "6667"; + + public int getPushPeriodInSecond() { + return pushPeriodInSecond; + } + + public void setPushPeriodInSecond(int pushPeriodInSecond) { + this.pushPeriodInSecond = pushPeriodInSecond; + } + + public boolean isEnabled() { + return isEnabled; + } + + public void setEnabled(boolean enabled) { + isEnabled = enabled; + } + + public String getPrometheusExporterPort() { + return prometheusExporterPort; + } + + public void setPrometheusExporterPort(String prometheusExporterPort) { + this.prometheusExporterPort = prometheusExporterPort; + } + + public List<String> getReporterList() { + return reporterList; + } + + public void setReporterList(List<String> reporterList) { + this.reporterList = reporterList; + } + + public String getIotdbSg() { + return iotdbSg; + } + + public void setIotdbSg(String iotdbSg) { + this.iotdbSg = iotdbSg; + } + + public String getIotdbUser() { + return iotdbUser; + } + + public void setIotdbUser(String iotdbUser) { + this.iotdbUser = iotdbUser; + } + + public String getIotdbPasswd() { + return iotdbPasswd; + } + + public void setIotdbPasswd(String iotdbPasswd) { + this.iotdbPasswd = iotdbPasswd; + } + + public String getIotdbIp() { + return iotdbIp; + } + + public void setIotdbIp(String iotdbIp) { + this.iotdbIp = iotdbIp; + } + + public String getIotdbPort() { + return iotdbPort; + } + + public void setIotdbPort(String iotdbPort) { + this.iotdbPort = iotdbPort; + } +} diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java new file mode 100644 index 0000000..7990a24 --- /dev/null +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +public class MetricConfigDescriptor { + private static final Logger logger = LoggerFactory.getLogger(MetricConfigDescriptor.class); + private final MetricConfig metricConfig = new MetricConfig(); + + public MetricConfig getMetricConfig() { + return metricConfig; + } + + private MetricConfigDescriptor() { + loadProps(); + } + + public static MetricConfigDescriptor getInstance() { + return MetricConfigDescriptorHolder.INSTANCE; + } + + public String getPropsUrl() { + String url = System.getProperty(MetricConstant.METRIC_CONF, null); + if (url == null) { + url = System.getProperty(MetricConstant.IOTDB_HOME, null); + if (url != null) { + url = url + File.separatorChar + "conf" + File.separatorChar + MetricConfig.CONFIG_NAME; + } else { + logger.warn( + "Cannot find IOTDB_HOME or METRIC_CONF environment variable when loading " + + "config file {}, use default configuration", + MetricConfig.CONFIG_NAME); + // update all data seriesPath + return null; + } + } else { + url += (File.separatorChar + MetricConfig.CONFIG_NAME); + } + return url; + } + + /** load an property file and set TsfileDBConfig variables. */ + private void loadProps() { + + String url = getPropsUrl(); + Properties properties = System.getProperties(); + if (url != null) { + try (InputStream inputStream = new FileInputStream(new File(url))) { + logger.info("Start to read config file {}", url); + properties.load(inputStream); + } catch (IOException e) { + logger.warn("Fail to find config file {}", url, e); + } + } + + metricConfig.setEnabled( + Boolean.parseBoolean( + properties.getProperty("enable_metric", Boolean.toString(metricConfig.isEnabled())))); + + String reporterList = properties.getProperty("metric_reporter_list"); + if (reporterList != null) { + metricConfig.setReporterList(getReporterList(reporterList)); + } + + metricConfig.setPushPeriodInSecond( + Integer.parseInt( + properties.getProperty( + "push_period_in_second", Integer.toString(metricConfig.getPushPeriodInSecond())))); + + metricConfig.setPrometheusExporterPort( + properties.getProperty( + "prometheus_exporter_port", metricConfig.getPrometheusExporterPort())); + + metricConfig.setIotdbIp(properties.getProperty("iotdb_ip", metricConfig.getIotdbIp())); + + metricConfig.setIotdbPort(properties.getProperty("iotdb_port", metricConfig.getIotdbPort())); + + metricConfig.setIotdbSg(properties.getProperty("iotdb_sg", metricConfig.getIotdbSg())); + metricConfig.setIotdbUser(properties.getProperty("iotdb_user", metricConfig.getIotdbUser())); + metricConfig.setIotdbPasswd( + properties.getProperty("iotdb_passwd", metricConfig.getIotdbPasswd())); + } + + private List<String> getReporterList(String reporterList) { + if (reporterList == null) { + return Collections.emptyList(); + } + List<String> reporters = new ArrayList<>(); + String[] split = reporterList.split(","); + for (String reporter : split) { + reporter = reporter.trim(); + if ("".equals(reporter)) { + continue; + } + reporters.add(reporter); + } + return reporters; + } + + private static class MetricConfigDescriptorHolder { + + private static final MetricConfigDescriptor INSTANCE = new MetricConfigDescriptor(); + } +} diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricReporter.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConstant.java similarity index 82% copy from metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricReporter.java copy to metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConstant.java index a1af228..9bbfd37 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricReporter.java +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConstant.java @@ -16,12 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.metrics; -public interface MetricReporter { - boolean start(); +package org.apache.iotdb.metrics.config; - void setMetricFactory(MetricFactory metricFactory); - - boolean stop(); +public class MetricConstant { + static final String METRIC_CONF = "METRIC_CONF"; + public static final String IOTDB_HOME = "IOTDB_HOME"; } diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/impl/DoNothingMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/impl/DoNothingMetricManager.java index 5f0480c..6b46b34 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/impl/DoNothingMetricManager.java +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/impl/DoNothingMetricManager.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.metrics.impl; +import org.apache.iotdb.metrics.KnownMetric; import org.apache.iotdb.metrics.MetricManager; import org.apache.iotdb.metrics.type.Counter; import org.apache.iotdb.metrics.type.Gauge; @@ -25,6 +26,7 @@ import org.apache.iotdb.metrics.type.Histogram; import org.apache.iotdb.metrics.type.Rate; import org.apache.iotdb.metrics.type.Timer; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -74,22 +76,16 @@ public class DoNothingMetricManager implements MetricManager { public void gauge(long value, String metric, String... tags) {} @Override - public void meter(int value, String metric, String... tags) {} + public void rate(int value, String metric, String... tags) {} @Override - public void meter(long value, String metric, String... tags) {} + public void rate(long value, String metric, String... tags) {} @Override public void timer(long delta, TimeUnit timeUnit, String metric, String... tags) {} @Override - public void timerStart(String metric, String... tags) {} - - @Override - public void timerEnd(String metric, String... tags) {} - - @Override - public Map<String, String[]> getAllMetricKeys() { + public List<String[]> getAllMetricKeys() { return null; } @@ -104,7 +100,7 @@ public class DoNothingMetricManager implements MetricManager { } @Override - public Map<String[], Rate> getAllMeters() { + public Map<String[], Rate> getAllRates() { return null; } @@ -117,4 +113,17 @@ public class DoNothingMetricManager implements MetricManager { public Map<String[], Timer> getAllTimers() { return null; } + + @Override + public boolean isEnable() { + return false; + } + + @Override + public void enableKnownMetric(KnownMetric metric) {} + + @Override + public boolean init() { + return false; + } } diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/type/HistogramSnapshot.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/type/HistogramSnapshot.java index 54adeec..430adf8 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/type/HistogramSnapshot.java +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/type/HistogramSnapshot.java @@ -37,8 +37,6 @@ public interface HistogramSnapshot { public abstract long getMin(); - public abstract double getStdDev(); - /** * Writes the values of the snapshot to the given stream. * diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/type/Timer.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/type/Timer.java index 2f05b2a..fb1952e 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/type/Timer.java +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/type/Timer.java @@ -37,5 +37,10 @@ public interface Timer extends IMetric { HistogramSnapshot takeSnapshot(); - Rate getMeter(); + /** + * It's not safe to use the update interface + * + * @return the rate related with the timer + */ + Rate getImmutableRate(); } diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricFactory.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/ReporterType.java similarity index 59% copy from metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricFactory.java copy to metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/ReporterType.java index 4896aaa..b13884c 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricFactory.java +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/ReporterType.java @@ -16,23 +16,36 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.metrics; +package org.apache.iotdb.metrics.utils; + +import java.util.HashMap; import java.util.Map; -public interface MetricFactory { +public enum ReporterType { + JMX("jmx"), + PROMETHEUS("prometheus"), + IOTDB("iotdb"); + + private String name; + + ReporterType(String name) { + this.name = name; + } - /** - * repeated calling the method will return the same Object instance. - * - * @param namespace - * @return - */ - MetricManager getMetric(String namespace); + public String getName() { + return name; + } - void enableKnownMetric(KnownMetric metric); + private static final Map<String, ReporterType> lookup = new HashMap<>(); - Map<String, MetricManager> getAllMetrics(); + static { + for (ReporterType reporterType : ReporterType.values()) { + lookup.put(reporterType.getName(), reporterType); + } + } - boolean isEnable(); + public static ReporterType get(String name) { + return lookup.get(name); + } } diff --git a/metrics/micrometer-metrics/pom.xml b/metrics/micrometer-metrics/pom.xml index 1c51742..319fc04 100644 --- a/metrics/micrometer-metrics/pom.xml +++ b/metrics/micrometer-metrics/pom.xml @@ -39,5 +39,15 @@ <artifactId>micrometer-registry-prometheus</artifactId> <version>1.6.2</version> </dependency> + <dependency> + <groupId>io.micrometer</groupId> + <artifactId>micrometer-registry-jmx</artifactId> + <version>1.6.2</version> + </dependency> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> + <version>4.1.2</version> + </dependency> </dependencies> </project> diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricFactory.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MeterIdUtils.java similarity index 68% rename from metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricFactory.java rename to metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MeterIdUtils.java index 4896aaa..2112b22 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/MetricFactory.java +++ b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MeterIdUtils.java @@ -16,23 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.metrics; -import java.util.Map; +package org.apache.iotdb.metrics.micrometer; -public interface MetricFactory { +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.Tags; - /** - * repeated calling the method will return the same Object instance. - * - * @param namespace - * @return - */ - MetricManager getMetric(String namespace); +public class MeterIdUtils { - void enableKnownMetric(KnownMetric metric); + private MeterIdUtils() {} - Map<String, MetricManager> getAllMetrics(); - - boolean isEnable(); + public static Meter.Id fromMetricName(String metricName, Meter.Type type, String... tags) { + return new Meter.Id(metricName, Tags.of(tags), null, null, type); + } } diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricFactory.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricFactory.java deleted file mode 100644 index 67aba66..0000000 --- a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricFactory.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.metrics.micrometer; - -import org.apache.iotdb.metrics.KnownMetric; -import org.apache.iotdb.metrics.MetricFactory; -import org.apache.iotdb.metrics.MetricManager; - -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmCompilationMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmHeapPressureMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; - -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class MicrometerMetricFactory implements MetricFactory { - boolean isEnable; - Map<String, MetricManager> currentMetricManagers = new ConcurrentHashMap<String, MetricManager>(); - - @Override - public MetricManager getMetric(String namespace) { - if (!isEnable) { - return null; - } - currentMetricManagers.putIfAbsent(namespace, new MicrometerMetricManager()); - return currentMetricManagers.get(namespace); - } - - @Override - public void enableKnownMetric(KnownMetric metric) { - if (!isEnable) { - return; - } - switch (metric) { - case JVM: - enableJVMMetrics(); - break; - case SYSTEM: - break; - case THREAD: - break; - default: - // ignore; - } - } - - private void enableJVMMetrics() { - MeterRegistry meterRegistry = (MeterRegistry) currentMetricManagers.get("iotdb"); - ClassLoaderMetrics classLoaderMetrics = new ClassLoaderMetrics(); - classLoaderMetrics.bindTo(meterRegistry); - JvmCompilationMetrics jvmCompilationMetrics = new JvmCompilationMetrics(); - jvmCompilationMetrics.bindTo(meterRegistry); - try (JvmGcMetrics jvmGcMetrics = new JvmGcMetrics(); - JvmHeapPressureMetrics jvmHeapPressureMetrics = new JvmHeapPressureMetrics()) { - jvmGcMetrics.bindTo(meterRegistry); - jvmHeapPressureMetrics.bindTo(meterRegistry); - } - JvmMemoryMetrics jvmMemoryMetrics = new JvmMemoryMetrics(); - jvmMemoryMetrics.bindTo(meterRegistry); - JvmThreadMetrics jvmThreadMetrics = new JvmThreadMetrics(); - jvmThreadMetrics.bindTo(meterRegistry); - } - - @Override - public Map<String, MetricManager> getAllMetrics() { - if (!isEnable) { - return Collections.emptyMap(); - } - return currentMetricManagers; - } - - @Override - public boolean isEnable() { - return isEnable; - } -} diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricManager.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricManager.java index 5eb65e4..249902c 100644 --- a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricManager.java +++ b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricManager.java @@ -19,42 +19,119 @@ package org.apache.iotdb.metrics.micrometer; +import org.apache.iotdb.metrics.KnownMetric; import org.apache.iotdb.metrics.MetricManager; +import org.apache.iotdb.metrics.config.MetricConfig; +import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import org.apache.iotdb.metrics.micrometer.type.MicrometerCounter; +import org.apache.iotdb.metrics.micrometer.type.MicrometerGauge; +import org.apache.iotdb.metrics.micrometer.type.MicrometerHistogram; +import org.apache.iotdb.metrics.micrometer.type.MicrometerRate; +import org.apache.iotdb.metrics.micrometer.type.MicrometerTimer; import org.apache.iotdb.metrics.type.Counter; import org.apache.iotdb.metrics.type.Gauge; import org.apache.iotdb.metrics.type.Histogram; import org.apache.iotdb.metrics.type.IMetric; import org.apache.iotdb.metrics.type.Rate; import org.apache.iotdb.metrics.type.Timer; +import org.apache.iotdb.metrics.utils.ReporterType; +import io.micrometer.core.instrument.Clock; import io.micrometer.core.instrument.Meter; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmCompilationMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmHeapPressureMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; +import io.micrometer.jmx.JmxConfig; +import io.micrometer.jmx.JmxMeterRegistry; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; public class MicrometerMetricManager implements MetricManager { + private static final Logger logger = LoggerFactory.getLogger(MicrometerMetricManager.class); + Map<Meter.Id, IMetric> currentMeters; + boolean isEnable; - io.micrometer.prometheus.PrometheusMeterRegistry prometheusMeterRegistry; + io.micrometer.core.instrument.MeterRegistry meterRegistry; + MetricConfig metricConfig = MetricConfigDescriptor.getInstance().getMetricConfig(); public MicrometerMetricManager() { - prometheusMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + meterRegistry = Metrics.globalRegistry; currentMeters = new ConcurrentHashMap<>(); + isEnable = metricConfig.isEnabled(); + } + + @Override + public boolean init() { + logger.debug("micrometer init registry"); + List<String> reporters = metricConfig.getReporterList(); + for (String reporter : reporters) { + switch (ReporterType.get(reporter)) { + case JMX: + Metrics.addRegistry(new JmxMeterRegistry(JmxConfig.DEFAULT, Clock.SYSTEM)); + break; + case PROMETHEUS: + Metrics.addRegistry(new PrometheusMeterRegistry(PrometheusConfig.DEFAULT)); + break; + case IOTDB: + break; + default: + logger.warn("Unsupported report type {}, please check the config.", reporter); + return false; + } + } + return true; } public MeterRegistry getMeterRegistry() { - return prometheusMeterRegistry; + return meterRegistry; + } + + public PrometheusMeterRegistry getPrometheusMeterRegistry() { + Set<MeterRegistry> meterRegistrySet = Metrics.globalRegistry.getRegistries(); + for (MeterRegistry childMeterRegistry : meterRegistrySet) { + if (childMeterRegistry instanceof PrometheusMeterRegistry) { + return (PrometheusMeterRegistry) childMeterRegistry; + } + } + return null; + } + + public JmxMeterRegistry getJmxMeterRegistry() { + Set<MeterRegistry> meterRegistrySet = Metrics.globalRegistry.getRegistries(); + for (MeterRegistry childMeterRegistry : meterRegistrySet) { + if (childMeterRegistry instanceof JmxMeterRegistry) { + return (JmxMeterRegistry) childMeterRegistry; + } + } + return null; } @Override public Counter counter(String metric, String... tags) { - io.micrometer.core.instrument.Counter innerCounter = - prometheusMeterRegistry.counter(metric, tags); + if (!isEnable) { + return null; + } + io.micrometer.core.instrument.Counter innerCounter = meterRegistry.counter(metric, tags); return (Counter) currentMeters.computeIfAbsent( innerCounter.getId(), key -> new MicrometerCounter(innerCounter)); @@ -62,88 +139,312 @@ public class MicrometerMetricManager implements MetricManager { @Override public void count(int delta, String metric, String... tags) { - io.micrometer.core.instrument.Counter innerCounter = - prometheusMeterRegistry.counter(metric, tags); + io.micrometer.core.instrument.Counter innerCounter = meterRegistry.counter(metric, tags); innerCounter.increment(delta); } @Override - public void count(long delta, String metric, String... tags) {} + public void count(long delta, String metric, String... tags) { + io.micrometer.core.instrument.Counter innerCounter = meterRegistry.counter(metric, tags); + innerCounter.increment(delta); + } @Override public Gauge gauge(String metric, String... tags) { - return null; + if (!isEnable) { + return null; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.GAUGE, tags); + return (Gauge) + currentMeters.computeIfAbsent(id, key -> new MicrometerGauge(meterRegistry, metric, tags)); } @Override public Histogram histogram(String metric, String... tags) { - return null; + if (!isEnable) { + return null; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.DISTRIBUTION_SUMMARY, tags); + return (Histogram) + currentMeters.computeIfAbsent( + id, + key -> { + io.micrometer.core.instrument.DistributionSummary distributionSummary = + io.micrometer.core.instrument.DistributionSummary.builder(metric) + .tags(tags) + .register(meterRegistry); + return new MicrometerHistogram(distributionSummary); + }); } + /** + * We only create a gauge(AtomicLong) to record the raw value, because we assume that the backend + * metrics system has the ability to calculate rate + * + * @param metric + * @param tags + * @return + */ @Override + @Deprecated public Rate rate(String metric, String... tags) { - return null; + if (!isEnable) { + return null; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.GAUGE, tags); + return (Rate) + currentMeters.computeIfAbsent( + id, + key -> + new MicrometerRate(meterRegistry.gauge(metric, Tags.of(tags), new AtomicLong(0)))); } @Override public Timer timer(String metric, String... tags) { - return null; + if (!isEnable) { + return null; + } + logger.info(metric + Arrays.toString(tags)); + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.TIMER, tags); + return (Timer) + currentMeters.computeIfAbsent( + id, + key -> { + io.micrometer.core.instrument.Timer timer = + io.micrometer.core.instrument.Timer.builder(metric) + .tags(tags) + .register(meterRegistry); + logger.info("create timer {}", metric); + return new MicrometerTimer(timer); + }); } @Override - public void histogram(int value, String metric, String... tags) {} - - @Override - public void histogram(long value, String metric, String... tags) {} - - @Override - public void gauge(int value, String metric, String... tags) {} + public void histogram(int value, String metric, String... tags) { + if (!isEnable) { + return; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.DISTRIBUTION_SUMMARY, tags); + ((Histogram) + currentMeters.computeIfAbsent( + id, + key -> { + io.micrometer.core.instrument.DistributionSummary distributionSummary = + io.micrometer.core.instrument.DistributionSummary.builder(metric) + .tags(tags) + .publishPercentileHistogram() + .publishPercentiles(0) + .register(meterRegistry); + return new MicrometerHistogram(distributionSummary); + })) + .update(value); + } @Override - public void gauge(long value, String metric, String... tags) {} + public void histogram(long value, String metric, String... tags) { + if (!isEnable) { + return; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.DISTRIBUTION_SUMMARY, tags); + ((Histogram) + currentMeters.computeIfAbsent( + id, + key -> { + io.micrometer.core.instrument.DistributionSummary distributionSummary = + io.micrometer.core.instrument.DistributionSummary.builder(metric) + .tags(tags) + .publishPercentileHistogram() + .publishPercentiles(0) + .register(meterRegistry); + return new MicrometerHistogram(distributionSummary); + })) + .update(value); + } @Override - public void meter(int value, String metric, String... tags) {} + public void gauge(int value, String metric, String... tags) { + if (!isEnable) { + return; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.GAUGE, tags); + ((Gauge) + (currentMeters.computeIfAbsent( + id, key -> new MicrometerGauge(meterRegistry, metric, tags)))) + .set(value); + } @Override - public void meter(long value, String metric, String... tags) {} + public void gauge(long value, String metric, String... tags) { + if (!isEnable) { + return; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.GAUGE, tags); + ((Gauge) + (currentMeters.computeIfAbsent( + id, key -> new MicrometerGauge(meterRegistry, metric, tags)))) + .set(value); + } @Override - public void timer(long delta, TimeUnit timeUnit, String metric, String... tags) {} + public void rate(int value, String metric, String... tags) { + if (!isEnable) { + return; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.GAUGE, tags); + ((Rate) + currentMeters.computeIfAbsent( + id, + key -> + new MicrometerRate( + meterRegistry.gauge(metric, Tags.of(tags), new AtomicLong(0))))) + .mark(value); + } @Override - public void timerStart(String metric, String... tags) {} + public void rate(long value, String metric, String... tags) { + if (!isEnable) { + return; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.GAUGE, tags); + ((Rate) + currentMeters.computeIfAbsent( + id, + key -> + new MicrometerRate( + meterRegistry.gauge(metric, Tags.of(tags), new AtomicLong(0))))) + .mark(value); + } @Override - public void timerEnd(String metric, String... tags) {} + public synchronized void timer(long delta, TimeUnit timeUnit, String metric, String... tags) { + if (!isEnable) { + return; + } + Meter.Id id = MeterIdUtils.fromMetricName(metric, Meter.Type.TIMER, tags); + ((Timer) + currentMeters.computeIfAbsent( + id, + key -> { + io.micrometer.core.instrument.Timer timer = + io.micrometer.core.instrument.Timer.builder(metric) + .tags(tags) + .register(meterRegistry); + return new MicrometerTimer(timer); + })) + .update(delta, timeUnit); + } @Override - public Map<String, String[]> getAllMetricKeys() { - return null; + public List<String[]> getAllMetricKeys() { + List<String[]> keys = new ArrayList<>(currentMeters.size()); + List<Meter> meterList = meterRegistry.getMeters(); + for (Meter meter : meterList) { + List<String> tags = new ArrayList<>(meter.getId().getTags().size() * 2 + 1); + tags.add(meter.getId().getName()); + for (Tag tag : meter.getId().getTags()) { + tags.add(tag.getKey()); + tags.add(tag.getValue()); + } + keys.add(tags.toArray(new String[0])); + } + return keys; } @Override public Map<String[], Counter> getAllCounters() { - return null; + Map<String[], IMetric> iMetricMap = getMetricByType(Meter.Type.COUNTER); + Map<String[], Counter> counterMap = new HashMap<>(); + iMetricMap.forEach((k, v) -> counterMap.put(k, (Counter) v)); + return counterMap; } @Override public Map<String[], Gauge> getAllGauges() { - return null; + Map<String[], IMetric> iMetricMap = getMetricByType(Meter.Type.GAUGE); + Map<String[], Gauge> gaugeMap = new HashMap<>(); + iMetricMap.forEach((k, v) -> gaugeMap.put(k, (Gauge) v)); + return gaugeMap; } @Override - public Map<String[], Rate> getAllMeters() { - return null; + public Map<String[], Rate> getAllRates() { + Map<String[], IMetric> iMetricMap = getMetricByType(Meter.Type.OTHER); + Map<String[], Rate> rateMap = new HashMap<>(); + iMetricMap.forEach((k, v) -> rateMap.put(k, (Rate) v)); + return rateMap; } @Override public Map<String[], Histogram> getAllHistograms() { - return null; + Map<String[], IMetric> iMetricMap = getMetricByType(Meter.Type.DISTRIBUTION_SUMMARY); + Map<String[], Histogram> histogramMap = new HashMap<>(); + iMetricMap.forEach((k, v) -> histogramMap.put(k, (Histogram) v)); + return histogramMap; } @Override public Map<String[], Timer> getAllTimers() { - return null; + Map<String[], IMetric> iMetricMap = getMetricByType(Meter.Type.TIMER); + Map<String[], Timer> timerMap = new HashMap<>(); + iMetricMap.forEach((k, v) -> timerMap.put(k, (Timer) v)); + return timerMap; + } + + private Map<String[], IMetric> getMetricByType(Meter.Type type) { + Map<String[], IMetric> iMetricMap = new HashMap<>(); + for (Map.Entry<Meter.Id, IMetric> entry : currentMeters.entrySet()) { + if (entry.getKey().getType() == type) { + List<String> tags = new ArrayList<>(entry.getKey().getTags().size() * 2); + tags.add(entry.getKey().getName()); + for (Tag tag : entry.getKey().getTags()) { + tags.add(tag.getKey()); + tags.add(tag.getValue()); + } + iMetricMap.put(tags.toArray(new String[0]), entry.getValue()); + } + } + return iMetricMap; + } + + @Override + public void enableKnownMetric(KnownMetric metric) { + if (!isEnable) { + return; + } + switch (metric) { + case JVM: + enableJVMMetrics(); + break; + case SYSTEM: + break; + case THREAD: + break; + default: + // ignore; + } + } + + private void enableJVMMetrics() { + if (!isEnable) { + return; + } + ClassLoaderMetrics classLoaderMetrics = new ClassLoaderMetrics(); + classLoaderMetrics.bindTo(meterRegistry); + JvmCompilationMetrics jvmCompilationMetrics = new JvmCompilationMetrics(); + jvmCompilationMetrics.bindTo(meterRegistry); + try (JvmGcMetrics jvmGcMetrics = new JvmGcMetrics(); + JvmHeapPressureMetrics jvmHeapPressureMetrics = new JvmHeapPressureMetrics()) { + jvmGcMetrics.bindTo(meterRegistry); + jvmHeapPressureMetrics.bindTo(meterRegistry); + } + JvmMemoryMetrics jvmMemoryMetrics = new JvmMemoryMetrics(); + jvmMemoryMetrics.bindTo(meterRegistry); + JvmThreadMetrics jvmThreadMetrics = new JvmThreadMetrics(); + jvmThreadMetrics.bindTo(meterRegistry); + } + + @Override + public boolean isEnable() { + return isEnable; } } diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricReporter.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricReporter.java index 518a4d0..3639dbe 100644 --- a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricReporter.java +++ b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricReporter.java @@ -19,32 +19,63 @@ package org.apache.iotdb.metrics.micrometer; -import org.apache.iotdb.metrics.MetricFactory; +import org.apache.iotdb.metrics.MetricManager; import org.apache.iotdb.metrics.MetricReporter; +import org.apache.iotdb.metrics.config.MetricConfig; +import org.apache.iotdb.metrics.config.MetricConfigDescriptor; +import org.apache.iotdb.metrics.utils.ReporterType; import com.sun.net.httpserver.HttpServer; +import io.micrometer.jmx.JmxMeterRegistry; import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.OutputStream; import java.net.InetSocketAddress; +import java.util.List; public class MicrometerMetricReporter implements MetricReporter { - MetricFactory micrometerMetricFactory; - Thread runThread; + private static final Logger logger = LoggerFactory.getLogger(MicrometerMetricReporter.class); + private MetricManager micrometerMetricManager; + private final MetricConfig metricConfig = MetricConfigDescriptor.getInstance().getMetricConfig(); + private Thread runThread; + + private JmxMeterRegistry jmxMeterRegistry; @Override public boolean start() { + List<String> reporters = metricConfig.getReporterList(); + for (String reporter : reporters) { + switch (ReporterType.get(reporter)) { + case JMX: + startJmxReporter( + ((MicrometerMetricManager) micrometerMetricManager).getJmxMeterRegistry()); + break; + case IOTDB: + break; + case PROMETHEUS: + startPrometheusReporter( + ((MicrometerMetricManager) micrometerMetricManager).getPrometheusMeterRegistry()); + break; + default: + logger.warn("Dropwizard don't support reporter type {}", reporter); + } + } + + return true; + } + + private void startPrometheusReporter(PrometheusMeterRegistry prometheusMeterRegistry) { try { - HttpServer server = HttpServer.create(new InetSocketAddress(8080), 0); + HttpServer server = + HttpServer.create( + new InetSocketAddress(Integer.parseInt(metricConfig.getPrometheusExporterPort())), 0); server.createContext( "/prometheus", httpExchange -> { - String response = - ((PrometheusMeterRegistry) - ((MicrometerMetricManager) micrometerMetricFactory.getMetric("iotdb")) - .getMeterRegistry()) - .scrape(); + String response = prometheusMeterRegistry.scrape(); httpExchange.sendResponseHeaders(200, response.getBytes().length); try (OutputStream os = httpExchange.getResponseBody()) { os.write(response.getBytes()); @@ -56,12 +87,16 @@ public class MicrometerMetricReporter implements MetricReporter { } catch (IOException e) { throw new RuntimeException(e); } - return true; + } + + private void startJmxReporter(JmxMeterRegistry jmxMeterRegistry) { + logger.debug("start jmx reporter from micrometer"); + jmxMeterRegistry.start(); } @Override - public void setMetricFactory(MetricFactory metricFactory) { - micrometerMetricFactory = metricFactory; + public void setMetricManager(MetricManager metricManager) { + micrometerMetricManager = metricManager; } @Override @@ -69,8 +104,10 @@ public class MicrometerMetricReporter implements MetricReporter { try { runThread.join(); } catch (InterruptedException e) { - e.printStackTrace(); + logger.warn("Failed to stop prometheus reporter", e); } + + ((MicrometerMetricManager) micrometerMetricManager).getJmxMeterRegistry().stop(); return true; } } diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java index 9fbfa6f..cdeb40a 100644 --- a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java +++ b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java @@ -22,13 +22,12 @@ package org.apache.iotdb.metrics.micrometer.type; import org.apache.iotdb.metrics.type.Counter; public class MicrometerCounter implements Counter { + io.micrometer.core.instrument.Counter counter; public MicrometerCounter(io.micrometer.core.instrument.Counter counter) { this.counter = counter; } - io.micrometer.core.instrument.Counter counter; - @Override public void inc() { counter.increment(); diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/impl/DoNothingFactory.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerGauge.java similarity index 56% rename from metrics/interface/src/main/java/org/apache/iotdb/metrics/impl/DoNothingFactory.java rename to metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerGauge.java index bce4ec8..e2c80b9 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/impl/DoNothingFactory.java +++ b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerGauge.java @@ -16,33 +16,32 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.metrics.impl; -import org.apache.iotdb.metrics.KnownMetric; -import org.apache.iotdb.metrics.MetricFactory; -import org.apache.iotdb.metrics.MetricManager; +package org.apache.iotdb.metrics.micrometer.type; -import java.util.Collections; -import java.util.Map; +import org.apache.iotdb.metrics.type.Gauge; -public class DoNothingFactory implements MetricFactory { - private DoNothingMetricManager metric = new DoNothingMetricManager(); +import io.micrometer.core.instrument.Tags; - @Override - public MetricManager getMetric(String namespace) { - return metric; - } +import java.util.concurrent.atomic.AtomicLong; - @Override - public void enableKnownMetric(KnownMetric metric) {} +public class MicrometerGauge implements Gauge { + private final AtomicLong atomicLong; + + public MicrometerGauge( + io.micrometer.core.instrument.MeterRegistry meterRegistry, + String metricName, + String... tags) { + atomicLong = meterRegistry.gauge(metricName, Tags.of(tags), new AtomicLong(0)); + } @Override - public Map<String, MetricManager> getAllMetrics() { - return Collections.emptyMap(); + public long value() { + return atomicLong.get(); } @Override - public boolean isEnable() { - return true; + public void set(long value) { + atomicLong.set(value); } } diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerHistogram.java similarity index 57% copy from metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java copy to metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerHistogram.java index 9fbfa6f..f843fa1 100644 --- a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java +++ b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerHistogram.java @@ -19,28 +19,34 @@ package org.apache.iotdb.metrics.micrometer.type; -import org.apache.iotdb.metrics.type.Counter; +import org.apache.iotdb.metrics.type.Histogram; -public class MicrometerCounter implements Counter { +public class MicrometerHistogram implements Histogram { - public MicrometerCounter(io.micrometer.core.instrument.Counter counter) { - this.counter = counter; - } + io.micrometer.core.instrument.DistributionSummary distributionSummary; - io.micrometer.core.instrument.Counter counter; + public MicrometerHistogram( + io.micrometer.core.instrument.DistributionSummary distributionSummary) { + this.distributionSummary = distributionSummary; + } @Override - public void inc() { - counter.increment(); + public void update(int value) { + distributionSummary.record(value); } @Override - public void inc(long n) { - counter.increment(n); + public void update(long value) { + distributionSummary.record(value); } @Override public long count() { - return (long) counter.count(); + return distributionSummary.count(); + } + + @Override + public org.apache.iotdb.metrics.type.HistogramSnapshot takeSnapshot() { + return new MicrometerHistogramSnapshot(distributionSummary.takeSnapshot()); } } diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerHistogramSnapshot.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerHistogramSnapshot.java new file mode 100644 index 0000000..3c3aedb --- /dev/null +++ b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerHistogramSnapshot.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.micrometer.type; + +import org.apache.iotdb.metrics.type.HistogramSnapshot; + +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Arrays; + +public class MicrometerHistogramSnapshot implements HistogramSnapshot { + + io.micrometer.core.instrument.distribution.HistogramSnapshot histogramSnapshot; + + public MicrometerHistogramSnapshot( + io.micrometer.core.instrument.distribution.HistogramSnapshot histogramSnapshot) { + this.histogramSnapshot = histogramSnapshot; + } + + @Override + public double getValue(double quantile) { + int prevIndex = 0; + int i = 0; + for (i = 0; i < this.histogramSnapshot.percentileValues().length; i++) { + if (this.histogramSnapshot.percentileValues()[i].percentile() == quantile) { + prevIndex = i; + break; + } else if (this.histogramSnapshot.percentileValues()[i].percentile() > quantile) { + break; + } + prevIndex = i; + } + + return this.histogramSnapshot.percentileValues()[prevIndex].value(); + } + + @Override + public long[] getValues() { + return Arrays.stream(this.histogramSnapshot.percentileValues()) + .mapToLong(k -> (long) k.value()) + .toArray(); + } + + @Override + public int size() { + return this.histogramSnapshot.percentileValues().length; + } + + @Override + public double getMedian() { + return getValue(0.5); + } + + @Override + public long getMax() { + return (long) this.histogramSnapshot.max(); + } + + @Override + public double getMean() { + return this.histogramSnapshot.mean(); + } + + @Override + public long getMin() { + // need distributionSummary to push 0 percentiles + return (long) getValue(0.0); + } + + @Override + public void dump(OutputStream output) { + this.histogramSnapshot.outputSummary((PrintStream) output, 100); + } +} diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerRate.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerRate.java new file mode 100644 index 0000000..0789f1c --- /dev/null +++ b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerRate.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metrics.micrometer.type; + +import org.apache.iotdb.metrics.type.Rate; + +import com.codahale.metrics.Meter; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * could not publish to other metrics system exclude jmx and csv, because micrometer assumes that + * other metrics system have the ability to calculate rate. Details is at + * https://github.com/micrometer-metrics/micrometer/issues/1935. + * + * <p>Now, we only record a gauge for the rate record in micrometer, and we use dropwizard meter to + * calculate the meter. + */ +public class MicrometerRate implements Rate { + AtomicLong atomicLong; + Meter meter; + + public MicrometerRate(AtomicLong atomicLong) { + this.atomicLong = atomicLong; + meter = new Meter(); + } + + @Override + public long getCount() { + return meter.getCount(); + } + + @Override + public double getOneMinuteRate() { + return meter.getOneMinuteRate(); + } + + @Override + public double getMeanRate() { + return meter.getMeanRate(); + } + + @Override + public double getFiveMinuteRate() { + return meter.getFiveMinuteRate(); + } + + @Override + public double getFifteenMinuteRate() { + return meter.getFifteenMinuteRate(); + } + + @Override + public void mark() { + atomicLong.set(1); + meter.mark(); + } + + @Override + public void mark(long n) { + atomicLong.set(n); + meter.mark(n); + } +} diff --git a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerTimer.java similarity index 53% copy from metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java copy to metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerTimer.java index 9fbfa6f..07c68e4 100644 --- a/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java +++ b/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerTimer.java @@ -19,28 +19,36 @@ package org.apache.iotdb.metrics.micrometer.type; -import org.apache.iotdb.metrics.type.Counter; +import org.apache.iotdb.metrics.type.HistogramSnapshot; +import org.apache.iotdb.metrics.type.Rate; +import org.apache.iotdb.metrics.type.Timer; -public class MicrometerCounter implements Counter { +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; - public MicrometerCounter(io.micrometer.core.instrument.Counter counter) { - this.counter = counter; - } +public class MicrometerTimer implements Timer { + + io.micrometer.core.instrument.Timer timer; + MicrometerRate micrometerRate; - io.micrometer.core.instrument.Counter counter; + public MicrometerTimer(io.micrometer.core.instrument.Timer timer) { + this.timer = timer; + micrometerRate = new MicrometerRate(new AtomicLong(0)); + } @Override - public void inc() { - counter.increment(); + public void update(long duration, TimeUnit unit) { + timer.record(duration, unit); + micrometerRate.mark(duration); } @Override - public void inc(long n) { - counter.increment(n); + public HistogramSnapshot takeSnapshot() { + return new MicrometerHistogramSnapshot(timer.takeSnapshot()); } @Override - public long count() { - return (long) counter.count(); + public Rate getImmutableRate() { + return micrometerRate; } } diff --git a/metrics/micrometer-metrics/src/main/resources/META-INF.services/org.apache.iotdb.metrics.micrometer.MicrometerMetricFactory b/metrics/micrometer-metrics/src/main/resources/META-INF.services/org.apache.iotdb.metrics.micrometer.MicrometerMetricFactory deleted file mode 100644 index 6860085..0000000 --- a/metrics/micrometer-metrics/src/main/resources/META-INF.services/org.apache.iotdb.metrics.micrometer.MicrometerMetricFactory +++ /dev/null @@ -1 +0,0 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by a [...] \ No newline at end of file diff --git a/metrics/micrometer-metrics/src/main/resources/META-INF.services/org.apache.iotdb.metrics.micrometer.MicrometerMetricReporter b/metrics/micrometer-metrics/src/main/resources/META-INF.services/org.apache.iotdb.metrics.micrometer.MicrometerMetricReporter deleted file mode 100644 index 3138baf..0000000 --- a/metrics/micrometer-metrics/src/main/resources/META-INF.services/org.apache.iotdb.metrics.micrometer.MicrometerMetricReporter +++ /dev/null @@ -1 +0,0 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by a [...] \ No newline at end of file diff --git a/metrics/micrometer-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricManager b/metrics/micrometer-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricManager new file mode 100644 index 0000000..235ab43 --- /dev/null +++ b/metrics/micrometer-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricManager @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.iotdb.metrics.micrometer.MicrometerMetricManager \ No newline at end of file diff --git a/metrics/micrometer-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricReporter b/metrics/micrometer-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricReporter new file mode 100644 index 0000000..4a14a74 --- /dev/null +++ b/metrics/micrometer-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.MetricReporter @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.iotdb.metrics.micrometer.MicrometerMetricReporter \ No newline at end of file diff --git a/metrics/pom.xml b/metrics/pom.xml index b7b9b10..6666aeb 100644 --- a/metrics/pom.xml +++ b/metrics/pom.xml @@ -35,6 +35,6 @@ <modules> <module>interface</module> <module>micrometer-metrics</module> + <module>dropwizard-metrics</module> </modules> - </project> diff --git a/server/pom.xml b/server/pom.xml index d1ce0ef..fb23f01 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -196,6 +196,16 @@ <artifactId>netty-buffer</artifactId> <version>4.1.27.Final</version> </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>metrics-interface</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>micrometer-metrics</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> <build> <plugins> diff --git a/server/src/assembly/resources/conf/iotdb-metric.properties b/server/src/assembly/resources/conf/iotdb-metric.properties new file mode 100644 index 0000000..4fbc6bb --- /dev/null +++ b/server/src/assembly/resources/conf/iotdb-metric.properties @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +enable_metric=true +metric_reporter_list=jmx +push_period_in_second=5 +prometheus_exporter_port=8090 +iotdb_ip=127.0.0.1 +iotdb_port=6667 +iotdb_sg=iotdb +iotdb_user=root +iotdb_passwd=root + + diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java index 8d97690..9625efa 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java @@ -66,6 +66,7 @@ import org.apache.iotdb.db.utils.MmapUtil; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.db.utils.UpgradeUtils; import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer; +import org.apache.iotdb.metrics.type.Timer; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TSStatus; @@ -273,6 +274,10 @@ public class StorageGroupProcessor { // DEFAULT_POOL_TRIM_INTERVAL_MILLIS private long timeWhenPoolNotEmpty = Long.MAX_VALUE; + private Timer write_total_timer; + + private Timer read_total_timer; + /** get the direct byte buffer from pool, each fetch contains two ByteBuffer */ public ByteBuffer[] getWalDirectByteBuffer() { ByteBuffer[] res = new ByteBuffer[2]; @@ -384,6 +389,24 @@ public class StorageGroupProcessor { DEFAULT_POOL_TRIM_INTERVAL_MILLIS, TimeUnit.MILLISECONDS); recover(); + read_total_timer = + IoTDB.serverMetricManager.timer( + "read_latency", + "sg", + logicalStorageGroupName, + "user", + "total", + "host", + config.getRpcAddress()); + write_total_timer = + IoTDB.serverMetricManager.timer( + "write_latency", + "sg", + logicalStorageGroupName, + "user", + "total", + "host", + config.getRpcAddress()); } public String getLogicalStorageGroupName() { @@ -783,6 +806,7 @@ public class StorageGroupProcessor { if (!isAlive(insertRowPlan.getTime())) { throw new OutOfTTLException(insertRowPlan.getTime(), (System.currentTimeMillis() - dataTTL)); } + long startTime = System.currentTimeMillis(); if (enableMemControl) { StorageEngine.blockInsertionIfReject(); } @@ -813,6 +837,18 @@ public class StorageGroupProcessor { } finally { writeUnlock(); } + long end = System.currentTimeMillis(); + logger.info("timer insert cost {} millis", end - startTime); + IoTDB.serverMetricManager.timer( + end - startTime, + TimeUnit.MILLISECONDS, + "insert_row_latency", + "sg", + logicalStorageGroupName, + "user", + insertRowPlan.getLoginUserName(), + "host", + config.getRpcAddress()); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java index c35ae48..f55532d 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.auth.authorizer.IAuthorizer; import org.apache.iotdb.db.auth.entity.PathPrivilege; import org.apache.iotdb.db.auth.entity.Role; import org.apache.iotdb.db.auth.entity.User; +import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngine; @@ -147,6 +148,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CANCELLED; import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_CHILD_PATHS; @@ -185,6 +187,7 @@ public class PlanExecutor implements IPlanExecutor { private IAuthorizer authorizer; private static final String INSERT_MEASUREMENTS_FAILED_MESSAGE = "failed to insert measurements "; + private IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); public PlanExecutor() throws QueryProcessException { queryRouter = new QueryRouter(); @@ -1099,6 +1102,7 @@ public class PlanExecutor implements IPlanExecutor { @Override public void insert(InsertRowPlan insertRowPlan) throws QueryProcessException { + long startTime = System.currentTimeMillis(); try { insertRowPlan.setMeasurementMNodes( new MeasurementMNode[insertRowPlan.getMeasurements().length]); @@ -1109,6 +1113,16 @@ public class PlanExecutor implements IPlanExecutor { if (insertRowPlan.getFailedMeasurements() != null) { checkFailedMeasurments(insertRowPlan); } + IoTDB.serverMetricManager.timer( + System.currentTimeMillis() - startTime, + TimeUnit.MILLISECONDS, + "insert_latency", + "sg", + "root", // TODO infer from insertRowPlan.getDeviceId() + "user", + insertRowPlan.getLoginUserName(), + "host", + config.getRpcAddress()); } catch (StorageEngineException | MetadataException e) { if (IoTDBDescriptor.getInstance().getConfig().isEnableStatMonitor()) { StatMonitor.getInstance().updateFailedStatValue(); @@ -1135,6 +1149,7 @@ public class PlanExecutor implements IPlanExecutor { @Override public void insertTablet(InsertTabletPlan insertTabletPlan) throws QueryProcessException { + long startTime = System.currentTimeMillis(); try { insertTabletPlan.setMeasurementMNodes( new MeasurementMNode[insertTabletPlan.getMeasurements().length]); @@ -1143,6 +1158,16 @@ public class PlanExecutor implements IPlanExecutor { if (insertTabletPlan.getFailedMeasurements() != null) { checkFailedMeasurments(insertTabletPlan); } + IoTDB.serverMetricManager.timer( + System.currentTimeMillis() - startTime, + TimeUnit.MILLISECONDS, + "insert_tablet_latency", + "sg", + "root", // TODO infer from insertTabletPlan.getDeviceId() + "user", + insertTabletPlan.getLoginUserName(), + "host", + config.getRpcAddress()); } catch (StorageEngineException | MetadataException e) { if (IoTDBDescriptor.getInstance().getConfig().isEnableStatMonitor()) { StatMonitor.getInstance().updateFailedStatValue(); diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index b3442a3..7ad371a 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@ -53,14 +53,14 @@ public class IoTDB implements IoTDBMBean { private static final Logger logger = LoggerFactory.getLogger(IoTDB.class); private final String mbeanName = String.format("%s:%s=%s", IoTDBConstant.IOTDB_PACKAGE, IoTDBConstant.JMX_TYPE, "IoTDB"); - private RegisterManager registerManager = new RegisterManager(); + private final RegisterManager registerManager = new RegisterManager(); public static MManager metaManager = MManager.getInstance(); public static IoTDB getInstance() { return IoTDBHolder.INSTANCE; } - public static MetricManager serverMetricManager = MetricService.getMetric("iotdb"); + public static final MetricManager serverMetricManager = MetricService.getMetricManager(); public static void main(String[] args) { if (args.length > 0) { @@ -153,6 +153,7 @@ public class IoTDB implements IoTDBMBean { logger.info("Deactivating IoTDB..."); registerManager.deregisterAll(); JMXService.deregisterMBean(mbeanName); + MetricService.stop(); logger.info("IoTDB is deactivated."); } diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index e4fd01c..3a02261 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@ -84,7 +84,6 @@ import org.apache.iotdb.db.tools.watermark.WatermarkEncoder; import org.apache.iotdb.db.utils.FilePathUtils; import org.apache.iotdb.db.utils.QueryDataSetUtils; import org.apache.iotdb.db.utils.SchemaUtils; -import org.apache.iotdb.metrics.type.Counter; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.ServerProperties; @@ -212,8 +211,6 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { private QueryTimeManager queryTimeManager = QueryTimeManager.getInstance(); - Counter counter = IoTDB.serverMetricManager.counter("request_total", "user", "root"); - public TSServiceImpl() throws QueryProcessException { processor = new Planner(); executor = new PlanExecutor(); @@ -502,6 +499,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { processor.parseSQLToPhysicalPlan( statement, sessionIdZoneIdMap.get(req.getSessionId()), req.fetchSize); + physicalPlan.setLoginUserName(sessionIdUsernameMap.get(req.getSessionId())); + return physicalPlan.isQuery() ? internalExecuteQueryStatement( statement, @@ -528,6 +527,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { processor.parseSQLToPhysicalPlan( statement, sessionIdZoneIdMap.get(req.getSessionId()), req.fetchSize); + physicalPlan.setLoginUserName(sessionIdUsernameMap.get(req.getSessionId())); + return physicalPlan.isQuery() ? internalExecuteQueryStatement( statement, @@ -553,6 +554,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { PhysicalPlan physicalPlan = processor.rawDataQueryReqToPhysicalPlan(req, sessionIdZoneIdMap.get(req.getSessionId())); + physicalPlan.setLoginUserName(sessionIdUsernameMap.get(req.getSessionId())); return physicalPlan.isQuery() ? internalExecuteQueryStatement( "", @@ -697,6 +699,25 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { if (!(plan instanceof ShowQueryProcesslistPlan)) { queryTimeManager.unRegisterQuery(queryId); } + + LOGGER.info( + "{}, {}, {}", + plan.getOperatorType().name(), + plan.getLoginUserName(), + config.getRpcAddress()); + + IoTDB.serverMetricManager.timer( + System.currentTimeMillis() - startTime, + TimeUnit.MILLISECONDS, + "query_latency", + "sg", + "root", + "query_type", + plan.getOperatorType().name(), + "user", + plan.getLoginUserName(), + "host", + config.getRpcAddress()); return resp; } catch (Exception e) { releaseQueryResourceNoExceptions(queryId); @@ -1148,7 +1169,6 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { req.deviceIds.get(0), req.getTimestamps().get(0)); } - counter.inc(); List<TSStatus> statusList = new ArrayList<>(); @@ -1176,6 +1196,18 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } } + long startTime = System.currentTimeMillis(); + IoTDB.serverMetricManager.timer( + System.currentTimeMillis() - startTime, + TimeUnit.MILLISECONDS, + "insert_records_latency", + "sg", + "root", + "user", + sessionIdUsernameMap.get(req.getSessionId()), + "host", + config.getRpcAddress()); + return isAllSuccessful ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) : RpcUtils.getStatus(statusList); @@ -1657,6 +1689,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { private TSStatus checkAuthority(PhysicalPlan plan, long sessionId) { List<PartialPath> paths = plan.getPaths(); + plan.setLoginUserName(sessionIdUsernameMap.get(sessionId)); try { if (!checkAuthorization(paths, plan, sessionIdUsernameMap.get(sessionId))) { return RpcUtils.getStatus(
