This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new e9dd92d13b6 Optimize Prometheus reporter snapshot updates (#18562)
e9dd92d13b6 is described below
commit e9dd92d13b65f417890d352cda4e8294f55c875d
Author: Jiang Tian <[email protected]>
AuthorDate: Mon Sep 7 09:45:25 2026 +0800
Optimize Prometheus reporter snapshot updates (#18562)
* Optimize Prometheus reporter snapshot updates
* fix compilation
* Fix Prometheus reporter initial snapshot race
* Fix Prometheus scheduler lifecycle and duplicate allocation
---
iotdb-core/metrics/ReadMe.md | 1 +
.../apache/iotdb/metrics/i18n/MetricsMessages.java | 1 +
.../apache/iotdb/metrics/i18n/MetricsMessages.java | 1 +
.../apache/iotdb/metrics/config/MetricConfig.java | 14 +++
.../metrics/config/MetricConfigDescriptor.java | 21 ++++
.../reporter/prometheus/PrometheusReporter.java | 120 ++++++++++++++++++++-
.../iotdb/metrics/config/MetricConfigTest.java | 16 +++
.../prometheus/PrometheusReporterTest.java | 75 +++++++++++++
.../conf/iotdb-system.properties.template | 6 ++
.../iotdb/commons/concurrent/ThreadName.java | 2 +
.../commons/service/metric/MetricService.java | 15 ++-
11 files changed, 267 insertions(+), 5 deletions(-)
diff --git a/iotdb-core/metrics/ReadMe.md b/iotdb-core/metrics/ReadMe.md
index f0eb361f0a2..6ab575abe09 100644
--- a/iotdb-core/metrics/ReadMe.md
+++ b/iotdb-core/metrics/ReadMe.md
@@ -63,6 +63,7 @@ Configure the metrics module through
`iotdb-system.properties`. The main options
| `dn(cn)_metric_level` | Initial metric level. | `OFF`, `CORE`, `IMPORTANT`,
`NORMAL`, `ALL` |
| `cn_metric_prometheus_reporter_port` | Prometheus HTTP port for ConfigNode.
| `9091` |
| `dn_metric_prometheus_reporter_port` | Prometheus HTTP port for DataNode. |
`9092` |
+| `prometheus_reporter_async_update` | Serve a cached Prometheus snapshot
refreshed every 15 seconds. | `true` |
More details, see the User Guide and the `iotdb-system.properties.template`
file.
diff --git
a/iotdb-core/metrics/interface/src/main/i18n/en/org/apache/iotdb/metrics/i18n/MetricsMessages.java
b/iotdb-core/metrics/interface/src/main/i18n/en/org/apache/iotdb/metrics/i18n/MetricsMessages.java
index 778da2b32bf..4cd7a7fd0a1 100644
---
a/iotdb-core/metrics/interface/src/main/i18n/en/org/apache/iotdb/metrics/i18n/MetricsMessages.java
+++
b/iotdb-core/metrics/interface/src/main/i18n/en/org/apache/iotdb/metrics/i18n/MetricsMessages.java
@@ -128,5 +128,6 @@ public final class MetricsMessages {
public static final String
LOG_IOTDBSESSIONREPORTER_START_WRITE_ARG_ARG_E79CDDAE = "IoTDBSessionReporter
start, write to {}:{}";
public static final String
LOG_PROMETHEUSREPORTER_STARTED_USE_PORT_ARG_A688FFC8 = "PrometheusReporter
started, use port {}";
public static final String
LOG_DETECTED_ERROR_TAKING_METRIC_TIMER_SNAPSHOT_WILL_DISCARD_METRIC_B7154169 =
"Detected an error when taking metric timer snapshot, will discard this metric";
+ public static final String
LOG_PROMETHEUSREPORTER_FAILED_TO_UPDATE_METRICS_SNAPSHOT_ASYNCHRONOUSLY_F19FE4E3
= "PrometheusReporter failed to update metrics snapshot asynchronously";
}
diff --git
a/iotdb-core/metrics/interface/src/main/i18n/zh/org/apache/iotdb/metrics/i18n/MetricsMessages.java
b/iotdb-core/metrics/interface/src/main/i18n/zh/org/apache/iotdb/metrics/i18n/MetricsMessages.java
index fd68351a939..968694f6eb0 100644
---
a/iotdb-core/metrics/interface/src/main/i18n/zh/org/apache/iotdb/metrics/i18n/MetricsMessages.java
+++
b/iotdb-core/metrics/interface/src/main/i18n/zh/org/apache/iotdb/metrics/i18n/MetricsMessages.java
@@ -124,5 +124,6 @@ public final class MetricsMessages {
public static final String
LOG_IOTDBSESSIONREPORTER_START_WRITE_ARG_ARG_E79CDDAE = "IoTDBSessionReporter
启动，写入 {}:{}";
public static final String
LOG_PROMETHEUSREPORTER_STARTED_USE_PORT_ARG_A688FFC8 = "PrometheusReporter
已启动，使用端口 {}";
public static final String
LOG_DETECTED_ERROR_TAKING_METRIC_TIMER_SNAPSHOT_WILL_DISCARD_METRIC_B7154169 =
"获取 metric timer 快照时检测到错误，将丢弃该 metric";
+ public static final String
LOG_PROMETHEUSREPORTER_FAILED_TO_UPDATE_METRICS_SNAPSHOT_ASYNCHRONOUSLY_F19FE4E3
= "PrometheusReporter 异步更新监控项快照失败";
}
diff --git
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java
index 2e0eaf5cb0b..c5d55162f8c 100644
---
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java
+++
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java
@@ -52,6 +52,9 @@ public class MetricConfig {
/** The export port for prometheus to get metrics. */
private Integer prometheusReporterPort = 9091;
+ /** Whether Prometheus metrics are collected asynchronously into a cached
snapshot. */
+ private boolean prometheusReporterAsyncUpdate = true;
+
private String prometheusReporterUsername = "";
private String prometheusReporterPassword = "";
@@ -140,6 +143,14 @@ public class MetricConfig {
this.prometheusReporterPort = prometheusReporterPort;
}
+ public boolean isPrometheusReporterAsyncUpdate() {
+ return prometheusReporterAsyncUpdate;
+ }
+
+ public void setPrometheusReporterAsyncUpdate(boolean
prometheusReporterAsyncUpdate) {
+ this.prometheusReporterAsyncUpdate = prometheusReporterAsyncUpdate;
+ }
+
public boolean prometheusNeedAuth() {
return prometheusReporterUsername != null &&
!prometheusReporterUsername.isEmpty();
}
@@ -264,6 +275,7 @@ public class MetricConfig {
metricLevel = newMetricConfig.getMetricLevel();
asyncCollectPeriodInSecond =
newMetricConfig.getAsyncCollectPeriodInSecond();
prometheusReporterPort = newMetricConfig.getPrometheusReporterPort();
+ prometheusReporterAsyncUpdate =
newMetricConfig.isPrometheusReporterAsyncUpdate();
prometheusReporterUsername =
newMetricConfig.getPrometheusReporterUsername();
prometheusReporterPassword =
newMetricConfig.getPrometheusReporterPassword();
internalReporterType = newMetricConfig.getInternalReportType();
@@ -287,6 +299,7 @@ public class MetricConfig {
&& metricLevel.equals(anotherMetricConfig.getMetricLevel())
&&
asyncCollectPeriodInSecond.equals(anotherMetricConfig.getAsyncCollectPeriodInSecond())
&&
prometheusReporterPort.equals(anotherMetricConfig.getPrometheusReporterPort())
+ && prometheusReporterAsyncUpdate ==
anotherMetricConfig.isPrometheusReporterAsyncUpdate()
&&
iotdbReporterConfig.equals(anotherMetricConfig.getIoTDBReporterConfig())
&&
internalReporterType.equals(anotherMetricConfig.getInternalReportType());
}
@@ -298,6 +311,7 @@ public class MetricConfig {
metricLevel,
asyncCollectPeriodInSecond,
prometheusReporterPort,
+ prometheusReporterAsyncUpdate,
iotdbReporterConfig,
internalReporterType);
}
diff --git
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java
index 0d83d387409..113bff7ebcc 100644
---
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java
+++
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java
@@ -129,6 +129,11 @@ public class MetricConfigDescriptor {
properties,
prefix)));
+ loadConfig.setPrometheusReporterAsyncUpdate(
+ Boolean.parseBoolean(
+ getPrometheusReporterAsyncUpdateProperty(
+ properties, prefix,
loadConfig.isPrometheusReporterAsyncUpdate())));
+
loadConfig.setPrometheusReporterUsername(
getPropertyWithoutPrefix(
"metric_prometheus_reporter_username",
@@ -222,6 +227,22 @@ public class MetricConfigDescriptor {
.orElse(defaultValue);
}
+ private String getPrometheusReporterAsyncUpdateProperty(
+ Properties properties, String prefix, boolean defaultValue) {
+ String value = properties.getProperty("prometheus_reporter_async_update");
+ if (value == null) {
+ // Keep accepting the metric-prefixed forms for compatibility with
node-specific configs.
+ value =
properties.getProperty("metric_prometheus_reporter_async_update");
+ }
+ if (value == null) {
+ value = properties.getProperty(prefix +
"prometheus_reporter_async_update");
+ }
+ if (value == null) {
+ value = properties.getProperty(prefix +
"metric_prometheus_reporter_async_update");
+ }
+ return value == null ? String.valueOf(defaultValue) : value.trim();
+ }
+
private static class MetricConfigDescriptorHolder {
private static final MetricConfigDescriptor INSTANCE = new
MetricConfigDescriptor();
}
diff --git
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/reporter/prometheus/PrometheusReporter.java
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/reporter/prometheus/PrometheusReporter.java
index 7846e056f8c..00ae6c6b988 100644
---
a/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/reporter/prometheus/PrometheusReporter.java
+++
b/iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/reporter/prometheus/PrometheusReporter.java
@@ -66,29 +66,68 @@ import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
public class PrometheusReporter implements Reporter {
private static final Logger LOGGER =
LoggerFactory.getLogger(PrometheusReporter.class);
private static final MetricConfig METRIC_CONFIG =
MetricConfigDescriptor.getInstance().getMetricConfig();
+ private static final long PROMETHEUS_DEFAULT_SCRAPE_INTERVAL_SECONDS = 15;
private final AbstractMetricManager metricManager;
- private DisposableServer httpServer;
+ private final Supplier<ScheduledExecutorService>
snapshotUpdateExecutorSupplier;
+ private volatile ScheduledExecutorService snapshotUpdateExecutor;
+ private volatile DisposableServer httpServer;
+
+ /** A null snapshot means that no complete scrape has been published yet. */
+ private volatile String metricsSnapshot;
+
+ private volatile ScheduledFuture<?> snapshotUpdateFuture;
private static final String REALM = "metrics";
public static final String BASIC_AUTH_PREFIX = "Basic ";
public static final char DIVIDER_BETWEEN_USERNAME_AND_DIVIDER = ':';
+ /**
+ * Creates a reporter with a self-managed scheduler for compatibility with
standalone users.
+ * Server-side code should use the constructor accepting a scheduler factory.
+ */
public PrometheusReporter(AbstractMetricManager metricManager) {
+ this(metricManager,
PrometheusReporter::newStandaloneSnapshotUpdateExecutor);
+ }
+
+ /**
+ * Creates a reporter with a scheduler factory. The factory is invoked on
every start to obtain a
+ * fresh executor for the reporter lifecycle.
+ */
+ public PrometheusReporter(
+ AbstractMetricManager metricManager,
+ Supplier<ScheduledExecutorService> snapshotUpdateExecutorSupplier) {
this.metricManager = metricManager;
+ this.snapshotUpdateExecutorSupplier =
Objects.requireNonNull(snapshotUpdateExecutorSupplier);
+ }
+
+ private static ScheduledExecutorService
newStandaloneSnapshotUpdateExecutor() {
+ return Executors.newSingleThreadScheduledExecutor(
+ runnable -> {
+ Thread thread = new Thread(runnable,
"prometheus-reporter-snapshot-updater");
+ thread.setDaemon(true);
+ return thread;
+ });
}
@Override
@SuppressWarnings("java:S1181")
- public boolean start() {
+ public synchronized boolean start() {
if (httpServer != null) {
LOGGER.warn(MetricsMessages.PROMETHEUS_REPORTER_ALREADY_START);
return false;
}
+ // A reporter can be started again after its metric manager has been reset.
+ metricsSnapshot = null;
try {
HttpServer serverTransport =
HttpServer.create()
@@ -104,8 +143,12 @@ public class PrometheusReporter implements Reporter {
// authenticate not pass
return Mono.empty();
}
+ String metrics =
+ METRIC_CONFIG.isPrometheusReporterAsyncUpdate()
+ ? getMetricsSnapshot()
+ : scrape();
return res.header(HttpHeaderNames.CONTENT_TYPE,
"text/plain")
- .sendString(Mono.just(scrape()));
+ .sendString(Mono.just(metrics));
}));
if (METRIC_CONFIG.isEnableSSL()) {
SslContext sslContext;
@@ -122,9 +165,20 @@ public class PrometheusReporter implements Reporter {
serverTransport = serverTransport.secure(spec ->
spec.sslContext(sslContext));
}
httpServer = serverTransport.bindNow();
+ if (METRIC_CONFIG.isPrometheusReporterAsyncUpdate()) {
+ startSnapshotUpdater();
+ }
} catch (Throwable e) {
// catch Throwable rather than Exception here because the code above
might cause a
// NoClassDefFoundError
+ stopSnapshotUpdater();
+ if (httpServer != null) {
+ try {
+ httpServer.disposeNow(Duration.ofSeconds(10));
+ } catch (Exception ignored) {
+ // do nothing
+ }
+ }
httpServer = null;
LOGGER.warn(MetricsMessages.PROMETHEUS_REPORTER_START_FAILED, e);
return false;
@@ -135,6 +189,63 @@ public class PrometheusReporter implements Reporter {
return true;
}
+ @SuppressWarnings("unsafeThreadSchedule")
+ private void startSnapshotUpdater() {
+ // Keep metric collection off Reactor HTTP threads and avoid overlapping
scrapes.
+ if (snapshotUpdateExecutor == null || snapshotUpdateExecutor.isShutdown())
{
+ // Create a fresh executor for every start so a stopped reporter can be
started again with
+ // the same managed thread-pool factory.
+ snapshotUpdateExecutor =
Objects.requireNonNull(snapshotUpdateExecutorSupplier.get());
+ }
+ // Delay the first background scrape until metric sets have been bound by
the metric service.
+ snapshotUpdateFuture =
+ snapshotUpdateExecutor.scheduleAtFixedRate(
+ this::updateSnapshot,
+ PROMETHEUS_DEFAULT_SCRAPE_INTERVAL_SECONDS,
+ PROMETHEUS_DEFAULT_SCRAPE_INTERVAL_SECONDS,
+ TimeUnit.SECONDS);
+ }
+
+ private void updateSnapshot() {
+ try {
+ String snapshot = scrape();
+ // Do not publish an empty scrape taken before metric sets are bound.
The request path will
+ // synchronously scrape until the first complete snapshot is available.
Empty snapshots are
+ // published after initialization so removed metrics are not kept in the
cache indefinitely.
+ if (!snapshot.isEmpty() || metricsSnapshot != null) {
+ metricsSnapshot = snapshot;
+ }
+ } catch (Throwable t) {
+ LOGGER.error(
+ MetricsMessages
+
.LOG_PROMETHEUSREPORTER_FAILED_TO_UPDATE_METRICS_SNAPSHOT_ASYNCHRONOUSLY_F19FE4E3,
+ t);
+ }
+ }
+
+ private String getMetricsSnapshot() {
+ String snapshot = metricsSnapshot;
+ if (snapshot == null) {
+ snapshot = scrape();
+ if (!snapshot.isEmpty()) {
+ metricsSnapshot = snapshot;
+ }
+ }
+ return snapshot;
+ }
+
+ private void stopSnapshotUpdater() {
+ if (snapshotUpdateFuture != null) {
+ snapshotUpdateFuture.cancel(false);
+ snapshotUpdateFuture = null;
+ }
+ ScheduledExecutorService executor = snapshotUpdateExecutor;
+ snapshotUpdateExecutor = null;
+ if (executor != null) {
+ executor.shutdownNow();
+ }
+ }
+
private boolean authenticate(HttpServerRequest req, HttpServerResponse res) {
if (!METRIC_CONFIG.prometheusNeedAuth()) {
return true;
@@ -318,7 +429,8 @@ public class PrometheusReporter implements Reporter {
}
@Override
- public boolean stop() {
+ public synchronized boolean stop() {
+ stopSnapshotUpdater();
if (httpServer != null) {
try {
httpServer.disposeNow(Duration.ofSeconds(10));
diff --git
a/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java
b/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java
index 30d3bd1ff37..3d1d3a0b0cf 100644
---
a/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java
+++
b/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/config/MetricConfigTest.java
@@ -39,6 +39,7 @@ public class MetricConfigTest {
properties.setProperty("cn_metric_level", "ALL");
properties.setProperty("cn_metric_async_collect_period", "10");
properties.setProperty("cn_metric_prometheus_reporter_port", "9090");
+ properties.setProperty("prometheus_reporter_async_update", "false");
properties.setProperty("cn_metric_iotdb_reporter_host", "0.0.0.0");
properties.setProperty("cn_metric_iotdb_reporter_port", "6669");
properties.setProperty("cn_metric_iotdb_reporter_username", "user");
@@ -55,6 +56,7 @@ public class MetricConfigTest {
assertEquals(MetricLevel.ALL, metricConfig.getMetricLevel());
assertEquals(10, (int) metricConfig.getAsyncCollectPeriodInSecond());
assertEquals(9090, (int) metricConfig.getPrometheusReporterPort());
+ assertEquals(false, metricConfig.isPrometheusReporterAsyncUpdate());
IoTDBReporterConfig reporterConfig = metricConfig.getIoTDBReporterConfig();
assertEquals("0.0.0.0", reporterConfig.getHost());
@@ -75,6 +77,7 @@ public class MetricConfigTest {
properties.setProperty("dn_metric_level", "ALL");
properties.setProperty("dn_metric_async_collect_period", "10");
properties.setProperty("dn_metric_prometheus_reporter_port", "9090");
+ properties.setProperty("metric_prometheus_reporter_async_update", "true");
properties.setProperty("dn_metric_iotdb_reporter_host", "0.0.0.0");
properties.setProperty("dn_metric_iotdb_reporter_port", "6669");
properties.setProperty("dn_metric_iotdb_reporter_username", "user");
@@ -92,6 +95,7 @@ public class MetricConfigTest {
assertEquals(MetricLevel.ALL, metricConfig.getMetricLevel());
assertEquals(10, (int) metricConfig.getAsyncCollectPeriodInSecond());
assertEquals(9090, (int) metricConfig.getPrometheusReporterPort());
+ assertEquals(true, metricConfig.isPrometheusReporterAsyncUpdate());
IoTDBReporterConfig reporterConfig = metricConfig.getIoTDBReporterConfig();
assertEquals("0.0.0.0", reporterConfig.getHost());
@@ -103,4 +107,16 @@ public class MetricConfigTest {
assertEquals(5, (int) reporterConfig.getPushPeriodInSecond());
assertEquals(InternalReporterType.IOTDB,
metricConfig.getInternalReportType());
}
+
+ @Test
+ public void testMetricConfigWithCustomNodePrefix() {
+ Properties properties = new Properties();
+ properties.setProperty("sn_metric_prometheus_reporter_async_update",
"false");
+
+ MetricConfigDescriptor.getInstance().loadProps(properties, "sn_");
+
+ assertEquals(
+ false,
+
MetricConfigDescriptor.getInstance().getMetricConfig().isPrometheusReporterAsyncUpdate());
+ }
}
diff --git
a/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/reporter/prometheus/PrometheusReporterTest.java
b/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/reporter/prometheus/PrometheusReporterTest.java
new file mode 100644
index 00000000000..7072777ac66
--- /dev/null
+++
b/iotdb-core/metrics/interface/src/test/java/org/apache/iotdb/metrics/reporter/prometheus/PrometheusReporterTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.reporter.prometheus;
+
+import org.apache.iotdb.metrics.config.MetricConfig;
+import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
+import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class PrometheusReporterTest {
+
+ @Test
+ public void testManagedExecutorRecreatedAfterRestart() {
+ MetricConfig metricConfig =
MetricConfigDescriptor.getInstance().getMetricConfig();
+ boolean originalAsyncUpdate =
metricConfig.isPrometheusReporterAsyncUpdate();
+ Integer originalPort = metricConfig.getPrometheusReporterPort();
+ metricConfig.setPrometheusReporterAsyncUpdate(true);
+ metricConfig.setPrometheusReporterPort(0);
+
+ AtomicInteger factoryCalls = new AtomicInteger();
+ List<ScheduledExecutorService> executors = new ArrayList<>();
+ PrometheusReporter reporter =
+ new PrometheusReporter(
+ new DoNothingMetricManager(),
+ () -> {
+ factoryCalls.incrementAndGet();
+ ScheduledExecutorService executor =
Executors.newSingleThreadScheduledExecutor();
+ executors.add(executor);
+ return executor;
+ });
+ try {
+ assertTrue(reporter.start());
+ assertEquals(1, factoryCalls.get());
+ assertTrue(reporter.stop());
+ assertTrue(executors.get(0).isShutdown());
+
+ assertTrue(reporter.start());
+ assertEquals(2, factoryCalls.get());
+ assertTrue(reporter.stop());
+ assertTrue(executors.get(1).isShutdown());
+ } finally {
+ reporter.stop();
+ metricConfig.setPrometheusReporterAsyncUpdate(originalAsyncUpdate);
+ metricConfig.setPrometheusReporterPort(originalPort);
+ executors.forEach(ScheduledExecutorService::shutdownNow);
+ }
+ }
+}
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 95a8154bba4..866dde0a94e 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -382,6 +382,12 @@ metric_prometheus_reporter_username=
# Datatype: String
metric_prometheus_reporter_password=
+# Whether Prometheus metrics are collected asynchronously and served from a
cached snapshot.
+# The snapshot is refreshed every 15 seconds, matching Prometheus's default
scrape interval.
+# effectiveMode: restart
+# Datatype: boolean
+prometheus_reporter_async_update=true
+
# The reporters of metric module to report metrics
# If there are more than one reporter, please separate them by commas ",".
# Options: [JMX, PROMETHEUS]
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
index 606a08bde33..a0b62cd6a62 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
@@ -191,6 +191,7 @@ public enum ThreadName {
// -------------------------- Metrics --------------------------
SYSTEM_SCHEDULE_METRICS("SystemScheduleMetrics"),
RESOURCE_CONTROL_DISK_STATISTIC("ResourceControl-DataRegionDiskStatistics"),
+ PROMETHEUS_REPORTER_SNAPSHOT_UPDATER("PrometheusReporter-Snapshot-Updater"),
PROMETHEUS_REACTOR_HTTP_EPOLL("reactor-http-epoll"),
PROMETHEUS_REACTOR_HTTP_NIO("reactor-http-nio"),
PROMETHEUS_BOUNDED_ELASTIC("boundedElastic-evictor"),
@@ -394,6 +395,7 @@ public enum ThreadName {
Arrays.asList(
SYSTEM_SCHEDULE_METRICS,
RESOURCE_CONTROL_DISK_STATISTIC,
+ PROMETHEUS_REPORTER_SNAPSHOT_UPDATER,
PROMETHEUS_REACTOR_HTTP_EPOLL,
PROMETHEUS_REACTOR_HTTP_NIO,
PROMETHEUS_REACTOR_HTTP_EPOLL,
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/MetricService.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/MetricService.java
index 51357fe8998..4af78e3d9cf 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/MetricService.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/MetricService.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.commons.service.metric;
+import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.StartupException;
import org.apache.iotdb.commons.i18n.ServiceMessages;
@@ -74,7 +76,18 @@ public class MetricService extends AbstractMetricService
implements MetricServic
hasJmxReporter = true;
break;
case PROMETHEUS:
- reporter = new PrometheusReporter(metricManager);
+ if (METRIC_CONFIG.isPrometheusReporterAsyncUpdate()) {
+ // Defer pool creation until start so duplicate reporters rejected
below do not
+ // register an unused pool.
+ reporter =
+ new PrometheusReporter(
+ metricManager,
+ () ->
+
IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor(
+
ThreadName.PROMETHEUS_REPORTER_SNAPSHOT_UPDATER.getName()));
+ } else {
+ reporter = new PrometheusReporter(metricManager);
+ }
break;
case IOTDB:
reporter = new IoTDBSessionReporter(metricManager);
