This is an automated email from the ASF dual-hosted git repository. HTHou pushed a commit to branch codex/prometheus in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 283b6fe3e42679cd89c7eb8a1176040ef790863a Author: HTHou <[email protected]> AuthorDate: Fri May 8 12:33:54 2026 +0800 Add metric scrape service to distribution --- distribution/pom.xml | 6 + distribution/src/assembly/all.xml | 4 + distribution/src/assembly/datanode.xml | 4 + .../src/assembly/external-service-impl.xml | 4 + external-service-impl/metric-scrape/pom.xml | 99 ++++++++ .../iotdb/metricscrape/MetricScrapeConfig.java | 122 +++++++++ .../iotdb/metricscrape/MetricScrapeHttpClient.java | 60 +++++ .../iotdb/metricscrape/MetricScrapeService.java | 125 +++++++++ .../iotdb/metricscrape/MetricScrapeTarget.java | 38 +++ .../iotdb/metricscrape/MetricTableWriter.java | 280 +++++++++++++++++++++ .../iotdb/metricscrape/PrometheusSample.java | 56 +++++ .../iotdb/metricscrape/PrometheusTextParser.java | 255 +++++++++++++++++++ .../metricscrape/PrometheusTextParserTest.java | 91 +++++++ external-service-impl/pom.xml | 1 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 66 +++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 37 +++ .../externalservice/BuiltinExternalServices.java | 6 +- .../conf/iotdb-system.properties.template | 34 +++ .../apache/iotdb/commons/conf/IoTDBConstant.java | 8 + 19 files changed, 1295 insertions(+), 1 deletion(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index f4773f6afad..22c862aef62 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -59,6 +59,12 @@ <version>2.0.7-SNAPSHOT</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>metric-scrape</artifactId> + <version>2.0.7-SNAPSHOT</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.apache.iotdb</groupId> <artifactId>rest</artifactId> diff --git a/distribution/src/assembly/all.xml b/distribution/src/assembly/all.xml index 1b2e1be054f..4a5d820bfde 100644 --- a/distribution/src/assembly/all.xml +++ b/distribution/src/assembly/all.xml @@ -96,6 +96,10 @@ <source>${maven.multiModuleProjectDirectory}/external-service-impl/mqtt/target/mqtt-${project.version}-jar-with-dependencies.jar</source> <outputDirectory>lib</outputDirectory> </file> + <file> + <source>${maven.multiModuleProjectDirectory}/external-service-impl/metric-scrape/target/metric-scrape-${project.version}-jar-with-dependencies.jar</source> + <outputDirectory>lib</outputDirectory> + </file> <file> <source>${maven.multiModuleProjectDirectory}/external-service-impl/rest/target/rest-${project.version}-jar-with-dependencies.jar</source> <outputDirectory>lib</outputDirectory> diff --git a/distribution/src/assembly/datanode.xml b/distribution/src/assembly/datanode.xml index 225fa5a7e7d..65e0e95ec60 100644 --- a/distribution/src/assembly/datanode.xml +++ b/distribution/src/assembly/datanode.xml @@ -79,6 +79,10 @@ <source>${maven.multiModuleProjectDirectory}/external-service-impl/mqtt/target/mqtt-${project.version}-jar-with-dependencies.jar</source> <outputDirectory>lib</outputDirectory> </file> + <file> + <source>${maven.multiModuleProjectDirectory}/external-service-impl/metric-scrape/target/metric-scrape-${project.version}-jar-with-dependencies.jar</source> + <outputDirectory>lib</outputDirectory> + </file> <file> <source>${maven.multiModuleProjectDirectory}/external-service-impl/rest/target/rest-${project.version}-jar-with-dependencies.jar</source> <outputDirectory>lib</outputDirectory> diff --git a/distribution/src/assembly/external-service-impl.xml b/distribution/src/assembly/external-service-impl.xml index c743fa85597..ff8098cdf0a 100644 --- a/distribution/src/assembly/external-service-impl.xml +++ b/distribution/src/assembly/external-service-impl.xml @@ -47,6 +47,10 @@ <source>${maven.multiModuleProjectDirectory}/external-service-impl/mqtt/target/mqtt-${project.version}-jar-with-dependencies.jar</source> <outputDirectory>/</outputDirectory> </file> + <file> + <source>${maven.multiModuleProjectDirectory}/external-service-impl/metric-scrape/target/metric-scrape-${project.version}-jar-with-dependencies.jar</source> + <outputDirectory>/</outputDirectory> + </file> <file> <source>${maven.multiModuleProjectDirectory}/external-service-impl/rest/target/rest-${project.version}-jar-with-dependencies.jar</source> <outputDirectory>/</outputDirectory> diff --git a/external-service-impl/metric-scrape/pom.xml b/external-service-impl/metric-scrape/pom.xml new file mode 100644 index 00000000000..4e956f55f97 --- /dev/null +++ b/external-service-impl/metric-scrape/pom.xml @@ -0,0 +1,99 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.iotdb</groupId> + <artifactId>external-service-impl</artifactId> + <version>2.0.7-SNAPSHOT</version> + </parent> + <artifactId>metric-scrape</artifactId> + <name>IoTDB: External-Service-Impl: Metric Scrape</name> + <properties> + <maven.compiler.source>8</maven.compiler.source> + <maven.compiler.target>8</maven.compiler.target> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>node-commons</artifactId> + <version>2.0.7-SNAPSHOT</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-server</artifactId> + <version>2.0.7-SNAPSHOT</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tsfile</groupId> + <artifactId>tsfile</artifactId> + <version>${tsfile.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tsfile</groupId> + <artifactId>common</artifactId> + <version>${tsfile.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>provided</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <version>${maven.assembly.version}</version> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <goals> + <goal>single</goal> + </goals> + <phase>package</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <ignoredDependencies> + <ignoredDependency>org.apache.tsfile:common</ignoredDependency> + <ignoredDependency>junit:junit</ignoredDependency> + </ignoredDependencies> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfig.java b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfig.java new file mode 100644 index 00000000000..2ea52c7f91e --- /dev/null +++ b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfig.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.apache.iotdb.db.conf.IoTDBConfig; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class MetricScrapeConfig { + + private final List<MetricScrapeTarget> targets; + private final int intervalSeconds; + private final String database; + private final String table; + private final int httpTimeoutMs; + + private MetricScrapeConfig( + List<MetricScrapeTarget> targets, + int intervalSeconds, + String database, + String table, + int httpTimeoutMs) { + this.targets = targets; + this.intervalSeconds = intervalSeconds; + this.database = database; + this.table = table; + this.httpTimeoutMs = httpTimeoutMs; + } + + public static MetricScrapeConfig from(IoTDBConfig config) { + if (config.getMetricScrapeIntervalSeconds() <= 0) { + throw new IllegalArgumentException("metric_scrape_interval_seconds should be positive"); + } + if (config.getMetricScrapeHttpTimeoutMs() <= 0) { + throw new IllegalArgumentException("metric_scrape_http_timeout_ms should be positive"); + } + if (config.getMetricScrapeDatabase() == null + || config.getMetricScrapeDatabase().trim().isEmpty()) { + throw new IllegalArgumentException("metric_scrape_database should not be empty"); + } + if (config.getMetricScrapeTable() == null || config.getMetricScrapeTable().trim().isEmpty()) { + throw new IllegalArgumentException("metric_scrape_table should not be empty"); + } + return new MetricScrapeConfig( + parseTargets(config.getMetricScrapeTargets()), + config.getMetricScrapeIntervalSeconds(), + config.getMetricScrapeDatabase().trim(), + config.getMetricScrapeTable().trim(), + config.getMetricScrapeHttpTimeoutMs()); + } + + private static List<MetricScrapeTarget> parseTargets(String rawTargets) { + if (rawTargets == null || rawTargets.trim().isEmpty()) { + return Collections.emptyList(); + } + List<MetricScrapeTarget> targets = new ArrayList<>(); + String[] targetItems = rawTargets.split(","); + for (String targetItem : targetItems) { + String target = targetItem.trim(); + if (target.isEmpty()) { + continue; + } + validateTarget(target); + targets.add(new MetricScrapeTarget(target)); + } + return Collections.unmodifiableList(targets); + } + + private static void validateTarget(String target) { + try { + URL url = new URL(target); + if (!"http".equalsIgnoreCase(url.getProtocol()) + && !"https".equalsIgnoreCase(url.getProtocol())) { + throw new IllegalArgumentException( + "Metric scrape target only supports http and https: " + target); + } + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Illegal metric scrape target: " + target, e); + } + } + + public List<MetricScrapeTarget> getTargets() { + return targets; + } + + public int getIntervalSeconds() { + return intervalSeconds; + } + + public String getDatabase() { + return database; + } + + public String getTable() { + return table; + } + + public int getHttpTimeoutMs() { + return httpTimeoutMs; + } +} diff --git a/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeHttpClient.java b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeHttpClient.java new file mode 100644 index 00000000000..4153afe4cb4 --- /dev/null +++ b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeHttpClient.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metricscrape; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; + +public class MetricScrapeHttpClient { + + public String get(String url, int timeoutMs) throws IOException { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setRequestMethod("GET"); + connection.setConnectTimeout(timeoutMs); + connection.setReadTimeout(timeoutMs); + connection.setRequestProperty( + "Accept", "text/plain; version=0.0.4, application/openmetrics-text, */*"); + connection.setRequestProperty("User-Agent", "IoTDB-MetricScrapeService"); + + int statusCode = connection.getResponseCode(); + if (statusCode < 200 || statusCode >= 300) { + throw new IOException("Metric scrape target " + url + " returns HTTP " + statusCode); + } + try (InputStream inputStream = connection.getInputStream()) { + return readToString(inputStream); + } finally { + connection.disconnect(); + } + } + + private String readToString(InputStream inputStream) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int length; + while ((length = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, length); + } + return new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + } +} diff --git a/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeService.java b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeService.java new file mode 100644 index 00000000000..da1c98d93e4 --- /dev/null +++ b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeService.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.externalservice.api.IExternalService; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class MetricScrapeService implements IExternalService { + + private static final Logger LOGGER = LoggerFactory.getLogger(MetricScrapeService.class); + + private final PrometheusTextParser parser = new PrometheusTextParser(); + private final MetricScrapeHttpClient httpClient = new MetricScrapeHttpClient(); + + private ScheduledExecutorService executorService; + private MetricTableWriter writer; + + @Override + public synchronized void start() { + if (executorService != null) { + return; + } + + MetricScrapeConfig config = MetricScrapeConfig.from(IoTDBDescriptor.getInstance().getConfig()); + if (config.getTargets().isEmpty()) { + LOGGER.warn("Metric scrape service is enabled but metric_scrape_targets is empty."); + return; + } + + writer = new MetricTableWriter(config.getDatabase(), config.getTable()); + writer.initializeSchema(); + + executorService = + Executors.newScheduledThreadPool( + config.getTargets().size(), new MetricScrapeThreadFactory()); + for (MetricScrapeTarget target : config.getTargets()) { + ScheduledExecutorUtil.safelyScheduleWithFixedDelay( + executorService, + () -> scrapeTarget(target, config), + 0, + config.getIntervalSeconds(), + TimeUnit.SECONDS); + } + + LOGGER.info( + "Start MetricScrapeService successfully, targets={}, interval={}s, table={}.{}", + config.getTargets(), + config.getIntervalSeconds(), + config.getDatabase(), + config.getTable()); + } + + @Override + public synchronized void stop() { + if (executorService != null) { + executorService.shutdownNow(); + executorService = null; + } + if (writer != null) { + writer.close(); + writer = null; + } + LOGGER.info("MetricScrapeService stopped."); + } + + private void scrapeTarget(MetricScrapeTarget target, MetricScrapeConfig config) { + try { + MetricTableWriter currentWriter = writer; + if (currentWriter == null) { + return; + } + long scrapeTimestamp = CommonDateTimeUtils.currentTime(); + String prometheusText = httpClient.get(target.getUrl(), config.getHttpTimeoutMs()); + List<PrometheusSample> samples = parser.parse(prometheusText, scrapeTimestamp); + if (samples.isEmpty()) { + LOGGER.debug("Metric scrape target {} returns no samples.", target); + return; + } + currentWriter.write(samples); + LOGGER.debug("Metric scrape target {} writes {} samples.", target, samples.size()); + } catch (Exception e) { + LOGGER.warn("Failed to scrape metric target {}.", target, e); + } + } + + private static class MetricScrapeThreadFactory implements ThreadFactory { + private final AtomicInteger index = new AtomicInteger(); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(runnable, "metric-scrape-service-" + index.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeTarget.java b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeTarget.java new file mode 100644 index 00000000000..88dec77c6d1 --- /dev/null +++ b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeTarget.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metricscrape; + +public class MetricScrapeTarget { + + private final String url; + + public MetricScrapeTarget(String url) { + this.url = url; + } + + public String getUrl() { + return url; + } + + @Override + public String toString() { + return url; + } +} diff --git a/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricTableWriter.java b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricTableWriter.java new file mode 100644 index 00000000000..514c6e29518 --- /dev/null +++ b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricTableWriter.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.common.SqlDialect; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.auth.AuthorityChecker; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.InternalClientSession; +import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.plan.Coordinator; +import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class MetricTableWriter { + + public static final String METRIC_NAME_COLUMN = "metric_name"; + public static final String VALUE_COLUMN = "value"; + private static final String TIME_COLUMN = "time"; + + private static final Logger LOGGER = LoggerFactory.getLogger(MetricTableWriter.class); + + private static final Coordinator COORDINATOR = Coordinator.getInstance(); + private static final SessionManager SESSION_MANAGER = SessionManager.getInstance(); + private static final IoTDBConfig IOTDB_CONFIG = IoTDBDescriptor.getInstance().getConfig(); + + private final String database; + private final String table; + private final IClientSession session; + private final SqlParser sqlParser = new SqlParser(); + private final Metadata metadata = LocalExecutionPlanner.getInstance().metadata; + + public MetricTableWriter(String database, String table) { + this.database = database; + this.table = table; + this.session = new InternalClientSession("METRIC_SCRAPE"); + this.session.setDatabaseName(database); + this.session.setSqlDialect(SqlDialect.TABLE); + SESSION_MANAGER.supplySession( + session, + AuthorityChecker.SUPER_USER_ID, + AuthorityChecker.SUPER_USER, + ZoneId.systemDefault(), + ClientVersion.V_1_0); + } + + public synchronized void initializeSchema() { + executeStatement("CREATE DATABASE IF NOT EXISTS " + quoteIdentifier(database)); + session.setDatabaseName(database); + executeStatement( + "CREATE TABLE IF NOT EXISTS " + + quoteIdentifier(table) + + " (" + + METRIC_NAME_COLUMN + + " STRING TAG, " + + VALUE_COLUMN + + " DOUBLE FIELD)"); + } + + public synchronized void write(List<PrometheusSample> samples) { + if (samples.isEmpty()) { + return; + } + Map<List<String>, TabletBuilder> builders = new LinkedHashMap<>(); + for (PrometheusSample sample : samples) { + List<String> labelKeys = new ArrayList<>(sample.getLabels().keySet()); + Collections.sort(labelKeys); + validateLabelKeys(labelKeys); + TabletBuilder builder = builders.get(labelKeys); + if (builder == null) { + builder = new TabletBuilder(labelKeys); + builders.put(labelKeys, builder); + } + builder.add(sample); + } + + for (TabletBuilder builder : builders.values()) { + executeInsert(builder.build()); + } + } + + public void close() { + SESSION_MANAGER.closeSession(session, COORDINATOR::cleanupQueryExecution, false); + } + + private void validateLabelKeys(List<String> labelKeys) { + for (String labelKey : labelKeys) { + if (METRIC_NAME_COLUMN.equalsIgnoreCase(labelKey) + || VALUE_COLUMN.equalsIgnoreCase(labelKey) + || TIME_COLUMN.equalsIgnoreCase(labelKey)) { + throw new IllegalArgumentException( + "Prometheus label conflicts with reserved IoTDB table column name: " + labelKey); + } + } + } + + private void executeInsert(InsertTabletStatement statement) { + long queryId = SESSION_MANAGER.requestQueryId(); + try { + ExecutionResult result = + COORDINATOR.executeForTableModel( + statement, + sqlParser, + session, + queryId, + SESSION_MANAGER.getSessionInfo(session), + "", + metadata, + IOTDB_CONFIG.getQueryTimeoutThreshold()); + warnIfFailed(result.status); + } finally { + COORDINATOR.cleanupQueryExecution(queryId); + } + } + + private void executeStatement(String sql) { + long queryId = SESSION_MANAGER.requestQueryId(); + try { + Statement statement = sqlParser.createStatement(sql, session.getZoneId(), session); + ExecutionResult result = + COORDINATOR.executeForTableModel( + statement, + sqlParser, + session, + queryId, + SESSION_MANAGER.getSessionInfo(session), + sql, + metadata, + IOTDB_CONFIG.getQueryTimeoutThreshold(), + false); + TSStatus status = result.status; + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + throw new IllegalStateException( + "Failed to execute metric scrape schema statement: " + sql + ", status: " + status); + } + } finally { + COORDINATOR.cleanupQueryExecution(queryId); + } + } + + private void warnIfFailed(TSStatus status) { + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + || status.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { + return; + } + LOGGER.warn( + "Metric scrape write failed, code={}, message={}", status.getCode(), status.getMessage()); + } + + private static String quoteIdentifier(String identifier) { + return "\"" + identifier.replace("\"", "\"\"") + "\""; + } + + private class TabletBuilder { + private final List<String> labelKeys; + private final List<PrometheusSample> samples = new ArrayList<>(); + + private TabletBuilder(List<String> labelKeys) { + this.labelKeys = new ArrayList<>(labelKeys); + } + + private void add(PrometheusSample sample) { + samples.add(sample); + } + + private InsertTabletStatement build() { + InsertTabletStatement statement = new InsertTabletStatement(); + statement.setDevicePath(new PartialPath(table, false)); + statement.setWriteToTable(true); + statement.setAligned(false); + statement.setRowCount(samples.size()); + + List<String> measurements = new ArrayList<>(labelKeys.size() + 2); + measurements.add(METRIC_NAME_COLUMN); + measurements.addAll(labelKeys); + measurements.add(VALUE_COLUMN); + statement.setMeasurements(measurements.toArray(new String[0])); + statement.setTimes(buildTimes()); + statement.setDataTypes(buildDataTypes(measurements.size())); + statement.setColumnCategories(buildColumnCategories(measurements.size())); + statement.setColumns(buildColumns()); + statement.setBitMaps(new BitMap[measurements.size()]); + return statement; + } + + private long[] buildTimes() { + long[] times = new long[samples.size()]; + for (int i = 0; i < samples.size(); i++) { + times[i] = samples.get(i).getTimestamp(); + } + return times; + } + + private TSDataType[] buildDataTypes(int columnSize) { + TSDataType[] dataTypes = new TSDataType[columnSize]; + for (int i = 0; i < columnSize - 1; i++) { + dataTypes[i] = TSDataType.STRING; + } + dataTypes[columnSize - 1] = TSDataType.DOUBLE; + return dataTypes; + } + + private TsTableColumnCategory[] buildColumnCategories(int columnSize) { + TsTableColumnCategory[] columnCategories = new TsTableColumnCategory[columnSize]; + for (int i = 0; i < columnSize - 1; i++) { + columnCategories[i] = TsTableColumnCategory.TAG; + } + columnCategories[columnSize - 1] = TsTableColumnCategory.FIELD; + return columnCategories; + } + + private Object[] buildColumns() { + Object[] columns = new Object[labelKeys.size() + 2]; + columns[0] = buildStringColumn(METRIC_NAME_COLUMN); + for (int i = 0; i < labelKeys.size(); i++) { + columns[i + 1] = buildStringColumn(labelKeys.get(i)); + } + double[] values = new double[samples.size()]; + for (int i = 0; i < samples.size(); i++) { + values[i] = samples.get(i).getValue(); + } + columns[columns.length - 1] = values; + return columns; + } + + private Binary[] buildStringColumn(String columnName) { + Binary[] values = new Binary[samples.size()]; + for (int i = 0; i < samples.size(); i++) { + String value = + METRIC_NAME_COLUMN.equals(columnName) + ? samples.get(i).getMetricName() + : samples.get(i).getLabels().get(columnName); + values[i] = new Binary(value.getBytes(StandardCharsets.UTF_8)); + } + return values; + } + } +} diff --git a/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusSample.java b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusSample.java new file mode 100644 index 00000000000..664429f792a --- /dev/null +++ b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusSample.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metricscrape; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +public class PrometheusSample { + + private final String metricName; + private final Map<String, String> labels; + private final double value; + private final long timestamp; + + public PrometheusSample( + String metricName, Map<String, String> labels, double value, long timestamp) { + this.metricName = metricName; + this.labels = Collections.unmodifiableMap(new LinkedHashMap<>(labels)); + this.value = value; + this.timestamp = timestamp; + } + + public String getMetricName() { + return metricName; + } + + public Map<String, String> getLabels() { + return labels; + } + + public double getValue() { + return value; + } + + public long getTimestamp() { + return timestamp; + } +} diff --git a/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusTextParser.java b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusTextParser.java new file mode 100644 index 00000000000..949b0901243 --- /dev/null +++ b/external-service-impl/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusTextParser.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class PrometheusTextParser { + + public List<PrometheusSample> parse(String text, long defaultTimestamp) { + if (text == null) { + throw new IllegalArgumentException("Prometheus text should not be null"); + } + List<PrometheusSample> samples = new ArrayList<>(); + String[] lines = text.split("\\r\\n|\\n|\\r"); + for (int i = 0; i < lines.length; i++) { + String line = lines[i].trim(); + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + samples.add(parseSample(line, i + 1, defaultTimestamp)); + } + return samples; + } + + private PrometheusSample parseSample(String line, int lineNumber, long defaultTimestamp) { + Parser parser = new Parser(line, lineNumber); + String metricName = parser.parseMetricName(); + Map<String, String> labels = parser.parseLabels(); + parser.skipWhitespace(); + double value = parser.parseValue(); + parser.skipWhitespace(); + long timestamp = parser.parseOptionalTimestamp(defaultTimestamp); + parser.skipWhitespace(); + parser.expectEnd(); + return new PrometheusSample(metricName, labels, value, timestamp); + } + + private static double parsePrometheusDouble(String token, int lineNumber) { + if ("+Inf".equals(token) || "Inf".equals(token)) { + return Double.POSITIVE_INFINITY; + } + if ("-Inf".equals(token)) { + return Double.NEGATIVE_INFINITY; + } + try { + return Double.parseDouble(token); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Illegal Prometheus sample value " + token + " at line " + lineNumber, e); + } + } + + private static long convertTimestamp(long prometheusTimestampMillis, int lineNumber) { + long timestamp = + TimestampPrecisionUtils.convertToCurrPrecision( + prometheusTimestampMillis, TimeUnit.MILLISECONDS); + try { + TimestampPrecisionUtils.checkTimestampPrecision(timestamp); + } catch (RuntimeException e) { + throw new IllegalArgumentException( + "Illegal Prometheus sample timestamp " + + prometheusTimestampMillis + + " at line " + + lineNumber, + e); + } + return timestamp; + } + + private static class Parser { + private final String line; + private final int lineNumber; + private int position; + + private Parser(String line, int lineNumber) { + this.line = line; + this.lineNumber = lineNumber; + } + + private String parseMetricName() { + if (position >= line.length() || Character.isWhitespace(line.charAt(position))) { + throw new IllegalArgumentException("Illegal Prometheus metric name at line " + lineNumber); + } + int start = position; + while (position < line.length() + && !Character.isWhitespace(line.charAt(position)) + && line.charAt(position) != '{') { + position++; + } + if (start == position) { + throw new IllegalArgumentException("Illegal Prometheus metric name at line " + lineNumber); + } + return line.substring(start, position); + } + + private Map<String, String> parseLabels() { + Map<String, String> labels = new LinkedHashMap<>(); + if (position >= line.length() || line.charAt(position) != '{') { + return labels; + } + position++; + skipWhitespace(); + if (position < line.length() && line.charAt(position) == '}') { + position++; + return labels; + } + + while (position < line.length()) { + String name = parseLabelName(); + skipWhitespace(); + expect('='); + skipWhitespace(); + String value = parseLabelValue(); + if (labels.put(name, value) != null) { + throw new IllegalArgumentException( + "Duplicate Prometheus label " + name + " at line " + lineNumber); + } + skipWhitespace(); + if (position < line.length() && line.charAt(position) == ',') { + position++; + skipWhitespace(); + if (position < line.length() && line.charAt(position) == '}') { + position++; + return labels; + } + continue; + } + expect('}'); + return labels; + } + throw new IllegalArgumentException("Unclosed Prometheus label set at line " + lineNumber); + } + + private String parseLabelName() { + int start = position; + while (position < line.length() && line.charAt(position) != '=') { + if (line.charAt(position) == ',' || line.charAt(position) == '}') { + break; + } + position++; + } + String name = line.substring(start, position).trim(); + if (name.isEmpty()) { + throw new IllegalArgumentException("Illegal Prometheus label name at line " + lineNumber); + } + return name; + } + + private String parseLabelValue() { + expect('"'); + StringBuilder builder = new StringBuilder(); + while (position < line.length()) { + char c = line.charAt(position++); + if (c == '"') { + return builder.toString(); + } + if (c == '\\') { + if (position >= line.length()) { + throw new IllegalArgumentException( + "Illegal Prometheus label escape at line " + lineNumber); + } + char escaped = line.charAt(position++); + switch (escaped) { + case 'n': + builder.append('\n'); + break; + case '\\': + case '"': + builder.append(escaped); + break; + default: + builder.append(escaped); + break; + } + } else { + builder.append(c); + } + } + throw new IllegalArgumentException("Unclosed Prometheus label value at line " + lineNumber); + } + + private double parseValue() { + String token = parseToken("sample value"); + return parsePrometheusDouble(token, lineNumber); + } + + private long parseOptionalTimestamp(long defaultTimestamp) { + if (position >= line.length()) { + return defaultTimestamp; + } + String token = parseToken("sample timestamp"); + try { + return convertTimestamp(Long.parseLong(token), lineNumber); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Illegal Prometheus sample timestamp " + token + " at line " + lineNumber, e); + } + } + + private String parseToken(String name) { + if (position >= line.length() || Character.isWhitespace(line.charAt(position))) { + throw new IllegalArgumentException("Missing Prometheus " + name + " at line " + lineNumber); + } + int start = position; + while (position < line.length() && !Character.isWhitespace(line.charAt(position))) { + position++; + } + return line.substring(start, position); + } + + private void skipWhitespace() { + while (position < line.length() && Character.isWhitespace(line.charAt(position))) { + position++; + } + } + + private void expect(char expected) { + if (position >= line.length() || line.charAt(position) != expected) { + throw new IllegalArgumentException( + "Expected '" + expected + "' in Prometheus sample at line " + lineNumber); + } + position++; + } + + private void expectEnd() { + if (position != line.length()) { + throw new IllegalArgumentException( + "Unexpected Prometheus sample content at line " + lineNumber); + } + } + } +} diff --git a/external-service-impl/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/PrometheusTextParserTest.java b/external-service-impl/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/PrometheusTextParserTest.java new file mode 100644 index 00000000000..15f36474bac --- /dev/null +++ b/external-service-impl/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/PrometheusTextParserTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class PrometheusTextParserTest { + + @Test + public void testParseSamples() { + String text = + "# HELP up Whether the target is up.\n" + + "# TYPE up gauge\n" + + "up{job=\"iotdb\",instance=\"127.0.0.1:6667\"} 1 1635232143960\n" + + "rpc:latency_seconds{method=\"query\",escaped=\"a\\\\b\\\"c\",} +Inf\n"; + + List<PrometheusSample> samples = new PrometheusTextParser().parse(text, 100); + + assertEquals(2, samples.size()); + PrometheusSample first = samples.get(0); + assertEquals("up", first.getMetricName()); + assertEquals("iotdb", first.getLabels().get("job")); + assertEquals("127.0.0.1:6667", first.getLabels().get("instance")); + assertEquals(1.0, first.getValue(), 0.0); + assertEquals(1635232143960L, first.getTimestamp()); + + PrometheusSample second = samples.get(1); + assertEquals("rpc:latency_seconds", second.getMetricName()); + assertEquals("query", second.getLabels().get("method")); + assertEquals("a\\b\"c", second.getLabels().get("escaped")); + assertTrue(Double.isInfinite(second.getValue())); + assertEquals(100, second.getTimestamp()); + } + + @Test + public void testParseIoTDBPrometheusReporterSamples() { + String text = + "# HELP file_count\n" + + "# TYPE file_count gauge\n" + + "file_count{cluster=\"defaultCluster\",nodeType=\"DataNode\",nodeId=\"1\",} 1\n"; + + List<PrometheusSample> samples = new PrometheusTextParser().parse(text, 100); + + assertEquals(1, samples.size()); + PrometheusSample sample = samples.get(0); + assertEquals("file_count", sample.getMetricName()); + assertEquals("defaultCluster", sample.getLabels().get("cluster")); + assertEquals("DataNode", sample.getLabels().get("nodeType")); + assertEquals("1", sample.getLabels().get("nodeId")); + assertEquals(1.0, sample.getValue(), 0.0); + assertEquals(100, sample.getTimestamp()); + } + + @Test + public void testParseIoTDBMetricNameAndTagKey() { + String text = "metric[name]{tag-key=\"value\",} 1\n"; + + List<PrometheusSample> samples = new PrometheusTextParser().parse(text, 100); + + assertEquals(1, samples.size()); + assertEquals("metric[name]", samples.get(0).getMetricName()); + assertEquals("value", samples.get(0).getLabels().get("tag-key")); + } + + @Test(expected = IllegalArgumentException.class) + public void testRejectDuplicateLabel() { + new PrometheusTextParser().parse("up{job=\"a\",job=\"b\"} 1", 100); + } +} diff --git a/external-service-impl/pom.xml b/external-service-impl/pom.xml index 04483b289f3..749e1efbba2 100644 --- a/external-service-impl/pom.xml +++ b/external-service-impl/pom.xml @@ -31,6 +31,7 @@ <packaging>pom</packaging> <modules> <module>mqtt</module> + <module>metric-scrape</module> <module>rest</module> <module>rest-openapi</module> </modules> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 9b177cffcfa..0f814c9cc90 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -125,6 +125,24 @@ public class IoTDBConfig { /** Max mqtt message size. Unit: byte */ private int mqttMaxMessageSize = 1048576; + /** Whether to enable the metric scrape service. */ + private boolean enableMetricScrapeService = false; + + /** The comma separated metric scrape target URLs. */ + private String metricScrapeTargets = ""; + + /** The metric scrape interval in seconds. */ + private int metricScrapeIntervalSeconds = 15; + + /** The table model database used by metric scrape service. */ + private String metricScrapeDatabase = "metrics"; + + /** The table used by metric scrape service. */ + private String metricScrapeTable = "iotdb_metrics"; + + /** The HTTP timeout used by metric scrape service. Unit: millisecond */ + private int metricScrapeHttpTimeoutMs = 10000; + /** Rpc binding address. */ private String rpcAddress = "127.0.0.1"; @@ -2608,6 +2626,54 @@ public class IoTDBConfig { this.mqttMaxMessageSize = mqttMaxMessageSize; } + public boolean isEnableMetricScrapeService() { + return enableMetricScrapeService; + } + + public void setEnableMetricScrapeService(boolean enableMetricScrapeService) { + this.enableMetricScrapeService = enableMetricScrapeService; + } + + public String getMetricScrapeTargets() { + return metricScrapeTargets; + } + + public void setMetricScrapeTargets(String metricScrapeTargets) { + this.metricScrapeTargets = metricScrapeTargets; + } + + public int getMetricScrapeIntervalSeconds() { + return metricScrapeIntervalSeconds; + } + + public void setMetricScrapeIntervalSeconds(int metricScrapeIntervalSeconds) { + this.metricScrapeIntervalSeconds = metricScrapeIntervalSeconds; + } + + public String getMetricScrapeDatabase() { + return metricScrapeDatabase; + } + + public void setMetricScrapeDatabase(String metricScrapeDatabase) { + this.metricScrapeDatabase = metricScrapeDatabase; + } + + public String getMetricScrapeTable() { + return metricScrapeTable; + } + + public void setMetricScrapeTable(String metricScrapeTable) { + this.metricScrapeTable = metricScrapeTable; + } + + public int getMetricScrapeHttpTimeoutMs() { + return metricScrapeHttpTimeoutMs; + } + + public void setMetricScrapeHttpTimeoutMs(int metricScrapeHttpTimeoutMs) { + this.metricScrapeHttpTimeoutMs = metricScrapeHttpTimeoutMs; + } + public int getTagAttributeFlushInterval() { return tagAttributeFlushInterval; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 7193fd27c58..23f79d1f676 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -882,6 +882,9 @@ public class IoTDBDescriptor { // mqtt loadMqttProps(properties); + // metric scrape + loadMetricScrapeProps(properties); + conf.setIntoOperationBufferSizeInByte( Long.parseLong( properties.getProperty( @@ -2028,6 +2031,40 @@ public class IoTDBDescriptor { } } + // metric scrape related + private void loadMetricScrapeProps(TrimProperties properties) { + if (properties.getProperty(IoTDBConstant.ENABLE_METRIC_SCRAPE) != null) { + conf.setEnableMetricScrapeService( + Boolean.parseBoolean(properties.getProperty(IoTDBConstant.ENABLE_METRIC_SCRAPE).trim())); + } + + if (properties.getProperty(IoTDBConstant.METRIC_SCRAPE_TARGETS) != null) { + conf.setMetricScrapeTargets( + properties.getProperty(IoTDBConstant.METRIC_SCRAPE_TARGETS).trim()); + } + + if (properties.getProperty(IoTDBConstant.METRIC_SCRAPE_INTERVAL_SECONDS) != null) { + conf.setMetricScrapeIntervalSeconds( + Integer.parseInt( + properties.getProperty(IoTDBConstant.METRIC_SCRAPE_INTERVAL_SECONDS).trim())); + } + + if (properties.getProperty(IoTDBConstant.METRIC_SCRAPE_DATABASE) != null) { + conf.setMetricScrapeDatabase( + properties.getProperty(IoTDBConstant.METRIC_SCRAPE_DATABASE).trim()); + } + + if (properties.getProperty(IoTDBConstant.METRIC_SCRAPE_TABLE) != null) { + conf.setMetricScrapeTable(properties.getProperty(IoTDBConstant.METRIC_SCRAPE_TABLE).trim()); + } + + if (properties.getProperty(IoTDBConstant.METRIC_SCRAPE_HTTP_TIMEOUT_MS) != null) { + conf.setMetricScrapeHttpTimeoutMs( + Integer.parseInt( + properties.getProperty(IoTDBConstant.METRIC_SCRAPE_HTTP_TIMEOUT_MS).trim())); + } + } + // timed flush memtable private void loadTimedService(TrimProperties properties) throws IOException { conf.setEnableTimedFlushSeqMemtable( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/externalservice/BuiltinExternalServices.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/externalservice/BuiltinExternalServices.java index 157734aef56..302a392aa41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/externalservice/BuiltinExternalServices.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/externalservice/BuiltinExternalServices.java @@ -32,7 +32,11 @@ public enum BuiltinExternalServices { REST( "REST", "org.apache.iotdb.rest.RestService", - IoTDBRestServiceDescriptor.getInstance().getConfig()::isEnableRestService); + IoTDBRestServiceDescriptor.getInstance().getConfig()::isEnableRestService), + METRIC_SCRAPE( + "METRIC_SCRAPE", + "org.apache.iotdb.metricscrape.MetricScrapeService", + IoTDBDescriptor.getInstance().getConfig()::isEnableMetricScrapeService); private final String serviceName; private final String className; diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 54d9ccaf5ce..80006bf4832 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -2141,6 +2141,40 @@ mqtt_payload_formatter=json # Datatype: int mqtt_max_message_size=1048576 +#################### +### Metric Scrape Configuration +#################### + +# whether to enable the metric scrape service. +# effectiveMode: restart +# Datatype: boolean +enable_metric_scrape_service=false + +# the comma separated Prometheus text exposition target URLs to scrape. +# effectiveMode: restart +# Datatype: String +metric_scrape_targets= + +# the interval between two scrapes of the same target in seconds. +# effectiveMode: restart +# Datatype: int +metric_scrape_interval_seconds=15 + +# the table model database used by the metric scrape service. +# effectiveMode: restart +# Datatype: String +metric_scrape_database=metrics + +# the table used by the metric scrape service. +# effectiveMode: restart +# Datatype: String +metric_scrape_table=iotdb_metrics + +# the HTTP connect and read timeout used by the metric scrape service in milliseconds. +# effectiveMode: restart +# Datatype: int +metric_scrape_http_timeout_ms=10000 + #################### ### IoTDB-AI Configuration #################### diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index 03e9a53db60..5d9332cff98 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -290,6 +290,14 @@ public class IoTDBConstant { public static final String MQTT_DATA_PATH = "mqtt_data_path"; public static final String MQTT_MAX_MESSAGE_SIZE = "mqtt_max_message_size"; + // metric scrape + public static final String ENABLE_METRIC_SCRAPE = "enable_metric_scrape_service"; + public static final String METRIC_SCRAPE_TARGETS = "metric_scrape_targets"; + public static final String METRIC_SCRAPE_INTERVAL_SECONDS = "metric_scrape_interval_seconds"; + public static final String METRIC_SCRAPE_DATABASE = "metric_scrape_database"; + public static final String METRIC_SCRAPE_TABLE = "metric_scrape_table"; + public static final String METRIC_SCRAPE_HTTP_TIMEOUT_MS = "metric_scrape_http_timeout_ms"; + // thrift public static final int DEFAULT_FETCH_SIZE = 5000; public static final int DEFAULT_CONNECTION_TIMEOUT_MS = 0;
