This is an automated email from the ASF dual-hosted git repository. HTHou pushed a commit to branch codex/standalone-metric-scrape in repository https://gitbox.apache.org/repos/asf/iotdb-extras.git
commit 81192b1928571d422b9420ea34b90e2ea5a3e771 Author: HTHou <[email protected]> AuthorDate: Thu Jun 18 17:51:44 2026 +0800 Add metric scrape module --- README-zh.md | 7 +- README.md | 7 +- metric-scrape/Dockerfile | 26 ++ metric-scrape/README.md | 146 ++++++++ metric-scrape/conf/metric-scrape.yml | 61 ++++ metric-scrape/k8s/configmap.yaml | 60 ++++ metric-scrape/k8s/deployment.yaml | 64 ++++ metric-scrape/pom.xml | 112 ++++++ .../apache/iotdb/metricscrape/IdentifierUtils.java | 50 +++ .../iotdb/metricscrape/MetricScrapeConfig.java | 195 +++++++++++ .../metricscrape/MetricScrapeConfigLoader.java | 256 ++++++++++++++ .../iotdb/metricscrape/MetricScrapeHttpClient.java | 59 ++++ .../iotdb/metricscrape/MetricScrapeMain.java | 63 ++++ .../iotdb/metricscrape/MetricScrapePlanner.java | 73 ++++ .../iotdb/metricscrape/MetricScrapeService.java | 119 +++++++ .../iotdb/metricscrape/MetricScrapeTarget.java | 62 ++++ .../iotdb/metricscrape/MetricTableWriter.java | 211 +++++++++++ .../iotdb/metricscrape/PrometheusSample.java | 69 ++++ .../iotdb/metricscrape/PrometheusTextParser.java | 389 +++++++++++++++++++++ .../iotdb/metricscrape/ScheduledExecutorUtil.java | 53 +++ .../metricscrape/MetricScrapeConfigLoaderTest.java | 41 +++ .../metricscrape/PrometheusTextParserTest.java | 112 ++++++ pom.xml | 1 + 23 files changed, 2232 insertions(+), 4 deletions(-) diff --git a/README-zh.md b/README-zh.md index eaea49f..ffe425f 100644 --- a/README-zh.md +++ b/README-zh.md @@ -61,7 +61,9 @@ IoTDB (物联网数据库) 是一个专为物联网 (IoT) 场景设计的时序 5. **MyBatis 生成器**：数据库访问代码生成工具 -6. **示例**：展示 IoTDB 与各种技术结合使用的示例应用程序和代码示例 +6. **Metric Scrape**：用于 IoTDB 表模型的独立 Prometheus 文本格式指标抓取工具 + +7. **示例**：展示 IoTDB 与各种技术结合使用的示例应用程序和代码示例 ## 环境要求 @@ -83,7 +85,7 @@ IoTDB (物联网数据库) 是一个专为物联网 (IoT) 场景设计的时序 2. 使用 Maven 构建项目： ```bash - # 构建整个项目（包括 distributions、iotdb-collector、mybatis-generator） + # 构建整个项目（包括 distributions、iotdb-collector、metric-scrape、mybatis-generator） mvn clean package -DskipTests # 或者构建所有组件 @@ -310,6 +312,7 @@ mvn clean package -Pwith-all-connectors,with-examples,with-springboot -DskipTest - [Flink IoTDB 连接器](/connectors/flink-iotdb-connector/README.md) - [Grafana 插件](/connectors/grafana-plugin/README.md) - [IoTDB Spring Boot Starter](/iotdb-spring-boot-starter/README.md) +- [Metric Scrape](/metric-scrape/README.md) - [Kubernetes Helm Charts](/helm/README.md) - [IoTDB Operator](/iotdb-operator/README.md) diff --git a/README.md b/README.md index 4e2ad89..270ead1 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,9 @@ This repository includes: 5. **MyBatis Generator**: Code generation tools for database access -6. **Examples**: Sample applications and code examples demonstrating the use of IoTDB with various technologies +6. **Metric Scrape**: Standalone Prometheus text exposition scraper for the IoTDB table model + +7. **Examples**: Sample applications and code examples demonstrating the use of IoTDB with various technologies ## Prerequisites @@ -83,7 +85,7 @@ To build the project from source, follow these steps: 2. Build the project with Maven: ```bash - # Build the entire project (includes distributions,iotdb-collector,mybatis-generator) + # Build the entire project (includes distributions,iotdb-collector,metric-scrape,mybatis-generator) mvn clean package -DskipTests # Or build all @@ -312,6 +314,7 @@ You can also refer to module-specific documentation: - [Flink IoTDB Connector](/connectors/flink-iotdb-connector/README.md) - [Grafana Plugin](/connectors/grafana-plugin/README.md) - [IoTDB Spring Boot Starter](/iotdb-spring-boot-starter/README.md) +- [Metric Scrape](/metric-scrape/README.md) - [Kubernetes Helm Charts](/helm/README.md) - [IoTDB Operator](/iotdb-operator/README.md) diff --git a/metric-scrape/Dockerfile b/metric-scrape/Dockerfile new file mode 100644 index 0000000..4df24f6 --- /dev/null +++ b/metric-scrape/Dockerfile @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM eclipse-temurin:17-jre-jammy + +WORKDIR /app + +COPY target/metric-scrape-2.0.4-SNAPSHOT.jar /app/metric-scrape.jar + +ENTRYPOINT ["java", "-jar", "/app/metric-scrape.jar", "-c", "/app/conf/metric-scrape.yml"] diff --git a/metric-scrape/README.md b/metric-scrape/README.md new file mode 100644 index 0000000..13cab17 --- /dev/null +++ b/metric-scrape/README.md @@ -0,0 +1,146 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +# Metric Scrape + +Metric Scrape is a standalone service that periodically scrapes Prometheus text exposition +endpoints and writes metrics into the Apache IoTDB table model through the +`org.apache.iotdb:iotdb-session` client. + +It is a pull scraper, not a Prometheus remote write receiver. + +## Build + +```bash +mvn -pl metric-scrape -am clean package -DskipTests +``` + +The runnable jar is generated at: + +```text +metric-scrape/target/metric-scrape-2.0.4-SNAPSHOT.jar +``` + +## Run + +```bash +java -jar metric-scrape/target/metric-scrape-2.0.4-SNAPSHOT.jar \ + -c metric-scrape/conf/metric-scrape.yml +``` + +If `-c` is omitted, the service reads `conf/metric-scrape.yml` from the current working +directory. + +## Configuration + +```yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + iotdb_database: 'metrics' + iotdb_username: 'root' + iotdb_password: 'root' + iotdb_urls: ['127.0.0.1:6667'] + +scrape_configs: + - job_name: 'iotdbModel' + static_configs: + - targets: ['localhost:9090'] + metrics_path: /metrics + relabel_configs: + - target_label: k8s_cluster + replacement: example-cluster + - target_label: k8s_namespace + replacement: default + - target_label: instance_id + replacement: model-service-local +``` + +Supported fields: + +| Field | Description | +| --- | --- | +| `global.scrape_interval` | Default scrape interval. Supports `ms`, `s`, `m`, `h`; no suffix means seconds. | +| `global.scrape_timeout` | Default HTTP timeout. Supports the same duration format. | +| `global.iotdb_database` | Target IoTDB table-model database. | +| `global.iotdb_username` | Session username. | +| `global.iotdb_password` | Session password. | +| `global.iotdb_urls` | IoTDB node URLs, for example `127.0.0.1:6667`. | +| `global.write_batch_size` | Optional tablet batch size. Default is `1000`. | +| `scrape_configs[].job_name` | Scrape job name. It is written as tag column `job_name`. | +| `scrape_configs[].scheme` | Optional target scheme. Default is `http`. | +| `scrape_configs[].metrics_path` | Metrics path. Default is `/metrics`. | +| `scrape_configs[].scrape_interval` | Optional interval override for this job. | +| `scrape_configs[].scrape_timeout` | Optional timeout override for this job. | +| `scrape_configs[].static_configs[].targets` | Target host list. | +| `scrape_configs[].static_configs[].labels` | Extra labels for targets. They are written as tag columns. | +| `scrape_configs[].relabel_configs[].target_label` | Constant target label name. | +| `scrape_configs[].relabel_configs[].replacement` | Constant target label value. | + +Only constant relabeling with `target_label` and `replacement` is supported for now. + +## Data Model + +Metric Scrape writes into the IoTDB table model: + +| Prometheus content | Table model mapping | +| --- | --- | +| `global.iotdb_database` | Database name. | +| `# HELP` metric family name | Table name. | +| Sample metric name | Field column name. | +| Sample value | Field column value, `DOUBLE`. | +| `job_name` | Tag column. | +| Target address | Tag column `instance`. | +| Sample labels, static labels, relabel replacement labels | Tag columns, `STRING`. | +| Sample timestamp | Row time. If missing, the current scrape time in milliseconds is used. | + +For histogram and summary samples, the longest matching `# HELP` metric family is used as the +table name. For example, `request_duration_seconds_sum` and +`request_duration_seconds_count` are written into table `request_duration_seconds` if the text +contains `# HELP request_duration_seconds ...`. + +Metric names and label names are normalized before being used as table or column identifiers: +characters other than letters, digits, and `_` are replaced by `_`, and identifiers starting with +a digit get a leading `_`. + +## Docker And Kubernetes + +The module includes a `Dockerfile` and example Kubernetes manifests under `k8s/`. + +```bash +docker build -t metric-scrape:latest metric-scrape +kubectl apply -f metric-scrape/k8s/configmap.yaml +kubectl apply -f metric-scrape/k8s/deployment.yaml +``` + +Update the image name, IoTDB URLs, credentials, and scrape targets before deploying to a real +cluster. + +## Example Query + +```sql +USE metrics; + +SELECT time, job_name, instance, env, http_server_requests_seconds_count +FROM http_server_requests_seconds +ORDER BY time DESC +LIMIT 10; +``` diff --git a/metric-scrape/conf/metric-scrape.yml b/metric-scrape/conf/metric-scrape.yml new file mode 100644 index 0000000..645d62d --- /dev/null +++ b/metric-scrape/conf/metric-scrape.yml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +global: + scrape_interval: 15s + scrape_timeout: 10s + iotdb_database: 'metrics' + iotdb_username: 'root' + iotdb_password: 'root' + iotdb_urls: ['127.0.0.1:6667'] + +scrape_configs: + - job_name: 'iotdbModel' + static_configs: + - targets: ['localhost:9090'] + metrics_path: /metrics + relabel_configs: + - target_label: k8s_cluster + replacement: example-cluster + - target_label: k8s_namespace + replacement: default + - target_label: instance_id + replacement: model-service-local + + - job_name: 'iotdbWeb' + metrics_path: /actuator/prometheus + static_configs: + - targets: ['localhost:8080'] + labels: {env: local, instance_id: web-service-local} + relabel_configs: + - target_label: k8s_cluster + replacement: example-cluster + - target_label: k8s_namespace + replacement: default + + - job_name: 'iotdbAdmin' + metrics_path: /actuator/prometheus + static_configs: + - targets: ['localhost:8081'] + labels: {env: local, instance_id: admin-service-local} + relabel_configs: + - target_label: k8s_cluster + replacement: example-cluster + - target_label: k8s_namespace + replacement: default diff --git a/metric-scrape/k8s/configmap.yaml b/metric-scrape/k8s/configmap.yaml new file mode 100644 index 0000000..00fcd59 --- /dev/null +++ b/metric-scrape/k8s/configmap.yaml @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: metric-scrape-config + namespace: default + labels: + app.kubernetes.io/name: metric-scrape + app.kubernetes.io/part-of: iotdb-monitoring +data: + metric-scrape.yml: | + global: + scrape_interval: 30s + scrape_timeout: 10s + iotdb_database: 'metrics' + iotdb_username: 'root' + iotdb_password: 'root' + iotdb_urls: ['iotdb-service.default.svc.cluster.local:6667'] + + scrape_configs: + - job_name: 'iotdbModel' + metrics_path: /metrics + static_configs: + - targets: ['model-service.default.svc.cluster.local:9090'] + relabel_configs: + - target_label: k8s_cluster + replacement: example-cluster + - target_label: k8s_namespace + replacement: default + - target_label: instance_id + replacement: model-service + + - job_name: 'iotdbWeb' + metrics_path: /actuator/prometheus + static_configs: + - targets: ['web-service.default.svc.cluster.local:8080'] + labels: {env: example, instance_id: web-service} + relabel_configs: + - target_label: k8s_cluster + replacement: example-cluster + - target_label: k8s_namespace + replacement: default diff --git a/metric-scrape/k8s/deployment.yaml b/metric-scrape/k8s/deployment.yaml new file mode 100644 index 0000000..cbd23bd --- /dev/null +++ b/metric-scrape/k8s/deployment.yaml @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metric-scrape + namespace: default + labels: + app.kubernetes.io/name: metric-scrape + app.kubernetes.io/part-of: iotdb-monitoring +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: metric-scrape + template: + metadata: + labels: + app.kubernetes.io/name: metric-scrape + app.kubernetes.io/part-of: iotdb-monitoring + annotations: + configmap/checksum: "v1" + spec: + containers: + - name: metric-scrape + image: metric-scrape:latest + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 768Mi + volumeMounts: + - mountPath: /app/conf + name: config + readOnly: true + volumes: + - name: config + configMap: + name: metric-scrape-config + items: + - key: metric-scrape.yml + path: metric-scrape.yml diff --git a/metric-scrape/pom.xml b/metric-scrape/pom.xml new file mode 100644 index 0000000..b527546 --- /dev/null +++ b/metric-scrape/pom.xml @@ -0,0 +1,112 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-extras-parent</artifactId> + <version>2.0.4-SNAPSHOT</version> + </parent> + <artifactId>metric-scrape</artifactId> + <name>IoTDB Extras: Metric Scrape</name> + <description>Standalone Prometheus text exposition scraper for the IoTDB table model.</description> + <dependencies> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>isession</artifactId> + <version>${iotdb.version}</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-session</artifactId> + <version>${iotdb.version}</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>service-rpc</artifactId> + <version>${iotdb.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tsfile</groupId> + <artifactId>common</artifactId> + <version>${tsfile.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tsfile</groupId> + <artifactId>tsfile</artifactId> + <version>${tsfile.version}</version> + </dependency> + <dependency> + <groupId>org.yaml</groupId> + <artifactId>snakeyaml</artifactId> + <version>2.0</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <version>${slf4j.version}</version> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <createDependencyReducedPom>false</createDependencyReducedPom> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>org.apache.iotdb.metricscrape.MetricScrapeMain</mainClass> + </transformer> + </transformers> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <ignoredUnusedDeclaredDependencies> + <ignoredUnusedDeclaredDependency>org.slf4j:slf4j-simple</ignoredUnusedDeclaredDependency> + </ignoredUnusedDeclaredDependencies> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/IdentifierUtils.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/IdentifierUtils.java new file mode 100644 index 0000000..21dc5fb --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/IdentifierUtils.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +public class IdentifierUtils { + + private IdentifierUtils() {} + + public static String sanitize(String identifier) { + if (identifier == null || identifier.trim().isEmpty()) { + return "unnamed"; + } + StringBuilder builder = new StringBuilder(identifier.length()); + for (int i = 0; i < identifier.length(); i++) { + char c = identifier.charAt(i); + if (Character.isLetterOrDigit(c) || c == '_') { + builder.append(c); + } else { + builder.append('_'); + } + } + if (builder.length() == 0) { + return "unnamed"; + } + if (Character.isDigit(builder.charAt(0))) { + builder.insert(0, '_'); + } + return builder.toString(); + } + + public static String quoteSqlIdentifier(String identifier) { + return "\"" + identifier.replace("\"", "\"\"") + "\""; + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfig.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfig.java new file mode 100644 index 0000000..116afe9 --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfig.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class MetricScrapeConfig { + + private final GlobalConfig global; + private final List<ScrapeConfig> scrapeConfigs; + + public MetricScrapeConfig(GlobalConfig global, List<ScrapeConfig> scrapeConfigs) { + this.global = global; + this.scrapeConfigs = copyList(scrapeConfigs); + } + + public GlobalConfig getGlobal() { + return global; + } + + public List<ScrapeConfig> getScrapeConfigs() { + return scrapeConfigs; + } + + public static class GlobalConfig { + private final Duration scrapeInterval; + private final Duration scrapeTimeout; + private final String databaseName; + private final String username; + private final String password; + private final List<String> nodeUrls; + private final int writeBatchSize; + + public GlobalConfig( + Duration scrapeInterval, + Duration scrapeTimeout, + String databaseName, + String username, + String password, + List<String> nodeUrls, + int writeBatchSize) { + this.scrapeInterval = scrapeInterval; + this.scrapeTimeout = scrapeTimeout; + this.databaseName = databaseName; + this.username = username; + this.password = password; + this.nodeUrls = copyList(nodeUrls); + this.writeBatchSize = writeBatchSize; + } + + public Duration getScrapeInterval() { + return scrapeInterval; + } + + public Duration getScrapeTimeout() { + return scrapeTimeout; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public List<String> getNodeUrls() { + return nodeUrls; + } + + public int getWriteBatchSize() { + return writeBatchSize; + } + } + + public static class ScrapeConfig { + private final String jobName; + private final String scheme; + private final String metricsPath; + private final Duration scrapeInterval; + private final Duration scrapeTimeout; + private final List<StaticConfig> staticConfigs; + private final List<RelabelConfig> relabelConfigs; + + public ScrapeConfig( + String jobName, + String scheme, + String metricsPath, + Duration scrapeInterval, + Duration scrapeTimeout, + List<StaticConfig> staticConfigs, + List<RelabelConfig> relabelConfigs) { + this.jobName = jobName; + this.scheme = scheme; + this.metricsPath = metricsPath; + this.scrapeInterval = scrapeInterval; + this.scrapeTimeout = scrapeTimeout; + this.staticConfigs = copyList(staticConfigs); + this.relabelConfigs = copyList(relabelConfigs); + } + + public String getJobName() { + return jobName; + } + + public String getScheme() { + return scheme; + } + + public String getMetricsPath() { + return metricsPath; + } + + public Duration getScrapeInterval() { + return scrapeInterval; + } + + public Duration getScrapeTimeout() { + return scrapeTimeout; + } + + public List<StaticConfig> getStaticConfigs() { + return staticConfigs; + } + + public List<RelabelConfig> getRelabelConfigs() { + return relabelConfigs; + } + } + + public static class StaticConfig { + private final List<String> targets; + private final Map<String, String> labels; + + public StaticConfig(List<String> targets, Map<String, String> labels) { + this.targets = copyList(targets); + this.labels = Collections.unmodifiableMap(new LinkedHashMap<>(labels)); + } + + public List<String> getTargets() { + return targets; + } + + public Map<String, String> getLabels() { + return labels; + } + } + + public static class RelabelConfig { + private final String targetLabel; + private final String replacement; + + public RelabelConfig(String targetLabel, String replacement) { + this.targetLabel = targetLabel; + this.replacement = replacement; + } + + public String getTargetLabel() { + return targetLabel; + } + + public String getReplacement() { + return replacement; + } + } + + private static <T> List<T> copyList(List<T> values) { + return Collections.unmodifiableList(new ArrayList<>(values)); + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoader.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoader.java new file mode 100644 index 0000000..ea0d5e6 --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoader.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.apache.iotdb.metricscrape.MetricScrapeConfig.GlobalConfig; +import org.apache.iotdb.metricscrape.MetricScrapeConfig.RelabelConfig; +import org.apache.iotdb.metricscrape.MetricScrapeConfig.ScrapeConfig; +import org.apache.iotdb.metricscrape.MetricScrapeConfig.StaticConfig; + +import org.yaml.snakeyaml.Yaml; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +public class MetricScrapeConfigLoader { + + private static final Duration DEFAULT_SCRAPE_INTERVAL = Duration.ofSeconds(15); + private static final Duration DEFAULT_SCRAPE_TIMEOUT = Duration.ofSeconds(10); + private static final String DEFAULT_METRICS_PATH = "/metrics"; + private static final String DEFAULT_SCHEME = "http"; + private static final int DEFAULT_WRITE_BATCH_SIZE = 1000; + + private MetricScrapeConfigLoader() {} + + public static MetricScrapeConfig load(Path path) throws IOException { + try (InputStream inputStream = Files.newInputStream(path)) { + Object loaded = new Yaml().load(inputStream); + if (!(loaded instanceof Map)) { + throw new IllegalArgumentException("YAML root should be a map"); + } + return parseConfig(castMap(loaded, "root")); + } + } + + private static MetricScrapeConfig parseConfig(Map<String, Object> root) { + Map<String, Object> globalMap = castMap(required(root, "global"), "global"); + GlobalConfig global = parseGlobal(globalMap); + List<ScrapeConfig> scrapeConfigs = parseScrapeConfigs(required(root, "scrape_configs"), global); + if (scrapeConfigs.isEmpty()) { + throw new IllegalArgumentException("scrape_configs should not be empty"); + } + return new MetricScrapeConfig(global, scrapeConfigs); + } + + private static GlobalConfig parseGlobal(Map<String, Object> globalMap) { + Duration interval = + parseDuration(optional(globalMap, "scrape_interval"), DEFAULT_SCRAPE_INTERVAL); + Duration timeout = parseDuration(optional(globalMap, "scrape_timeout"), DEFAULT_SCRAPE_TIMEOUT); + String databaseName = requiredString(globalMap, "iotdb_database"); + String username = stringValue(optional(globalMap, "iotdb_username"), "root"); + String password = stringValue(optional(globalMap, "iotdb_password"), "root"); + List<String> nodeUrls = parseStringList(required(globalMap, "iotdb_urls"), "iotdb_urls"); + int batchSize = + parsePositiveInt(optional(globalMap, "write_batch_size"), DEFAULT_WRITE_BATCH_SIZE); + return new GlobalConfig( + interval, timeout, databaseName, username, password, nodeUrls, batchSize); + } + + private static List<ScrapeConfig> parseScrapeConfigs(Object value, GlobalConfig global) { + List<Object> configs = castList(value, "scrape_configs"); + List<ScrapeConfig> result = new ArrayList<>(); + for (int i = 0; i < configs.size(); i++) { + Map<String, Object> map = castMap(configs.get(i), "scrape_configs[" + i + "]"); + String jobName = requiredString(map, "job_name"); + String scheme = stringValue(optional(map, "scheme"), DEFAULT_SCHEME); + String metricsPath = + normalizeMetricsPath(stringValue(optional(map, "metrics_path"), DEFAULT_METRICS_PATH)); + Duration interval = + parseDuration(optional(map, "scrape_interval"), global.getScrapeInterval()); + Duration timeout = parseDuration(optional(map, "scrape_timeout"), global.getScrapeTimeout()); + List<StaticConfig> staticConfigs = parseStaticConfigs(required(map, "static_configs")); + List<RelabelConfig> relabelConfigs = parseRelabelConfigs(optional(map, "relabel_configs")); + result.add( + new ScrapeConfig( + jobName, scheme, metricsPath, interval, timeout, staticConfigs, relabelConfigs)); + } + return result; + } + + private static List<StaticConfig> parseStaticConfigs(Object value) { + List<Object> configs = castList(value, "static_configs"); + List<StaticConfig> result = new ArrayList<>(); + for (int i = 0; i < configs.size(); i++) { + Map<String, Object> map = castMap(configs.get(i), "static_configs[" + i + "]"); + List<String> targets = parseStringList(required(map, "targets"), "targets"); + Map<String, String> labels = parseStringMap(optional(map, "labels")); + result.add(new StaticConfig(targets, labels)); + } + return result; + } + + private static List<RelabelConfig> parseRelabelConfigs(Object value) { + if (value == null) { + return Collections.emptyList(); + } + List<Object> configs = castList(value, "relabel_configs"); + List<RelabelConfig> result = new ArrayList<>(); + for (int i = 0; i < configs.size(); i++) { + Map<String, Object> map = castMap(configs.get(i), "relabel_configs[" + i + "]"); + String targetLabel = requiredString(map, "target_label"); + String replacement = stringValue(optional(map, "replacement"), ""); + result.add(new RelabelConfig(targetLabel, replacement)); + } + return result; + } + + private static Object required(Map<String, Object> map, String key) { + Object value = map.get(key); + if (value == null) { + throw new IllegalArgumentException("Missing required config: " + key); + } + return value; + } + + private static Object optional(Map<String, Object> map, String key) { + return map.get(key); + } + + private static String requiredString(Map<String, Object> map, String key) { + String value = stringValue(required(map, key), null); + if (isBlank(value)) { + throw new IllegalArgumentException("Config " + key + " should not be empty"); + } + return value; + } + + private static String stringValue(Object value, String defaultValue) { + return value == null ? defaultValue : String.valueOf(value); + } + + private static String normalizeMetricsPath(String path) { + if (isBlank(path)) { + return DEFAULT_METRICS_PATH; + } + return path.startsWith("/") ? path : "/" + path; + } + + private static Duration parseDuration(Object value, Duration defaultValue) { + if (value == null) { + return defaultValue; + } + String text = String.valueOf(value).trim().toLowerCase(Locale.ROOT); + if (text.isEmpty()) { + return defaultValue; + } + long multiplier; + String number; + if (text.endsWith("ms")) { + multiplier = 1; + number = text.substring(0, text.length() - 2); + } else if (text.endsWith("s")) { + multiplier = 1000; + number = text.substring(0, text.length() - 1); + } else if (text.endsWith("m")) { + multiplier = 60_000; + number = text.substring(0, text.length() - 1); + } else if (text.endsWith("h")) { + multiplier = 3_600_000; + number = text.substring(0, text.length() - 1); + } else { + multiplier = 1000; + number = text; + } + try { + long amount = Long.parseLong(number); + if (amount <= 0) { + throw new IllegalArgumentException("Duration should be positive: " + value); + } + return Duration.ofMillis(amount * multiplier); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Illegal duration: " + value, e); + } + } + + private static int parsePositiveInt(Object value, int defaultValue) { + if (value == null) { + return defaultValue; + } + int parsed = Integer.parseInt(String.valueOf(value)); + if (parsed <= 0) { + throw new IllegalArgumentException("Value should be positive: " + value); + } + return parsed; + } + + private static List<String> parseStringList(Object value, String name) { + List<Object> rawList = castList(value, name); + List<String> result = new ArrayList<>(); + for (Object item : rawList) { + String text = String.valueOf(item); + if (isBlank(text)) { + throw new IllegalArgumentException(name + " contains empty value"); + } + result.add(text); + } + return result; + } + + private static Map<String, String> parseStringMap(Object value) { + if (value == null) { + return Collections.emptyMap(); + } + Map<String, Object> rawMap = castMap(value, "labels"); + Map<String, String> result = new LinkedHashMap<>(); + for (Map.Entry<String, Object> entry : rawMap.entrySet()) { + result.put(entry.getKey(), String.valueOf(entry.getValue())); + } + return result; + } + + @SuppressWarnings("unchecked") + private static Map<String, Object> castMap(Object value, String name) { + if (!(value instanceof Map)) { + throw new IllegalArgumentException(name + " should be a map"); + } + Map<String, Object> result = new LinkedHashMap<>(); + ((Map<?, ?>) value).forEach((key, item) -> result.put(String.valueOf(key), item)); + return result; + } + + private static List<Object> castList(Object value, String name) { + if (!(value instanceof List)) { + throw new IllegalArgumentException(name + " should be a list"); + } + return new ArrayList<>((List<?>) value); + } + + private static boolean isBlank(String value) { + return value == null || value.trim().isEmpty(); + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeHttpClient.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeHttpClient.java new file mode 100644 index 0000000..267c55b --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeHttpClient.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; + +public class MetricScrapeHttpClient { + + public String get(String url, int timeoutMs) throws IOException { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setRequestMethod("GET"); + connection.setConnectTimeout(timeoutMs); + connection.setReadTimeout(timeoutMs); + connection.setRequestProperty( + "Accept", "text/plain; version=0.0.4, application/openmetrics-text, */*"); + connection.setRequestProperty("User-Agent", "Apache-IoTDB-MetricScrape/1.0"); + + int statusCode = connection.getResponseCode(); + if (statusCode < 200 || statusCode >= 300) { + throw new IOException("Metric scrape target " + url + " returns HTTP " + statusCode); + } + try (InputStream inputStream = connection.getInputStream()) { + return readToString(inputStream); + } finally { + connection.disconnect(); + } + } + + private String readToString(InputStream inputStream) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int length; + while ((length = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, length); + } + return new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeMain.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeMain.java new file mode 100644 index 0000000..2885f05 --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeMain.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; + +public class MetricScrapeMain { + + private static final Logger LOGGER = LoggerFactory.getLogger(MetricScrapeMain.class); + private static final String DEFAULT_CONFIG = "conf/metric-scrape.yml"; + + public static void main(String[] args) throws Exception { + Path configPath = parseConfigPath(args); + MetricScrapeConfig config = MetricScrapeConfigLoader.load(configPath); + MetricScrapeService service = new MetricScrapeService(config); + CountDownLatch stopLatch = new CountDownLatch(1); + + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + LOGGER.info("Stopping metric scrape service."); + service.stop(); + stopLatch.countDown(); + }, + "metric-scrape-shutdown")); + + service.start(); + LOGGER.info("Metric scrape service started with config {}.", configPath); + stopLatch.await(); + } + + private static Path parseConfigPath(String[] args) { + if (args.length == 0) { + return Paths.get(DEFAULT_CONFIG); + } + if (args.length == 2 && ("-c".equals(args[0]) || "--config".equals(args[0]))) { + return Paths.get(args[1]); + } + throw new IllegalArgumentException("Usage: java -jar metric-scrape.jar [-c config.yml]"); + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapePlanner.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapePlanner.java new file mode 100644 index 0000000..42e48af --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapePlanner.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.apache.iotdb.metricscrape.MetricScrapeConfig.RelabelConfig; +import org.apache.iotdb.metricscrape.MetricScrapeConfig.ScrapeConfig; +import org.apache.iotdb.metricscrape.MetricScrapeConfig.StaticConfig; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class MetricScrapePlanner { + + public List<MetricScrapeTarget> buildTargets(MetricScrapeConfig config) { + List<MetricScrapeTarget> result = new ArrayList<>(); + for (ScrapeConfig scrapeConfig : config.getScrapeConfigs()) { + for (StaticConfig staticConfig : scrapeConfig.getStaticConfigs()) { + for (String target : staticConfig.getTargets()) { + Map<String, String> labels = new LinkedHashMap<>(); + labels.put("job_name", scrapeConfig.getJobName()); + labels.put("instance", target); + labels.putAll(staticConfig.getLabels()); + for (RelabelConfig relabelConfig : scrapeConfig.getRelabelConfigs()) { + labels.put(relabelConfig.getTargetLabel(), relabelConfig.getReplacement()); + } + result.add( + new MetricScrapeTarget( + scrapeConfig.getJobName(), + buildUrl(scrapeConfig, target), + scrapeConfig.getScrapeInterval(), + scrapeConfig.getScrapeTimeout(), + labels)); + } + } + } + return result; + } + + private String buildUrl(ScrapeConfig scrapeConfig, String target) { + if (target.startsWith("http://";) || target.startsWith("https://";)) { + return appendMetricsPath(target, scrapeConfig.getMetricsPath()); + } + return scrapeConfig.getScheme() + "://" + target + scrapeConfig.getMetricsPath(); + } + + private String appendMetricsPath(String target, String metricsPath) { + if (target.endsWith(metricsPath)) { + return target; + } + if (target.endsWith("/")) { + return target.substring(0, target.length() - 1) + metricsPath; + } + return target + metricsPath; + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeService.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeService.java new file mode 100644 index 0000000..466469b --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeService.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class MetricScrapeService { + + private static final Logger LOGGER = LoggerFactory.getLogger(MetricScrapeService.class); + + private final MetricScrapeConfig config; + private final MetricScrapeHttpClient httpClient = new MetricScrapeHttpClient(); + private final PrometheusTextParser parser = new PrometheusTextParser(); + private final MetricScrapePlanner planner = new MetricScrapePlanner(); + private MetricTableWriter writer; + private ScheduledExecutorService executorService; + + public MetricScrapeService(MetricScrapeConfig config) { + this.config = config; + } + + public synchronized void start() throws Exception { + List<MetricScrapeTarget> targets = planner.buildTargets(config); + if (targets.isEmpty()) { + throw new IllegalArgumentException("No metric scrape target configured"); + } + writer = new MetricTableWriter(config.getGlobal()); + writer.initializeDatabase(); + AtomicInteger threadIndex = new AtomicInteger(); + executorService = + Executors.newScheduledThreadPool( + Math.min(targets.size(), Runtime.getRuntime().availableProcessors() * 2), + runnable -> { + Thread thread = + new Thread(runnable, "metric-scrape-" + threadIndex.incrementAndGet()); + thread.setDaemon(false); + return thread; + }); + for (MetricScrapeTarget target : targets) { + ScheduledExecutorUtil.safelyScheduleWithFixedDelay( + executorService, + () -> scrapeTarget(target), + 0, + target.getInterval().toMillis(), + TimeUnit.MILLISECONDS); + LOGGER.info( + "Scheduled metric target {} with interval {}.", target.getUrl(), target.getInterval()); + } + } + + public synchronized void stop() { + if (executorService != null) { + executorService.shutdownNow(); + executorService = null; + } + if (writer != null) { + writer.close(); + writer = null; + } + } + + private void scrapeTarget(MetricScrapeTarget target) { + try { + String text = + httpClient.get(target.getUrl(), Math.toIntExact(target.getTimeout().toMillis())); + List<PrometheusSample> samples = parser.parse(text, System.currentTimeMillis()); + writer.write(enrichSamples(samples, target)); + LOGGER.debug("Scraped {} samples from {}.", samples.size(), target.getUrl()); + } catch (IOException | RuntimeException e) { + LOGGER.warn("Failed to scrape metric target {}.", target.getUrl(), e); + } catch (Exception e) { + LOGGER.warn("Failed to write metric samples from {}.", target.getUrl(), e); + } + } + + private List<PrometheusSample> enrichSamples( + List<PrometheusSample> samples, MetricScrapeTarget target) { + List<PrometheusSample> result = new ArrayList<>(samples.size()); + for (PrometheusSample sample : samples) { + result.add(sample.withLabels(mergeLabels(target.getLabels(), sample.getLabels()))); + } + return result; + } + + private Map<String, String> mergeLabels( + Map<String, String> targetLabels, Map<String, String> sampleLabels) { + java.util.LinkedHashMap<String, String> labels = new java.util.LinkedHashMap<>(targetLabels); + for (Map.Entry<String, String> entry : sampleLabels.entrySet()) { + labels.putIfAbsent(entry.getKey(), entry.getValue()); + } + return labels; + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeTarget.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeTarget.java new file mode 100644 index 0000000..2519987 --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricScrapeTarget.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import java.time.Duration; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +public class MetricScrapeTarget { + + private final String jobName; + private final String url; + private final Duration interval; + private final Duration timeout; + private final Map<String, String> labels; + + public MetricScrapeTarget( + String jobName, String url, Duration interval, Duration timeout, Map<String, String> labels) { + this.jobName = jobName; + this.url = url; + this.interval = interval; + this.timeout = timeout; + this.labels = Collections.unmodifiableMap(new LinkedHashMap<>(labels)); + } + + public String getJobName() { + return jobName; + } + + public String getUrl() { + return url; + } + + public Duration getInterval() { + return interval; + } + + public Duration getTimeout() { + return timeout; + } + + public Map<String, String> getLabels() { + return labels; + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricTableWriter.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricTableWriter.java new file mode 100644 index 0000000..a0df465 --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/MetricTableWriter.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.metricscrape.MetricScrapeConfig.GlobalConfig; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.TableSessionBuilder; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class MetricTableWriter { + + private static final Logger LOGGER = LoggerFactory.getLogger(MetricTableWriter.class); + + private final GlobalConfig config; + private final ITableSession session; + + public MetricTableWriter(GlobalConfig config) throws IoTDBConnectionException { + this.config = config; + this.session = + new TableSessionBuilder() + .nodeUrls(config.getNodeUrls()) + .username(config.getUsername()) + .password(config.getPassword()) + .database(config.getDatabaseName()) + .build(); + } + + public synchronized void initializeDatabase() + throws IoTDBConnectionException, StatementExecutionException { + session.executeNonQueryStatement( + "CREATE DATABASE IF NOT EXISTS " + + IdentifierUtils.quoteSqlIdentifier(config.getDatabaseName())); + } + + public synchronized void write(List<PrometheusSample> samples) + throws IoTDBConnectionException, StatementExecutionException { + if (samples.isEmpty()) { + return; + } + Map<TabletKey, TabletBuilder> builders = new LinkedHashMap<>(); + for (PrometheusSample sample : samples) { + TabletKey key = TabletKey.from(sample); + TabletBuilder builder = + builders.computeIfAbsent( + key, ignored -> new TabletBuilder(key, config.getWriteBatchSize())); + builder.add(sample); + if (builder.isFull()) { + flush(builder); + builder.reset(); + } + } + for (TabletBuilder builder : builders.values()) { + if (!builder.isEmpty()) { + flush(builder); + } + } + } + + public void close() { + try { + session.close(); + } catch (IoTDBConnectionException e) { + LOGGER.warn("Failed to close IoTDB session.", e); + } + } + + private void flush(TabletBuilder builder) + throws IoTDBConnectionException, StatementExecutionException { + session.insert(builder.tablet); + } + + private static class TabletKey { + private final String table; + private final String field; + private final List<String> labelKeys; + private final Map<String, String> sanitizedToOriginalLabel; + + private static TabletKey from(PrometheusSample sample) { + Map<String, String> sanitizedToOriginalLabel = sanitizeLabels(sample); + List<String> labelKeys = new ArrayList<>(sanitizedToOriginalLabel.keySet()); + return new TabletKey( + IdentifierUtils.sanitize(sample.getMetricFamilyName()), + IdentifierUtils.sanitize(sample.getMetricName()), + labelKeys, + sanitizedToOriginalLabel); + } + + private static Map<String, String> sanitizeLabels(PrometheusSample sample) { + List<String> originalLabels = new ArrayList<>(sample.getLabels().keySet()); + Collections.sort(originalLabels); + Map<String, String> result = new LinkedHashMap<>(); + for (String originalLabel : originalLabels) { + String sanitizedLabel = IdentifierUtils.sanitize(originalLabel); + String previous = result.put(sanitizedLabel, originalLabel); + if (previous != null && !previous.equals(originalLabel)) { + throw new IllegalArgumentException( + "Prometheus labels " + + previous + + " and " + + originalLabel + + " map to the same IoTDB column " + + sanitizedLabel); + } + } + return result; + } + + private TabletKey( + String table, + String field, + List<String> labelKeys, + Map<String, String> sanitizedToOriginalLabel) { + this.table = table; + this.field = field; + this.labelKeys = Collections.unmodifiableList(new ArrayList<>(labelKeys)); + this.sanitizedToOriginalLabel = + Collections.unmodifiableMap(new LinkedHashMap<>(sanitizedToOriginalLabel)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TabletKey)) { + return false; + } + TabletKey tabletKey = (TabletKey) o; + return Objects.equals(table, tabletKey.table) + && Objects.equals(field, tabletKey.field) + && Objects.equals(labelKeys, tabletKey.labelKeys); + } + + @Override + public int hashCode() { + return Objects.hash(table, field, labelKeys); + } + } + + private static class TabletBuilder { + private final TabletKey key; + private final Tablet tablet; + + private TabletBuilder(TabletKey key, int maxRows) { + this.key = key; + List<String> columnNames = new ArrayList<>(key.labelKeys); + columnNames.add(key.field); + List<TSDataType> dataTypes = new ArrayList<>(columnNames.size()); + List<ColumnCategory> columnCategories = new ArrayList<>(columnNames.size()); + for (int i = 0; i < key.labelKeys.size(); i++) { + dataTypes.add(TSDataType.STRING); + columnCategories.add(ColumnCategory.TAG); + } + dataTypes.add(TSDataType.DOUBLE); + columnCategories.add(ColumnCategory.FIELD); + this.tablet = new Tablet(key.table, columnNames, dataTypes, columnCategories, maxRows); + } + + private boolean isFull() { + return tablet.getRowSize() == tablet.getMaxRowNumber(); + } + + private boolean isEmpty() { + return tablet.getRowSize() == 0; + } + + private void reset() { + tablet.reset(); + } + + private void add(PrometheusSample sample) { + int row = tablet.getRowSize(); + tablet.addTimestamp(row, sample.getTimestamp()); + for (String labelKey : key.labelKeys) { + String originalLabel = key.sanitizedToOriginalLabel.get(labelKey); + tablet.addValue(labelKey, row, sample.getLabels().get(originalLabel)); + } + tablet.addValue(key.field, row, sample.getValue()); + } + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusSample.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusSample.java new file mode 100644 index 0000000..7e8a5f8 --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusSample.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +public class PrometheusSample { + + private final String metricFamilyName; + private final String metricName; + private final Map<String, String> labels; + private final double value; + private final long timestamp; + + public PrometheusSample( + String metricFamilyName, + String metricName, + Map<String, String> labels, + double value, + long timestamp) { + this.metricFamilyName = metricFamilyName; + this.metricName = metricName; + this.labels = Collections.unmodifiableMap(new LinkedHashMap<>(labels)); + this.value = value; + this.timestamp = timestamp; + } + + public String getMetricFamilyName() { + return metricFamilyName; + } + + public String getMetricName() { + return metricName; + } + + public Map<String, String> getLabels() { + return labels; + } + + public double getValue() { + return value; + } + + public long getTimestamp() { + return timestamp; + } + + public PrometheusSample withLabels(Map<String, String> newLabels) { + return new PrometheusSample(metricFamilyName, metricName, newLabels, value, timestamp); + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusTextParser.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusTextParser.java new file mode 100644 index 0000000..c328dd1 --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/PrometheusTextParser.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +public class PrometheusTextParser { + + private static final String[] METRIC_FAMILY_SAMPLE_SUFFIXES = { + "_sum", "_count", "_bucket", "_created" + }; + + public List<PrometheusSample> parse(String text, long defaultTimestamp) { + if (text == null) { + throw new IllegalArgumentException("Prometheus text should not be null"); + } + List<PrometheusSample> samples = new ArrayList<>(); + String[] lines = text.split("\\r\\n|\\n|\\r"); + Set<String> helpMetricNames = collectHelpMetricNames(lines); + for (int i = 0; i < lines.length; i++) { + String line = lines[i].trim(); + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + samples.add(parseSample(line, i + 1, defaultTimestamp, helpMetricNames)); + } + return samples; + } + + private PrometheusSample parseSample( + String line, int lineNumber, long defaultTimestamp, Set<String> helpMetricNames) { + Parser parser = new Parser(line, lineNumber); + String metricName = parser.parseMetricName(); + Map<String, String> labels = parser.parseLabels(); + parser.skipWhitespace(); + double value = parser.parseValue(); + parser.skipWhitespace(); + long timestamp = parser.parseOptionalTimestamp(defaultTimestamp); + parser.skipWhitespace(); + parser.expectEnd(); + return new PrometheusSample( + resolveMetricFamilyName(metricName, helpMetricNames), metricName, labels, value, timestamp); + } + + private static Set<String> collectHelpMetricNames(String[] lines) { + Set<String> helpMetricNames = new LinkedHashSet<>(); + for (String rawLine : lines) { + String line = rawLine.trim(); + if (!line.startsWith("# HELP")) { + continue; + } + int position = "# HELP".length(); + while (position < line.length() && Character.isWhitespace(line.charAt(position))) { + position++; + } + int start = position; + while (position < line.length() && !Character.isWhitespace(line.charAt(position))) { + position++; + } + if (start < position) { + helpMetricNames.add(line.substring(start, position)); + } + } + return helpMetricNames; + } + + private static String resolveMetricFamilyName(String metricName, Set<String> helpMetricNames) { + if (helpMetricNames.contains(metricName)) { + return metricName; + } + + String result = null; + for (String helpMetricName : helpMetricNames) { + if (isMetricFamilyComponent(metricName, helpMetricName) + && (result == null || helpMetricName.length() > result.length())) { + result = helpMetricName; + } + } + return result == null ? metricName : result; + } + + private static boolean isMetricFamilyComponent(String metricName, String helpMetricName) { + for (String suffix : METRIC_FAMILY_SAMPLE_SUFFIXES) { + if (metricName.equals(helpMetricName + suffix)) { + return true; + } + } + return false; + } + + private static double parsePrometheusDouble(String token, int lineNumber) { + String lowerToken = token.toLowerCase(Locale.ROOT); + switch (lowerToken) { + case "inf": + case "+inf": + case "infinity": + case "+infinity": + return Double.POSITIVE_INFINITY; + case "-inf": + case "-infinity": + return Double.NEGATIVE_INFINITY; + case "nan": + case "+nan": + case "-nan": + return Double.NaN; + default: + break; + } + if (token.endsWith("d") || token.endsWith("D") || token.endsWith("f") || token.endsWith("F")) { + throw new IllegalArgumentException( + "Illegal Prometheus sample value " + token + " at line " + lineNumber); + } + try { + return Double.parseDouble(removeValidNumericUnderscores(token, lineNumber)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Illegal Prometheus sample value " + token + " at line " + lineNumber, e); + } + } + + private static String removeValidNumericUnderscores(String token, int lineNumber) { + if (token.indexOf('_') < 0) { + return token; + } + for (int i = 0; i < token.length(); i++) { + if (token.charAt(i) == '_') { + if (i == 0 + || i == token.length() - 1 + || !Character.isDigit(token.charAt(i - 1)) + || !Character.isDigit(token.charAt(i + 1))) { + throw new IllegalArgumentException( + "Illegal Prometheus sample value " + token + " at line " + lineNumber); + } + } + } + return token.replace("_", ""); + } + + private static class Parser { + private final String line; + private final int lineNumber; + private int position; + + private Parser(String line, int lineNumber) { + this.line = line; + this.lineNumber = lineNumber; + } + + private String parseMetricName() { + if (position >= line.length() || Character.isWhitespace(line.charAt(position))) { + throw new IllegalArgumentException("Illegal Prometheus metric name at line " + lineNumber); + } + int start = position; + while (position < line.length() + && !Character.isWhitespace(line.charAt(position)) + && line.charAt(position) != '{') { + position++; + } + if (start == position) { + throw new IllegalArgumentException("Illegal Prometheus metric name at line " + lineNumber); + } + return line.substring(start, position); + } + + private Map<String, String> parseLabels() { + Map<String, String> labels = new LinkedHashMap<>(); + if (position >= line.length() || line.charAt(position) != '{') { + return labels; + } + position++; + skipWhitespace(); + if (position < line.length() && line.charAt(position) == '}') { + position++; + return labels; + } + + while (position < line.length()) { + String name = parseLabelName(); + skipWhitespace(); + expect('='); + skipWhitespace(); + String value = parseLabelValue(); + if (labels.put(name, value) != null) { + throw new IllegalArgumentException( + "Duplicate Prometheus label " + name + " at line " + lineNumber); + } + skipWhitespace(); + if (position < line.length() && line.charAt(position) == ',') { + position++; + skipWhitespace(); + if (position < line.length() && line.charAt(position) == '}') { + position++; + return labels; + } + continue; + } + expect('}'); + return labels; + } + throw new IllegalArgumentException("Unclosed Prometheus label set at line " + lineNumber); + } + + private String parseLabelName() { + int start = position; + while (position < line.length() && line.charAt(position) != '=') { + if (line.charAt(position) == ',' || line.charAt(position) == '}') { + break; + } + position++; + } + String name = line.substring(start, position).trim(); + if (name.isEmpty()) { + throw new IllegalArgumentException("Illegal Prometheus label name at line " + lineNumber); + } + return name; + } + + private String parseLabelValue() { + expect('"'); + StringBuilder builder = new StringBuilder(); + int delimiterDepth = 0; + while (position < line.length()) { + char c = line.charAt(position++); + if (c == '"') { + if (delimiterDepth == 0 && isLabelValueTerminator()) { + return builder.toString(); + } + builder.append(c); + continue; + } + if (c == '\\') { + if (position >= line.length()) { + throw new IllegalArgumentException( + "Illegal Prometheus label escape at line " + lineNumber); + } + char escaped = line.charAt(position++); + switch (escaped) { + case 'n': + builder.append('\n'); + break; + case '\\': + case '"': + builder.append(escaped); + break; + default: + throw new IllegalArgumentException( + "Illegal Prometheus label escape at line " + lineNumber); + } + } else { + builder.append(c); + delimiterDepth = updateDelimiterDepth(delimiterDepth, c); + } + } + throw new IllegalArgumentException("Unclosed Prometheus label value at line " + lineNumber); + } + + private int updateDelimiterDepth(int delimiterDepth, char c) { + switch (c) { + case '{': + case '[': + case '(': + return delimiterDepth + 1; + case '}': + case ']': + case ')': + return Math.max(0, delimiterDepth - 1); + default: + return delimiterDepth; + } + } + + private boolean isLabelValueTerminator() { + int next = skipWhitespace(position); + if (next >= line.length()) { + return true; + } + if (line.charAt(next) == '}') { + return next + 1 >= line.length() || Character.isWhitespace(line.charAt(next + 1)); + } + if (line.charAt(next) != ',') { + return false; + } + int afterComma = skipWhitespace(next + 1); + return afterComma >= line.length() + || line.charAt(afterComma) == '}' + || isLabelAssignmentStart(afterComma); + } + + private boolean isLabelAssignmentStart(int start) { + if (start >= line.length() || line.charAt(start) == '"') { + return false; + } + int scan = start; + while (scan < line.length()) { + char c = line.charAt(scan); + if (c == '=') { + String labelName = line.substring(start, scan).trim(); + int valueStart = skipWhitespace(scan + 1); + return !labelName.isEmpty() + && valueStart < line.length() + && line.charAt(valueStart) == '"'; + } + if (c == ',' || c == '}') { + return false; + } + scan++; + } + return false; + } + + private double parseValue() { + return parsePrometheusDouble(parseToken("sample value"), lineNumber); + } + + private long parseOptionalTimestamp(long defaultTimestamp) { + if (position >= line.length()) { + return defaultTimestamp; + } + String token = parseToken("sample timestamp"); + try { + return Long.parseLong(token); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Illegal Prometheus sample timestamp " + token + " at line " + lineNumber, e); + } + } + + private String parseToken(String name) { + if (position >= line.length() || Character.isWhitespace(line.charAt(position))) { + throw new IllegalArgumentException("Missing Prometheus " + name + " at line " + lineNumber); + } + int start = position; + while (position < line.length() && !Character.isWhitespace(line.charAt(position))) { + position++; + } + return line.substring(start, position); + } + + private void skipWhitespace() { + while (position < line.length() && Character.isWhitespace(line.charAt(position))) { + position++; + } + } + + private int skipWhitespace(int from) { + while (from < line.length() && Character.isWhitespace(line.charAt(from))) { + from++; + } + return from; + } + + private void expect(char expected) { + if (position >= line.length() || line.charAt(position) != expected) { + throw new IllegalArgumentException( + "Expected '" + expected + "' in Prometheus sample at line " + lineNumber); + } + position++; + } + + private void expectEnd() { + if (position != line.length()) { + throw new IllegalArgumentException( + "Unexpected Prometheus sample content at line " + lineNumber); + } + } + } +} diff --git a/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/ScheduledExecutorUtil.java b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/ScheduledExecutorUtil.java new file mode 100644 index 0000000..fbbd6d9 --- /dev/null +++ b/metric-scrape/src/main/java/org/apache/iotdb/metricscrape/ScheduledExecutorUtil.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +final class ScheduledExecutorUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(ScheduledExecutorUtil.class); + + private ScheduledExecutorUtil() {} + + @SuppressWarnings("unsafeThreadSchedule") + static ScheduledFuture<?> safelyScheduleWithFixedDelay( + ScheduledExecutorService executor, + Runnable command, + long initialDelay, + long delay, + TimeUnit unit) { + return executor.scheduleWithFixedDelay( + () -> { + try { + command.run(); + } catch (Throwable t) { + LOGGER.error("Scheduled metric scrape task failed.", t); + } + }, + initialDelay, + delay, + unit); + } +} diff --git a/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoaderTest.java b/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoaderTest.java new file mode 100644 index 0000000..648c5fc --- /dev/null +++ b/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/MetricScrapeConfigLoaderTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.junit.Test; + +import java.nio.file.Paths; +import java.time.Duration; + +import static org.junit.Assert.assertEquals; + +public class MetricScrapeConfigLoaderTest { + + @Test + public void loadExampleConfig() throws Exception { + MetricScrapeConfig config = MetricScrapeConfigLoader.load(Paths.get("conf/metric-scrape.yml")); + + assertEquals(Duration.ofSeconds(15), config.getGlobal().getScrapeInterval()); + assertEquals(Duration.ofSeconds(10), config.getGlobal().getScrapeTimeout()); + assertEquals("metrics", config.getGlobal().getDatabaseName()); + assertEquals(3, config.getScrapeConfigs().size()); + assertEquals("iotdbModel", config.getScrapeConfigs().get(0).getJobName()); + assertEquals("/metrics", config.getScrapeConfigs().get(0).getMetricsPath()); + } +} diff --git a/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/PrometheusTextParserTest.java b/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/PrometheusTextParserTest.java new file mode 100644 index 0000000..3677f94 --- /dev/null +++ b/metric-scrape/src/test/java/org/apache/iotdb/metricscrape/PrometheusTextParserTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.metricscrape; + +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class PrometheusTextParserTest { + + @Test + public void parseSummarySamplesIntoHelpFamily() { + String text = + "# HELP request_duration_seconds Request duration.\n" + + "# TYPE request_duration_seconds summary\n" + + "request_duration_seconds_sum{method=\"query\",status=\"ok\"} 10.5\n" + + "request_duration_seconds_count{method=\"query\",status=\"ok\"} 3 1000\n"; + + List<PrometheusSample> samples = new PrometheusTextParser().parse(text, 123); + + assertEquals(2, samples.size()); + assertEquals("request_duration_seconds", samples.get(0).getMetricFamilyName()); + assertEquals("request_duration_seconds_sum", samples.get(0).getMetricName()); + assertEquals(10.5, samples.get(0).getValue(), 0.0); + assertEquals(123, samples.get(0).getTimestamp()); + assertEquals(1000, samples.get(1).getTimestamp()); + } + + @Test + public void parseLabelValueContainingQueryText() { + String text = "statement_cost{type=\"select * from root where a=\\\"b\\\", x=1\"} 1.0"; + + List<PrometheusSample> samples = new PrometheusTextParser().parse(text, 123); + + assertEquals(1, samples.size()); + assertEquals("select * from root where a=\"b\", x=1", samples.get(0).getLabels().get("type")); + } + + @Test + public void parseUnescapedQuoteInNestedLabelValueWithoutGhostLabels() { + String text = "m{type=\"Filter{predicate=\"x\", op=\"AND\"}\",quantile=\"0.5\"} 1"; + + List<PrometheusSample> samples = new PrometheusTextParser().parse(text, 123); + + assertEquals(1, samples.size()); + assertEquals("Filter{predicate=\"x\", op=\"AND\"}", samples.get(0).getLabels().get("type")); + assertEquals("0.5", samples.get(0).getLabels().get("quantile")); + assertFalse(samples.get(0).getLabels().containsKey("op")); + } + + @Test + public void onlyRouteKnownFamilyComponentSuffixesToHelpFamily() { + String text = + "# HELP http_requests Total requests.\n" + + "http_requests 5\n" + + "http_requests_duration_seconds 0.3\n" + + "http_requests_sum 7\n"; + + List<PrometheusSample> samples = new PrometheusTextParser().parse(text, 123); + + assertEquals("http_requests", samples.get(0).getMetricFamilyName()); + assertEquals("http_requests_duration_seconds", samples.get(1).getMetricFamilyName()); + assertEquals("http_requests", samples.get(2).getMetricFamilyName()); + } + + @Test + public void parsePrometheusFloatTokens() { + List<PrometheusSample> samples = + new PrometheusTextParser().parse("a inf\nb -inf\nc nan\nd 1_000\n", 123); + + assertTrue(Double.isInfinite(samples.get(0).getValue())); + assertTrue(Double.isInfinite(samples.get(1).getValue())); + assertTrue(Double.isNaN(samples.get(2).getValue())); + assertEquals(1000.0, samples.get(3).getValue(), 0.0); + } + + @Test + public void rejectJavaSpecificFloatSuffixes() { + assertThrows( + IllegalArgumentException.class, () -> new PrometheusTextParser().parse("a 1d\n", 123)); + assertThrows( + IllegalArgumentException.class, () -> new PrometheusTextParser().parse("a 2f\n", 123)); + } + + @Test + public void rejectUnknownLabelEscapes() { + assertThrows( + IllegalArgumentException.class, + () -> new PrometheusTextParser().parse("a{label=\"\\t\"} 1\n", 123)); + } +} diff --git a/pom.xml b/pom.xml index ca88f39..4de8c34 100644 --- a/pom.xml +++ b/pom.xml @@ -36,6 +36,7 @@ <module>connectors</module> <module>distributions</module> <module>iotdb-collector</module> + <module>metric-scrape</module> <module>mybatis-generator</module> </modules> <properties>
