This is an automated email from the ASF dual-hosted git repository. zhaoqingran pushed a commit to branch hertzbeat-log in repository https://gitbox.apache.org/repos/asf/hertzbeat.git
commit 0b60c2f026da2ad0ee679a036fc6d52a068be535 Author: Logic <[email protected]> AuthorDate: Mon Apr 7 11:23:01 2025 +0800 feat(extend-http-xmlpath): add system info metric and improve XML parsing - Add new metric 'system_info' to collect device information - Update 'status' metric to use more accurate XML path - Introduce aliasFields and calculates sections for better data handling- Improve field naming consistency across languages --- .../common/constants/ConfigConstants.java | 2 + hertzbeat-log/pom.xml | 74 ++++++++++++ .../hertzbeat/log/config/LogAutoConfiguration.java | 34 ++++++ .../hertzbeat/log/config/OpenTelemetryConfig.java | 126 +++++++++++++++++++++ .../src/main/resources/META-INF/spring.factories | 17 +++ hertzbeat-manager/pom.xml | 5 + .../src/main/resources/logback-spring.xml | 27 ++++- pom.xml | 7 ++ 8 files changed, 291 insertions(+), 1 deletion(-) diff --git a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java index 727d9d7d12..d71e7dc2e6 100644 --- a/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java +++ b/hertzbeat-common/src/main/java/org/apache/hertzbeat/common/constants/ConfigConstants.java @@ -65,6 +65,8 @@ public interface ConfigConstants { String INFO = "info"; String GRAFANA = "grafana"; + + String LOG = "log"; } } diff --git a/hertzbeat-log/pom.xml b/hertzbeat-log/pom.xml new file mode 100644 index 0000000000..871c6a57fa --- /dev/null +++ b/hertzbeat-log/pom.xml @@ -0,0 +1,74 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.hertzbeat</groupId> + <artifactId>hertzbeat</artifactId> + <version>2.0-SNAPSHOT</version> + </parent> + + <artifactId>hertzbeat-log</artifactId> + <name>${project.artifactId}</name> + <properties> + <maven-jar-plugin.version>3.2.0</maven-jar-plugin.version> + <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version> + </properties> + + <dependencies> + <!-- hertzbeat common --> + <dependency> + <groupId>org.apache.hertzbeat</groupId> + <artifactId>hertzbeat-common</artifactId> + </dependency> + <!-- hertzbeat warehouse --> + <dependency> + <groupId>org.apache.hertzbeat</groupId> + <artifactId>hertzbeat-warehouse</artifactId> + </dependency> + <!-- OpenTelemetry --> + <dependency> + <groupId>io.opentelemetry</groupId> + <artifactId>opentelemetry-api</artifactId> + <version>1.33.0</version> + </dependency> + + <dependency> + <groupId>io.opentelemetry</groupId> + <artifactId>opentelemetry-sdk</artifactId> + <version>1.33.0</version> + </dependency> + + <dependency> + <groupId>io.opentelemetry</groupId> + <artifactId>opentelemetry-sdk-logs</artifactId> + <version>1.33.0</version> + </dependency> + <dependency> + <groupId>io.opentelemetry</groupId> + <artifactId>opentelemetry-exporter-otlp</artifactId> + <version>1.33.0</version> + </dependency> + <dependency> + <groupId>io.opentelemetry.instrumentation</groupId> + <artifactId>opentelemetry-logback-appender-1.0</artifactId> + <version>2.14.0-alpha</version> + </dependency> + </dependencies> +</project> diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/LogAutoConfiguration.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/LogAutoConfiguration.java new file mode 100644 index 0000000000..bd661c21d9 --- /dev/null +++ b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/LogAutoConfiguration.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.log.config; + +import org.apache.hertzbeat.common.constants.ConfigConstants; +import org.apache.hertzbeat.common.constants.SignConstants; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.ComponentScan; + +/** + * Log auto configuration. + */ +@ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG + + SignConstants.DOT + + ConfigConstants.FunctionModuleConstants.LOG +) +@EnableConfigurationProperties(LogAutoConfiguration.class) +public class LogAutoConfiguration { +} diff --git a/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/OpenTelemetryConfig.java b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/OpenTelemetryConfig.java new file mode 100644 index 0000000000..e342214204 --- /dev/null +++ b/hertzbeat-log/src/main/java/org/apache/hertzbeat/log/config/OpenTelemetryConfig.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hertzbeat.log.config; + +import static io.opentelemetry.semconv.ServiceAttributes.SERVICE_NAME; +import static io.opentelemetry.semconv.ServiceAttributes.SERVICE_VERSION; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter; +import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; +import io.opentelemetry.sdk.resources.Resource; + +import jakarta.annotation.PostConstruct; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hertzbeat.warehouse.store.history.greptime.GreptimeProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +/** + * OpenTelemetryConfig is responsible for initializing OpenTelemetry with the specified service name and GrepTimeDB endpoint. + * It ensures that the initialization is done in a thread-safe manner and includes authentication for GrepTimeDB. + */ +@Configuration +public class OpenTelemetryConfig { + + private static final Logger log = LoggerFactory.getLogger(OpenTelemetryConfig.class); + + @Value("${hzb.version}") + private String version; + + @Autowired + private GreptimeProperties greptimeProperties; + + /** + * Initializes OpenTelemetry with the given service name and GrepTimeDB endpoint. + * Includes authentication if configured in GreptimeProperties. + */ + @PostConstruct + public void initializeOpenTelemetry() { + if (greptimeProperties == null || !greptimeProperties.enabled()) { + log.info("GrepTimeDB logging is disabled, skipping OpenTelemetry configuration."); + return; + } + + try { + Resource resource = Resource.getDefault() + .merge(Resource.builder() + .put(SERVICE_NAME, "HertzBeat") + .put(SERVICE_VERSION, version) + .build()); + + Map<String, String> headers = new HashMap<>(); + + headers.put("X-Greptime-DB-Name", "public"); + headers.put("X-Greptime-Log-Table-Name", "HertzBeat"); + headers.put("X-Greptime-Log-Extract-Keys", version); + + addAuthenticationHeaders(headers); + + OtlpHttpLogRecordExporter logExporter = OtlpHttpLogRecordExporter.builder() + .setEndpoint(greptimeProperties.httpEndpoint() + "/v1/otlp/v1/logs") + .setHeaders(()-> headers) + .setTimeout(10, TimeUnit.SECONDS) + .build(); + + SdkLoggerProvider loggerProvider = SdkLoggerProvider.builder() + .setResource(resource) + .addLogRecordProcessor( + BatchLogRecordProcessor.builder(logExporter) + .setScheduleDelay(1000, TimeUnit.MILLISECONDS) + .setMaxExportBatchSize(512) + .build()) + .build(); + + OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() + .setLoggerProvider(loggerProvider) + .build(); + + OpenTelemetryAppender.install(openTelemetry); + log.info("OpenTelemetry successfully configured with GrepTimeDB exporter."); + } catch (Exception e) { + log.error("Failed to initialize OpenTelemetry with GrepTimeDB", e); + } + } + + /** + * Adds authentication headers to the provided map if username and password are configured. + * + * @param headers the map to which authentication headers will be added + */ + private void addAuthenticationHeaders(Map<String, String> headers) { + if (StringUtils.isNotBlank(greptimeProperties.username()) && + StringUtils.isNotBlank(greptimeProperties.password())) { + String credentials = greptimeProperties.username() + ":" + greptimeProperties.password(); + String encodedCredentials = Base64.getEncoder().encodeToString(credentials.getBytes()); + headers.put("Authorization", "Basic " + encodedCredentials); + } + + } +} \ No newline at end of file diff --git a/hertzbeat-log/src/main/resources/META-INF/spring.factories b/hertzbeat-log/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..07b683cf39 --- /dev/null +++ b/hertzbeat-log/src/main/resources/META-INF/spring.factories @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.apache.hertzbeat.log.config.LogAutoConfiguration \ No newline at end of file diff --git a/hertzbeat-manager/pom.xml b/hertzbeat-manager/pom.xml index 3d20aef1e9..d5351ad9cf 100644 --- a/hertzbeat-manager/pom.xml +++ b/hertzbeat-manager/pom.xml @@ -89,6 +89,11 @@ <groupId>org.apache.hertzbeat</groupId> <artifactId>hertzbeat-grafana</artifactId> </dependency> + <!-- log --> + <dependency> + <groupId>org.apache.hertzbeat</groupId> + <artifactId>hertzbeat-log</artifactId> + </dependency> <!-- spring --> <dependency> <groupId>org.springframework.boot</groupId> diff --git a/hertzbeat-manager/src/main/resources/logback-spring.xml b/hertzbeat-manager/src/main/resources/logback-spring.xml index 950db70962..ed526637f3 100644 --- a/hertzbeat-manager/src/main/resources/logback-spring.xml +++ b/hertzbeat-manager/src/main/resources/logback-spring.xml @@ -76,6 +76,26 @@ </filter> </appender> + <!-- OpenTelemetry Appender for shipping logs to GrepTimeDB --> + <appender name="OpenTelemetryAppender" class="io.opentelemetry.instrumentation.logback.v1_0.OpenTelemetryAppender"> + <!-- Capture source code information (class name, method name, line number) --> + <captureCodeAttributes>true</captureCodeAttributes> + + <!-- Capture MDC context values --> + <captureMdcAttributes> + <pattern>.*</pattern> + </captureMdcAttributes> + + <!-- Capture experimental attributes like exception details --> + <captureExperimentalAttributes>true</captureExperimentalAttributes> + + <!-- Capture marker attributes --> + <captureMarkerAttribute>true</captureMarkerAttribute> + + <!-- Capture logger context attributes --> + <captureLoggerContext>true</captureLoggerContext> + </appender> + <!-- Settings for this logger: for example, all output logs under the org.springframework package must be at level info or above to be output! --> <!-- This can avoid outputting many common debug information of the spring framework! --> <logger name="org.springframework" level="info"/> @@ -90,12 +110,14 @@ <logger name="org.mongodb" level="warn"/> <logger name="io.greptime" level="warn"/> <logger name="org.apache.kafka" level="warn"/> + <logger name="io.opentelemetry" level="info"/> <!-- Production environment configuration --> <springProfile name="prod"> <root level="INFO"> <appender-ref ref="SystemOutFileAppender"/> <appender-ref ref="ErrOutFileAppender"/> + <appender-ref ref="OpenTelemetryAppender"/> </root> <!-- North log --> <Logger name="com.obs.services.AbstractClient" level="OFF" @@ -118,6 +140,7 @@ <appender-ref ref="ConsoleAppender"/> <appender-ref ref="SystemOutFileAppender"/> <appender-ref ref="ErrOutFileAppender"/> + <appender-ref ref="OpenTelemetryAppender"/> </root> </springProfile> @@ -127,6 +150,7 @@ <appender-ref ref="ConsoleAppender"/> <appender-ref ref="SystemOutFileAppender"/> <appender-ref ref="ErrOutFileAppender"/> + <appender-ref ref="OpenTelemetryAppender"/> </root> </springProfile> @@ -136,7 +160,8 @@ <appender-ref ref="ConsoleAppender"/> <appender-ref ref="SystemOutFileAppender"/> <appender-ref ref="ErrOutFileAppender"/> + <appender-ref ref="OpenTelemetryAppender"/> </root> </springProfile> -</configuration> +</configuration> \ No newline at end of file diff --git a/pom.xml b/pom.xml index bec9f62c07..df170e64e7 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,7 @@ <module>hertzbeat-push</module> <module>hertzbeat-plugin</module> <module>hertzbeat-grafana</module> + <module>hertzbeat-log</module> <module>hertzbeat-e2e</module> <module>hertzbeat-base</module> </modules> @@ -231,6 +232,12 @@ <artifactId>hertzbeat-grafana</artifactId> <version>${hertzbeat.version}</version> </dependency> + <!-- log --> + <dependency> + <groupId>org.apache.hertzbeat</groupId> + <artifactId>hertzbeat-log</artifactId> + <version>${hertzbeat.version}</version> + </dependency> <!-- collector-basic --> <dependency> <groupId>org.apache.hertzbeat</groupId> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
