This is an automated email from the ASF dual-hosted git repository. zhaoqingran pushed a commit to branch duckdb in repository https://gitbox.apache.org/repos/asf/hertzbeat.git
commit 25fe67f77e92ebda66d5991d2116c21346eb2b5f Author: Logic <[email protected]> AuthorDate: Mon Dec 16 23:27:36 2024 +0800 init --- hertzbeat-manager/pom.xml | 5 + .../src/main/resources/application.yml | 4 + .../warehouse/constants/WarehouseConstants.java | 2 + .../warehouse/store/history/HistoryDataReader.java | 4 +- .../store/history/duckdb/DuckDBDataStorage.java | 119 +++++++++++++++++++++ .../store/history/duckdb/DuckDBProperties.java | 17 +++ pom.xml | 2 +- 7 files changed, 150 insertions(+), 3 deletions(-) diff --git a/hertzbeat-manager/pom.xml b/hertzbeat-manager/pom.xml index 8b459bb86..f6a510f4b 100644 --- a/hertzbeat-manager/pom.xml +++ b/hertzbeat-manager/pom.xml @@ -175,6 +175,11 @@ <artifactId>h2</artifactId> <scope>runtime</scope> </dependency> + <dependency> + <groupId>org.duckdb</groupId> + <artifactId>duckdb_jdbc</artifactId> + <version>1.1.3</version> + </dependency> <dependency> <groupId>cn.afterturn</groupId> <artifactId>easypoi-spring-boot-starter</artifactId> diff --git a/hertzbeat-manager/src/main/resources/application.yml b/hertzbeat-manager/src/main/resources/application.yml index 4f3ea8821..544abc6cf 100644 --- a/hertzbeat-manager/src/main/resources/application.yml +++ b/hertzbeat-manager/src/main/resources/application.yml @@ -131,6 +131,10 @@ warehouse: # The maximum number of history records retained, if this number is exceeded, half of the data in this configuration item will be deleted # (please set this configuration reasonably as history records can affect performance when it is large) max-history-record-num: 6000 + duckdb: + enabled: false + driver-class-name: org.duckdb.DuckDBDriver + url: jdbc:duckdb:./data/hertzbeat.duckdb victoria-metrics: enabled: false url: http://localhost:8428 diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/constants/WarehouseConstants.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/constants/WarehouseConstants.java index aee6e4ff9..117799738 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/constants/WarehouseConstants.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/constants/WarehouseConstants.java @@ -31,6 +31,8 @@ public interface WarehouseConstants { * History database name. */ interface HistoryName { + String DUCKDB = "duckdb"; + String GREPTIME = "greptime"; String INFLUXDB = "influxdb"; diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/HistoryDataReader.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/HistoryDataReader.java index b98a6e0f8..845d6d156 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/HistoryDataReader.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/HistoryDataReader.java @@ -32,7 +32,7 @@ public interface HistoryDataReader { boolean isServerAvailable(); /** - * query history range metrics data from tsdb + * query history range metrics data from time-series db * @param monitorId monitor id * @param app monitor type * @param metrics metrics @@ -45,7 +45,7 @@ public interface HistoryDataReader { String label, String history); /** - * query history range interval metrics data from tsdb + * query history range interval metrics data from time-series db * max min mean metrics value * @param monitorId monitor id * @param app monitor type diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/duckdb/DuckDBDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/duckdb/DuckDBDataStorage.java new file mode 100644 index 000000000..86a60278c --- /dev/null +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/duckdb/DuckDBDataStorage.java @@ -0,0 +1,119 @@ +package org.apache.hertzbeat.warehouse.store.history.duckdb; + +import com.zaxxer.hikari.HikariDataSource; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.entity.dto.Value; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.apache.hertzbeat.warehouse.store.history.AbstractHistoryDataStorage; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +/** + * duckdb data storage + */ +@Primary +@Component +@ConditionalOnProperty(prefix = "warehouse.store.duckdb", + name = "enabled", havingValue = "true") +@Slf4j +public class DuckDBDataStorage extends AbstractHistoryDataStorage { + + private static final String CONSTANTS_URL_PREFIX = "jdbc:TAOS-RS://"; + private static final Pattern SQL_SPECIAL_STRING_PATTERN = Pattern.compile("(\\\\)|(')"); + private static final String INSTANCE_NULL = "''"; + private static final String CONSTANTS_CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS %s"; + private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO `%s` USING `%s` TAGS (%s) VALUES %s"; + private static final String CREATE_SUPER_TABLE_SQL = "CREATE STABLE IF NOT EXISTS `%s` %s TAGS (monitor BIGINT)"; + private static final String NO_SUPER_TABLE_ERROR = "Table does not exist"; + + private static final String QUERY_HISTORY_WITH_INSTANCE_SQL = + "SELECT ts, instance, `%s` FROM `%s` WHERE instance = '%s' AND ts >= now - %s order by ts desc"; + private static final String QUERY_HISTORY_SQL = + "SELECT ts, instance, `%s` FROM `%s` WHERE ts >= now - %s order by ts desc"; + private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = + "SELECT first(ts), first(`%s`), avg(`%s`), min(`%s`), max(`%s`) FROM `%s` WHERE instance = '%s' AND ts >= now - %s interval(4h)"; + private static final String QUERY_INSTANCE_SQL = + "SELECT DISTINCT instance FROM `%s` WHERE ts >= now - 1w"; + + private static final String TABLE_NOT_EXIST = "Table does not exist"; + + private static final String HERTZBEAT = "hertzbeat"; + + public DuckDBDataStorage(DuckDBProperties duckDBProperties) { + if (duckDBProperties == null) { + log.error("init error, please config Warehouse DuckDB props in application.yml"); + throw new IllegalArgumentException("please config Warehouse DuckDB props"); + } + } + + private HikariDataSource hikariDataSource; + + /** + * init duckdb data storage + * + * @param duckDBProperties duckdb properties + */ + private void initDuckDBDatabase(final DuckDBProperties duckDBProperties) throws SQLException { + try ( + final Connection tempConnection = DriverManager.getConnection(duckDBProperties.url()) + ) { + tempConnection.prepareStatement(String.format(CONSTANTS_CREATE_DATABASE, HERTZBEAT)) + .execute(); + } + } + + /** + * query history range metrics data from time-series db + * + * @param monitorId monitor id + * @param app monitor type + * @param metrics metrics + * @param metric metric + * @param label label + * @param history range + * @return metrics data + */ + @Override + public Map<String, List<Value>> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { + return Map.of(); + } + + /** + * query history range interval metrics data from time-series db + * max min mean metrics value + * + * @param monitorId monitor id + * @param app monitor type + * @param metrics metrics + * @param metric metric + * @param label label + * @param history history range + * @return metrics data + */ + @Override + public Map<String, List<Value>> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { + return Map.of(); + } + + /** + * save metrics data + * + * @param metricsData metrics data + */ + @Override + public void saveData(CollectRep.MetricsData metricsData) { + + } + + @Override + public void destroy() throws Exception { + + } +} diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/duckdb/DuckDBProperties.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/duckdb/DuckDBProperties.java new file mode 100644 index 000000000..703d158a0 --- /dev/null +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/duckdb/DuckDBProperties.java @@ -0,0 +1,17 @@ +package org.apache.hertzbeat.warehouse.store.history.duckdb; + +import org.apache.hertzbeat.common.constants.ConfigConstants; +import org.apache.hertzbeat.common.constants.SignConstants; +import org.apache.hertzbeat.warehouse.constants.WarehouseConstants; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.bind.DefaultValue; + +@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE + + SignConstants.DOT + + WarehouseConstants.STORE + + SignConstants.DOT + + WarehouseConstants.HistoryName.DUCKDB) +public record DuckDBProperties(@DefaultValue("false") boolean enabled, +@DefaultValue("jdbc:duckdb:./data/hertzbeat.duckdb") String url, +@DefaultValue("org.duckdb.DuckDBDriver") String driverClassName) { +} diff --git a/pom.xml b/pom.xml index 05974f113..0fa3dcf61 100644 --- a/pom.xml +++ b/pom.xml @@ -157,7 +157,7 @@ <consul-api.version>1.4.5</consul-api.version> <nacos-client.version>2.2.1</nacos-client.version> <vesoft-client.version>3.6.0</vesoft-client.version> - <eclipse-jpa.version>4.0.2</eclipse-jpa.version> + <eclipse-jpa.version>4.0.4</eclipse-jpa.version> <jutf7.version>1.0.0</jutf7.version> <spring-boot-starter-sureness.version>1.1.0</spring-boot-starter-sureness.version> <huawei.sdk.version>3.1.37</huawei.sdk.version> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
