This is an automated email from the ASF dual-hosted git repository.
zhaoqingran pushed a commit to branch doris
in repository https://gitbox.apache.org/repos/asf/hertzbeat.git
The following commit(s) were added to refs/heads/doris by this push:
new bb34a39c4 ```feat(warehouse): create table dynamically in
DorisDataStorage
bb34a39c4 is described below
commit bb34a39c456714d6c9c846a277c7ce89b16e036c
Author: zqr10159 <[email protected]>
AuthorDate: Mon Aug 26 00:16:53 2024 +0800
```feat(warehouse): create table dynamically in DorisDataStorage
DorisDataStorage now creates the database and table dynamically based on
the incoming metrics data.
This change removes the need for manual database and table creation in the
Doris database.
The connection validation and table creation have been adapted to enhance
the auto-configuration
capabilities.
```
---
manager/src/main/resources/application.yml | 1 +
.../store/history/doris/DorisDataStorage.java | 40 ++++++++++++++++++----
2 files changed, 35 insertions(+), 6 deletions(-)
diff --git a/manager/src/main/resources/application.yml
b/manager/src/main/resources/application.yml
index 10f389150..f62b2547d 100644
--- a/manager/src/main/resources/application.yml
+++ b/manager/src/main/resources/application.yml
@@ -169,6 +169,7 @@ warehouse:
enabled: false
host: 127.0.0.1
httpPort: 8030
+ jdbcPort: 9030
username: root
password:
database: hertzbeat
diff --git
a/warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/doris/DorisDataStorage.java
b/warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/doris/DorisDataStorage.java
index c732247c3..27637ba25 100644
---
a/warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/doris/DorisDataStorage.java
+++
b/warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/doris/DorisDataStorage.java
@@ -38,6 +38,30 @@ import org.springframework.stereotype.Component;
@ConditionalOnProperty(prefix = "warehouse.store.doris", name = "enabled",
havingValue = "true")
@Slf4j
public class DorisDataStorage extends AbstractHistoryDataStorage {
+ private final static String CREATE_TABLE_SQL = """
+ CREATE TABLE IF NOT EXISTS %s (
+ ts datetime,
+ v VARIANT,
+ INDEX idx_var(v) USING INVERTED PROPERTIES("parser" =
"unicode")
+ )
+ ENGINE = OLAP
+ DUPLICATE KEY(`ts`)
+ PARTITION BY RANGE(`ts`) ()
+ DISTRIBUTED BY RANDOM BUCKETS 16
+ PROPERTIES (
+ "compaction_policy" = "time_series",
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.create_history_partition" = "true",
+ "dynamic_partition.time_unit" = "DAY",
+ "dynamic_partition.start" = "-30",
+ "dynamic_partition.end" = "1",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "16",
+ "dynamic_partition.replication_num" = "1",
+ "replication_num" = "1",
+ "enable_single_replica_compaction" = "true"
+ );
+ """;
private HikariDataSource hikariDataSource;
@@ -49,7 +73,7 @@ public class DorisDataStorage extends
AbstractHistoryDataStorage {
return;
}
this.hikariDataSource = new HikariDataSource();
- this.hikariDataSource.setJdbcUrl("jdbc:mysql://" +
dorisProperties.host() + ":" + dorisProperties.jdbcPort() + "/" +
dorisProperties.database());
+ this.hikariDataSource.setJdbcUrl("jdbc:mysql://" +
dorisProperties.host() + ":" + dorisProperties.jdbcPort() + "/" + "mysql");
this.hikariDataSource.setUsername(dorisProperties.username());
this.hikariDataSource.setPassword(dorisProperties.password());
this.hikariDataSource.setDriverClassName("com.mysql.cj.jdbc.Driver");
@@ -66,20 +90,23 @@ public class DorisDataStorage extends
AbstractHistoryDataStorage {
// validation query
this.hikariDataSource.setConnectionTestQuery("select 1");
createDatabase(dorisProperties.database());
-
+ if (hikariDataSource.isRunning()) {
+ this.serverAvailable = true;
+ }
}
private void createDatabase(String database) {
try (Connection connection = hikariDataSource.getConnection()) {
connection.createStatement().execute("CREATE DATABASE IF NOT
EXISTS " + database);
+ log.info("[warehouse doris]--Create database {} successful",
database);
} catch (SQLException e) {
log.error("[warehouse doris]--Error: {}", e.getMessage(), e);
}
}
- private void createTable(String database, String tableName) {
+ private void createTable(String tableName) {
try (Connection connection = hikariDataSource.getConnection()) {
- connection.createStatement().execute("USE " + database);
+ connection.createStatement().execute("USE " +
dorisProperties.database());
connection.createStatement().execute("CREATE TABLE IF NOT EXISTS "
+ tableName);
} catch (SQLException e) {
log.error("[warehouse doris]--Error: {}", e.getMessage(), e);
@@ -140,7 +167,7 @@ public class DorisDataStorage extends
AbstractHistoryDataStorage {
}
String monitorId = String.valueOf(metricsData.getId());
String tableName = getTableName(metricsData.getApp(),
metricsData.getMetrics());
- createTable(dorisProperties.database(), tableName);
+// createTable(tableName);
List<CollectRep.Field> fieldsList = metricsData.getFieldsList();
StringBuilder jsonData = new StringBuilder();
jsonData.append("[");
@@ -171,6 +198,7 @@ public class DorisDataStorage extends
AbstractHistoryDataStorage {
} else {
jsonData.append(']');
}
+ log.info("[warehouse doris]--Write data: {}", jsonData);
try {
DorisStreamLoadUtil.sendData(
@@ -182,7 +210,7 @@ public class DorisDataStorage extends
AbstractHistoryDataStorage {
tableName,
jsonData.toString()
);
- log.debug("[warehouse doris]-Write successful");
+ log.info("[warehouse doris]-Write successful");
} catch (Exception e) {
log.error("[warehouse doris]--Error: {}", e.getMessage(), e);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
