This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 8d164d3 Add Tidb provider module. (#5844)
8d164d3 is described below
commit 8d164d3dd8c35cdafe683abc7fd9f4e4b530808f
Author: Jared Tan <[email protected]>
AuthorDate: Sun Nov 22 23:28:54 2020 +0800
Add Tidb provider module. (#5844)
* fix transaction too large error when use TiDB as storage
* add CHANGES.md
* fix typo
* add tidb storage provider module.
* add e2e.
* revert ui submodule.
* fix license.
* Fix wrong health check test
* add tidb ttl e2e.
Co-authored-by: moonsphere <[email protected]>
Co-authored-by: kezhenxu94 <[email protected]>
Co-authored-by: kezhenxu94 <[email protected]>
---
.github/workflows/e2e.storages.yaml | 2 +-
.github/workflows/e2e.ttl.yaml | 2 +-
CHANGES.md | 1 +
docs/en/setup/backend/backend-storage.md | 15 +-
oap-server/server-bootstrap/pom.xml | 5 +
.../src/main/resources/application.yml | 13 ++
.../client/jdbc/hikaricp/JDBCHikariCPClient.java | 6 +-
oap-server/server-storage-plugin/pom.xml | 1 +
.../plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java | 2 +-
.../plugin/jdbc/mysql/MySQLStorageConfig.java | 2 +-
.../{ => storage-tidb-plugin}/pom.xml | 38 ++-
.../plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java} | 12 +-
.../plugin/jdbc/tidb/TiDBStorageConfig.java} | 21 +-
.../plugin/jdbc/tidb/TiDBStorageProvider.java | 177 ++++++++++++++
...alking.oap.server.library.module.ModuleProvider | 19 ++
.../docker/storage/docker-compose.tidb.yml | 50 ++++
.../e2e-test/docker/storage/tidbconfig/tidb.toml | 254 +++++++++++++++++++++
.../e2e-test/docker/ttl/docker-compose.tidb.yml | 51 +++++
18 files changed, 623 insertions(+), 48 deletions(-)
diff --git a/.github/workflows/e2e.storages.yaml
b/.github/workflows/e2e.storages.yaml
index e82aca5..370c4ee 100644
--- a/.github/workflows/e2e.storages.yaml
+++ b/.github/workflows/e2e.storages.yaml
@@ -34,7 +34,7 @@ jobs:
timeout-minutes: 90
strategy:
matrix:
- storage: ['mysql', 'es6', 'es7.0', 'es7.9', 'influxdb']
+ storage: ['mysql', 'es6', 'es7.0', 'es7.9', 'influxdb', 'tidb']
env:
SW_STORAGE: ${{ matrix.storage }}
steps:
diff --git a/.github/workflows/e2e.ttl.yaml b/.github/workflows/e2e.ttl.yaml
index 645fb88..89c3b1c 100644
--- a/.github/workflows/e2e.ttl.yaml
+++ b/.github/workflows/e2e.ttl.yaml
@@ -37,7 +37,7 @@ jobs:
timeout-minutes: 90
strategy:
matrix:
- storage: ['es6', 'es7', 'influxdb']
+ storage: ['es6', 'es7', 'influxdb', 'tidb']
env:
SW_STORAGE: ${{ matrix.storage }}
steps:
diff --git a/CHANGES.md b/CHANGES.md
index 72fd2a9..91a19a6 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -50,6 +50,7 @@ Release Notes.
* Fix that chunked string is incorrect while the tag contains colon.
* Fix the incorrect dynamic configuration key bug of `endpoint-name-grouping`.
* Remove unused min date timebucket in jdbc deletehistory logical
+* Fix "transaction too large error" when use TiDB as storage.
#### UI
* Fix incorrect label in radial chart in topology.
diff --git a/docs/en/setup/backend/backend-storage.md
b/docs/en/setup/backend/backend-storage.md
index a189afc..70297be 100644
--- a/docs/en/setup/backend/backend-storage.md
+++ b/docs/en/setup/backend/backend-storage.md
@@ -223,22 +223,25 @@ All connection related settings including link url,
username and password are in
Here are some of the settings, please follow
[HikariCP](https://github.com/brettwooldridge/HikariCP) connection pool
document for all the settings.
## TiDB
-Currently tested TiDB in version 2.0.9, and Mysql Client driver in version
8.0.13.
-Active TiDB as storage, set storage provider to **mysql**.
+Tested TiDB Server 4.0.8 version and Mysql Client driver 8.0.13 version
currently.
+Active TiDB as storage, set storage provider to **tidb**.
```yaml
storage:
- selector: ${SW_STORAGE:mysql}
- mysql:
+ selector: ${SW_STORAGE:tidb}
+ tidb:
properties:
- jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"}
+ jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:4000/swtest"}
dataSource.user: ${SW_DATA_SOURCE_USER:root}
- dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234}
+ dataSource.password: ${SW_DATA_SOURCE_PASSWORD:""}
dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true}
dataSource.prepStmtCacheSize:
${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250}
dataSource.prepStmtCacheSqlLimit:
${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048}
dataSource.useServerPrepStmts:
${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true}
+ dataSource.useAffectedRows: ${SW_DATA_SOURCE_USE_AFFECTED_ROWS:true}
metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000}
+ maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20}
+ numOfSearchableValuesPerTag:
${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2}
```
All connection related settings including link url, username and password are
in `application.yml`.
These settings can refer to the configuration of *MySQL* above.
diff --git a/oap-server/server-bootstrap/pom.xml
b/oap-server/server-bootstrap/pom.xml
index 954426b..6c8fb3a 100644
--- a/oap-server/server-bootstrap/pom.xml
+++ b/oap-server/server-bootstrap/pom.xml
@@ -157,6 +157,11 @@
<artifactId>storage-influxdb-plugin</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.skywalking</groupId>
+ <artifactId>storage-tidb-plugin</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- storage module -->
<!-- queryBuild module -->
diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml
b/oap-server/server-bootstrap/src/main/resources/application.yml
index 5cc427e..050e3ee 100755
--- a/oap-server/server-bootstrap/src/main/resources/application.yml
+++ b/oap-server/server-bootstrap/src/main/resources/application.yml
@@ -171,6 +171,19 @@ storage:
metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000}
maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20}
numOfSearchableValuesPerTag:
${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2}
+ tidb:
+ properties:
+ jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:4000/tidbswtest"}
+ dataSource.user: ${SW_DATA_SOURCE_USER:root}
+ dataSource.password: ${SW_DATA_SOURCE_PASSWORD:""}
+ dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true}
+ dataSource.prepStmtCacheSize:
${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250}
+ dataSource.prepStmtCacheSqlLimit:
${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048}
+ dataSource.useServerPrepStmts:
${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true}
+ dataSource.useAffectedRows: ${SW_DATA_SOURCE_USE_AFFECTED_ROWS:true}
+ metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000}
+ maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20}
+ numOfSearchableValuesPerTag:
${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2}
influxdb:
# InfluxDB configuration
url: ${SW_STORAGE_INFLUXDB_URL:http://localhost:8086}
diff --git
a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java
b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java
index 502eed3..03bf6ac 100644
---
a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java
+++
b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/jdbc/hikaricp/JDBCHikariCPClient.java
@@ -90,14 +90,14 @@ public class JDBCHikariCPClient implements Client,
HealthCheckable {
}
}
- public boolean execute(Connection connection, String sql, Object...
params) throws JDBCClientException {
+ public int executeUpdate(Connection connection, String sql, Object...
params) throws JDBCClientException {
LOGGER.debug("execute query with result: {}", sql);
- boolean result;
+ int result;
PreparedStatement statement = null;
try {
statement = connection.prepareStatement(sql);
setStatementParam(statement, params);
- result = statement.execute();
+ result = statement.executeUpdate();
statement.closeOnCompletion();
healthChecker.health();
} catch (SQLException e) {
diff --git a/oap-server/server-storage-plugin/pom.xml
b/oap-server/server-storage-plugin/pom.xml
index 1c2c38b..16b7d1d 100644
--- a/oap-server/server-storage-plugin/pom.xml
+++ b/oap-server/server-storage-plugin/pom.xml
@@ -34,6 +34,7 @@
<module>storage-zipkin-plugin</module>
<module>storage-jaeger-plugin</module>
<module>storage-influxdb-plugin</module>
+ <module>storage-tidb-plugin</module>
</modules>
</project>
\ No newline at end of file
diff --git
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
index 4f9dcf7..d3367f7 100644
---
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
+++
b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
@@ -60,7 +60,7 @@ public class H2HistoryDeleteDAO implements IHistoryDeleteDAO {
return;
}
}
- client.execute(connection, dataDeleteSQL.toString(), deadline);
+ client.executeUpdate(connection, dataDeleteSQL.toString(),
deadline);
} catch (JDBCClientException | SQLException e) {
throw new IOException(e.getMessage(), e);
}
diff --git
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java
b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java
index eb9b1ae..bf0cccb 100644
---
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java
+++
b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java
@@ -25,7 +25,7 @@ import
org.apache.skywalking.oap.server.library.module.ModuleConfig;
@Setter
@Getter
-public final class MySQLStorageConfig extends ModuleConfig {
+public class MySQLStorageConfig extends ModuleConfig {
private int metadataQueryMaxSize = 5000;
/**
* Inherit from {@link
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageConfig#getMaxSizeOfArrayColumn()}
diff --git a/oap-server/server-storage-plugin/pom.xml
b/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml
similarity index 50%
copy from oap-server/server-storage-plugin/pom.xml
copy to oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml
index 1c2c38b..1c97331 100644
--- a/oap-server/server-storage-plugin/pom.xml
+++ b/oap-server/server-storage-plugin/storage-tidb-plugin/pom.xml
@@ -17,23 +17,37 @@
~
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<parent>
- <artifactId>oap-server</artifactId>
+ <artifactId>server-storage-plugin</artifactId>
<groupId>org.apache.skywalking</groupId>
<version>8.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>server-storage-plugin</artifactId>
- <packaging>pom</packaging>
- <modules>
- <module>storage-jdbc-hikaricp-plugin</module>
- <module>storage-elasticsearch-plugin</module>
- <module>storage-elasticsearch7-plugin</module>
- <module>storage-zipkin-plugin</module>
- <module>storage-jaeger-plugin</module>
- <module>storage-influxdb-plugin</module>
- </modules>
+ <artifactId>storage-tidb-plugin</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.skywalking</groupId>
+ <artifactId>server-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.skywalking</groupId>
+ <artifactId>library-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.skywalking</groupId>
+ <artifactId>storage-jdbc-hikaricp-plugin</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
</project>
\ No newline at end of file
diff --git
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java
similarity index 86%
copy from
oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
copy to
oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java
index 4f9dcf7..be3208b 100644
---
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java
+++
b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBHistoryDeleteDAO.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao;
+package org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb;
import java.io.IOException;
import java.sql.Connection;
@@ -28,18 +28,19 @@ import
org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariC
import org.apache.skywalking.oap.server.storage.plugin.jdbc.SQLBuilder;
import org.joda.time.DateTime;
-public class H2HistoryDeleteDAO implements IHistoryDeleteDAO {
+public class TiDBHistoryDeleteDAO implements IHistoryDeleteDAO {
private final JDBCHikariCPClient client;
- public H2HistoryDeleteDAO(JDBCHikariCPClient client) {
+ public TiDBHistoryDeleteDAO(JDBCHikariCPClient client) {
this.client = client;
}
@Override
public void deleteHistory(Model model, String timeBucketColumnName, int
ttl) throws IOException {
SQLBuilder dataDeleteSQL = new SQLBuilder("delete from " +
model.getName() + " where ")
- .append(timeBucketColumnName).append("<= ? ");
+ .append(timeBucketColumnName).append("<= ? ")
+ .append(" limit 10000");
try (Connection connection = client.getConnection()) {
long deadline;
@@ -60,7 +61,8 @@ public class H2HistoryDeleteDAO implements IHistoryDeleteDAO {
return;
}
}
- client.execute(connection, dataDeleteSQL.toString(), deadline);
+ while (client.executeUpdate(connection, dataDeleteSQL.toString(),
deadline) > 0) {
+ }
} catch (JDBCClientException | SQLException e) {
throw new IOException(e.getMessage(), e);
}
diff --git
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java
b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageConfig.java
similarity index 53%
copy from
oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java
copy to
oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageConfig.java
index eb9b1ae..197187c 100644
---
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java
+++
b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageConfig.java
@@ -16,28 +16,13 @@
*
*/
-package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql;
+package org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb;
-import java.util.Properties;
import lombok.Getter;
import lombok.Setter;
-import org.apache.skywalking.oap.server.library.module.ModuleConfig;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLStorageConfig;
@Setter
@Getter
-public final class MySQLStorageConfig extends ModuleConfig {
- private int metadataQueryMaxSize = 5000;
- /**
- * Inherit from {@link
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageConfig#getMaxSizeOfArrayColumn()}
- *
- * @since 8.2.0
- */
- private int maxSizeOfArrayColumn = 20;
- /**
- * Inherit from {@link
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageConfig#getNumOfSearchableValuesPerTag()}
- *
- * @since 8.2.0
- */
- private int numOfSearchableValuesPerTag = 2;
- private Properties properties;
+public class TiDBStorageConfig extends MySQLStorageConfig {
}
diff --git
a/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageProvider.java
b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageProvider.java
new file mode 100644
index 0000000..30d4fa8
--- /dev/null
+++
b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/tidb/TiDBStorageProvider.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.oap.server.core.Const;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.config.ConfigService;
+import org.apache.skywalking.oap.server.core.storage.IBatchDAO;
+import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO;
+import org.apache.skywalking.oap.server.core.storage.StorageDAO;
+import org.apache.skywalking.oap.server.core.storage.StorageException;
+import org.apache.skywalking.oap.server.core.storage.StorageModule;
+import
org.apache.skywalking.oap.server.core.storage.cache.INetworkAddressAliasDAO;
+import
org.apache.skywalking.oap.server.core.storage.management.UITemplateManagementDAO;
+import org.apache.skywalking.oap.server.core.storage.model.ModelCreator;
+import
org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskLogQueryDAO;
+import
org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDAO;
+import
org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO;
+import
org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO;
+import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO;
+import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO;
+import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO;
+import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO;
+import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO;
+import
org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO;
+import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO;
+import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO;
+import
org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient;
+import org.apache.skywalking.oap.server.library.module.ModuleConfig;
+import org.apache.skywalking.oap.server.library.module.ModuleDefine;
+import org.apache.skywalking.oap.server.library.module.ModuleProvider;
+import org.apache.skywalking.oap.server.library.module.ModuleStartException;
+import
org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException;
+import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2BatchDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2HistoryDeleteDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2MetadataQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2MetricsQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2NetworkAddressAliasDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ProfileTaskLogQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ProfileTaskQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ProfileThreadSnapshotQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2StorageDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2TopNRecordsQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2TopologyQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2UITemplateManagementDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLAggregationQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLAlarmQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLLogQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLTableInstaller;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLTraceQueryDAO;
+import
org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MysqlBrowserLogQueryDAO;
+
+/**
+ * TiDB storage enhanced and came from MySQLStorageProvider to support TiDB.
+ *
+ * caution: need add "useAffectedRows=true" to jdbc url.
+ */
+@Slf4j
+public class TiDBStorageProvider extends ModuleProvider {
+
+ private TiDBStorageConfig config;
+ private JDBCHikariCPClient mysqlClient;
+
+ public TiDBStorageProvider() {
+ config = new TiDBStorageConfig();
+ }
+
+ @Override
+ public String name() {
+ return "tidb";
+ }
+
+ @Override
+ public Class<? extends ModuleDefine> module() {
+ return StorageModule.class;
+ }
+
+ @Override
+ public ModuleConfig createConfigBeanIfAbsent() {
+ return config;
+ }
+
+ @Override
+ public void prepare() throws ServiceNotProvidedException {
+ mysqlClient = new JDBCHikariCPClient(config.getProperties());
+
+ this.registerServiceImplementation(IBatchDAO.class, new
H2BatchDAO(mysqlClient));
+ this.registerServiceImplementation(
+ StorageDAO.class,
+ new H2StorageDAO(
+ getManager(), mysqlClient, config.getMaxSizeOfArrayColumn(),
config.getNumOfSearchableValuesPerTag())
+ );
+ this.registerServiceImplementation(
+ INetworkAddressAliasDAO.class, new
H2NetworkAddressAliasDAO(mysqlClient));
+
+ this.registerServiceImplementation(ITopologyQueryDAO.class, new
H2TopologyQueryDAO(mysqlClient));
+ this.registerServiceImplementation(IMetricsQueryDAO.class, new
H2MetricsQueryDAO(mysqlClient));
+ this.registerServiceImplementation(
+ ITraceQueryDAO.class,
+ new MySQLTraceQueryDAO(
+ getManager(),
+ mysqlClient,
+ config.getMaxSizeOfArrayColumn(),
+ config.getNumOfSearchableValuesPerTag()
+ )
+ );
+ this.registerServiceImplementation(IBrowserLogQueryDAO.class, new
MysqlBrowserLogQueryDAO(mysqlClient));
+ this.registerServiceImplementation(
+ IMetadataQueryDAO.class, new H2MetadataQueryDAO(mysqlClient,
config.getMetadataQueryMaxSize()));
+ this.registerServiceImplementation(IAggregationQueryDAO.class, new
MySQLAggregationQueryDAO(mysqlClient));
+ this.registerServiceImplementation(IAlarmQueryDAO.class, new
MySQLAlarmQueryDAO(mysqlClient));
+ this.registerServiceImplementation(
+ IHistoryDeleteDAO.class, new H2HistoryDeleteDAO(mysqlClient));
+ this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new
H2TopNRecordsQueryDAO(mysqlClient));
+ this.registerServiceImplementation(ILogQueryDAO.class, new
MySQLLogQueryDAO(mysqlClient));
+
+ this.registerServiceImplementation(IProfileTaskQueryDAO.class, new
H2ProfileTaskQueryDAO(mysqlClient));
+ this.registerServiceImplementation(IProfileTaskLogQueryDAO.class, new
H2ProfileTaskLogQueryDAO(mysqlClient));
+ this.registerServiceImplementation(
+ IProfileThreadSnapshotQueryDAO.class, new
H2ProfileThreadSnapshotQueryDAO(mysqlClient));
+ this.registerServiceImplementation(UITemplateManagementDAO.class, new
H2UITemplateManagementDAO(mysqlClient));
+
+ this.registerServiceImplementation(IHistoryDeleteDAO.class, new
TiDBHistoryDeleteDAO(mysqlClient));
+ }
+
+ @Override
+ public void start() throws ServiceNotProvidedException,
ModuleStartException {
+ final ConfigService configService = getManager().find(CoreModule.NAME)
+ .provider()
+
.getService(ConfigService.class);
+ final int numOfSearchableTags =
configService.getSearchableTracesTags().split(Const.COMMA).length;
+ if (numOfSearchableTags * config.getNumOfSearchableValuesPerTag() >
config.getMaxSizeOfArrayColumn()) {
+ throw new ModuleStartException("Size of searchableTracesTags[" +
numOfSearchableTags
+ + "] *
numOfSearchableValuesPerTag[" + config.getNumOfSearchableValuesPerTag()
+ + "] > maxSizeOfArrayColumn[" +
config.getMaxSizeOfArrayColumn()
+ + "]. Potential out of bound in
the runtime.");
+ }
+
+ try {
+ mysqlClient.connect();
+
+ MySQLTableInstaller installer = new MySQLTableInstaller(
+ mysqlClient, getManager(), config.getMaxSizeOfArrayColumn(),
config.getNumOfSearchableValuesPerTag()
+ );
+
getManager().find(CoreModule.NAME).provider().getService(ModelCreator.class).addModelListener(installer);
+ } catch (StorageException e) {
+ throw new ModuleStartException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void notifyAfterCompleted() throws ServiceNotProvidedException,
ModuleStartException {
+
+ }
+
+ @Override
+ public String[] requiredModules() {
+ return new String[] {CoreModule.NAME};
+ }
+}
diff --git
a/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider
b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider
new file mode 100644
index 0000000..cfe7e5f
--- /dev/null
+++
b/oap-server/server-storage-plugin/storage-tidb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+org.apache.skywalking.oap.server.storage.plugin.jdbc.tidb.TiDBStorageProvider
\ No newline at end of file
diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml
b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml
new file mode 100644
index 0000000..a570ef8
--- /dev/null
+++ b/test/e2e/e2e-test/docker/storage/docker-compose.tidb.yml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+ tidb:
+ image: pingcap/tidb:latest
+ expose:
+ - 4000
+ volumes:
+ - ./tidbconfig/tidb.toml:/tidb.toml:ro
+ restart: on-failure
+ healthcheck:
+ test: ["CMD", "sh", "-c", "nc -zn 127.0.0.1 4000"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ networks:
+ - e2e
+
+ oap:
+ extends:
+ file: ../base-compose.yml
+ service: oap
+ environment:
+ SW_STORAGE: tidb
+ SW_PROMETHEUS_FETCHER_ACTIVE: "true"
+ SW_TELEMETRY: prometheus
+ SW_JDBC_URL: "jdbc:mysql://tidb:4000/test"
+ SW_DATA_SOURCE_PASSWORD: ""
+ depends_on:
+ tidb:
+ condition: service_healthy
+ entrypoint: ['sh', '-c', '/download-mysql.sh &&
/skywalking/docker-entrypoint.sh']
+
+networks:
+ e2e:
diff --git a/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml
b/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml
new file mode 100644
index 0000000..693ba1f
--- /dev/null
+++ b/test/e2e/e2e-test/docker/storage/tidbconfig/tidb.toml
@@ -0,0 +1,254 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TiDB Configuration.
+
+# TiDB server host.
+host = "0.0.0.0"
+
+# TiDB server port.
+port = 4000
+
+# Registered store name, [tikv, mocktikv]
+store = "mocktikv"
+
+# TiDB storage path.
+path = "/tmp/tidb"
+
+# The socket file to use for connection.
+socket = ""
+
+# Run ddl worker on this tidb-server.
+run-ddl = true
+
+# Schema lease duration, very dangerous to change only if you know what you do.
+lease = "0"
+
+# When create table, split a separated region for it. It is recommended to
+# turn off this option if there will be a large number of tables created.
+split-table = true
+
+# The limit of concurrent executed sessions.
+token-limit = 1000
+
+# Only print a log when out of memory quota.
+# Valid options: ["log", "cancel"]
+oom-action = "log"
+
+# Set the memory quota for a query in bytes. Default: 32GB
+mem-quota-query = 34359738368
+
+# Enable coprocessor streaming.
+enable-streaming = false
+
+# Set system variable 'lower_case_table_names'
+lower-case-table-names = 2
+
+[log]
+# Log level: debug, info, warn, error, fatal.
+level = "error"
+
+# Log format, one of json, text, console.
+format = "text"
+
+# Disable automatic timestamp in output
+disable-timestamp = false
+
+# Stores slow query log into separated files.
+slow-query-file = ""
+
+# Queries with execution time greater than this value will be logged.
(Milliseconds)
+slow-threshold = 300
+
+# Queries with internal result greater than this value will be logged.
+expensive-threshold = 10000
+
+# Maximum query length recorded in log.
+query-log-max-len = 2048
+
+# File logging.
+[log.file]
+# Log file name.
+filename = ""
+
+# Max log file size in MB (upper limit to 4096MB).
+max-size = 300
+
+# Max log file keep days. No clean up by default.
+max-days = 0
+
+# Maximum number of old log files to retain. No clean up by default.
+max-backups = 0
+
+# Rotate log by day
+log-rotate = true
+
+[security]
+# Path of file that contains list of trusted SSL CAs for connection with mysql
client.
+ssl-ca = ""
+
+# Path of file that contains X509 certificate in PEM format for connection
with mysql client.
+ssl-cert = ""
+
+# Path of file that contains X509 key in PEM format for connection with mysql
client.
+ssl-key = ""
+
+# Path of file that contains list of trusted SSL CAs for connection with
cluster components.
+cluster-ssl-ca = ""
+
+# Path of file that contains X509 certificate in PEM format for connection
with cluster components.
+cluster-ssl-cert = ""
+
+# Path of file that contains X509 key in PEM format for connection with
cluster components.
+cluster-ssl-key = ""
+
+[status]
+# If enable status report HTTP service.
+report-status = true
+
+# TiDB status port.
+status-port = 10080
+
+# Prometheus pushgateway address, leaves it empty will disable prometheus push.
+metrics-addr = "pushgateway:9091"
+
+# Prometheus client push interval in second, set \"0\" to disable prometheus
push.
+metrics-interval = 15
+
+[performance]
+# Max CPUs to use, 0 use number of CPUs in the machine.
+max-procs = 0
+# StmtCountLimit limits the max count of statement inside a transaction.
+stmt-count-limit = 5000
+
+# Set keep alive option for tcp connection.
+tcp-keep-alive = true
+
+# The maximum number of retries when commit a transaction.
+retry-limit = 10
+
+# Whether support cartesian product.
+cross-join = true
+
+# Stats lease duration, which influences the time of analyze and stats load.
+stats-lease = "3s"
+
+# Run auto analyze worker on this tidb-server.
+run-auto-analyze = true
+
+# Probability to use the query feedback to update stats, 0 or 1 for always
false/true.
+feedback-probability = 0.0
+
+# The max number of query feedback that cache in memory.
+query-feedback-limit = 1024
+
+# Pseudo stats will be used if the ratio between the modify count and
+# row count in statistics of a table is greater than it.
+pseudo-estimate-ratio = 0.7
+
+[proxy-protocol]
+# PROXY protocol acceptable client networks.
+# Empty string means disable PROXY protocol, * means all networks.
+networks = ""
+
+# PROXY protocol header read timeout, unit is second
+header-timeout = 5
+
+[plan-cache]
+enabled = false
+capacity = 2560
+shards = 256
+
+[prepared-plan-cache]
+enabled = false
+capacity = 100
+
+[opentracing]
+# Enable opentracing.
+enable = false
+
+# Whether to enable the rpc metrics.
+rpc-metrics = false
+
+[opentracing.sampler]
+# Type specifies the type of the sampler: const, probabilistic, rateLimiting,
or remote
+type = "const"
+
+# Param is a value passed to the sampler.
+# Valid values for Param field are:
+# - for "const" sampler, 0 or 1 for always false/true respectively
+# - for "probabilistic" sampler, a probability between 0 and 1
+# - for "rateLimiting" sampler, the number of spans per second
+# - for "remote" sampler, param is the same as for "probabilistic"
+# and indicates the initial sampling rate before the actual one
+# is received from the mothership
+param = 1.0
+
+# SamplingServerURL is the address of jaeger-agent's HTTP sampling server
+sampling-server-url = ""
+
+# MaxOperations is the maximum number of operations that the sampler
+# will keep track of. If an operation is not tracked, a default probabilistic
+# sampler will be used rather than the per operation specific sampler.
+max-operations = 0
+
+# SamplingRefreshInterval controls how often the remotely controlled sampler
will poll
+# jaeger-agent for the appropriate sampling strategy.
+sampling-refresh-interval = 0
+
+[opentracing.reporter]
+# QueueSize controls how many spans the reporter can keep in memory before it
starts dropping
+# new spans. The queue is continuously drained by a background go-routine, as
fast as spans
+# can be sent out of process.
+queue-size = 0
+
+# BufferFlushInterval controls how often the buffer is force-flushed, even if
it's not full.
+# It is generally not useful, as it only matters for very low traffic services.
+buffer-flush-interval = 0
+
+# LogSpans, when true, enables LoggingReporter that runs in parallel with the
main reporter
+# and logs all submitted spans. Main Configuration.Logger must be initialized
in the code
+# for this option to have any effect.
+log-spans = false
+
+# LocalAgentHostPort instructs reporter to send spans to jaeger-agent at this
address
+local-agent-host-port = ""
+
+[tikv-client]
+# Max gRPC connections that will be established with each tikv-server.
+grpc-connection-count = 16
+
+# After a duration of this time in seconds if the client doesn't see any
activity it pings
+# the server to see if the transport is still alive.
+grpc-keepalive-time = 10
+
+# After having pinged for keepalive check, the client waits for a duration of
Timeout in seconds
+# and if no activity is seen even after that the connection is closed.
+grpc-keepalive-timeout = 3
+
+# max time for commit command, must be twice bigger than raft election timeout.
+commit-timeout = "41s"
+
+[binlog]
+
+# Socket file to write binlog.
+binlog-socket = ""
+
+# WriteTimeout specifies how long it will wait for writing binlog to pump.
+write-timeout = "15s"
+
+# If IgnoreError is true, when writting binlog meets error, TiDB would stop
writting binlog,
+# but still provide service.
+ignore-error = false
diff --git a/test/e2e/e2e-test/docker/ttl/docker-compose.tidb.yml
b/test/e2e/e2e-test/docker/ttl/docker-compose.tidb.yml
new file mode 100644
index 0000000..9746298
--- /dev/null
+++ b/test/e2e/e2e-test/docker/ttl/docker-compose.tidb.yml
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+ tidb:
+ image: pingcap/tidb:latest
+ expose:
+ - 4000
+ volumes:
+ - ./tidbconfig/tidb.toml:/tidb.toml:ro
+ restart: on-failure
+ healthcheck:
+ test: ["CMD", "sh", "-c", "nc -zn 127.0.0.1 4000"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ networks:
+ - e2e
+
+ oap:
+ extends:
+ file: ../base-compose.yml
+ service: oap
+ environment:
+ SW_STORAGE: tidb
+ SW_PROMETHEUS_FETCHER_ACTIVE: "true"
+ SW_TELEMETRY: prometheus
+ SW_JDBC_URL: "jdbc:mysql://tidb:4000/test"
+ SW_DATA_SOURCE_PASSWORD: ""
+ SW_CORE_DATA_KEEPER_EXECUTE_PERIOD: 1
+ depends_on:
+ tidb:
+ condition: service_healthy
+ entrypoint: ['sh', '-c', '/download-mysql.sh &&
/skywalking/docker-entrypoint.sh']
+
+networks:
+ e2e:
