caishunfeng commented on code in PR #10276:
URL: https://github.com/apache/dolphinscheduler/pull/10276#discussion_r884236520
##########
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java:
##########
@@ -43,5 +43,7 @@ public static void main(String[] args) {
public void run(ApplicationReadyEvent readyEvent) {
// install task plugin
taskPluginManager.installPlugin();
+ DataSourceUtils dataSourceUtils = new DataSourceUtils();
Review Comment:
It‘s better to use sprint bean, and maybe `DataSourceManager` is better,
WDYT?
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java:
##########
@@ -17,43 +17,55 @@
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
+import com.fasterxml.jackson.databind.JsonNode;
import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickHouseDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2DataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift.RedshiftDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SQLServerDataSourceProcessor;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.sql.Connection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static java.lang.String.format;
public class DataSourceUtils {
- private DataSourceUtils() {
+ public DataSourceUtils() {
}
private static final Logger logger =
LoggerFactory.getLogger(DataSourceUtils.class);
- private static final DataSourceProcessor mysqlProcessor = new
MySQLDataSourceProcessor();
- private static final DataSourceProcessor postgreSqlProcessor = new
PostgreSQLDataSourceProcessor();
- private static final DataSourceProcessor hiveProcessor = new
HiveDataSourceProcessor();
- private static final DataSourceProcessor sparkProcessor = new
SparkDataSourceProcessor();
- private static final DataSourceProcessor clickhouseProcessor = new
ClickHouseDataSourceProcessor();
- private static final DataSourceProcessor oracleProcessor = new
OracleDataSourceProcessor();
- private static final DataSourceProcessor sqlServerProcessor = new
SQLServerDataSourceProcessor();
- private static final DataSourceProcessor db2PROCESSOR = new
Db2DataSourceProcessor();
- private static final DataSourceProcessor prestoPROCESSOR = new
PrestoDataSourceProcessor();
- private static final DataSourceProcessor redshiftProcessor = new
RedshiftDataSourceProcessor();
+ private static final Map<String, DataSourceProcessor>
dataSourceProcessorMap = new ConcurrentHashMap<>();
+
+ public void installProcessor() {
+ final Set<String> names = new HashSet<>();
+
+ ServiceLoader.load(DataSourceProcessor.class).forEach(factory -> {
+ final String name = factory.getDbType().name();
+
+ logger.info("start register processor: " + name);
+ if (!names.add(name)) {
+ throw new IllegalStateException(format("Duplicate datasource
plugins named '%s'", name));
+ }
+ loadDatasourceClient(factory);
+
+ logger.info("done register processor: " + name);
Review Comment:
```suggestion
logger.info("done register processor: {}", name);
```
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java:
##########
@@ -17,43 +17,55 @@
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
+import com.fasterxml.jackson.databind.JsonNode;
import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickHouseDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2DataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift.RedshiftDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SQLServerDataSourceProcessor;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.sql.Connection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static java.lang.String.format;
public class DataSourceUtils {
- private DataSourceUtils() {
+ public DataSourceUtils() {
}
private static final Logger logger =
LoggerFactory.getLogger(DataSourceUtils.class);
- private static final DataSourceProcessor mysqlProcessor = new
MySQLDataSourceProcessor();
- private static final DataSourceProcessor postgreSqlProcessor = new
PostgreSQLDataSourceProcessor();
- private static final DataSourceProcessor hiveProcessor = new
HiveDataSourceProcessor();
- private static final DataSourceProcessor sparkProcessor = new
SparkDataSourceProcessor();
- private static final DataSourceProcessor clickhouseProcessor = new
ClickHouseDataSourceProcessor();
- private static final DataSourceProcessor oracleProcessor = new
OracleDataSourceProcessor();
- private static final DataSourceProcessor sqlServerProcessor = new
SQLServerDataSourceProcessor();
- private static final DataSourceProcessor db2PROCESSOR = new
Db2DataSourceProcessor();
- private static final DataSourceProcessor prestoPROCESSOR = new
PrestoDataSourceProcessor();
- private static final DataSourceProcessor redshiftProcessor = new
RedshiftDataSourceProcessor();
+ private static final Map<String, DataSourceProcessor>
dataSourceProcessorMap = new ConcurrentHashMap<>();
+
+ public void installProcessor() {
+ final Set<String> names = new HashSet<>();
+
+ ServiceLoader.load(DataSourceProcessor.class).forEach(factory -> {
+ final String name = factory.getDbType().name();
+
+ logger.info("start register processor: " + name);
+ if (!names.add(name)) {
+ throw new IllegalStateException(format("Duplicate datasource
plugins named '%s'", name));
+ }
+ loadDatasourceClient(factory);
+
+ logger.info("done register processor: " + name);
+
+ });
+ int i = 1;
Review Comment:
remove if not use.
##########
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java:
##########
@@ -92,7 +90,8 @@ public class DataSourceController extends BaseController {
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result createDataSource(@ApiIgnore @RequestAttribute(value =
Constants.SESSION_USER) User loginUser,
@ApiParam(name = "dataSourceParam", value =
"DATA_SOURCE_PARAM", required = true)
- @RequestBody BaseDataSourceParamDTO
dataSourceParam) {
+ @RequestBody String jsonStr) {
Review Comment:
It's better to provide json example.
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java:
##########
@@ -17,43 +17,55 @@
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
+import com.fasterxml.jackson.databind.JsonNode;
import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickHouseDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2DataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift.RedshiftDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDataSourceProcessor;
-import
org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SQLServerDataSourceProcessor;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.sql.Connection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static java.lang.String.format;
public class DataSourceUtils {
- private DataSourceUtils() {
+ public DataSourceUtils() {
}
private static final Logger logger =
LoggerFactory.getLogger(DataSourceUtils.class);
- private static final DataSourceProcessor mysqlProcessor = new
MySQLDataSourceProcessor();
- private static final DataSourceProcessor postgreSqlProcessor = new
PostgreSQLDataSourceProcessor();
- private static final DataSourceProcessor hiveProcessor = new
HiveDataSourceProcessor();
- private static final DataSourceProcessor sparkProcessor = new
SparkDataSourceProcessor();
- private static final DataSourceProcessor clickhouseProcessor = new
ClickHouseDataSourceProcessor();
- private static final DataSourceProcessor oracleProcessor = new
OracleDataSourceProcessor();
- private static final DataSourceProcessor sqlServerProcessor = new
SQLServerDataSourceProcessor();
- private static final DataSourceProcessor db2PROCESSOR = new
Db2DataSourceProcessor();
- private static final DataSourceProcessor prestoPROCESSOR = new
PrestoDataSourceProcessor();
- private static final DataSourceProcessor redshiftProcessor = new
RedshiftDataSourceProcessor();
+ private static final Map<String, DataSourceProcessor>
dataSourceProcessorMap = new ConcurrentHashMap<>();
+
+ public void installProcessor() {
+ final Set<String> names = new HashSet<>();
+
+ ServiceLoader.load(DataSourceProcessor.class).forEach(factory -> {
+ final String name = factory.getDbType().name();
+
+ logger.info("start register processor: " + name);
Review Comment:
```suggestion
logger.info("start register processor: {}", name);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
