This is an automated email from the ASF dual-hosted git repository.
wenjun pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 3b1de41acb Remove dolphinscheduler-data-quality from
dolphinscheduler-task-dataquality (#15791)
3b1de41acb is described below
commit 3b1de41acbee827b320370b14c6c73c7158f1aec
Author: Wenjun Ruan <[email protected]>
AuthorDate: Thu Apr 4 23:44:12 2024 +0800
Remove dolphinscheduler-data-quality from dolphinscheduler-task-dataquality
(#15791)
---
dolphinscheduler-api/pom.xml | 5 +++
.../data/quality/flow/batch/reader/JdbcReader.java | 8 +++--
.../data/quality/flow/batch/writer/JdbcWriter.java | 8 +++--
.../data/quality/utils/ParserUtilsTest.java | 39 ----------------------
.../dolphinscheduler-task-dataquality/pom.xml | 5 ---
.../plugin/task/dq/utils/RuleParserUtils.java | 23 ++++++++-----
6 files changed, 32 insertions(+), 56 deletions(-)
diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml
index 681a85318a..40d556a17e 100644
--- a/dolphinscheduler-api/pom.xml
+++ b/dolphinscheduler-api/pom.xml
@@ -61,6 +61,11 @@
<artifactId>dolphinscheduler-meter</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-data-quality</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-all</artifactId>
diff --git
a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java
b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java
index 274d4f793a..97ae414051 100644
---
a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java
+++
b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.data.quality.flow.batch.reader;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.dolphinscheduler.data.quality.Constants.DATABASE;
import static org.apache.dolphinscheduler.data.quality.Constants.DB_TABLE;
import static org.apache.dolphinscheduler.data.quality.Constants.DOTS;
@@ -32,17 +33,19 @@ import
org.apache.dolphinscheduler.data.quality.config.ValidateResult;
import
org.apache.dolphinscheduler.data.quality.execution.SparkRuntimeEnvironment;
import org.apache.dolphinscheduler.data.quality.flow.batch.BatchReader;
import org.apache.dolphinscheduler.data.quality.utils.ConfigUtils;
-import org.apache.dolphinscheduler.data.quality.utils.ParserUtils;
import org.apache.spark.sql.DataFrameReader;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
+import java.net.URLDecoder;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
+import lombok.SneakyThrows;
+
/**
* AbstractJdbcSource
*/
@@ -74,6 +77,7 @@ public class JdbcReader implements BatchReader {
return jdbcReader(env.sparkSession()).load();
}
+ @SneakyThrows
private DataFrameReader jdbcReader(SparkSession sparkSession) {
DataFrameReader reader = sparkSession.read()
@@ -81,7 +85,7 @@ public class JdbcReader implements BatchReader {
.option(URL, config.getString(URL))
.option(DB_TABLE, config.getString(DATABASE) + "." +
config.getString(TABLE))
.option(USER, config.getString(USER))
- .option(PASSWORD,
ParserUtils.decode(config.getString(PASSWORD)))
+ .option(PASSWORD,
URLDecoder.decode(config.getString(PASSWORD), UTF_8.name()))
.option(DRIVER, config.getString(DRIVER));
Config jdbcConfig = ConfigUtils.extractSubConfig(config, JDBC + DOTS,
false);
diff --git
a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java
b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java
index 07b2bd60d5..b737567f21 100644
---
a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java
+++
b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java
@@ -33,13 +33,16 @@ import
org.apache.dolphinscheduler.data.quality.config.Config;
import org.apache.dolphinscheduler.data.quality.config.ValidateResult;
import
org.apache.dolphinscheduler.data.quality.execution.SparkRuntimeEnvironment;
import org.apache.dolphinscheduler.data.quality.flow.batch.BatchWriter;
-import org.apache.dolphinscheduler.data.quality.utils.ParserUtils;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import lombok.SneakyThrows;
+
import com.google.common.base.Strings;
/**
@@ -70,6 +73,7 @@ public class JdbcWriter implements BatchWriter {
}
}
+ @SneakyThrows
@Override
public void write(Dataset<Row> data, SparkRuntimeEnvironment env) {
if (!Strings.isNullOrEmpty(config.getString(SQL))) {
@@ -82,7 +86,7 @@ public class JdbcWriter implements BatchWriter {
.option(URL, config.getString(URL))
.option(DB_TABLE, config.getString(DATABASE) + "." +
config.getString(TABLE))
.option(USER, config.getString(USER))
- .option(PASSWORD,
ParserUtils.decode(config.getString(PASSWORD)))
+ .option(PASSWORD,
URLDecoder.decode(config.getString(PASSWORD), StandardCharsets.UTF_8.name()))
.mode(config.getString(SAVE_MODE))
.save();
}
diff --git
a/dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java
b/dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java
deleted file mode 100644
index 328316cc39..0000000000
---
a/dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.data.quality.utils;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-public class ParserUtilsTest {
-
- @Test
- public void testParserUtils() {
- String testStr = "aaa$bbb$ccc%ddd^eee#fff";
- String encode = ParserUtils.encode(testStr);
- String decode = ParserUtils.decode(encode);
- Assertions.assertEquals(testStr, decode);
-
- String blank = "";
- Assertions.assertEquals(ParserUtils.encode(blank), blank);
- Assertions.assertEquals(ParserUtils.decode(blank), blank);
-
- Assertions.assertNull(ParserUtils.encode(null));
- Assertions.assertNull(ParserUtils.decode(null));
- }
-}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml
b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml
index cc64b6cdcb..6626234a19 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml
@@ -43,11 +43,6 @@
<artifactId>dolphinscheduler-datasource-all</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-data-quality</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java
index e99bb3dfaf..185573f66e 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.dq.utils;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_BUSINESS_DATE;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_CURRENT_DATE;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_DATETIME;
@@ -62,7 +63,6 @@ import static
org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConst
import static
org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.USER;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.data.quality.utils.ParserUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
import
org.apache.dolphinscheduler.plugin.task.api.DataQualityTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ExecuteSqlType;
@@ -80,12 +80,15 @@ import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import lombok.SneakyThrows;
+
import com.fasterxml.jackson.databind.node.ArrayNode;
/**
@@ -102,9 +105,10 @@ public class RuleParserUtils {
private static final String AND_TARGET_FILTER = "AND (${target_filter})";
private static final String WHERE_TARGET_FILTER = "WHERE
(${target_filter})";
+ @SneakyThrows
public static List<BaseConfig> getReaderConfigList(
Map<String, String>
inputParameterValue,
-
DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws
DataQualityException {
+
DataQualityTaskExecutionContext dataQualityTaskExecutionContext) {
List<BaseConfig> readerConfigList = new ArrayList<>();
@@ -123,7 +127,7 @@ public class RuleParserUtils {
config.put(URL,
DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getSourceType()),
sourceDataSource));
config.put(USER, sourceDataSource.getUser());
- config.put(PASSWORD,
ParserUtils.encode(sourceDataSource.getPassword()));
+ config.put(PASSWORD,
URLEncoder.encode(sourceDataSource.getPassword(), UTF_8.name()));
config.put(DRIVER, DataSourceUtils
.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getSourceType())));
String outputTable = inputParameterValue.get(SRC_DATABASE) +
"_" + inputParameterValue.get(SRC_TABLE);
@@ -150,7 +154,7 @@ public class RuleParserUtils {
config.put(URL,
DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getTargetType()),
targetDataSource));
config.put(USER, targetDataSource.getUser());
- config.put(PASSWORD,
ParserUtils.encode(targetDataSource.getPassword()));
+ config.put(PASSWORD,
URLEncoder.encode(targetDataSource.getPassword(), UTF_8.name()));
config.put(DRIVER, DataSourceUtils
.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getTargetType())));
String outputTable =
@@ -264,9 +268,10 @@ public class RuleParserUtils {
return defaultInputParameterValue;
}
+ @SneakyThrows
public static List<BaseConfig> getWriterConfigList(
String sql,
-
DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws
DataQualityException {
+
DataQualityTaskExecutionContext dataQualityTaskExecutionContext) {
List<BaseConfig> writerConfigList = new ArrayList<>();
if
(StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getWriterConnectorType()))
{
@@ -284,7 +289,7 @@ public class RuleParserUtils {
config.put(URL,
DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getWriterType()),
writerDataSource));
config.put(USER, writerDataSource.getUser());
- config.put(PASSWORD,
ParserUtils.encode(writerDataSource.getPassword()));
+ config.put(PASSWORD,
URLEncoder.encode(writerDataSource.getPassword(), UTF_8.name()));
config.put(DRIVER, DataSourceUtils
.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType())));
config.put(SQL, sql);
@@ -336,8 +341,9 @@ public class RuleParserUtils {
return readerConfigList;
}
+ @SneakyThrows
public static BaseConfig getStatisticsValueConfig(
-
DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws
DataQualityException {
+
DataQualityTaskExecutionContext dataQualityTaskExecutionContext) {
BaseConfig baseConfig = null;
if
(StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getStatisticsValueConnectorType()))
{
BaseConnectionParam writerDataSource =
@@ -354,7 +360,7 @@ public class RuleParserUtils {
config.put(URL, DataSourceUtils.getJdbcUrl(
DbType.of(dataQualityTaskExecutionContext.getStatisticsValueType()),
writerDataSource));
config.put(USER, writerDataSource.getUser());
- config.put(PASSWORD,
ParserUtils.encode(writerDataSource.getPassword()));
+ config.put(PASSWORD,
URLEncoder.encode(writerDataSource.getPassword(), UTF_8.name()));
config.put(DRIVER, DataSourceUtils
.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType())));
}
@@ -544,6 +550,7 @@ public class RuleParserUtils {
/**
* the unique code use to get the same type and condition task statistics
value
+ *
* @param inputParameterValue
* @return
*/
