kezhenxu94 commented on a change in pull request #6718: URL: https://github.com/apache/dolphinscheduler/pull/6718#discussion_r744896261
########## File path: dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/ConfigUtils.java ########## @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.data.quality.utils; + +import org.apache.dolphinscheduler.data.quality.config.Config; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class ConfigUtils { + + private ConfigUtils() { + throw new IllegalStateException("Construct ConfigUtils"); + } + + /** + * Extract sub config with fixed prefix + * + * @param source config source + * @param prefix config prefix + * @param keepPrefix true if keep prefix + */ + public static Config extractSubConfig(Config source, String prefix, boolean keepPrefix) { + Map<String, Object> values = new LinkedHashMap<>(); + + for (Map.Entry<String, Object> entry : source.entrySet()) { + final String key = entry.getKey(); + final String value = String.valueOf(entry.getValue()); + + if (key.startsWith(prefix)) { + if (keepPrefix) { + values.put(key, value); + } else { + values.put(key.substring(prefix.length()), value); + } + } + } + + return new Config(values); + } Review comment: This method is only used in 2 places and the method can be written in a one-line statement, maybe just inline it and not to provide too many util classes? ```java new Config(source.entrySet() .stream() .filter(it -> it.getKey().startsWith(prefix)) .collect(toMap(it -> keepPrefix ? it.getKey() : it.getKey().substring(prefix.length()), Entry::getValue))); ``` ########## File path: dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/JsonUtils.java ########## @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.data.quality.utils; + +import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; +import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT; +import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; +import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; +import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; +import static com.fasterxml.jackson.databind.SerializationFeature.FAIL_ON_EMPTY_BEANS; + +import java.util.TimeZone; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * JsonUtil + */ +public class JsonUtils { + + private static final Logger logger = LoggerFactory.getLogger(JsonUtils.class); + + /** + * can use static singleton, inject: just make sure to reuse! + */ + private static final ObjectMapper MAPPER = new ObjectMapper() + .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) + .configure(ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,true) + .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) + .configure(REQUIRE_SETTERS_FOR_GETTERS, true) + .configure(FAIL_ON_EMPTY_BEANS,false) + .setTimeZone(TimeZone.getDefault()); + + private JsonUtils() { + throw new UnsupportedOperationException("Construct JSONUtils"); + } + + public static String toJson(Object object) { Review comment: This is not used, please remove ########## File path: dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/Preconditions.java ########## @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.data.quality.utils; + +/** + * utility methods for validating input + */ +public final class Preconditions { + + private Preconditions() { + throw new UnsupportedOperationException("Construct Preconditions"); + } + + /** + * if condition is false will throw an IllegalArgumentException with the given message + * + * @param condition condition + * @param errorMsg error message + * @throws IllegalArgumentException Thrown, if the condition is violated. + */ + public static void checkArgument(boolean condition, Object errorMsg) { Review comment: Why do you choose to write again instead of using `com.google.common.base.Preconditions.checkArgument`? ########## File path: dolphinscheduler-data-quality/pom.xml ########## @@ -0,0 +1,162 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>dolphinscheduler</artifactId> + <groupId>org.apache.dolphinscheduler</groupId> + <version>2.0.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>dolphinscheduler-data-quality</artifactId> + <name>dolphinscheduler-data-quality</name> + + <packaging>jar</packaging> + + <properties> + <jackson.version>2.9.0</jackson.version> + <scope>provided</scope> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_${scala.binary.version}</artifactId> + <scope>${scope}</scope> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-sql_${scala.binary.version}</artifactId> + <scope>${scope}</scope> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-hive_${scala.binary.version}</artifactId> + <scope>${scope}</scope> + <exclusions> + <exclusion> + <groupId>commons-httpclient</groupId> + <artifactId>commons-httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson.version}</version> + <scope>${scope}</scope> + </dependency> + + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> + </dependency> + + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + </dependency> + + <dependency> + <groupId>ru.yandex.clickhouse</groupId> + <artifactId>clickhouse-jdbc</artifactId> + </dependency> + + <dependency> + <groupId>com.microsoft.sqlserver</groupId> + <artifactId>mssql-jdbc</artifactId> + </dependency> + + <dependency> + <groupId>com.facebook.presto</groupId> + <artifactId>presto-jdbc</artifactId> + </dependency> + + <dependency> + <groupId>org.jacoco</groupId> + <artifactId>org.jacoco.agent</artifactId> + <classifier>runtime</classifier> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.2</version> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <archive> + <manifest> + <mainClass>org.apache.dolphinscheduler.data.quality.DataQualityApplication</mainClass> + </manifest> + </archive> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>assembly</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-source-plugin</artifactId> + <version>2.1</version> + <configuration> + <attach>true</attach> + </configuration> + <executions> + <execution> + <phase>compile</phase> + <goals> + <goal>jar</goal> + </goals> + </execution> + </executions> + </plugin> Review comment: I don't think you need this plugin? ########## File path: dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/flow/writer/WriterFactoryTest.java ########## @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.data.quality.flow.writer; + +import org.apache.dolphinscheduler.data.quality.config.WriterConfig; +import org.apache.dolphinscheduler.data.quality.flow.batch.BatchWriter; +import org.apache.dolphinscheduler.data.quality.flow.batch.writer.WriterFactory; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +/** + * WriterFactoryTest + */ +public class WriterFactoryTest { + + @Test + public void testWriterGenerate() { + + List<WriterConfig> writerConfigs = new ArrayList<>(); + WriterConfig writerConfig = new WriterConfig(); + writerConfig.setType("JDBC"); + writerConfig.setConfig(null); + writerConfigs.add(writerConfig); + + int flag = 0; + try { + List<BatchWriter> writers = WriterFactory.getInstance().getWriters(null,writerConfigs); + if (writers != null && writers.size() >= 1) { + flag = 1; + } + } catch (Exception e) { + e.printStackTrace(); + } + + Assert.assertEquals(1,flag); Review comment: Same here, remove the `try-catch` block ########## File path: dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/flow/reader/ReaderFactoryTest.java ########## @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.data.quality.flow.reader; + +import static org.apache.dolphinscheduler.data.quality.Constants.DATABASE; +import static org.apache.dolphinscheduler.data.quality.Constants.DRIVER; +import static org.apache.dolphinscheduler.data.quality.Constants.PASSWORD; +import static org.apache.dolphinscheduler.data.quality.Constants.TABLE; +import static org.apache.dolphinscheduler.data.quality.Constants.URL; +import static org.apache.dolphinscheduler.data.quality.Constants.USER; + +import org.apache.dolphinscheduler.data.quality.config.ReaderConfig; +import org.apache.dolphinscheduler.data.quality.flow.batch.BatchReader; +import org.apache.dolphinscheduler.data.quality.flow.batch.reader.ReaderFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +/** + * ConnectorFactoryTest + */ +public class ReaderFactoryTest { + + @Test + public void testConnectorGenerate() { + + List<ReaderConfig> readerConfigs = new ArrayList<>(); + ReaderConfig readerConfig = new ReaderConfig(); + readerConfig.setType("JDBC"); + Map<String,Object> config = new HashMap<>(); + config.put(DATABASE,"test"); + config.put(TABLE,"test1"); + config.put(URL,"jdbc:mysql://localhost:3306/test"); + config.put(USER,"test"); + config.put(PASSWORD,"123456"); + config.put(DRIVER,"com.mysql.jdbc.Driver"); + readerConfig.setConfig(config); + readerConfigs.add(readerConfig); + + int flag = 0; + try { + List<BatchReader> readers = ReaderFactory.getInstance().getReaders(null,readerConfigs); + if (readers != null && readers.size() >= 1) { + flag = 1; + } + } catch (Exception e) { + e.printStackTrace(); + } + + Assert.assertEquals(1,flag); Review comment: No need to write this kind of codes, just remove the `try-catch` and let it throw the exception to make the test fail ########## File path: dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.controller; + +import static org.apache.dolphinscheduler.api.enums.Status.GET_DATASOURCE_OPTIONS_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.GET_RULE_FORM_CREATE_JSON_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_EXECUTE_RESULT_LIST_PAGING_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RULE_LIST_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RULE_LIST_PAGING_ERROR; + +import org.apache.dolphinscheduler.api.exceptions.ApiException; +import org.apache.dolphinscheduler.api.service.DqExecuteResultService; +import org.apache.dolphinscheduler.api.service.DqRuleService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import springfox.documentation.annotations.ApiIgnore; + +/** + * data quality controller + */ Review comment: This kind of JavaDoc is meaningless, please remove them all or write more useful comments ########## File path: dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/configuration/ConfigurationParserTest.java ########## @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.data.quality.configuration; + +import org.apache.dolphinscheduler.data.quality.config.DataQualityConfiguration; +import org.apache.dolphinscheduler.data.quality.utils.JsonUtils; + +import org.junit.Assert; +import org.junit.Test; + +/** + * ConfigurationParserTest + */ +public class ConfigurationParserTest { + + @Test Review comment: You don't need to make it so complex, simply use this, if there is exception thrown, the test will fail ```java @Test public void testConfigurationValidate() { String parameterStr = "{\"name\":\"data quality test\",\"env\":{\"type\":\"batch\",\"config\":null}," + "\"readers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"test\",\"password\":\"Test@123!\"," + "\"driver\":\"com.mysql.jdbc.Driver\",\"user\":\"test\",\"output_table\":\"test1\",\"table\":\"test1\"," + "\"url\":\"jdbc:mysql://172.16.100.199:3306/test\"} }],\"transformers\":[{\"type\":\"sql\",\"config\":" + "{\"index\":1,\"output_table\":\"miss_count\",\"sql\":\"SELECT COUNT(*) AS miss FROM test1 WHERE (c1 is null or c1 = '') \"} }," + "{\"type\":\"sql\",\"config\":{\"index\":2,\"output_table\":\"total_count\",\"sql\":\"SELECT COUNT(*) AS total FROM test1 \"} }]," + "\"writers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"dolphinscheduler\",\"password\":\"test\"," + "\"driver\":\"org.postgresql.Driver\",\"user\":\"test\",\"table\":\"t_ds_dq_execute_result\"," + "\"url\":\"jdbc:postgresql://172.16.100.199:5432/dolphinscheduler?stringtype=unspecified\"," + "\"sql\":\"SELECT 0 as rule_type,'data quality test' as rule_name,7 as process_definition_id,80 as process_instance_id," + "80 as task_instance_id,miss_count.miss AS statistics_value, total_count.total AS comparison_value,2 as check_type,10 as" + " threshold, 3 as operator, 0 as failure_strategy, '2021-06-29 10:18:59' as create_time,'2021-06-29 10:18:59' as update_time " + "from miss_count FULL JOIN total_count\"} }]}"; DataQualityConfiguration dataQualityConfiguration = JsonUtils.fromJson(parameterStr, DataQualityConfiguration.class); dataQualityConfiguration.validate(); } ``` ########## File path: dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/flow/reader/JdbcReaderTest.java ########## @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.data.quality.flow.reader; + +import static org.apache.dolphinscheduler.data.quality.Constants.DATABASE; +import static org.apache.dolphinscheduler.data.quality.Constants.DRIVER; +import static org.apache.dolphinscheduler.data.quality.Constants.PASSWORD; +import static org.apache.dolphinscheduler.data.quality.Constants.TABLE; +import static org.apache.dolphinscheduler.data.quality.Constants.URL; +import static org.apache.dolphinscheduler.data.quality.Constants.USER; + +import org.apache.dolphinscheduler.data.quality.config.Config; +import org.apache.dolphinscheduler.data.quality.flow.FlowTestBase; +import org.apache.dolphinscheduler.data.quality.flow.batch.reader.JdbcReader; + +import java.sql.Connection; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * JdbcConnectorTest + */ +public class JdbcReaderTest extends FlowTestBase { + + @Before + public void before() { Review comment: Missing '@Override' annotation on 'before()' ########## File path: dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/flow/writer/JdbcWriterTest.java ########## @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.data.quality.flow.writer; + +import static org.apache.dolphinscheduler.data.quality.Constants.DATABASE; +import static org.apache.dolphinscheduler.data.quality.Constants.DRIVER; +import static org.apache.dolphinscheduler.data.quality.Constants.PASSWORD; +import static org.apache.dolphinscheduler.data.quality.Constants.TABLE; +import static org.apache.dolphinscheduler.data.quality.Constants.URL; +import static org.apache.dolphinscheduler.data.quality.Constants.USER; + +import org.apache.dolphinscheduler.data.quality.config.Config; +import org.apache.dolphinscheduler.data.quality.flow.FlowTestBase; +import org.apache.dolphinscheduler.data.quality.flow.batch.reader.JdbcReader; +import org.apache.dolphinscheduler.data.quality.flow.batch.writer.JdbcWriter; + +import java.sql.Connection; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; + +/** + * JdbcWriterTest + */ +public class JdbcWriterTest extends FlowTestBase { + + @Before + public void before() { Review comment: Missing '@Override' annotation on 'before()' ########## File path: dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml ########## @@ -131,4 +131,9 @@ <unpack/> </artifact> </artifactSet> + <artifactSet to="lib/plugin/task/dataquality"> Review comment: If this is a task plugin, why don't you move the module to `dolphinscheduler-task-plugin/dolphinscheduler-data-quality`? ########## File path: dolphinscheduler-data-quality/pom.xml ########## @@ -0,0 +1,162 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>dolphinscheduler</artifactId> + <groupId>org.apache.dolphinscheduler</groupId> + <version>2.0.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>dolphinscheduler-data-quality</artifactId> + <name>dolphinscheduler-data-quality</name> + + <packaging>jar</packaging> + + <properties> + <jackson.version>2.9.0</jackson.version> + <scope>provided</scope> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_${scala.binary.version}</artifactId> + <scope>${scope}</scope> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-sql_${scala.binary.version}</artifactId> + <scope>${scope}</scope> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-hive_${scala.binary.version}</artifactId> + <scope>${scope}</scope> + <exclusions> + <exclusion> + <groupId>commons-httpclient</groupId> + <artifactId>commons-httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson.version}</version> + <scope>${scope}</scope> + </dependency> + + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>mysql</groupId> + <artifactId>mysql-connector-java</artifactId> + </dependency> + + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + </dependency> + + <dependency> + <groupId>ru.yandex.clickhouse</groupId> + <artifactId>clickhouse-jdbc</artifactId> + </dependency> + + <dependency> + <groupId>com.microsoft.sqlserver</groupId> + <artifactId>mssql-jdbc</artifactId> + </dependency> + + <dependency> + <groupId>com.facebook.presto</groupId> + <artifactId>presto-jdbc</artifactId> + </dependency> + + <dependency> + <groupId>org.jacoco</groupId> + <artifactId>org.jacoco.agent</artifactId> + <classifier>runtime</classifier> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.2</version> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> Review comment: Please don't use this kind of package style, you will hide a lot of dependencies that might bring license issues, FYI @CalvinKirs ########## File path: dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/StringUtils.java ########## @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.data.quality.utils; + +/** + * StringUtils + */ +public class StringUtils { Review comment: Why do you choose to write again instead of using `com.google.common.base.Strings`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
