This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch cleanup-dependencies in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 21c90dfcd45c8cd66ae29281ee5dec84e682c358 Author: kezhenxu94 <[email protected]> AuthorDate: Fri May 6 15:38:08 2022 +0800 Clean up unused dependencies and packaging issues --- .github/workflows/e2e.yml | 1 + README.md | 4 +- README_zh_CN.md | 2 +- .../dolphinscheduler-alert-dingtalk/pom.xml | 5 - .../dolphinscheduler-alert-email/pom.xml | 5 - .../plugin/alert/email/MailUtilsTest.java | 2 + .../dolphinscheduler-alert-feishu/pom.xml | 4 - .../dolphinscheduler-alert-http/pom.xml | 5 - .../dolphinscheduler-alert-script/pom.xml | 7 - .../dolphinscheduler-alert-telegram/pom.xml | 5 - .../dolphinscheduler-alert-wechat/pom.xml | 10 - .../dolphinscheduler-alert-server/pom.xml | 22 -- dolphinscheduler-alert/pom.xml | 7 - dolphinscheduler-api/pom.xml | 110 ++-------- .../api/service/TaskDefinitionServiceImplTest.java | 7 +- dolphinscheduler-common/pom.xml | 235 +-------------------- .../common/utils/HiveConfUtils.java | 87 -------- .../dolphinscheduler/common/utils/S3Utils.java | 5 +- .../common/utils/HiveConfUtilsTest.java | 47 ----- dolphinscheduler-dao/pom.xml | 30 --- .../org/apache/dolphinscheduler/dao/AlertDao.java | 12 +- .../dao/entity/ProcessDefinition.java | 5 +- .../dao/entity/ProcessInstance.java | 7 +- .../dao/entity/TaskDefinition.java | 8 +- .../dolphinscheduler/dao/entity/UdfFunc.java | 4 +- .../dolphinscheduler/dao/upgrade/SchemaUtils.java | 4 +- dolphinscheduler-data-quality/pom.xml | 32 +-- .../dolphinscheduler-datasource-clickhouse/pom.xml | 2 - .../dolphinscheduler-datasource-hive/pom.xml | 2 - .../dolphinscheduler-datasource-postgresql/pom.xml | 5 - dolphinscheduler-dist/release-docs/LICENSE | 5 - dolphinscheduler-master/pom.xml | 19 +- .../server/master/dispatch/ExecutorDispatcher.java | 2 +- .../master/dispatch/host/CommonHostManager.java | 2 +- .../master/registry/MasterRegistryClient.java | 34 +-- .../server/master/registry/ServerNodeManager.java | 2 +- .../master/runner/WorkflowExecuteThread.java | 4 +- .../master/runner/WorkflowExecuteThreadPool.java | 6 +- .../master/runner/task/CommonTaskProcessor.java | 2 +- .../master/runner/task/SubTaskProcessor.java | 2 +- .../master/runner/task/SwitchTaskProcessor.java | 4 +- .../server/master/service/FailoverService.java | 2 +- .../dispatch/host/RoundRobinHostManagerTest.java | 8 +- .../dolphinscheduler/registry/api/Registry.java | 1 - .../dolphinscheduler-registry-zookeeper/pom.xml | 5 - dolphinscheduler-server/pom.xml | 5 +- .../server/log/SensitiveDataConverter.java | 8 +- .../server/utils/ProcessUtils.java | 5 +- .../server/utils/ProcessUtilsTest.java | 4 +- dolphinscheduler-service/pom.xml | 12 +- .../service/process/ProcessServiceImpl.java | 32 +-- .../service/quartz/impl/QuartzExecutorImpl.java | 6 +- dolphinscheduler-spi/pom.xml | 6 +- dolphinscheduler-standalone-server/pom.xml | 5 + .../dolphinscheduler-task-api/pom.xml | 22 -- .../plugin/task/api/ShellCommandExecutor.java | 7 +- .../task/api/parameters/AbstractParameters.java | 3 +- .../dolphinscheduler-task-dataquality/pom.xml | 6 +- .../dolphinscheduler-task-datax/pom.xml | 10 +- .../plugin/task/datax/DataxTask.java | 2 +- .../dolphinscheduler-task-emr/pom.xml | 34 +-- .../plugin/task/flink/FlinkParametersTest.java | 4 +- .../dolphinscheduler-task-procedure/pom.xml | 10 +- .../plugin/task/procedure/ProcedureParameters.java | 2 +- .../plugin/task/python/PythonTask.java | 1 - .../plugin/task/spark/SparkParametersTest.java | 4 +- .../plugin/task/spark/SparkTaskTest.java | 6 +- .../dolphinscheduler-task-sql/pom.xml | 9 +- .../dolphinscheduler/plugin/task/sql/SqlTask.java | 2 +- .../tools/datasource/dao/ResourceDao.java | 5 +- .../tools/datasource/dao/UpgradeDao.java | 11 +- dolphinscheduler-worker/pom.xml | 10 +- .../server/worker/processor/TaskKillProcessor.java | 10 +- .../worker/registry/WorkerRegistryClient.java | 6 +- .../server/worker/runner/TaskExecuteThread.java | 5 +- pom.xml | 55 +++-- tools/dependencies/known-dependencies.txt | 10 - 77 files changed, 216 insertions(+), 880 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c8f7ef7cf6..55b90a9036 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,6 +20,7 @@ on: push: branches: - dev + - cleanup-dependencies name: E2E diff --git a/README.md b/README.md index c014dd390e..419b4764ec 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Dolphin Scheduler Official Website DolphinScheduler is a distributed and extensible workflow scheduler platform with powerful DAG visual interfaces, dedicated to solving complex job dependencies in the data pipeline and providing various types of jobs available `out of the box`. Its main objectives are as follows: - - Highly Reliable, + - Highly Reliable, DolphinScheduler adopts a decentralized multi-master and multi-worker architecture design, which naturally supports easy expansion and high availability (not restricted by a single point of bottleneck), and its performance increases linearly with the increase of machines - High performance, supporting tens of millions of tasks every day - Support multi-tenant. @@ -86,7 +86,7 @@ dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-s ## Thanks -DolphinScheduler is based on a lot of excellent open-source projects, such as Google guava, guice, grpc, netty, quartz, and many open-source projects of Apache and so on. +DolphinScheduler is based on a lot of excellent open-source projects, such as Google guava, grpc, netty, quartz, and many open-source projects of Apache and so on. We would like to express our deep gratitude to all the open-source projects used in Dolphin Scheduler. We hope that we are not only the beneficiaries of open-source, but also give back to the community. Besides, we hope everyone who have the same enthusiasm and passion for open source could join in and contribute to the open-source community! ## Get Help diff --git a/README_zh_CN.md b/README_zh_CN.md index 8decfe0348..6352b8e2c6 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -82,7 +82,7 @@ dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-s ## 感谢 -Dolphin Scheduler使用了很多优秀的开源项目，比如google的guava、guice、grpc，netty，quartz，以及apache的众多开源项目等等， +Dolphin Scheduler使用了很多优秀的开源项目，比如google的guava、grpc，netty，quartz，以及apache的众多开源项目等等， 正是由于站在这些开源项目的肩膀上，才有Dolphin Scheduler的诞生的可能。对此我们对使用的所有开源软件表示非常的感谢！我们也希望自己不仅是开源的受益者，也能成为开源的贡献者,也希望对开源有同样热情和信念的伙伴加入进来，一起为开源献出一份力！ ## 获得帮助 diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml index 4108a1736b..34b1ef3335 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml @@ -32,10 +32,5 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </dependency> - - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> </dependencies> </project> diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml index 4391ca4c2a..1ec2302022 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml @@ -37,11 +37,6 @@ <artifactId>poi-ooxml</artifactId> </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-email</artifactId> diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java index 193ba1b7be..e4bb76dd27 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java @@ -30,10 +30,12 @@ import java.util.List; import java.util.Map; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Ignore("The test case makes no sense") public class MailUtilsTest { private static final Logger logger = LoggerFactory.getLogger(MailUtilsTest.class); static MailSender mailSender; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml index a166c5d30f..93450e08f6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml @@ -32,9 +32,5 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> </dependencies> </project> diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml index 974a023ca8..007c3cd835 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml @@ -28,11 +28,6 @@ <packaging>jar</packaging> <dependencies> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml index 2b265a7b7e..96038b970c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml @@ -26,11 +26,4 @@ <modelVersion>4.0.0</modelVersion> <artifactId>dolphinscheduler-alert-script</artifactId> <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - </dependencies> </project> diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml index 68e1e92daa..ed761d446a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml @@ -32,10 +32,5 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </dependency> - - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> </dependencies> </project> diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml index bd875423c3..b293ac6c1e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml @@ -28,16 +28,6 @@ <packaging>jar</packaging> <dependencies> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - </dependency> - <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index f6be237334..6ed92bf63d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -85,36 +85,14 @@ <artifactId>dolphinscheduler-dao</artifactId> </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - </dependency> - <dependency> <groupId>org.codehaus.janino</groupId> <artifactId>janino</artifactId> </dependency> - - <!--excel poi--> - <dependency> - <groupId>org.apache.poi</groupId> - <artifactId>poi</artifactId> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index 45a41b494b..64111ddd50 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -35,11 +35,4 @@ <module>dolphinscheduler-alert-plugins</module> <module>dolphinscheduler-alert-server</module> </modules> - - <dependencies> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - </dependencies> </project> diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index b5dd813843..074fcc6872 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -53,14 +53,26 @@ <dependency> <groupId>org.apache.dolphinscheduler</groupId> - <artifactId>dolphinscheduler-task-all</artifactId> + <artifactId>dolphinscheduler-task-api</artifactId> </dependency> <dependency> <groupId>org.apache.dolphinscheduler</groupId> <artifactId>dolphinscheduler-ui</artifactId> </dependency> + <dependency> + <groupId>org.apache.dolphinscheduler</groupId> + <artifactId>dolphinscheduler-registry-zookeeper</artifactId> + </dependency> + <dependency> + <groupId>org.codehaus.janino</groupId> + <artifactId>janino</artifactId> + </dependency> + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-s3</artifactId> + </dependency> <!--springboot--> <dependency> <groupId>org.springframework.boot</groupId> @@ -109,11 +121,6 @@ <artifactId>spring-context</artifactId> </dependency> - <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </dependency> - <dependency> <groupId>org.quartz-scheduler</groupId> <artifactId>quartz</artifactId> @@ -133,11 +140,6 @@ </exclusions> </dependency> - <dependency> - <groupId>org.quartz-scheduler</groupId> - <artifactId>quartz-jobs</artifactId> - </dependency> - <dependency> <groupId>io.springfox</groupId> <artifactId>springfox-swagger2</artifactId> @@ -164,96 +166,14 @@ <artifactId>swagger-bootstrap-ui</artifactId> </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-framework</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-recipes</artifactId> - <exclusions> - <exclusion> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </exclusion> - </exclusions> - </dependency> - - <!-- hadoop --> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <exclusions> - <exclusion> - <groupId>javax.servlet</groupId> - <artifactId>servlet-api</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.curator</groupId> - <artifactId>curator-client</artifactId> - </exclusion> - <exclusion> - <artifactId>jackson-core-asl</artifactId> - <groupId>org.codehaus.jackson</groupId> - </exclusion> - <exclusion> - <artifactId>jackson-mapper-asl</artifactId> - <groupId>org.codehaus.jackson</groupId> - </exclusion> - <exclusion> - <artifactId>jackson-jaxrs</artifactId> - <groupId>org.codehaus.jackson</groupId> - </exclusion> - <exclusion> - <artifactId>jackson-xc</artifactId> - <groupId>org.codehaus.jackson</groupId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - <exclusions> - <exclusion> - <artifactId>servlet-api</artifactId> - <groupId>javax.servlet</groupId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-common</artifactId> - <exclusions> - <exclusion> - <artifactId>servlet-api</artifactId> - <groupId>javax.servlet</groupId> - </exclusion> - </exclusions> - </dependency> - - <dependency> <groupId>org.hibernate.validator</groupId> <artifactId>hibernate-validator</artifactId> </dependency> <dependency> - <groupId>org.codehaus.janino</groupId> - <artifactId>janino</artifactId> + <groupId>io.fabric8</groupId> + <artifactId>kubernetes-client</artifactId> </dependency> <!-- just for test --> diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java index 9aade5555c..1e9ed3cbe5 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java @@ -32,7 +32,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; -import org.apache.dolphinscheduler.plugin.task.shell.ShellParameters; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.task.TaskPluginManager; @@ -279,10 +278,10 @@ public class TaskDefinitionServiceImplTest { String taskParams = "{\"resourceList\":[],\"localParams\":[{\"prop\":\"datetime\",\"direct\":\"IN\",\"type\":\"VARCHAR\"," + "\"value\":\"${system.datetime}\"}],\"rawScript\":\"echo ${datetime}\",\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"]," + "\\\"failedNode\\\":[\\\"\\\"]}\",\"dependence\":{}}"; - ShellParameters parameters = JSONUtils.parseObject(taskParams, ShellParameters.class); + Map parameters = JSONUtils.parseObject(taskParams, Map.class); Assert.assertNotNull(parameters); String params = "{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo 1\",\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{}}"; - ShellParameters parameters1 = JSONUtils.parseObject(params, ShellParameters.class); + Map parameters1 = JSONUtils.parseObject(params, Map.class); Assert.assertNotNull(parameters1); } @@ -333,4 +332,4 @@ public class TaskDefinitionServiceImplTest { Map<String, Object> failResult = taskDefinitionService.releaseTaskDefinition(loginUser, projectCode, taskCode, ReleaseState.getEnum(2)); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failResult.get(Constants.STATUS)); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index a2fe9ac4c3..bd4db8bf7a 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -43,10 +43,6 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </dependency> - <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -68,14 +64,6 @@ <artifactId>spring-boot-starter-aop</artifactId> </dependency> - <dependency> - <groupId>commons-configuration</groupId> - <artifactId>commons-configuration</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> @@ -91,6 +79,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> <exclusions> <exclusion> <groupId>org.slf4j</groupId> @@ -182,6 +171,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> + <scope>provided</scope> <exclusions> <exclusion> <groupId>org.slf4j</groupId> @@ -234,6 +224,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> + <scope>provided</scope> <exclusions> <exclusion> <groupId>javax.servlet</groupId> @@ -277,227 +268,9 @@ <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-s3</artifactId> + <scope>provided</scope> </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-jdbc</artifactId> - <exclusions> - <exclusion> - <artifactId>slf4j-log4j12</artifactId> - <groupId>org.slf4j</groupId> - </exclusion> - <exclusion> - <groupId>org.eclipse.jetty.aggregate</groupId> - <artifactId>jetty-all</artifactId> - </exclusion> - - <exclusion> - <groupId>org.apache.ant</groupId> - <artifactId>ant</artifactId> - </exclusion> - <exclusion> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-json</artifactId> - </exclusion> - <exclusion> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-jvm</artifactId> - </exclusion> - <exclusion> - <groupId>com.github.joshelser</groupId> - <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId> - </exclusion> - - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty-all</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - </exclusion> - <exclusion> - <groupId>io.dropwizard.metrics</groupId> - <artifactId>metrics-core</artifactId> - </exclusion> - <exclusion> - <groupId>javax.servlet</groupId> - <artifactId>servlet-api</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.curator</groupId> - <artifactId>curator-client</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-api</artifactId> - </exclusion> - - <exclusion> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </exclusion> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-jaxrs</artifactId> - </exclusion> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-xc</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - - <exclusion> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - </exclusion> - <exclusion> - <artifactId>log4j-slf4j-impl</artifactId> - <groupId>org.apache.logging.log4j</groupId> - </exclusion> - <exclusion> - <artifactId>javax.servlet</artifactId> - <groupId>org.eclipse.jetty.orbit</groupId> - </exclusion> - <exclusion> - <artifactId>servlet-api-2.5</artifactId> - <groupId>org.mortbay.jetty</groupId> - </exclusion> - <exclusion> - <artifactId>jasper-runtime</artifactId> - <groupId>tomcat</groupId> - </exclusion> - <exclusion> - <artifactId>slider-core</artifactId> - <groupId>org.apache.slider</groupId> - </exclusion> - <exclusion> - <artifactId>hbase-server</artifactId> - <groupId>org.apache.hbase</groupId> - </exclusion> - <exclusion> - <artifactId>jersey-client</artifactId> - <groupId>com.sun.jersey</groupId> - </exclusion> - <exclusion> - <artifactId>jersey-core</artifactId> - <groupId>com.sun.jersey</groupId> - </exclusion> - <exclusion> - <artifactId>jersey-json</artifactId> - <groupId>com.sun.jersey</groupId> - </exclusion> - <exclusion> - <artifactId>jersey-server</artifactId> - <groupId>com.sun.jersey</groupId> - </exclusion> - <exclusion> - <artifactId>jersey-guice</artifactId> - <groupId>com.sun.jersey.contribs</groupId> - </exclusion> - <exclusion> - <artifactId>hbase-common</artifactId> - <groupId>org.apache.hbase</groupId> - </exclusion> - <exclusion> - <artifactId>hbase-hadoop2-compat</artifactId> - <groupId>org.apache.hbase</groupId> - </exclusion> - <exclusion> - <artifactId>hbase-client</artifactId> - <groupId>org.apache.hbase</groupId> - </exclusion> - <exclusion> - <artifactId>hbase-hadoop-compat</artifactId> - <groupId>org.apache.hbase</groupId> - </exclusion> - <exclusion> - <artifactId>tephra-hbase-compat-1.0</artifactId> - <groupId>co.cask.tephra</groupId> - </exclusion> - <exclusion> - <artifactId>jaxb-api</artifactId> - <groupId>javax.xml.bind</groupId> - </exclusion> - <exclusion> - <artifactId>hive-llap-client</artifactId> - <groupId>org.apache.hive</groupId> - </exclusion> - <exclusion> - <artifactId>hive-llap-common</artifactId> - <groupId>org.apache.hive</groupId> - </exclusion> - <exclusion> - <artifactId>hive-llap-server</artifactId> - <groupId>org.apache.hive</groupId> - </exclusion> - <exclusion> - <artifactId>tephra-core</artifactId> - <groupId>co.cask.tephra</groupId> - </exclusion> - <exclusion> - <artifactId>ant</artifactId> - <groupId>ant</groupId> - </exclusion> - <exclusion> - <artifactId>stringtemplate</artifactId> - <groupId>org.antlr</groupId> - </exclusion> - <exclusion> - <artifactId>antlr-runtime</artifactId> - <groupId>org.antlr</groupId> - </exclusion> - <exclusion> - <artifactId>hive-shims</artifactId> - <groupId>org.apache.hive</groupId> - </exclusion> - <exclusion> - <artifactId>jsp-api</artifactId> - <groupId>javax.servlet</groupId> - </exclusion> - <exclusion> - <artifactId>log4j-api</artifactId> - <groupId>org.apache.logging.log4j</groupId> - </exclusion> - <exclusion> - <artifactId>log4j-core</artifactId> - <groupId>org.apache.logging.log4j</groupId> - </exclusion> - <exclusion> - <artifactId>log4j-web</artifactId> - <groupId>org.apache.logging.log4j</groupId> - </exclusion> - <exclusion> - <artifactId>jasper-compiler</artifactId> - <groupId>tomcat</groupId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - </dependency> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-core</artifactId> - </dependency> <dependency> <groupId>com.github.oshi</groupId> <artifactId>oshi-core</artifactId> diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HiveConfUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HiveConfUtils.java deleted file mode 100644 index b231b4e4b1..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HiveConfUtils.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.utils; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * hive conf utils - */ -public class HiveConfUtils { - - private HiveConfUtils() { - throw new UnsupportedOperationException("Construct HiveConfUtils"); - } - - private static class HiveConfHandler { - private static HiveConf singleton; - - private static Map<String,Object> hiveConfVars; - - static { - singleton = new HiveConf(); - hiveConfVars = new HashMap<>(); - Arrays.stream(ConfVars.values()).forEach(confVar -> hiveConfVars.put(confVar.varname,confVar)); - } - } - - /** - * get HiveConf instance - * @return HiveConf hiveConf - */ - public static HiveConf getInstance() { - return HiveConfHandler.singleton; - } - - /** - * get hive conf vars - * @return - */ - public static Map<String,Object> getHiveConfVars() { - return HiveConfHandler.hiveConfVars; - } - - /** - * Determine if it belongs to a hive conf property - * @param conf config - * @return boolean result - */ - public static boolean isHiveConfVar(String conf) { - // the default hive conf var name - String confKey = conf.split("=")[0]; - Map<String, Object> hiveConfVars = HiveConfUtils.getHiveConfVars(); - if (hiveConfVars.get(confKey) != null) { - return true; - } - - // the security authorization hive conf var name - HiveConf hiveConf = HiveConfUtils.getInstance(); - String hiveAuthorizationSqlStdAuthConfigWhitelist = hiveConf.getVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); - Pattern modWhiteListPattern = Pattern.compile(hiveAuthorizationSqlStdAuthConfigWhitelist); - Matcher matcher = modWhiteListPattern.matcher(confKey); - return matcher.matches(); - } - -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java index 13e17c3bba..3c93e9ea78 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java @@ -37,7 +37,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.spi.enums.ResourceType; -import org.jets3t.service.ServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -125,7 +124,7 @@ public class S3Utils implements Closeable, StorageOperate { } @Override - public void createTenantDirIfNotExists(String tenantCode) throws ServiceException { + public void createTenantDirIfNotExists(String tenantCode) throws Exception { createFolder(tenantCode+ FOLDER_SEPARATOR +RESOURCE_TYPE_UDF); createFolder(tenantCode+ FOLDER_SEPARATOR +RESOURCE_TYPE_FILE); } @@ -314,4 +313,4 @@ public class S3Utils implements Closeable, StorageOperate { public ResUploadType returnStorageType() { return ResUploadType.S3; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java deleted file mode 100644 index 85912d5ff6..0000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.utils; - -import org.junit.Assert; -import org.junit.Test; - -/** - * hive conf utils test - */ -public class HiveConfUtilsTest { - - /** - * test is hive conf var - */ - @Test - public void testIsHiveConfVar() { - - String conf = "hive.exec.script.wrapper=123"; - boolean hiveConfVar = HiveConfUtils.isHiveConfVar(conf); - Assert.assertTrue(hiveConfVar); - - conf = "hive.test.v1=v1"; - hiveConfVar = HiveConfUtils.isHiveConfVar(conf); - Assert.assertFalse(hiveConfVar); - - conf = "tez.queue.name=tezQueue"; - hiveConfVar = HiveConfUtils.isHiveConfVar(conf); - Assert.assertTrue(hiveConfVar); - - } -} diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 2b34034138..238033c732 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -78,15 +78,7 @@ <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> </dependency> - <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> - </dependency> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-annotations</artifactId> @@ -95,29 +87,7 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - </dependency> - <dependency> - <groupId>com.cronutils</groupId> - <artifactId>cron-utils</artifactId> - </dependency> - - <dependency> - <groupId>commons-configuration</groupId> - <artifactId>commons-configuration</artifactId> - </dependency> - - <dependency> - <groupId>org.yaml</groupId> - <artifactId>snakeyaml</artifactId> - </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index a2d2aacdf8..76fb17e143 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -37,8 +37,8 @@ import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.AlertSendStatusMapper; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; - +import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -46,13 +46,13 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import org.joda.time.DateTime; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.google.common.base.Strings; import com.google.common.collect.Lists; @Component @@ -113,7 +113,7 @@ public class AlertDao { .map(Alert::getContent) .map(DigestUtils::sha1Hex) .map(String::toLowerCase) - .orElse(StringUtils.EMPTY); + .orElse(""); } /** @@ -163,7 +163,7 @@ public class AlertDao { alert.setSign(generateSign(alert)); // we use this method to avoid insert duplicate alert(issue #5525) // we modified this method to optimize performance(issue #9174) - Date crashAlarmSuppressionStartTime = DateTime.now().plusMinutes(-crashAlarmSuppression).toDate(); + Date crashAlarmSuppressionStartTime = Date.from(LocalDateTime.now().plusMinutes(-crashAlarmSuppression).atZone(ZoneId.systemDefault()).toInstant()); alertMapper.insertAlertWhenServerCrash(alert, crashAlarmSuppressionStartTime); } @@ -274,7 +274,7 @@ public class AlertDao { */ public List<AlertPluginInstance> listInstanceByAlertGroupId(int alertGroupId) { String alertInstanceIdsParam = alertGroupMapper.queryAlertGroupInstanceIdsById(alertGroupId); - if (StringUtils.isNotBlank(alertInstanceIdsParam)) { + if (!Strings.isNullOrEmpty(alertInstanceIdsParam)) { String[] idsArray = alertInstanceIdsParam.split(","); List<Integer> ids = Arrays.stream(idsArray) .map(s -> Integer.parseInt(s.trim())) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index f68ad065b3..80331db34d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -23,8 +23,6 @@ import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.commons.lang.StringUtils; - import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -36,6 +34,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.google.common.base.Strings; /** * process definition @@ -309,7 +308,7 @@ public class ProcessDefinition { } public Map<String, String> getGlobalParamMap() { - if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { + if (globalParamMap == null && !Strings.isNullOrEmpty(globalParams)) { List<Property> propList = JSONUtils.toList(globalParams, Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index 8f639efc07..29971f94db 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -26,8 +26,6 @@ import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus; -import org.apache.commons.lang.StringUtils; - import java.util.Date; import java.util.Objects; @@ -35,6 +33,7 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.google.common.base.Strings; /** * process instance @@ -541,7 +540,7 @@ public class ProcessInstance { * @param cmd cmd */ public void addHistoryCmd(CommandType cmd) { - if (StringUtils.isNotEmpty(this.historyCmd)) { + if (!Strings.isNullOrEmpty(this.historyCmd)) { this.historyCmd = String.format("%s,%s", this.historyCmd, cmd.toString()); } else { this.historyCmd = cmd.toString(); @@ -554,7 +553,7 @@ public class ProcessInstance { * @return whether complement data */ public boolean isComplementData() { - if (StringUtils.isEmpty(this.historyCmd)) { + if (Strings.isNullOrEmpty(this.historyCmd)) { return false; } return historyCmd.startsWith(CommandType.COMPLEMENT_DATA.toString()); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java index 58e416e5ef..0206719a35 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java @@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.common.enums.TimeoutFlag; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.commons.lang.StringUtils; import org.apache.commons.collections4.CollectionUtils; import java.util.Date; import java.util.HashMap; @@ -41,6 +40,7 @@ import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.base.Strings; /** * task definition @@ -307,7 +307,7 @@ public class TaskDefinition { } public Map<String, String> getTaskParamMap() { - if (taskParamMap == null && StringUtils.isNotEmpty(taskParams)) { + if (taskParamMap == null && !Strings.isNullOrEmpty(taskParams)) { JsonNode localParams = JSONUtils.parseObject(taskParams).findValue("localParams"); //If a jsonNode is null, not only use !=null, but also it should use the isNull method to be estimated. @@ -477,8 +477,8 @@ public class TaskDefinition { && timeoutFlag == that.timeoutFlag && timeoutNotifyStrategy == that.timeoutNotifyStrategy && (Objects.equals(resourceIds, that.resourceIds) - || (StringUtils.EMPTY.equals(resourceIds) && that.resourceIds == null) - || (StringUtils.EMPTY.equals(that.resourceIds) && resourceIds == null)) + || ("".equals(resourceIds) && that.resourceIds == null) + || ("".equals(that.resourceIds) && resourceIds == null)) && environmentCode == that.environmentCode && taskGroupId == that.taskGroupId && taskGroupPriority == that.taskGroupPriority; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java index 949499d1e8..de1919809f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java @@ -23,7 +23,7 @@ import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.KeyDeserializer; -import org.apache.commons.lang.StringUtils; +import com.google.common.base.Strings; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -236,7 +236,7 @@ public class UdfFunc { @Override public Object deserializeKey(String key, DeserializationContext ctxt) throws IOException { - if (StringUtils.isBlank(key)) { + if (Strings.isNullOrEmpty(key)) { return null; } return JSONUtils.parseObject(key, UdfFunc.class); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java index bf6dc9ab59..625c3551f1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.commons.lang.StringUtils; - import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -81,7 +79,7 @@ public class SchemaUtils { * @return Determine whether schemaVersion is higher than version */ public static boolean isAGreatVersion(String schemaVersion, String version) { - if (StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) { + if (Strings.isNullOrEmpty(schemaVersion) || Strings.isNullOrEmpty(version)) { throw new RuntimeException("schemaVersion or version is empty"); } diff --git a/dolphinscheduler-data-quality/pom.xml b/dolphinscheduler-data-quality/pom.xml index 7419601c88..024739a23a 100644 --- a/dolphinscheduler-data-quality/pom.xml +++ b/dolphinscheduler-data-quality/pom.xml @@ -171,34 +171,4 @@ </dependency> </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <version>2.2</version> - <configuration> - <appendAssemblyId>false</appendAssemblyId> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - <archive> - <manifest> - <mainClass>org.apache.dolphinscheduler.data.quality.DataQualityApplication</mainClass> - </manifest> - </archive> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> \ No newline at end of file +</project> diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml index 46fd1ad376..a3f23ff7b9 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/pom.xml @@ -67,8 +67,6 @@ <version>${clickhouse.jdbc.version}</version> </dependency> - - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml index 9d0a35428e..4ba64c86d0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml @@ -320,8 +320,6 @@ </exclusions> </dependency> - - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml index f70e3e44f4..e2c20d7c93 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/pom.xml @@ -43,11 +43,6 @@ <version>${project.version}</version> </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> <groupId>org.postgresql</groupId> <artifactId>postgresql</artifactId> diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 30fb7ac3a5..fe51638edb 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -230,7 +230,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. commons-beanutils 1.9.4 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.9.4, Apache 2.0 commons-cli 1.2: https://mvnrepository.com/artifact/commons-cli/commons-cli/1.2, Apache 2.0 commons-codec 1.11: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.11, Apache 2.0 - commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0 commons-collections4 4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-collections4/4.1, Apache 2.0 commons-compress 1.21: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.21, Apache 2.0 commons-configuration 1.10: https://mvnrepository.com/artifact/commons-configuration/commons-configuration/1.10, Apache 2.0 @@ -259,8 +258,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. gson 2.8.8: https://github.com/google/gson, Apache 2.0 guava 24.1-jre: https://mvnrepository.com/artifact/com.google.guava/guava/24.1-jre, Apache 2.0 guava-retrying 2.0.0: https://mvnrepository.com/artifact/com.github.rholder/guava-retrying/2.0.0, Apache 2.0 - guice 3.0: https://mvnrepository.com/artifact/com.google.inject/guice/3.0, Apache 2.0 - guice-servlet 3.0: https://mvnrepository.com/artifact/com.google.inject.extensions/guice-servlet/3.0, Apache 2.0 hadoop-annotations 2.7.3:https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-annotations/2.7.3, Apache 2.0 hadoop-auth 2.7.3: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-auth/2.7.3, Apache 2.0 hadoop-client 2.7.3: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client/2.7.3, Apache 2.0 @@ -458,7 +455,6 @@ BSD licenses The following components are provided under a BSD license. See project link for details. The text of each license is also included at licenses/LICENSE-[project].txt. - asm 3.1: https://github.com/jdf/javalin/tree/master/lib/asm-3.1, BSD click 8.0: https://github.com/pallets/click, BSD 3-Clause curvesapi 1.06: https://mvnrepository.com/artifact/com.github.virtuald/curvesapi/1.06, BSD 3-clause javolution 5.5.1: https://mvnrepository.com/artifact/javolution/javolution/5.5.1, BSD @@ -492,7 +488,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. jaxb-impl 2.2.3-1: https://mvnrepository.com/artifact/com.sun.xml.bind/jaxb-impl/2.2.3-1, CDDL and GPL 1.1 jersey-client 1.9: https://mvnrepository.com/artifact/com.sun.jersey/jersey-client/1.9, CDDL 1.1 and GPL 1.1 jersey-core 1.9: https://mvnrepository.com/artifact/com.sun.jersey/jersey-core/1.9, CDDL 1.1 and GPL 1.1 - jersey-guice 1.9: https://mvnrepository.com/artifact/com.sun.jersey.contribs/jersey-guice/1.9, CDDL 1.1 and GPL 1.1 jersey-json 1.9: https://mvnrepository.com/artifact/com.sun.jersey/jersey-json/1.9, CDDL 1.1 and GPL 1.1 jersey-server 1.9: https://mvnrepository.com/artifact/com.sun.jersey/jersey-server/1.9, CDDL 1.1 and GPL 1.1 jta 1.1: https://mvnrepository.com/artifact/javax.transaction/jta/1.1, CDDL 1.0 diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index e32666e555..d35051dcab 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -47,6 +47,10 @@ <groupId>org.apache.dolphinscheduler</groupId> <artifactId>dolphinscheduler-server</artifactId> </dependency> + <dependency> + <groupId>org.apache.dolphinscheduler</groupId> + <artifactId>dolphinscheduler-registry-zookeeper</artifactId> + </dependency> <dependency> <groupId>org.apache.dolphinscheduler</groupId> @@ -67,11 +71,24 @@ </exclusion> </exclusions> </dependency> - <dependency> <groupId>org.codehaus.janino</groupId> <artifactId>janino</artifactId> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.github.ben-manes.caffeine</groupId> + <artifactId>caffeine</artifactId> + </dependency> <dependency> <groupId>org.mockito</groupId> diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java index 7c94144af8..e94bc75e44 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java @@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.server.master.dispatch.executor.ExecutorManag import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; import org.apache.dolphinscheduler.server.master.dispatch.host.HostManager; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.util.concurrent.ConcurrentHashMap; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java index e0753b72d6..4051068c53 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java @@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collection; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java index c79f9b7071..371075dfd4 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java @@ -20,50 +20,28 @@ package org.apache.dolphinscheduler.server.master.registry; import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_MASTERS; import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_NODE; import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; - +import java.util.Collections; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import com.google.common.collect.Sets; +import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.enums.NodeType; -import org.apache.dolphinscheduler.common.enums.StateEvent; -import org.apache.dolphinscheduler.common.enums.StateEventType; -import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus; import org.apache.dolphinscheduler.registry.api.ConnectionState; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; -import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; import org.apache.dolphinscheduler.server.master.service.FailoverService; import org.apache.dolphinscheduler.server.registry.HeartBeatTask; -import org.apache.dolphinscheduler.server.utils.ProcessUtils; -import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.registry.RegistryClient; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang.StringUtils; - -import java.time.Duration; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import com.google.common.collect.Sets; - /** * zookeeper master client * <p> diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java index a6599b6032..1993f6a30f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java @@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.service.queue.MasterPriorityQueue; import org.apache.dolphinscheduler.service.registry.RegistryClient; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.util.Collection; import java.util.Collections; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java index e4f09e5a9c..32794bae75 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java @@ -73,8 +73,8 @@ import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.NumberUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import java.util.ArrayList; import java.util.Arrays; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java index cd337b4c95..5920232855 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java @@ -30,13 +30,11 @@ import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheM import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.process.ProcessService; -import org.apache.commons.lang.StringUtils; - import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.PostConstruct; - +import com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -175,7 +173,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor { */ private void notifyProcess(ProcessInstance finishProcessInstance, ProcessInstance processInstance, TaskInstance taskInstance) { String host = processInstance.getHost(); - if (StringUtils.isEmpty(host)) { + if (Strings.isNullOrEmpty(host)) { logger.error("process {} host is empty, cannot notify task {} now", processInstance.getId(), taskInstance.getId()); return; } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java index 5833bc56c1..82138a83d5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java @@ -31,7 +31,7 @@ import org.apache.dolphinscheduler.service.queue.TaskPriority; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.util.Date; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SubTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SubTaskProcessor.java index 747c3dd77a..1ace5d373d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SubTaskProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SubTaskProcessor.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.master.runner.task; import com.fasterxml.jackson.core.type.TypeReference; import com.google.auto.service.AutoService; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java index 444d41622c..22d9df5b8c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java @@ -31,7 +31,7 @@ import org.apache.dolphinscheduler.server.utils.LogUtils; import org.apache.dolphinscheduler.server.utils.SwitchTaskUtils; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.util.Date; import java.util.HashMap; @@ -207,7 +207,7 @@ public class SwitchTaskProcessor extends BaseTaskProcessor { return ""; } String value = property.getValue(); - if (!org.apache.commons.lang.math.NumberUtils.isNumber(value)) { + if (!org.apache.commons.lang3.math.NumberUtils.isCreatable(value)) { value = "\"" + value + "\""; } logger.info("paramName:{}，paramValue:{}", paramName, value); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java index 4f25c78dad..9b40041ae9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java @@ -35,7 +35,7 @@ import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.registry.RegistryClient; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.util.Date; import java.util.HashMap; diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java index 0707eade28..c8121d494b 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java @@ -21,9 +21,7 @@ import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.ExecutionContextTestUtils; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; - -import org.apache.commons.lang.StringUtils; - +import org.assertj.core.util.Strings; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -51,7 +49,7 @@ public class RoundRobinHostManagerTest { Mockito.when(serverNodeManager.getWorkerGroupNodes("default")).thenReturn(null); ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); Host emptyHost = roundRobinHostManager.select(context); - Assert.assertTrue(StringUtils.isEmpty(emptyHost.getAddress())); + Assert.assertTrue(Strings.isNullOrEmpty(emptyHost.getAddress())); } @Test @@ -59,7 +57,7 @@ public class RoundRobinHostManagerTest { Mockito.when(serverNodeManager.getWorkerGroupNodes("default")).thenReturn(Sets.newHashSet("192.168.1.1:22")); ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); Host host = roundRobinHostManager.select(context); - Assert.assertTrue(!StringUtils.isEmpty(host.getAddress())); + Assert.assertTrue(!Strings.isNullOrEmpty(host.getAddress())); Assert.assertTrue(host.getAddress().equalsIgnoreCase("192.168.1.1:22")); } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java index a1d0b03bae..1eb6f93baf 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java @@ -22,7 +22,6 @@ package org.apache.dolphinscheduler.registry.api; import java.io.Closeable; import java.time.Duration; import java.util.Collection; -import java.util.Map; public interface Registry extends Closeable { boolean subscribe(String path, SubscribeListener listener); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml index 10a2b3adba..028e5d6cb7 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml @@ -52,11 +52,6 @@ <artifactId>curator-recipes</artifactId> </dependency> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - </dependency> - <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 44e98b955e..ad9770ed86 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -38,7 +38,10 @@ <groupId>org.apache.dolphinscheduler</groupId> <artifactId>dolphinscheduler-service</artifactId> </dependency> - + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java index ee37a871d0..67e471f9c7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java @@ -19,11 +19,9 @@ package org.apache.dolphinscheduler.server.log; import org.apache.dolphinscheduler.common.Constants; -import org.apache.commons.lang.StringUtils; - import java.util.regex.Matcher; import java.util.regex.Pattern; - +import com.google.common.base.Strings; import ch.qos.logback.classic.pattern.MessageConverter; import ch.qos.logback.classic.spi.ILoggingEvent; @@ -57,7 +55,7 @@ public class SensitiveDataConverter extends MessageConverter { String tempLogMsg = oriLogMsg; - if (!StringUtils.isEmpty(tempLogMsg)) { + if (!Strings.isNullOrEmpty(tempLogMsg)) { tempLogMsg = passwordHandler(pwdPattern, tempLogMsg); } return tempLogMsg; @@ -78,7 +76,7 @@ public class SensitiveDataConverter extends MessageConverter { String password = matcher.group(); - String maskPassword = StringUtils.repeat(Constants.STAR, StringUtils.length(password)); + String maskPassword = Strings.repeat(Constants.STAR, password.length()); matcher.appendReplacement(sb, maskPassword); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 4dff8ede58..2637d6a11a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -30,9 +30,8 @@ import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.service.log.LogClientService; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.SystemUtils; - +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.SystemUtils; import java.io.File; import java.nio.charset.StandardCharsets; import java.util.ArrayList; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java index a6243198b7..d3ed3ce959 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java @@ -18,15 +18,13 @@ package org.apache.dolphinscheduler.server.utils; import static org.powermock.api.mockito.PowerMockito.when; - +import org.apache.commons.lang3.SystemUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus; -import org.apache.commons.lang.SystemUtils; - import java.util.ArrayList; import java.util.List; diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 11e007af50..9e7d5d6016 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -44,9 +44,8 @@ </dependency> <dependency> <groupId>org.apache.dolphinscheduler</groupId> - <artifactId>dolphinscheduler-registry-zookeeper</artifactId> + <artifactId>dolphinscheduler-registry-api</artifactId> </dependency> - <dependency> <groupId>org.apache.dolphinscheduler</groupId> <artifactId>dolphinscheduler-task-api</artifactId> @@ -81,8 +80,8 @@ </exclusions> </dependency> <dependency> - <groupId>org.quartz-scheduler</groupId> - <artifactId>quartz-jobs</artifactId> + <groupId>com.cronutils</groupId> + <artifactId>cron-utils</artifactId> </dependency> <dependency> @@ -91,13 +90,10 @@ <scope>provided</scope> </dependency> - <dependency> - <groupId>com.github.ben-manes.caffeine</groupId> - <artifactId>caffeine</artifactId> - </dependency> <dependency> <groupId>io.fabric8</groupId> <artifactId>kubernetes-client</artifactId> + <scope>provided</scope> </dependency> </dependencies> </project> diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 53c20a5ae8..12ca7d32ab 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -133,8 +133,6 @@ import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.NumberUtils; import java.util.ArrayList; import java.util.Arrays; @@ -158,6 +156,8 @@ import org.springframework.transaction.annotation.Transactional; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Joiner; +import com.google.common.base.Strings; import com.google.common.collect.Lists; /** @@ -597,7 +597,7 @@ public class ProcessServiceImpl implements ProcessService { try (LogClientService logClient = new LogClientService()) { for (TaskInstance taskInstance : taskInstanceList) { String taskLogPath = taskInstance.getLogPath(); - if (StringUtils.isEmpty(taskInstance.getHost())) { + if (Strings.isNullOrEmpty(taskInstance.getHost())) { continue; } Host host = Host.of(taskInstance.getHost()); @@ -794,7 +794,7 @@ public class ProcessServiceImpl implements ProcessService { // set process instance priority processInstance.setProcessInstancePriority(command.getProcessInstancePriority()); - String workerGroup = StringUtils.isBlank(command.getWorkerGroup()) ? Constants.DEFAULT_WORKER_GROUP : command.getWorkerGroup(); + String workerGroup = Strings.isNullOrEmpty(command.getWorkerGroup()) ? Constants.DEFAULT_WORKER_GROUP : command.getWorkerGroup(); processInstance.setWorkerGroup(workerGroup); processInstance.setEnvironmentCode(Objects.isNull(command.getEnvironmentCode()) ? -1 : command.getEnvironmentCode()); processInstance.setTimeout(processDefinition.getTimeout()); @@ -1138,7 +1138,7 @@ public class ProcessServiceImpl implements ProcessService { @Override public void setSubProcessParam(ProcessInstance subProcessInstance) { String cmdParam = subProcessInstance.getCommandParam(); - if (StringUtils.isEmpty(cmdParam)) { + if (Strings.isNullOrEmpty(cmdParam)) { return; } Map<String, String> paramMap = JSONUtils.toMap(cmdParam); @@ -1153,7 +1153,7 @@ public class ProcessServiceImpl implements ProcessService { } // copy parent instance user def params to sub process.. String parentInstanceId = paramMap.get(CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID); - if (StringUtils.isNotEmpty(parentInstanceId)) { + if (!Strings.isNullOrEmpty(parentInstanceId)) { ProcessInstance parentInstance = findProcessInstanceDetailById(Integer.parseInt(parentInstanceId)); if (parentInstance != null) { subProcessInstance.setGlobalParams(joinGlobalParams(parentInstance.getGlobalParams(), subProcessInstance.getGlobalParams())); @@ -1404,7 +1404,7 @@ public class ProcessServiceImpl implements ProcessService { public Map<String, String> getGlobalParamMap(String globalParams) { List<Property> propList; Map<String, String> globalParamMap = new HashMap<>(); - if (StringUtils.isNotEmpty(globalParams)) { + if (!Strings.isNullOrEmpty(globalParams)) { propList = JSONUtils.toList(globalParams, Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } @@ -1424,7 +1424,7 @@ public class ProcessServiceImpl implements ProcessService { Map<String, Object> subProcessParam = JSONUtils.toMap(task.getTaskParams(), String.class, Object.class); long childDefineCode = 0L; if (subProcessParam.containsKey(Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE)) { - childDefineCode = NumberUtils.toLong(String.valueOf(subProcessParam.get(Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE))); + childDefineCode = Long.parseLong(String.valueOf(subProcessParam.get(Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE))); } ProcessDefinition subProcessDefinition = processDefineMapper.queryByCode(childDefineCode); @@ -1911,7 +1911,7 @@ public class ProcessServiceImpl implements ProcessService { */ @Override public void changeOutParam(TaskInstance taskInstance) { - if (StringUtils.isEmpty(taskInstance.getVarPool())) { + if (Strings.isNullOrEmpty(taskInstance.getVarPool())) { return; } List<Property> properties = JSONUtils.toList(taskInstance.getVarPool(), Property.class); @@ -2124,16 +2124,16 @@ public class ProcessServiceImpl implements ProcessService { List<Resource> resourceList = resourceMapper.queryResource(fullName, resourceType.ordinal()); if (CollectionUtils.isEmpty(resourceList)) { - return StringUtils.EMPTY; + return ""; } int userId = resourceList.get(0).getUserId(); User user = userMapper.selectById(userId); if (Objects.isNull(user)) { - return StringUtils.EMPTY; + return ""; } Tenant tenant = tenantMapper.queryById(user.getTenantId()); if (Objects.isNull(tenant)) { - return StringUtils.EMPTY; + return ""; } return tenant.getTenantCode(); } @@ -2234,7 +2234,7 @@ public class ProcessServiceImpl implements ProcessService { public String getTaskWorkerGroup(TaskInstance taskInstance) { String workerGroup = taskInstance.getWorkerGroup(); - if (StringUtils.isNotBlank(workerGroup)) { + if (!Strings.isNullOrEmpty(workerGroup)) { return workerGroup; } int processInstanceId = taskInstance.getProcessInstanceId(); @@ -2444,9 +2444,9 @@ public class ProcessServiceImpl implements ProcessService { .collect(toSet()); } if (CollectionUtils.isEmpty(resourceIds)) { - return StringUtils.EMPTY; + return ""; } - return StringUtils.join(resourceIds, ","); + return Joiner.on(",").join(resourceIds); } @Override @@ -3041,4 +3041,4 @@ public class ProcessServiceImpl implements ProcessService { throw new ServiceException("delete command fail, id:" + commandId); } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/impl/QuartzExecutorImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/impl/QuartzExecutorImpl.java index b4d5ee9246..c31de54643 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/impl/QuartzExecutorImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/impl/QuartzExecutorImpl.java @@ -34,14 +34,12 @@ import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.quartz.QuartzExecutor; -import org.apache.commons.lang.StringUtils; - import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; - +import com.google.common.base.Strings; import org.quartz.CronTrigger; import org.quartz.Job; import org.quartz.JobDetail; @@ -133,7 +131,7 @@ public class QuartzExecutorImpl implements QuartzExecutor { CronTrigger oldCronTrigger = (CronTrigger) scheduler.getTrigger(triggerKey); String oldCronExpression = oldCronTrigger.getCronExpression(); - if (!StringUtils.equalsIgnoreCase(cronExpression, oldCronExpression)) { + if (!Strings.nullToEmpty(cronExpression).equalsIgnoreCase(Strings.nullToEmpty(oldCronExpression))) { // reschedule job trigger scheduler.rescheduleJob(triggerKey, cronTrigger); logger.info("reschedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", diff --git a/dolphinscheduler-spi/pom.xml b/dolphinscheduler-spi/pom.xml index 5a259564c0..3a371858d9 100644 --- a/dolphinscheduler-spi/pom.xml +++ b/dolphinscheduler-spi/pom.xml @@ -35,11 +35,11 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-collections4</artifactId> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> </dependency> + <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml index be34fee4dd..f149296fdb 100644 --- a/dolphinscheduler-standalone-server/pom.xml +++ b/dolphinscheduler-standalone-server/pom.xml @@ -50,6 +50,11 @@ <artifactId>dolphinscheduler-log-server</artifactId> </dependency> + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + </dependency> + <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-test</artifactId> diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml index 64b2aa05d5..dd274a9ffc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml @@ -50,14 +50,6 @@ <groupId>ch.qos.logback</groupId> <artifactId>logback-classic</artifactId> </dependency> - <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> @@ -279,19 +271,5 @@ </exclusion> </exclusions> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <exclusions> - <exclusion> - <artifactId>slf4j-log4j12</artifactId> - <groupId>org.slf4j</groupId> - </exclusion> - <exclusion> - <artifactId>com.sun.jersey</artifactId> - <groupId>jersey-json</groupId> - </exclusion> - </exclusions> - </dependency> </dependencies> </project> diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java index 37a3e963b7..d6b0aaad8f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.plugin.task.api; import org.apache.dolphinscheduler.plugin.task.api.utils.OSUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; import java.io.File; import java.io.IOException; @@ -29,7 +28,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Consumer; - +import com.google.common.base.Strings; import org.slf4j.Logger; /** @@ -93,7 +92,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { if (OSUtils.isWindows()) { sb.append("@echo off\n"); sb.append("cd /d %~dp0\n"); - if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { + if (!Strings.isNullOrEmpty(taskRequest.getEnvironmentConfig())) { sb.append(taskRequest.getEnvironmentConfig()).append("\n"); } else { if (taskRequest.getEnvFile() != null) { @@ -104,7 +103,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { sb.append("#!/bin/sh\n"); sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); sb.append("cd $BASEDIR\n"); - if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { + if (!Strings.isNullOrEmpty(taskRequest.getEnvironmentConfig())) { sb.append(taskRequest.getEnvironmentConfig()).append("\n"); } else { if (taskRequest.getEnvFile() != null) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java index 4a7f1c83ca..fe535b46f1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; +import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; @@ -24,8 +25,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.apache.commons.collections4.CollectionUtils; - import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml index c49dc65676..d4a9db394a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml @@ -49,6 +49,10 @@ <artifactId>dolphinscheduler-data-quality</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> + </dependency> </dependencies> -</project> \ No newline at end of file +</project> diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml index 5228593532..02b4428e86 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml @@ -29,11 +29,6 @@ <packaging>jar</packaging> <dependencies> - <dependency> - <groupId>org.apache.dolphinscheduler</groupId> - <artifactId>dolphinscheduler-datasource-all</artifactId> - </dependency> - <dependency> <groupId>org.apache.dolphinscheduler</groupId> <artifactId>dolphinscheduler-spi</artifactId> @@ -50,6 +45,11 @@ <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> + </dependency> + <dependency> <groupId>com.alibaba</groupId> <artifactId>druid</artifactId> diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java index 5627edad18..0934d868cf 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -39,7 +39,7 @@ import org.apache.dolphinscheduler.spi.enums.Flag; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import java.io.File; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml index f7d547010b..d6bfbf3af8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml @@ -46,36 +46,4 @@ </dependency> </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <createDependencyReducedPom>false</createDependencyReducedPom> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <relocations> - <relocation> - <pattern>com.amazonaws</pattern> - <shadedPattern>com.amazonaws.dsemrtask.wrapper</shadedPattern> - </relocation> - </relocations> - <transformers> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> \ No newline at end of file +</project> diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkParametersTest.java index 001f177b82..52aa7dbc5d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkParametersTest.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import org.apache.commons.collections.CollectionUtils; - import java.util.LinkedList; import java.util.List; @@ -31,7 +29,7 @@ public class FlinkParametersTest { @Test public void getResourceFilesList() { FlinkParameters flinkParameters = new FlinkParameters(); - Assert.assertTrue(CollectionUtils.isEmpty(flinkParameters.getResourceFilesList())); + Assert.assertTrue(flinkParameters.getResourceFilesList().isEmpty()); ResourceInfo mainResource = new ResourceInfo(); mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml index b964cc64fd..586c47e3ec 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml @@ -29,11 +29,6 @@ <packaging>jar</packaging> <dependencies> - <dependency> - <groupId>org.apache.dolphinscheduler</groupId> - <artifactId>dolphinscheduler-datasource-all</artifactId> - </dependency> - <dependency> <groupId>org.apache.dolphinscheduler</groupId> <artifactId>dolphinscheduler-spi</artifactId> @@ -49,5 +44,10 @@ <artifactId>dolphinscheduler-datasource-api</artifactId> <version>${project.version}</version> </dependency> + + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> + </dependency> </dependencies> </project> diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java index 129b3120e8..b3a15a539e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java @@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourc import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; import java.util.HashMap; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java index c153847b18..5ce77202fb 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTask.java @@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; import java.io.File; import java.io.IOException; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java index b1c7806d12..286bcbc0ba 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkParametersTest.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.spark; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import org.apache.commons.collections.CollectionUtils; - import java.util.LinkedList; import java.util.List; @@ -32,7 +30,7 @@ public class SparkParametersTest { @Test public void getResourceFilesList() { SparkParameters sparkParameters = new SparkParameters(); - Assert.assertTrue(CollectionUtils.isEmpty(sparkParameters.getResourceFilesList())); + Assert.assertTrue(sparkParameters.getResourceFilesList().isEmpty()); ResourceInfo mainResource = new ResourceInfo(); mainResource.setRes("testSparkMain-1.0.0-SNAPSHOT.jar\""); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java index 17c2ff0c4b..9d5565ab95 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.spark; import java.util.Collections; -import org.apache.commons.lang.StringUtils; - import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.spi.utils.JSONUtils; @@ -71,10 +69,10 @@ public class SparkTaskTest { sparkParameters.setLocalParams(Collections.emptyList()); sparkParameters.setRawScript("selcet 11111;"); sparkParameters.setProgramType(ProgramType.SQL); - sparkParameters.setMainClass(StringUtils.EMPTY); + sparkParameters.setMainClass(""); sparkParameters.setDeployMode("client"); sparkParameters.setAppName("sparksql"); - sparkParameters.setOthers(StringUtils.EMPTY); + sparkParameters.setOthers(""); sparkParameters.setSparkVersion("SPARK2"); sparkParameters.setDriverCores(1); sparkParameters.setDriverMemory("512M"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml index 4c59bc8688..f358b0c97b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml @@ -29,11 +29,6 @@ <packaging>jar</packaging> <dependencies> - <dependency> - <groupId>org.apache.dolphinscheduler</groupId> - <artifactId>dolphinscheduler-datasource-all</artifactId> - </dependency> - <dependency> <groupId>org.apache.dolphinscheduler</groupId> <artifactId>dolphinscheduler-spi</artifactId> @@ -49,6 +44,10 @@ <artifactId>dolphinscheduler-datasource-api</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> + </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index ad31202004..60afd74fa0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -40,7 +40,7 @@ import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import java.sql.Connection; import java.sql.PreparedStatement; diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java index e4e8d130a0..d397794166 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java @@ -22,8 +22,6 @@ import java.util.Objects; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.apache.directory.api.util.Strings; - import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -35,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Joiner; +import com.google.common.base.Strings; /** * resource dao @@ -99,7 +98,7 @@ public class ResourceDao { String[] splits = fullName.split("/"); for (int i = 1; i < splits.length; i++) { String parentFullName = Joiner.on("/").join(Arrays.copyOfRange(splits,0, splits.length - i)); - if (Strings.isNotEmpty(parentFullName)) { + if (!Strings.isNullOrEmpty(parentFullName)) { long size = resourceSizeMap.getOrDefault(parentFullName, 0L); resourceSizeMap.put(parentFullName, size + fileSize); } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java index 8d08dd5e43..49092a6bdc 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java @@ -45,7 +45,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParamet import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; import java.io.FileNotFoundException; import java.io.IOException; @@ -75,6 +74,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Joiner; +import com.google.common.base.Strings; public abstract class UpgradeDao { public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); @@ -376,7 +377,7 @@ public abstract class UpgradeDao { ConnectionUtils.releaseResource(pstmt, conn); } } - + /** * update version * @@ -484,9 +485,9 @@ public abstract class UpgradeDao { if (resourceJsonNode != null && !resourceJsonNode.isEmpty()) { List<ResourceInfo> resourceList = JSONUtils.toList(param.get("resourceList").toString(), ResourceInfo.class); List<Integer> resourceIds = resourceList.stream().map(ResourceInfo::getId).collect(Collectors.toList()); - taskDefinitionLog.setResourceIds(StringUtils.join(resourceIds, Constants.COMMA)); + taskDefinitionLog.setResourceIds(Joiner.on(Constants.COMMA).join(resourceIds)); } else { - taskDefinitionLog.setResourceIds(StringUtils.EMPTY); + taskDefinitionLog.setResourceIds(""); } if (TASK_TYPE_SUB_PROCESS.equals(taskType)) { JsonNode jsonNodeDefinitionId = param.get("processDefinitionId"); @@ -585,7 +586,7 @@ public abstract class UpgradeDao { } private String convertLocations(String locations, Map<String, Long> taskIdCodeMap) { - if (StringUtils.isBlank(locations)) { + if (Strings.isNullOrEmpty(locations)) { return locations; } Map<String, ObjectNode> locationsMap = JSONUtils.parseObject(locations, new TypeReference<Map<String, ObjectNode>>() { diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index e16dddd155..c689349002 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -51,6 +51,10 @@ <groupId>org.apache.dolphinscheduler</groupId> <artifactId>dolphinscheduler-meter</artifactId> </dependency> + <dependency> + <groupId>org.apache.dolphinscheduler</groupId> + <artifactId>dolphinscheduler-registry-zookeeper</artifactId> + </dependency> <dependency> <groupId>org.apache.dolphinscheduler</groupId> @@ -71,12 +75,14 @@ </exclusion> </exclusions> </dependency> - <dependency> <groupId>org.codehaus.janino</groupId> <artifactId>janino</artifactId> </dependency> - + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-s3</artifactId> + </dependency> <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java index ecc1ffe755..ecbbe78128 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java @@ -37,8 +37,6 @@ import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.service.log.LogClientService; -import org.apache.commons.lang.StringUtils; - import java.util.Collections; import java.util.List; @@ -48,7 +46,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - +import com.google.common.base.Strings; import io.netty.channel.Channel; /** @@ -168,7 +166,7 @@ public class TaskKillProcessor implements NettyRequestProcessor { } try { String pidsStr = ProcessUtils.getPidsStr(processId); - if (!StringUtils.isEmpty(pidsStr)) { + if (!Strings.isNullOrEmpty(pidsStr)) { String cmd = String.format("kill -9 %s", pidsStr); cmd = OSUtils.getSudoCmd(tenantCode, cmd); logger.info("process id:{}, cmd:{}", processId, cmd); @@ -196,9 +194,9 @@ public class TaskKillProcessor implements NettyRequestProcessor { host.getPort()); String log = logClient.viewLog(host.getIp(), host.getPort(), logPath); List<String> appIds = Collections.emptyList(); - if (!StringUtils.isEmpty(log)) { + if (!Strings.isNullOrEmpty(log)) { appIds = LoggerUtils.getAppIds(log, logger); - if (StringUtils.isEmpty(executePath)) { + if (Strings.isNullOrEmpty(executePath)) { logger.error("task instance execute path is empty"); throw new RuntimeException("task instance execute path is empty"); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java index e218b0a4aa..6cf5aa8db8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java @@ -33,8 +33,6 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.service.registry.RegistryClient; -import org.apache.commons.lang.StringUtils; - import java.io.IOException; import java.util.Set; import java.util.StringJoiner; @@ -48,7 +46,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; - +import com.google.common.base.Strings; import com.google.common.collect.Sets; /** @@ -165,7 +163,7 @@ public class WorkerRegistryClient { for (String workGroup : this.workerGroups) { StringJoiner workerPathJoiner = new StringJoiner(SINGLE_SLASH); workerPathJoiner.add(REGISTRY_DOLPHINSCHEDULER_WORKERS); - if (StringUtils.isEmpty(workGroup)) { + if (Strings.isNullOrEmpty(workGroup)) { workGroup = DEFAULT_WORKER_GROUP; } // trim and lower case is need diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index cc15eb62f4..48c590cbd7 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -39,7 +39,6 @@ import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang.StringUtils; import java.io.File; import java.io.IOException; @@ -47,7 +46,7 @@ import java.util.*; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - +import com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -219,7 +218,7 @@ public class TaskExecuteThread implements Runnable, Delayed { // get exec dir String execLocalPath = taskExecutionContext.getExecutePath(); - if (StringUtils.isEmpty(execLocalPath)) { + if (Strings.isNullOrEmpty(execLocalPath)) { logger.warn("task: {} exec local path is empty.", taskExecutionContext.getTaskName()); return; } diff --git a/pom.xml b/pom.xml index 12f57d8a84..68afa1326a 100644 --- a/pom.xml +++ b/pom.xml @@ -67,17 +67,16 @@ <h2.version>1.4.200</h2.version> <commons.codec.version>1.11</commons.codec.version> <commons.logging.version>1.1.1</commons.logging.version> + <commons.lang3.version>3.12.0</commons.lang3.version> <httpclient.version>4.5.13</httpclient.version> <httpcore.version>4.4.1</httpcore.version> <junit.version>4.12</junit.version> <mysql.connector.version>8.0.16</mysql.connector.version> <slf4j.api.version>1.7.5</slf4j.api.version> <slf4j.log4j12.version>1.7.5</slf4j.log4j12.version> - <commons.collections.version>3.2.2</commons.collections.version> <commons.httpclient>3.0.1</commons.httpclient> <commons.beanutils.version>1.9.4</commons.beanutils.version> <commons.configuration.version>1.10</commons.configuration.version> - <commons.lang.version>2.6</commons.lang.version> <commons.email.version>1.5</commons.email.version> <poi.version>4.1.2</poi.version> <javax.servlet.api.version>3.1.0</javax.servlet.api.version> @@ -494,6 +493,11 @@ <artifactId>commons-logging</artifactId> <version>${commons.logging.version}</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <version>${commons.lang3.version}</version> + </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> @@ -573,12 +577,6 @@ <version>${slf4j.log4j12.version}</version> </dependency> - <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - <version>${commons.collections.version}</version> - </dependency> - <dependency> <groupId>commons-httpclient</groupId> <artifactId>commons-httpclient</artifactId> @@ -596,12 +594,6 @@ <version>${commons.configuration.version}</version> </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>${commons.lang.version}</version> - </dependency> - <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-classic</artifactId> @@ -649,6 +641,10 @@ <groupId>junit</groupId> <artifactId>junit</artifactId> </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -666,8 +662,6 @@ <artifactId>hadoop-yarn-common</artifactId> <version>${hadoop.version}</version> </dependency> - - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-collections4</artifactId> @@ -801,12 +795,6 @@ <version>${py4j.version}</version> </dependency> - <dependency> - <groupId>org.codehaus.janino</groupId> - <artifactId>janino</artifactId> - <version>${janino.version}</version> - </dependency> - <dependency> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> @@ -1161,6 +1149,29 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>${maven-dependency-plugin.version}</version> + <executions> + <execution> + <id>analyze-dependencies</id> + <goals> + <goal>analyze-only</goal> + </goals> + <configuration> + <ignoredDependencies> + <!-- Because of SpringBoot auto-configurations, the configuration is happening outside of your application code, so Maven believes these dependencies to be unused --> + <!-- Static code analysis tools like (maven-dependency-plugin) can not detect runtime dependencies, so you should instruct them about runtime dependencies --> + <!-- https://stackoverflow.com/questions/37528928/spring-boot-core-dependencies-seen-as-unused-by-maven-dependency-plugin --> + <ignoredDependency>org.springframework*:*</ignoredDependency> + <ignoredDependency>org.apache.dolphinscheduler:dolphinscheduler-meter</ignoredDependency> + </ignoredDependencies> + <ignoreNonCompile>true</ignoreNonCompile> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 65a370772a..a43e7c6a37 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -8,7 +8,6 @@ apacheds-i18n-2.0.0-M15.jar apacheds-kerberos-codec-2.0.0-M15.jar api-asn1-api-1.0.0-M20.jar api-util-1.0.0-M20.jar -asm-3.1.jar asm-6.2.1.jar aspectjweaver-1.9.7.jar audience-annotations-0.5.0.jar @@ -24,7 +23,6 @@ clickhouse-jdbc-0.1.52.jar commons-email-1.5.jar commons-cli-1.2.jar commons-codec-1.11.jar -commons-collections-3.2.2.jar commons-collections4-4.1.jar commons-compiler-3.1.6.jar commons-compress-1.21.jar @@ -57,8 +55,6 @@ druid-1.2.4.jar gson-2.8.8.jar guava-24.1-jre.jar guava-retrying-2.0.0.jar -guice-3.0.jar -guice-servlet-3.0.jar h2-1.4.200.jar hadoop-annotations-2.7.3.jar hadoop-auth-2.7.3.jar @@ -115,11 +111,6 @@ jaxb-api-2.3.1.jar jaxb-impl-2.2.3-1.jar jboss-logging-3.4.2.Final.jar jdo-api-3.0.1.jar -jersey-client-1.9.jar -jersey-core-1.9.jar -jersey-guice-1.9.jar -jersey-json-1.9.jar -jersey-server-1.9.jar jets3t-0.9.0.jar jettison-1.1.jar jetty-6.1.26.jar @@ -284,4 +275,3 @@ websocket-client-9.4.44.v20210927.jar websocket-common-9.4.44.v20210927.jar zeppelin-client-0.10.1.jar zeppelin-common-0.10.1.jar -
