This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new f6ef628030 Support reading application properties from kubernetes
config map (#11402)
f6ef628030 is described below
commit f6ef6280309b2130ba87499de2b992a1d4c9c9f5
Author: kezhenxu94 <[email protected]>
AuthorDate: Tue Aug 16 20:25:17 2022 +0800
Support reading application properties from kubernetes config map (#11402)
---
.../service/impl/ProcessDefinitionServiceImpl.java | 19 ++++
.../src/main/resources/application.yaml | 2 +-
dolphinscheduler-bom/pom.xml | 103 ++++++++++----------
dolphinscheduler-dist/release-docs/LICENSE | 104 ++++++++++----------
.../master/runner/StateWheelExecuteThread.java | 2 +
.../src/main/resources/application.yaml | 2 +-
.../service/process/ProcessServiceImpl.java | 5 +-
dolphinscheduler-standalone-server/pom.xml | 60 +++++++-----
.../src/main/resources/application.yaml | 2 +-
.../plugin/task/api/k8s/impl/K8sTaskExecutor.java | 106 +++++++++++----------
.../plugin/task/api/utils/K8sUtils.java | 47 ++++-----
.../plugin/task/api/k8s/K8sTaskExecutorTest.java | 15 +--
pom.xml | 2 +-
tools/dependencies/known-dependencies.txt | 60 ++++++------
14 files changed, 281 insertions(+), 248 deletions(-)
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
index ea8e7e6eae..1c7b2aba27 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
@@ -146,6 +146,24 @@ import static
org.apache.dolphinscheduler.common.Constants.EMPTY_STRING;
import static org.apache.dolphinscheduler.common.Constants.IMPORT_SUFFIX;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.COMPLEX_TASK_TYPES;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SQL;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletResponse;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.http.MediaType;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.multipart.MultipartFile;
+
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
/**
* process definition service impl
@@ -172,6 +190,7 @@ public class ProcessDefinitionServiceImpl extends
BaseServiceImpl implements Pro
@Autowired
private ProcessDefinitionMapper processDefinitionMapper;
+ @Lazy
@Autowired
private ProcessInstanceService processInstanceService;
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml
b/dolphinscheduler-api/src/main/resources/application.yaml
index 642b7acbbc..23db3389f1 100644
--- a/dolphinscheduler-api/src/main/resources/application.yaml
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -65,7 +65,7 @@ spring:
properties:
org.quartz.threadPool:threadPriority: 5
org.quartz.jobStore.isClustered: true
- org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
+ org.quartz.jobStore.class:
org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true
diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml
index 746eba31c9..8603c3b116 100644
--- a/dolphinscheduler-bom/pom.xml
+++ b/dolphinscheduler-bom/pom.xml
@@ -26,7 +26,7 @@
<artifactId>dolphinscheduler-bom</artifactId>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
-
+
<properties>
<netty.version>4.1.53.Final</netty.version>
<spring-boot.version>2.5.6</spring-boot.version>
@@ -48,7 +48,7 @@
<commons-collections4.version>4.1</commons-collections4.version>
<httpclient.version>4.5.13</httpclient.version>
<httpcore.version>4.4.15</httpcore.version>
- <jackson.version>2.10.5</jackson.version>
+ <jackson.version>2.13.0</jackson.version>
<protostuff.version>1.7.2</protostuff.version>
<byte-buddy.version>1.9.16</byte-buddy.version>
<logback.version>1.2.11</logback.version>
@@ -82,15 +82,14 @@
<commons-compress.version>1.21</commons-compress.version>
<commons-math3.version>3.1.1</commons-math3.version>
<error_prone_annotations.version>2.5.1</error_prone_annotations.version>
- <kubernetes.version>5.8.0</kubernetes.version>
<hibernate-validator.version>6.2.2.Final</hibernate-validator.version>
<aws-sdk.version>1.12.160</aws-sdk.version>
<joda-time.version>2.10.13</joda-time.version>
<okhttp.version>3.14.9</okhttp.version>
<json-path.version>2.7.0</json-path.version>
-
+
<spring-cloud-dependencies.version>2020.0.6</spring-cloud-dependencies.version>
</properties>
-
+
<dependencyManagement>
<dependencies>
<!-- netty -->
@@ -106,7 +105,7 @@
<artifactId>netty-all</artifactId>
<version>${netty.version}</version>
</dependency>
-
+
<!-- spring -->
<dependency>
<groupId>org.springframework.boot</groupId>
@@ -146,7 +145,7 @@
<version>${spring.version}</version>
<scope>test</scope>
</dependency>
-
+
<dependency>
<groupId>org.java-websocket</groupId>
<artifactId>Java-WebSocket</artifactId>
@@ -168,7 +167,7 @@
<artifactId>mybatis-plus-annotation</artifactId>
<version>${mybatis-plus.version}</version>
</dependency>
-
+
<!-- quartz-->
<dependency>
<groupId>org.quartz-scheduler</groupId>
@@ -180,13 +179,13 @@
<artifactId>cron-utils</artifactId>
<version>${cron-utils.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>${druid.version}</version>
</dependency>
-
+
<!-- Zookeeper -->
<dependency>
<groupId>org.apache.zookeeper</groupId>
@@ -245,7 +244,7 @@
<artifactId>curator-test</artifactId>
<version>${curator-test.version}</version>
</dependency>
-
+
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
@@ -281,7 +280,7 @@
<artifactId>commons-email</artifactId>
<version>${commons-email.version}</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
@@ -308,7 +307,7 @@
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
-
+
<!--protostuff-->
<dependency>
<groupId>io.protostuff</groupId>
@@ -320,33 +319,33 @@
<artifactId>protostuff-runtime</artifactId>
<version>${protostuff.version}</version>
</dependency>
-
+
<dependency>
<groupId>net.bytebuddy</groupId>
<artifactId>byte-buddy</artifactId>
<version>${byte-buddy.version}</version>
</dependency>
-
+
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
<version>${reflections.version}</version>
</dependency>
-
+
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql-connector.version}</version>
<scope>test</scope>
</dependency>
-
+
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc8</artifactId>
<version>${oracle-jdbc.version}</version>
<scope>test</scope>
</dependency>
-
+
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
@@ -374,7 +373,7 @@
<artifactId>logback-core</artifactId>
<version>${logback.version}</version>
</dependency>
-
+
<!--excel poi-->
<dependency>
<groupId>org.apache.poi</groupId>
@@ -386,7 +385,7 @@
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
-
+
<!-- hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -426,37 +425,37 @@
<artifactId>hadoop-yarn-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>${commons-collections4.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
-
+
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
<version>${hive-jdbc.version}</version>
</dependency>
-
+
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-core</artifactId>
@@ -476,31 +475,31 @@
</exclusion>
</exclusions>
</dependency>
-
+
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<version>${clickhouse-jdbc.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>
<version>${mssql-jdbc.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-jdbc</artifactId>
<version>${presto-jdbc.version}</version>
</dependency>
-
+
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<version>${servlet-api.version}</version>
</dependency>
-
+
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
@@ -511,49 +510,49 @@
<artifactId>springfox-swagger2</artifactId>
<version>${springfox.version}</version>
</dependency>
-
+
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>${springfox.version}</version>
</dependency>
-
+
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-models</artifactId>
<version>${swagger-models.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.github.xiaoymin</groupId>
<artifactId>swagger-bootstrap-ui</artifactId>
<version>${swagger.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
<version>${guava-retry.version}</version>
</dependency>
-
+
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
<version>${activation.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
<version>${javax-mail}</version>
</dependency>
-
+
<dependency>
<groupId>net.sf.py4j</groupId>
<artifactId>py4j</artifactId>
<version>${py4j.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
@@ -574,19 +573,14 @@
<artifactId>error_prone_annotations</artifactId>
<version>${error_prone_annotations.version}</version>
</dependency>
-
- <dependency>
- <groupId>io.fabric8</groupId>
- <artifactId>kubernetes-client</artifactId>
- <version>${kubernetes.version}</version>
- </dependency>
+
<!--
https://mvnrepository.com/artifact/org.hibernate.validator/hibernate-validator
-->
<dependency>
<groupId>org.hibernate.validator</groupId>
<artifactId>hibernate-validator</artifactId>
<version>${hibernate-validator.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-emr</artifactId>
@@ -597,26 +591,33 @@
<artifactId>joda-time</artifactId>
<version>${joda-time.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>${aws-sdk.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp.version}</version>
</dependency>
-
+
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>${json-path.version}</version>
</dependency>
-
+
+ <dependency>
+ <groupId>org.springframework.cloud</groupId>
+ <artifactId>spring-cloud-dependencies</artifactId>
+ <version>${spring-cloud-dependencies.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
</dependencies>
-
+
</dependencyManagement>
</project>
diff --git a/dolphinscheduler-dist/release-docs/LICENSE
b/dolphinscheduler-dist/release-docs/LICENSE
index e688d655dc..42e201d82c 100644
--- a/dolphinscheduler-dist/release-docs/LICENSE
+++ b/dolphinscheduler-dist/release-docs/LICENSE
@@ -218,7 +218,7 @@ The text of each license is also included at
licenses/LICENSE-[project].txt.
accessors-smart 2.4.7: https://github.com/netplex/json-smart-v2, Apache 2.0
apacheds-i18n 2.0.0-M15:
https://mvnrepository.com/artifact/org.apache.directory.server/apacheds-i18n/2.0.0-M15,
Apache 2.0
apacheds-kerberos-codec 2.0.0-M15:
https://mvnrepository.com/artifact/org.apache.directory.server/apacheds-kerberos-codec/2.0.0-M15,
Apache 2.0
- tomcat-embed-el 9.0.54:
https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.54,
Apache 2.0
+ tomcat-embed-el 9.0.55:
https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.55,
Apache 2.0
api-asn1-api 1.0.0-M20:
https://mvnrepository.com/artifact/org.apache.directory.api/api-asn1-api/1.0.0-M20,
Apache 2.0
api-util 1.0.0-M20:
https://mvnrepository.com/artifact/org.apache.directory.api/api-util/1.0.0-M20,
Apache 2.0
audience-annotations 0.5.0:
https://mvnrepository.com/artifact/org.apache.yetus/audience-annotations/0.5.0,
Apache 2.0
@@ -284,14 +284,14 @@ The text of each license is also included at
licenses/LICENSE-[project].txt.
httpclient 4.5.13:
https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient/4.5.13,
Apache 2.0
httpcore 4.4.15:
https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore/4.4.15,
Apache 2.0
httpmime 4.5.13:
https://mvnrepository.com/artifact/org.apache.httpcomponents/httpmime/4.5.13,
Apache 2.0
- jackson-annotations 2.10.5:
https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.10.5,
Apache 2.0
- jackson-core 2.10.5: https://github.com/FasterXML/jackson-core, Apache 2.0
+ jackson-annotations 2.13.0:
https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.13.0,
Apache 2.0
+ jackson-core 2.13.0: https://github.com/FasterXML/jackson-core, Apache 2.0
jackson-core-asl 1.9.13:
https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-core-asl/1.9.13,
Apache 2.0
- jackson-databind 2.10.5: https://github.com/FasterXML/jackson-databind,
Apache 2.0
- jackson-datatype-jdk8 2.12.5:
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.12.5,
Apache 2.0
- jackson-datatype-jsr310 2.12.5:
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.12.5,
Apache 2.0
+ jackson-databind 2.13.0: https://github.com/FasterXML/jackson-databind,
Apache 2.0
+ jackson-datatype-jdk8 2.13.0:
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.13.0,
Apache 2.0
+ jackson-datatype-jsr310 2.13.0:
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.13.0,
Apache 2.0
jackson-mapper-asl 1.9.13:
https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-mapper-asl/1.9.13,
Apache 2.0
- jackson-module-parameter-names 2.12.5:
https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-parameter-names/2.12.5,
Apache 2.0
+ jackson-module-parameter-names 2.13.0:
https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-parameter-names/2.13.0,
Apache 2.0
javax.jdo-3.2.0-m3:
https://mvnrepository.com/artifact/org.datanucleus/javax.jdo/3.2.0-m3, Apache
2.0
java-xmlbuilder 0.4 :
https://mvnrepository.com/artifact/com.jamesmurty.utils/java-xmlbuilder/0.4,
Apache 2.0
jdo-api 3.0.1: https://mvnrepository.com/artifact/javax.jdo/jdo-api/3.0.1,
Apache 2.0
@@ -346,27 +346,27 @@ The text of each license is also included at
licenses/LICENSE-[project].txt.
snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2,
Apache 2.0
snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0
SparseBitSet 1.2:
https://mvnrepository.com/artifact/com.zaxxer/SparseBitSet/1.2, Apache 2.0
- spring-aop 5.3.12:
https://mvnrepository.com/artifact/org.springframework/spring-aop/5.3.12,
Apache 2.0
+ spring-aop 5.3.13:
https://mvnrepository.com/artifact/org.springframework/spring-aop/5.3.13,
Apache 2.0
spring-beans 5.3.19:
https://mvnrepository.com/artifact/org.springframework/spring-beans/5.3.19,
Apache 2.0
- spring-boot 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.5.6,
Apache 2.0
- spring-boot-actuator 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator/2.5.6,
Apache 2.0
- spring-boot-actuator-autoconfigure 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator-autoconfigure/2.5.6,
Apache 2.0
- spring-boot-configuration-processor 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-configuration-processor/2.5.6,
Apache 2.0
- spring-boot-autoconfigure 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.5.6,
Apache 2.0
- spring-boot-starter 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.5.6,
Apache 2.0
- spring-boot-starter-actuator 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-actuator/2.5.6,
Apache 2.0
- spring-boot-starter-aop 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.5.6,
Apache 2.0
- spring-boot-starter-jdbc 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.5.6,
Apache 2.0
- spring-boot-starter-jetty 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.5.6,
Apache 2.0
- spring-boot-starter-json 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.5.6,
Apache 2.0
- spring-boot-starter-logging 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.5.6,
Apache 2.0
- spring-boot-starter-quartz 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-quartz/2.5.6,
Apache 2.0
- spring-boot-starter-web 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.5.6,
Apache 2.0
- spring-boot-starter-cache 2.5.6:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-cache/2.5.6,
Apache 2.0
+ spring-boot 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.6.1,
Apache 2.0
+ spring-boot-actuator 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator/2.6.1,
Apache 2.0
+ spring-boot-actuator-autoconfigure 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-actuator-autoconfigure/2.6.1,
Apache 2.0
+ spring-boot-configuration-processor 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-configuration-processor/2.6.1,
Apache 2.0
+ spring-boot-autoconfigure 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.6.1,
Apache 2.0
+ spring-boot-starter 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.6.1,
Apache 2.0
+ spring-boot-starter-actuator 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-actuator/2.6.1,
Apache 2.0
+ spring-boot-starter-aop 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.6.1,
Apache 2.0
+ spring-boot-starter-jdbc 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.6.1,
Apache 2.0
+ spring-boot-starter-jetty 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.6.1,
Apache 2.0
+ spring-boot-starter-json 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.6.1,
Apache 2.0
+ spring-boot-starter-logging 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.6.1,
Apache 2.0
+ spring-boot-starter-quartz 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-quartz/2.6.1,
Apache 2.0
+ spring-boot-starter-web 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.6.1,
Apache 2.0
+ spring-boot-starter-cache 2.6.1:
https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-cache/2.6.1,
Apache 2.0
spring-context 5.3.19:
https://mvnrepository.com/artifact/org.springframework/spring-context/5.3.19,
Apache 2.0
- spring-context-support 5.3.12:
https://mvnrepository.com/artifact/org.springframework/spring-context-support/5.3.12,
Apache 2.0
+ spring-context-support 5.3.13:
https://mvnrepository.com/artifact/org.springframework/spring-context-support/5.3.13,
Apache 2.0
spring-core 5.3.19:
https://mvnrepository.com/artifact/org.springframework/spring-core/5.3.19,
Apache 2.0
- spring-expression 5.3.12:
https://mvnrepository.com/artifact/org.springframework/spring-expression/5.3.12,
Apache 2.0
+ spring-expression 5.3.13:
https://mvnrepository.com/artifact/org.springframework/spring-expression/5.3.13,
Apache 2.0
springfox-core 2.9.2:
https://mvnrepository.com/artifact/io.springfox/springfox-core/2.9.2, Apache 2.0
springfox-schema 2.9.2:
https://mvnrepository.com/artifact/io.springfox/springfox-schema/2.9.2, Apache
2.0
springfox-spi 2.9.2:
https://mvnrepository.com/artifact/io.springfox/springfox-spi/2.9.2, Apache 2.0
@@ -374,18 +374,18 @@ The text of each license is also included at
licenses/LICENSE-[project].txt.
springfox-swagger2 2.9.2:
https://mvnrepository.com/artifact/io.springfox/springfox-swagger2/2.9.2,
Apache 2.0
springfox-swagger-common 2.9.2:
https://mvnrepository.com/artifact/io.springfox/springfox-swagger-common/2.9.2,
Apache 2.0
springfox-swagger-ui 2.9.2:
https://mvnrepository.com/artifact/io.springfox/springfox-swagger-ui/2.9.2,
Apache 2.0
- spring-jcl 5.3.12:
https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.3.12,
Apache 2.0
+ spring-jcl 5.3.13:
https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.3.13,
Apache 2.0
spring-jdbc 5.3.19:
https://mvnrepository.com/artifact/org.springframework/spring-jdbc/5.3.19,
Apache 2.0
spring-plugin-core 1.2.0.RELEASE:
https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-core/1.2.0.RELEASE,
Apache 2.0
spring-plugin-metadata 1.2.0.RELEASE:
https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-metadata/1.2.0.RELEASE,
Apache 2.0
spring-tx 5.3.19:
https://mvnrepository.com/artifact/org.springframework/spring-tx/5.3.19, Apache
2.0
- spring-web 5.3.12:
https://mvnrepository.com/artifact/org.springframework/spring-web/5.3.12,
Apache 2.0
- spring-webmvc 5.3.12:
https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.3.12,
Apache 2.0
+ spring-web 5.3.13:
https://mvnrepository.com/artifact/org.springframework/spring-web/5.3.13,
Apache 2.0
+ spring-webmvc 5.3.13:
https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.3.13,
Apache 2.0
swagger-annotations 1.5.20:
https://mvnrepository.com/artifact/io.swagger/swagger-annotations/1.5.20,
Apache 2.0
swagger-bootstrap-ui 1.9.3:
https://mvnrepository.com/artifact/com.github.xiaoymin/swagger-bootstrap-ui/1.9.3,
Apache 2.0
swagger-models 1.5.24:
https://mvnrepository.com/artifact/io.swagger/swagger-models/1.5.24, Apache 2.0
tephra-api 0.6.0:
https://mvnrepository.com/artifact/co.cask.tephra/tephra-api/0.6.0, Apache 2.0
- tomcat-embed-el 9.0.54:
https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.54,
Apache 2.0
+ tomcat-embed-el 9.0.55:
https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-el/9.0.55,
Apache 2.0
xercesImpl 2.9.1:
https://mvnrepository.com/artifact/xerces/xercesImpl/2.9.1, Apache 2.0
xmlbeans 3.1.0:
https://mvnrepository.com/artifact/org.apache.xmlbeans/xmlbeans/3.1.0, Apache
2.0
xml-apis 1.3.04:
https://mvnrepository.com/artifact/xml-apis/xml-apis/1.3.04, Apache 2.0 and W3C
@@ -397,30 +397,30 @@ The text of each license is also included at
licenses/LICENSE-[project].txt.
protostuff-collectionschema 1.7.2:
https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0
prometheus client_java(simpleclient) 0.12.0:
https://github.com/prometheus/client_java, Apache 2.0
snowflake snowflake-2010:
https://github.com/twitter-archive/snowflake/tree/snowflake-2010, Apache 2.0
- kubernetes-client 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-client/5.8.0, Apache
2.0
- kubernetes-model-admissionregistration 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-admissionregistration/5.8.0,
Apache 2.0
- kubernetes-model-apiextensions 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apiextensions/5.8.0,
Apache 2.0
- kubernetes-model-apps 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apps/5.8.0,
Apache 2.0
- kubernetes-model-autoscaling 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/5.8.0,
Apache 2.0
- kubernetes-model-batch 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/5.8.0,
Apache 2.0
- kubernetes-model-certificates 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-certificates/5.8.0,
Apache 2.0
- kubernetes-model-common 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-common/5.8.0,
Apache 2.0
- kubernetes-model-coordination 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-coordination/5.8.0,
Apache 2.0
- kubernetes-model-core 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-core/5.8.0,
Apache 2.0
- kubernetes-model-discovery 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-discovery/5.8.0,
Apache 2.0
- kubernetes-model-events 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-events/5.8.0,
Apache 2.0
- kubernetes-model-extensions 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-extensions/5.8.0,
Apache 2.0
- kubernetes-model-flowcontrol 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-flowcontrol/5.8.0,
Apache 2.0
- kubernetes-model-metrics 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-metrics/5.8.0,
Apache 2.0
- kubernetes-model-networking 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-networking/5.8.0,
Apache 2.0
- kubernetes-model-node 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-node/5.8.0,
Apache 2.0
- kubernetes-model-policy 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-policy/5.8.0,
Apache 2.0
- kubernetes-model-rbac 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-rbac/5.8.0,
Apache 2.0
- kubernetes-model-scheduling 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-scheduling/5.8.0,
Apache 2.0
- kubernetes-model-storageclass 5.8.0:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-storageclass/5.8.0,
Apache 2.0
+ kubernetes-client 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-client/4.13.3, Apache
2.0
+ kubernetes-model-admissionregistration 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-admissionregistration/4.13.3,
Apache 2.0
+ kubernetes-model-apiextensions 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apiextensions/4.13.3,
Apache 2.0
+ kubernetes-model-apps 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-apps/4.13.3,
Apache 2.0
+ kubernetes-model-autoscaling 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/4.13.3,
Apache 2.0
+ kubernetes-model-batch 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-autoscaling/4.13.3,
Apache 2.0
+ kubernetes-model-certificates 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-certificates/4.13.3,
Apache 2.0
+ kubernetes-model-common 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-common/4.13.3,
Apache 2.0
+ kubernetes-model-coordination 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-coordination/4.13.3,
Apache 2.0
+ kubernetes-model-core 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-core/4.13.3,
Apache 2.0
+ kubernetes-model-discovery 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-discovery/4.13.3,
Apache 2.0
+ kubernetes-model-events 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-events/4.13.3,
Apache 2.0
+ kubernetes-model-extensions 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-extensions/4.13.3,
Apache 2.0
+ kubernetes-model-flowcontrol 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-flowcontrol/4.13.3,
Apache 2.0
+ kubernetes-model-metrics 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-metrics/4.13.3,
Apache 2.0
+ kubernetes-model-networking 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-networking/4.13.3,
Apache 2.0
+ kubernetes-model-node 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-node/4.13.3,
Apache 2.0
+ kubernetes-model-policy 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-policy/4.13.3,
Apache 2.0
+ kubernetes-model-rbac 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-rbac/4.13.3,
Apache 2.0
+ kubernetes-model-scheduling 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-scheduling/4.13.3,
Apache 2.0
+ kubernetes-model-storageclass 4.13.3:
https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-storageclass/4.13.3,
Apache 2.0
zjsonpatch 0.3.0
https://mvnrepository.com/artifact/io.fabric8/zjsonpatch/0.3.0, Apache 2.0
generex 1.0.2
https://mvnrepository.com/artifact/com.github.mifmif/generex/1.0.2, Apache 2.0
- jackson-dataformat-yaml 2.12.5
https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.12.5,
Apache 2.0
+ jackson-dataformat-yaml 2.13.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.13.0,
Apache 2.0
logging-interceptor 3.14.9
https://mvnrepository.com/artifact/com.squareup.okhttp3/logging-interceptor/3.14.9,
Apache 2.0
okhttp 3.14.3
https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp/3.14.3, Apache
2.0
okio 1.17.2
https://mvnrepository.com/artifact/com.squareup.okio/okio/1.17.2, Apache 2.0
@@ -429,7 +429,7 @@ The text of each license is also included at
licenses/LICENSE-[project].txt.
jboss-logging:jar 3.4.2.Final
https://mvnrepository.com/artifact/org.jboss.logging/jboss-logging/3.4.2.Final,
Apache 2.0
ion-java 1.0.2
https://mvnrepository.com/artifact/software.amazon.ion/ion-java/1.0.2 Apache
2.0
jmespath-java 1.12.160
https://mvnrepository.com/artifact/com.amazonaws/jmespath-java/1.12.160 Apache
2.0
- jackson-dataformat-cbor 2.12.5
https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-cbor/2.12.5
Apache 2.0
+ jackson-dataformat-cbor 2.13.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-cbor/2.13.0
Apache 2.0
aws-java-sdk-emr 1.12.160
https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-emr/1.12.160
Apache 2.0
aws-java-sdk-core 1.12.160
https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-core/1.12.160
Apache 2.0
aws-java-sdk-s3 1.12.160
https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-s3/1.12.160
Apache 2.0
diff --git
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index 3c82a8adc9..2a19b1ca8d 100644
---
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -38,6 +38,7 @@ import
org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
@@ -79,6 +80,7 @@ public class StateWheelExecuteThread extends BaseDaemonThread
{
@Autowired
private MasterConfig masterConfig;
+ @Lazy
@Autowired
private WorkflowExecuteThreadPool workflowExecuteThreadPool;
diff --git a/dolphinscheduler-master/src/main/resources/application.yaml
b/dolphinscheduler-master/src/main/resources/application.yaml
index d4a82e1afc..c17bf12620 100644
--- a/dolphinscheduler-master/src/main/resources/application.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -56,7 +56,7 @@ spring:
properties:
org.quartz.threadPool:threadPriority: 5
org.quartz.jobStore.isClustered: true
- org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
+ org.quartz.jobStore.class:
org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true
diff --git
a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 4972ee393a..b4faf51168 100644
---
a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++
b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -266,9 +266,6 @@ public class ProcessServiceImpl implements ProcessService {
@Autowired
private CuringParamsService curingGlobalParamsService;
- @Autowired
- private ProcessService processService;
-
/**
* handle Command (construct ProcessInstance from Command) , wrapped in
transaction
*
@@ -1261,7 +1258,7 @@ public class ProcessServiceImpl implements ProcessService
{
try {
// submit task to db
// Only want to use transaction here
- task = processService.submitTask(processInstance,
taskInstance);
+ task = submitTask(processInstance, taskInstance);
if (task != null && task.getId() != 0) {
break;
}
diff --git a/dolphinscheduler-standalone-server/pom.xml
b/dolphinscheduler-standalone-server/pom.xml
index dd4d756065..fd90e39d24 100644
--- a/dolphinscheduler-standalone-server/pom.xml
+++ b/dolphinscheduler-standalone-server/pom.xml
@@ -15,18 +15,29 @@
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
<parent>
- <artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
- <modelVersion>4.0.0</modelVersion>
-
+
<artifactId>dolphinscheduler-standalone-server</artifactId>
-
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-bom</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
@@ -44,17 +55,17 @@
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-alert-server</artifactId>
</dependency>
-
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-log-server</artifactId>
</dependency>
-
+
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
-
+
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
@@ -65,20 +76,21 @@
</exclusion>
</exclusions>
</dependency>
- </dependencies>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-bom</artifactId>
- <version>${project.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
+ <dependency>
+ <groupId>org.springframework.cloud</groupId>
+ <artifactId>spring-cloud-context</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.cloud</groupId>
+ <artifactId>spring-cloud-kubernetes-commons</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.cloud</groupId>
+
<artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
+ </dependency>
+ </dependencies>
+
<build>
<plugins>
<plugin>
@@ -96,10 +108,10 @@
<executions>
<execution>
<id>dolphinscheduler-standalone-server</id>
- <phase>package</phase>
<goals>
<goal>single</goal>
</goals>
+ <phase>package</phase>
<configuration>
<finalName>standalone-server</finalName>
<descriptors>
@@ -112,7 +124,7 @@
</plugin>
</plugins>
</build>
-
+
<profiles>
<profile>
<id>docker</id>
diff --git
a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index 5640467616..d952cbb5b1 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -49,7 +49,7 @@ spring:
properties:
org.quartz.threadPool:threadPriority: 5
org.quartz.jobStore.isClustered: true
- org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
+ org.quartz.jobStore.class:
org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
index 79e1c48af3..7800d70bf0 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
@@ -32,6 +32,7 @@ import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAME_LAB
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RESTART_POLICY;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_INSTANCE_ID;
+import io.fabric8.kubernetes.client.KubernetesClientException;
import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
@@ -58,20 +59,20 @@ import org.slf4j.Logger;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.Quantity;
import io.fabric8.kubernetes.api.model.ResourceRequirements;
-import io.fabric8.kubernetes.api.model.batch.v1.Job;
-import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder;
-import io.fabric8.kubernetes.api.model.batch.v1.JobStatus;
+import io.fabric8.kubernetes.api.model.batch.Job;
+import io.fabric8.kubernetes.api.model.batch.JobBuilder;
+import io.fabric8.kubernetes.api.model.batch.JobStatus;
import io.fabric8.kubernetes.client.Watch;
import io.fabric8.kubernetes.client.Watcher;
-import io.fabric8.kubernetes.client.WatcherException;
/**
* K8sTaskExecutor used to submit k8s task to K8S
*/
public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
+
private Job job;
public K8sTaskExecutor(Logger logger, TaskExecutionContext taskRequest) {
- super(logger,taskRequest);
+ super(logger, taskRequest);
}
public Job buildK8sJob(K8sTaskMainParameters k8STaskMainParameters) {
@@ -99,7 +100,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor
{
List<EnvVar> envVars = new ArrayList<>();
envVars.add(taskInstanceIdVar);
if (MapUtils.isNotEmpty(otherParams)) {
- for (Map.Entry<String,String> entry : otherParams.entrySet()) {
+ for (Map.Entry<String, String> entry : otherParams.entrySet()) {
String param = entry.getKey();
String paramValue = entry.getValue();
EnvVar envVar = new EnvVar(param, paramValue, null);
@@ -107,60 +108,58 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
}
}
return new JobBuilder()
- .withApiVersion(API_VERSION)
- .withNewMetadata()
- .withName(k8sJobName)
- .withLabels(labelMap)
- .withNamespace(namespaceName)
- .endMetadata()
- .withNewSpec()
- .withTtlSecondsAfterFinished(JOB_TTL_SECONDS)
- .withNewTemplate()
- .withNewSpec()
- .addNewContainer()
- .withName(k8sJobName)
- .withImage(image)
- .withImagePullPolicy(IMAGE_PULL_POLICY)
- .withResources(new ResourceRequirements(limitRes, reqRes))
- .withEnv(envVars)
- .endContainer()
- .withRestartPolicy(RESTART_POLICY)
- .endSpec()
- .endTemplate()
- .withBackoffLimit(retryNum)
- .endSpec()
- .build();
+ .withApiVersion(API_VERSION)
+ .withNewMetadata()
+ .withName(k8sJobName)
+ .withLabels(labelMap)
+ .withNamespace(namespaceName)
+ .endMetadata()
+ .withNewSpec()
+ .withTtlSecondsAfterFinished(JOB_TTL_SECONDS)
+ .withNewTemplate()
+ .withNewSpec()
+ .addNewContainer()
+ .withName(k8sJobName)
+ .withImage(image)
+ .withImagePullPolicy(IMAGE_PULL_POLICY)
+ .withResources(new ResourceRequirements(limitRes, reqRes))
+ .withEnv(envVars)
+ .endContainer()
+ .withRestartPolicy(RESTART_POLICY)
+ .endSpec()
+ .endTemplate()
+ .withBackoffLimit(retryNum)
+ .endSpec()
+ .build();
}
- public void registerBatchJobWatcher(Job job, String taskInstanceId,
TaskResponse taskResponse, K8sTaskMainParameters k8STaskMainParameters) {
+ public void registerBatchJobWatcher(Job job, String taskInstanceId,
TaskResponse taskResponse,
+ K8sTaskMainParameters
k8STaskMainParameters) {
CountDownLatch countDownLatch = new CountDownLatch(1);
Watcher<Job> watcher = new Watcher<Job>() {
+
@Override
public void eventReceived(Action action, Job job) {
if (action != Action.ADDED) {
int jobStatus = getK8sJobStatus(job);
- setTaskStatus(jobStatus,taskInstanceId, taskResponse,
k8STaskMainParameters);
+ setTaskStatus(jobStatus, taskInstanceId, taskResponse,
k8STaskMainParameters);
countDownLatch.countDown();
- }
}
+ }
@Override
- public void onClose(WatcherException e) {
- logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail
in k8s: %s",job.getMetadata().getName(),e.getMessage()));
+ public void onClose(KubernetesClientException e) {
+ logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail
in k8s: %s", job.getMetadata().getName(),
+ e.getMessage()));
taskResponse.setExitStatusCode(EXIT_CODE_FAILURE);
countDownLatch.countDown();
}
-
- @Override
- public void onClose() {
- logger.warn("Watch gracefully closed");
- }
};
Watch watch = null;
try {
watch =
k8sUtils.createBatchJobWatcher(job.getMetadata().getName(), watcher);
boolean timeoutFlag = taskRequest.getTaskTimeoutStrategy() ==
TaskTimeoutStrategy.FAILED
- || taskRequest.getTaskTimeoutStrategy() ==
TaskTimeoutStrategy.WARNFAILED;
+ || taskRequest.getTaskTimeoutStrategy() ==
TaskTimeoutStrategy.WARNFAILED;
if (timeoutFlag) {
Boolean timeout =
!(countDownLatch.await(taskRequest.getTaskTimeout(), TimeUnit.SECONDS));
waitTimeout(timeout);
@@ -169,11 +168,11 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
}
flushLog(taskResponse);
} catch (InterruptedException e) {
- logger.error("job failed in k8s: {}",e.getMessage(), e);
+ logger.error("job failed in k8s: {}", e.getMessage(), e);
Thread.currentThread().interrupt();
taskResponse.setExitStatusCode(EXIT_CODE_FAILURE);
} catch (Exception e) {
- logger.error("job failed in k8s: {}",e.getMessage(), e);
+ logger.error("job failed in k8s: {}", e.getMessage(), e);
taskResponse.setExitStatusCode(EXIT_CODE_FAILURE);
} finally {
if (watch != null) {
@@ -186,7 +185,8 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
public TaskResponse run(String k8sParameterStr) throws Exception {
TaskResponse result = new TaskResponse();
int taskInstanceId = taskRequest.getTaskInstanceId();
- K8sTaskMainParameters k8STaskMainParameters =
JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
+ K8sTaskMainParameters k8STaskMainParameters =
+ JSONUtils.parseObject(k8sParameterStr,
K8sTaskMainParameters.class);
try {
if (null ==
TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId)) {
result.setExitStatusCode(EXIT_CODE_KILL);
@@ -197,7 +197,7 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
return result;
}
K8sTaskExecutionContext k8sTaskExecutionContext =
taskRequest.getK8sTaskExecutionContext();
- String configYaml = k8sTaskExecutionContext.getConfigYaml();
+ String configYaml = k8sTaskExecutionContext.getConfigYaml();
k8sUtils.buildClient(configYaml);
submitJob2k8s(k8sParameterStr);
registerBatchJobWatcher(job, Integer.toString(taskInstanceId),
result, k8STaskMainParameters);
@@ -220,7 +220,8 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
public void submitJob2k8s(String k8sParameterStr) {
int taskInstanceId = taskRequest.getTaskInstanceId();
String taskName = taskRequest.getTaskName().toLowerCase(Locale.ROOT);
- K8sTaskMainParameters k8STaskMainParameters =
JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
+ K8sTaskMainParameters k8STaskMainParameters =
+ JSONUtils.parseObject(k8sParameterStr,
K8sTaskMainParameters.class);
try {
logger.info("[K8sJobExecutor-{}-{}] start to submit job",
taskName, taskInstanceId);
job = buildK8sJob(k8STaskMainParameters);
@@ -236,7 +237,8 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
@Override
public void stopJobOnK8s(String k8sParameterStr) {
- K8sTaskMainParameters k8STaskMainParameters =
JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
+ K8sTaskMainParameters k8STaskMainParameters =
+ JSONUtils.parseObject(k8sParameterStr,
K8sTaskMainParameters.class);
String namespaceName = k8STaskMainParameters.getNamespaceName();
String jobName = job.getMetadata().getName();
try {
@@ -260,17 +262,21 @@ public class K8sTaskExecutor extends
AbstractK8sTaskExecutor {
}
}
- public void setTaskStatus(int jobStatus,String taskInstanceId,
TaskResponse taskResponse, K8sTaskMainParameters k8STaskMainParameters) {
+ public void setTaskStatus(int jobStatus, String taskInstanceId,
TaskResponse taskResponse,
+ K8sTaskMainParameters k8STaskMainParameters) {
if (jobStatus == EXIT_CODE_SUCCESS || jobStatus == EXIT_CODE_FAILURE) {
if (null ==
TaskExecutionContextCacheManager.getByTaskInstanceId(Integer.valueOf(taskInstanceId)))
{
logStringBuffer.append(String.format("[K8sJobExecutor-%s]
killed", job.getMetadata().getName()));
taskResponse.setExitStatusCode(EXIT_CODE_KILL);
} else if (jobStatus == EXIT_CODE_SUCCESS) {
- logStringBuffer.append(String.format("[K8sJobExecutor-%s]
succeed in k8s", job.getMetadata().getName()));
+ logStringBuffer
+ .append(String.format("[K8sJobExecutor-%s] succeed in
k8s", job.getMetadata().getName()));
taskResponse.setExitStatusCode(EXIT_CODE_SUCCESS);
} else {
- String errorMessage =
k8sUtils.getPodLog(job.getMetadata().getName(),
k8STaskMainParameters.getNamespaceName());
- logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail
in k8s: %s", job.getMetadata().getName(), errorMessage));
+ String errorMessage =
+ k8sUtils.getPodLog(job.getMetadata().getName(),
k8STaskMainParameters.getNamespaceName());
+ logStringBuffer.append(String.format("[K8sJobExecutor-%s] fail
in k8s: %s", job.getMetadata().getName(),
+ errorMessage));
taskResponse.setExitStatusCode(EXIT_CODE_FAILURE);
}
}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java
index f2b104875b..89fcbf8627 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/K8sUtils.java
@@ -28,8 +28,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.fabric8.kubernetes.api.model.Pod;
-import io.fabric8.kubernetes.api.model.batch.v1.Job;
-import io.fabric8.kubernetes.api.model.batch.v1.JobList;
+import io.fabric8.kubernetes.api.model.batch.Job;
+import io.fabric8.kubernetes.api.model.batch.JobList;
import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClient;
@@ -37,40 +37,41 @@ import io.fabric8.kubernetes.client.Watch;
import io.fabric8.kubernetes.client.Watcher;
public class K8sUtils {
+
private static final Logger log = LoggerFactory.getLogger(K8sUtils.class);
private KubernetesClient client;
public void createJob(String namespace, Job job) {
try {
- client.batch().v1()
- .jobs()
- .inNamespace(namespace)
- .create(job);
+ client.batch()
+ .jobs()
+ .inNamespace(namespace)
+ .create(job);
} catch (Exception e) {
- throw new TaskException("fail to create job",e);
+ throw new TaskException("fail to create job", e);
}
}
public void deleteJob(String jobName, String namespace) {
try {
- client.batch().v1()
- .jobs()
- .inNamespace(namespace)
- .withName(jobName)
- .delete();
+ client.batch()
+ .jobs()
+ .inNamespace(namespace)
+ .withName(jobName)
+ .delete();
} catch (Exception e) {
- throw new TaskException("fail to delete job",e);
+ throw new TaskException("fail to delete job", e);
}
}
public Boolean jobExist(String jobName, String namespace) {
Optional<Job> result;
try {
- JobList jobList =
client.batch().v1().jobs().inNamespace(namespace).list();
+ JobList jobList =
client.batch().jobs().inNamespace(namespace).list();
List<Job> jobs = jobList.getItems();
result = jobs.stream()
- .filter(job -> job.getMetadata().getName().equals(jobName))
- .findFirst();
+ .filter(job -> job.getMetadata().getName().equals(jobName))
+ .findFirst();
return result.isPresent();
} catch (Exception e) {
throw new TaskException("fail to check job: ", e);
@@ -79,10 +80,10 @@ public class K8sUtils {
public Watch createBatchJobWatcher(String jobName, Watcher<Job> watcher) {
try {
- return client.batch().v1()
- .jobs().withName(jobName).watch(watcher);
+ return client.batch()
+ .jobs().withName(jobName).watch(watcher);
} catch (Exception e) {
- throw new TaskException("fail to register batch job watcher",e);
+ throw new TaskException("fail to register batch job watcher", e);
}
}
@@ -97,9 +98,9 @@ public class K8sUtils {
}
}
return client.pods().inNamespace(namespace)
- .withName(podName)
- .tailingLines(LOG_LINES)
- .getLog(Boolean.TRUE);
+ .withName(podName)
+ .tailingLines(LOG_LINES)
+ .getLog(Boolean.TRUE);
} catch (Exception e) {
log.error("fail to getPodLog", e);
log.error("response bodies : {}", e.getMessage());
@@ -112,7 +113,7 @@ public class K8sUtils {
Config config = Config.fromKubeconfig(configYaml);
client = new DefaultKubernetesClient(config);
} catch (Exception e) {
- throw new TaskException("fail to build k8s ApiClient",e);
+ throw new TaskException("fail to build k8s ApiClient", e);
}
}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
index db30248487..f8af0b64e9 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
@@ -16,8 +16,8 @@
*/
package org.apache.dolphinscheduler.plugin.task.api.k8s;
-import io.fabric8.kubernetes.api.model.batch.v1.Job;
-import io.fabric8.kubernetes.api.model.batch.v1.JobStatus;
+import io.fabric8.kubernetes.api.model.batch.Job;
+import io.fabric8.kubernetes.api.model.batch.JobStatus;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL;
@@ -64,17 +64,6 @@ public class K8sTaskExecutorTest {
job = k8sTaskExecutor.buildK8sJob(k8sTaskMainParameters);
}
@Test
- public void testBuildK8sJobNormal() {
- String jobStr = "Job(apiVersion=batch/v1, kind=Job,
metadata=ObjectMeta(annotations=null, clusterName=null, creationTimestamp=null,
deletionGracePeriodSeconds=null, deletionTimestamp=null, finalizers=[],
generateName=null, generation=null, labels={k8s.cn/layer=batch,
k8s.cn/name=k8s_task_test-1000}, managedFields=[], name=k8s_task_test-1000,
namespace=default, ownerReferences=[], resourceVersion=null, selfLink=null,
uid=null, additionalProperties={}), spec=JobSpec(activeDeadlineS [...]
- Assert.assertEquals(jobStr, job.toString());
- }
- @Test
- public void testGetJobNormal() {
- k8sTaskExecutor.setJob(job);
- String jobStr = "Job(apiVersion=batch/v1, kind=Job,
metadata=ObjectMeta(annotations=null, clusterName=null, creationTimestamp=null,
deletionGracePeriodSeconds=null, deletionTimestamp=null, finalizers=[],
generateName=null, generation=null, labels={k8s.cn/layer=batch,
k8s.cn/name=k8s_task_test-1000}, managedFields=[], name=k8s_task_test-1000,
namespace=default, ownerReferences=[], resourceVersion=null, selfLink=null,
uid=null, additionalProperties={}), spec=JobSpec(activeDeadlineS [...]
- Assert.assertEquals(jobStr,k8sTaskExecutor.getJob().toString());
- }
- @Test
public void testGetK8sJobStatusNormal() {
JobStatus jobStatus = new JobStatus();
jobStatus.setSucceeded(1);
diff --git a/pom.xml b/pom.xml
index 7c0bc2cbd5..1084b0622b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,7 +62,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <spring.boot.version>2.5.6</spring.boot.version>
+ <spring.boot.version>2.6.1</spring.boot.version>
<java.version>1.8</java.version>
<junit.version>4.12</junit.version>
<spotbugs.version>3.1.12</spotbugs.version>
diff --git a/tools/dependencies/known-dependencies.txt
b/tools/dependencies/known-dependencies.txt
index 60bfb995d3..1380796af4 100755
--- a/tools/dependencies/known-dependencies.txt
+++ b/tools/dependencies/known-dependencies.txt
@@ -81,18 +81,19 @@ httpclient-4.5.13.jar
httpcore-4.4.15.jar
httpmime-4.5.13.jar
j2objc-annotations-1.1.jar
-jackson-annotations-2.10.5.jar
-jackson-core-2.10.5.jar
+jackson-annotations-2.13.0.jar
+jackson-core-2.13.0.jar
jackson-core-asl-1.9.13.jar
-jackson-databind-2.10.5.jar
+jackson-databind-2.13.0.jar
jackson-datatype-jdk8-2.12.5.jar
jackson-datatype-jsr310-2.12.5.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-parameter-names-2.12.5.jar
+jakarta.activation-api-1.2.2.jar
jakarta.annotation-api-1.3.5.jar
jakarta.servlet-api-4.0.4.jar
jakarta.validation-api-2.0.2.jar
jakarta.websocket-api-1.1.2.jar
+jakarta.xml.bind-api-2.3.3.jar
jamon-runtime-2.3.1.jar
java-xmlbuilder-0.4.jar
javassist-3.27.0-GA.jar
@@ -123,7 +124,7 @@ jna-5.10.0.jar
jna-platform-5.10.0.jar
joda-time-2.10.13.jar
jpam-1.1.jar
-jsch-0.1.42.jar
+jsch-0.1.55.jar
jsp-api-2.1.jar
jsqlparser-2.1.jar
jsr305-3.0.0.jar
@@ -179,7 +180,7 @@ spring-boot-2.5.6.jar
spring-boot-actuator-2.5.6.jar
spring-boot-actuator-autoconfigure-2.5.6.jar
spring-boot-autoconfigure-2.5.6.jar
-spring-boot-configuration-processor-2.5.6.jar
+spring-boot-configuration-processor-2.6.1.jar
spring-boot-starter-2.5.6.jar
spring-boot-starter-actuator-2.5.6.jar
spring-boot-starter-aop-2.5.6.jar
@@ -190,6 +191,9 @@ spring-boot-starter-logging-2.5.6.jar
spring-boot-starter-quartz-2.5.6.jar
spring-boot-starter-web-2.5.6.jar
spring-boot-starter-cache-2.5.6.jar
+spring-cloud-context-3.0.6.jar
+spring-cloud-kubernetes-commons-2.0.6.jar
+spring-cloud-starter-kubernetes-fabric8-config-2.0.6.jar
spring-context-5.3.19.jar
spring-context-support-5.3.12.jar
spring-core-5.3.19.jar
@@ -220,31 +224,33 @@ xmlbeans-3.1.0.jar
xmlenc-0.52.jar
zookeeper-3.4.14.jar
Java-WebSocket-1.5.1.jar
-kubernetes-client-5.8.0.jar
-kubernetes-model-admissionregistration-5.8.0.jar
-kubernetes-model-apiextensions-5.8.0.jar
-kubernetes-model-apps-5.8.0.jar
-kubernetes-model-autoscaling-5.8.0.jar
-kubernetes-model-batch-5.8.0.jar
-kubernetes-model-certificates-5.8.0.jar
-kubernetes-model-common-5.8.0.jar
-kubernetes-model-coordination-5.8.0.jar
-kubernetes-model-core-5.8.0.jar
-kubernetes-model-discovery-5.8.0.jar
-kubernetes-model-events-5.8.0.jar
-kubernetes-model-extensions-5.8.0.jar
-kubernetes-model-flowcontrol-5.8.0.jar
-kubernetes-model-metrics-5.8.0.jar
-kubernetes-model-networking-5.8.0.jar
-kubernetes-model-node-5.8.0.jar
-kubernetes-model-policy-5.8.0.jar
-kubernetes-model-rbac-5.8.0.jar
-kubernetes-model-scheduling-5.8.0.jar
-kubernetes-model-storageclass-5.8.0.jar
+kubernetes-client-4.13.3.jar
+kubernetes-model-admissionregistration-4.13.3.jar
+kubernetes-model-apiextensions-4.13.3.jar
+kubernetes-model-apps-4.13.3.jar
+kubernetes-model-autoscaling-4.13.3.jar
+kubernetes-model-batch-4.13.3.jar
+kubernetes-model-certificates-4.13.3.jar
+kubernetes-model-common-4.13.3.jar
+kubernetes-model-coordination-4.13.3.jar
+kubernetes-model-core-4.13.3.jar
+kubernetes-model-discovery-4.13.3.jar
+kubernetes-model-events-4.13.3.jar
+kubernetes-model-extensions-4.13.3.jar
+kubernetes-model-metrics-4.13.3.jar
+kubernetes-model-networking-4.13.3.jar
+kubernetes-model-node-4.13.3.jar
+kubernetes-model-policy-4.13.3.jar
+kubernetes-model-rbac-4.13.3.jar
+kubernetes-model-settings-4.13.3.jar
+kubernetes-model-scheduling-4.13.3.jar
+kubernetes-model-storageclass-4.13.3.jar
zjsonpatch-0.3.0.jar
automaton-1.11-8.jar
generex-1.0.2.jar
jackson-dataformat-yaml-2.12.5.jar
+jackson-module-jaxb-annotations-2.12.5.jar
+jackson-module-parameter-names-2.12.5.jar
logging-interceptor-3.14.9.jar
okhttp-3.14.9.jar
okio-1.17.2.jar
