This is an automated email from the ASF dual-hosted git repository.
wenjun pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 163f8f01f0 Fix jdbc registry cannot work (#15861)
163f8f01f0 is described below
commit 163f8f01f0b3726374c81d9fa87da6571b3c460c
Author: Wenjun Ruan <[email protected]>
AuthorDate: Thu Apr 18 15:43:28 2024 +0800
Fix jdbc registry cannot work (#15861)
---
.github/workflows/backend.yml | 12 +-
.../Dockerfile | 6 +-
.../{mysql => mysql_with_mysql_registry}/deploy.sh | 1 +
.../docker-compose-base.yaml} | 19 ++-
.../docker-compose-cluster.yaml | 0
.../dolphinscheduler_env.sh | 7 +-
.../install_env.sh | 3 -
.../running_test.sh | 0
.../start-job.sh | 8 +-
.../Dockerfile | 6 +-
.../deploy.sh | 0
.../docker-compose-base.yaml | 0
.../docker-compose-cluster.yaml | 0
.../dolphinscheduler_env.sh | 0
.../install_env.sh | 0
.../running_test.sh | 0
.../start-job.sh | 8 +-
.../Dockerfile | 6 +-
.../deploy.sh | 1 +
.../docker-compose-base.yaml} | 20 ++-
.../docker-compose-cluster.yaml | 0
.../dolphinscheduler_env.sh | 7 +-
.../install_env.sh | 3 -
.../running_test.sh | 0
.../start-job.sh | 8 +-
.../Dockerfile | 6 +-
.../deploy.sh | 0
.../docker-compose-base.yaml | 0
.../docker-compose-cluster.yaml | 0
.../dolphinscheduler_env.sh | 0
.../install_env.sh | 0
.../running_test.sh | 0
.../start-job.sh | 8 +-
.../apache/dolphinscheduler/alert/AlertServer.java | 19 +--
.../dolphinscheduler/api/ApiApplicationServer.java | 6 +-
.../org/apache/dolphinscheduler/dao/PluginDao.java | 4 +-
.../dao/{ => repository/impl}/AlertDaoTest.java | 4 +-
.../server/master/MasterServer.java | 6 +-
.../dolphinscheduler-registry-jdbc/README.md | 10 +-
.../dolphinscheduler-registry-jdbc/pom.xml | 10 ++
...ion.java => JdbcRegistryAutoConfiguration.java} | 30 +++-
.../src/main/resources/META-INF/spring.factories | 48 +++----
.../src/main/resources/mysql_registry_init.sql | 1 -
.../src/main/bin/initialize-jdbc-registry.sh | 22 ++-
.../tools/command/CommandApplication.java | 152 +++++++++++++++++++++
.../src/main/resources/application.yaml | 2 +
.../server/worker/WorkerServer.java | 6 +-
47 files changed, 325 insertions(+), 124 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index ea09a17fd2..671223d286 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -106,10 +106,14 @@ jobs:
strategy:
matrix:
case:
- - name: cluster-test-mysql
- script: .github/workflows/cluster-test/mysql/start-job.sh
- - name: cluster-test-postgresql
- script: .github/workflows/cluster-test/postgresql/start-job.sh
+ - name: cluster-test-mysql-with-zookeeper-registry
+ script:
.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh
+ - name: cluster-test-mysql-with-mysql-registry
+ script:
.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh
+ - name: cluster-test-postgresql-zookeeper-registry
+ script:
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh
+ - name: cluster-test-postgresql-with-postgresql-registry
+ script:
.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
steps:
- uses: actions/checkout@v2
with:
diff --git a/.github/workflows/cluster-test/mysql/Dockerfile
b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile
similarity index 85%
copy from .github/workflows/cluster-test/mysql/Dockerfile
copy to .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile
index c7d6abe889..12c7db3c18 100644
--- a/.github/workflows/cluster-test/mysql/Dockerfile
+++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile
@@ -28,10 +28,10 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin
/root/apache-dolphinschedule
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
#Setting install.sh
-COPY .github/workflows/cluster-test/mysql/install_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
+COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
#Setting dolphinscheduler_env.sh
-COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
+COPY
.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
#Download mysql jar
ENV MYSQL_URL
"https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar";
@@ -43,6 +43,6 @@ cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER
$DOLPHINSCHEDULER_HOME
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER
$DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
#Deploy
-COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh
+COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh
/root/deploy.sh
CMD [ "/bin/bash", "/root/deploy.sh" ]
diff --git a/.github/workflows/cluster-test/mysql/deploy.sh
b/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh
similarity index 95%
copy from .github/workflows/cluster-test/mysql/deploy.sh
copy to .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh
index 722a4b849f..72b2a630fa 100644
--- a/.github/workflows/cluster-test/mysql/deploy.sh
+++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh
@@ -35,6 +35,7 @@ service ssh start
#Init schema
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
+/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
#Start Cluster
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
diff --git a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml
similarity index 70%
copy from .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
copy to
.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml
index 7343c8eee7..d59e3c868c 100644
--- a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
+++
b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml
@@ -18,12 +18,17 @@
version: "3"
services:
- ds:
- container_name: ds
- image: jdk8:ds_mysql_cluster
+ mysql:
+ container_name: mysql
+ image: mysql:5.7.36
+ command: --default-authentication-plugin=mysql_native_password
restart: always
+ environment:
+ MYSQL_ROOT_PASSWORD: 123456
ports:
- - "12345:12345"
- - "5679:5679"
- - "1235:1235"
- - "50053:50053"
+ - "3306:3306"
+ healthcheck:
+ test: mysqladmin ping -h 127.0.0.1 -u root
--password=$$MYSQL_ROOT_PASSWORD
+ interval: 5s
+ timeout: 60s
+ retries: 120
diff --git a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
b/.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml
similarity index 100%
copy from .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
copy to
.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml
diff --git a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh
b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh
similarity index 91%
copy from .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh
copy to
.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh
index 671c70a5bb..58937e740c 100755
--- a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh
+++
b/.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh
@@ -31,9 +31,10 @@ export
SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
# Registry center configuration, determines the type and link of the registry
center
-export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
-export
REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183}
-export REGISTRY_ZOOKEEPER_BLOCK_UNTIL_CONNECTED=30000ms
+export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc}
+export
REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
+export REGISTRY_HIKARI_CONFIG_USERNAME=root
+export REGISTRY_HIKARI_CONFIG_PASSWORD=123456
# Tasks related configurations, need to change the configuration if you use
the related tasks.
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
diff --git a/.github/workflows/cluster-test/postgresql/install_env.sh
b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh
similarity index 96%
copy from .github/workflows/cluster-test/postgresql/install_env.sh
copy to .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh
index e43813479b..cd660febf8 100644
--- a/.github/workflows/cluster-test/postgresql/install_env.sh
+++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh
@@ -56,6 +56,3 @@
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
# script. The user needs to have sudo privileges and permissions to operate
hdfs. If hdfs is enabled than the root directory needs
# to be created by this user
deployUser=${deployUser:-"dolphinscheduler"}
-
-# The root of zookeeper, for now DolphinScheduler default registry server is
zookeeper.
-zkRoot=${zkRoot:-"/dolphinscheduler"}
diff --git a/.github/workflows/cluster-test/mysql/running_test.sh
b/.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
similarity index 100%
copy from .github/workflows/cluster-test/mysql/running_test.sh
copy to .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
diff --git a/.github/workflows/cluster-test/mysql/start-job.sh
b/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh
similarity index 74%
copy from .github/workflows/cluster-test/mysql/start-job.sh
copy to .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh
index ee67c5179b..0ce48c64ae 100644
--- a/.github/workflows/cluster-test/mysql/start-job.sh
+++ b/.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh
@@ -18,16 +18,16 @@
set -euox pipefail
#Start base service containers
-docker-compose -f
.github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d
+docker-compose -f
.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml
up -d
#Build ds mysql cluster image
-docker build -t jdk8:ds_mysql_cluster -f
.github/workflows/cluster-test/mysql/Dockerfile .
+docker build -t jdk8:ds_mysql_cluster -f
.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile .
#Start ds mysql cluster container
-docker-compose -f
.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d
+docker-compose -f
.github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml
up -d
#Running tests
-/bin/bash .github/workflows/cluster-test/mysql/running_test.sh
+/bin/bash
.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
#Cleanup
docker rm -f $(docker ps -aq)
diff --git a/.github/workflows/cluster-test/mysql/Dockerfile
b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile
similarity index 84%
rename from .github/workflows/cluster-test/mysql/Dockerfile
rename to
.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile
index c7d6abe889..574c059442 100644
--- a/.github/workflows/cluster-test/mysql/Dockerfile
+++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile
@@ -28,10 +28,10 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin
/root/apache-dolphinschedule
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
#Setting install.sh
-COPY .github/workflows/cluster-test/mysql/install_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
+COPY
.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
#Setting dolphinscheduler_env.sh
-COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
+COPY
.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
#Download mysql jar
ENV MYSQL_URL
"https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar";
@@ -43,6 +43,6 @@ cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER
$DOLPHINSCHEDULER_HOME
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER
$DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
#Deploy
-COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh
+COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh
/root/deploy.sh
CMD [ "/bin/bash", "/root/deploy.sh" ]
diff --git a/.github/workflows/cluster-test/mysql/deploy.sh
b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh
similarity index 100%
rename from .github/workflows/cluster-test/mysql/deploy.sh
rename to .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh
diff --git a/.github/workflows/cluster-test/mysql/docker-compose-base.yaml
b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml
similarity index 100%
rename from .github/workflows/cluster-test/mysql/docker-compose-base.yaml
rename to
.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml
diff --git a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml
similarity index 100%
copy from .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
copy to
.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml
diff --git a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh
b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh
similarity index 100%
rename from .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh
rename to
.github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh
diff --git a/.github/workflows/cluster-test/mysql/install_env.sh
b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh
similarity index 100%
rename from .github/workflows/cluster-test/mysql/install_env.sh
rename to
.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh
diff --git a/.github/workflows/cluster-test/mysql/running_test.sh
b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
similarity index 100%
rename from .github/workflows/cluster-test/mysql/running_test.sh
rename to
.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
diff --git a/.github/workflows/cluster-test/mysql/start-job.sh
b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh
similarity index 74%
rename from .github/workflows/cluster-test/mysql/start-job.sh
rename to
.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh
index ee67c5179b..db8d23147e 100644
--- a/.github/workflows/cluster-test/mysql/start-job.sh
+++ b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh
@@ -18,16 +18,16 @@
set -euox pipefail
#Start base service containers
-docker-compose -f
.github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d
+docker-compose -f
.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml
up -d
#Build ds mysql cluster image
-docker build -t jdk8:ds_mysql_cluster -f
.github/workflows/cluster-test/mysql/Dockerfile .
+docker build -t jdk8:ds_mysql_cluster -f
.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile .
#Start ds mysql cluster container
-docker-compose -f
.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d
+docker-compose -f
.github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml
up -d
#Running tests
-/bin/bash .github/workflows/cluster-test/mysql/running_test.sh
+/bin/bash
.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
#Cleanup
docker rm -f $(docker ps -aq)
diff --git a/.github/workflows/cluster-test/postgresql/Dockerfile
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
similarity index 77%
copy from .github/workflows/cluster-test/postgresql/Dockerfile
copy to
.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
index 38234ee7b3..bb2d9a5383 100644
--- a/.github/workflows/cluster-test/postgresql/Dockerfile
+++
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
@@ -28,12 +28,12 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin
/root/apache-dolphinschedule
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
#Setting install.sh
-COPY .github/workflows/cluster-test/postgresql/install_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
+COPY
.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
#Setting dolphinscheduler_env.sh
-COPY .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
+COPY
.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
#Deploy
-COPY .github/workflows/cluster-test/postgresql/deploy.sh /root/deploy.sh
+COPY
.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh
/root/deploy.sh
CMD [ "/bin/bash", "/root/deploy.sh" ]
diff --git a/.github/workflows/cluster-test/postgresql/deploy.sh
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh
similarity index 94%
copy from .github/workflows/cluster-test/postgresql/deploy.sh
copy to
.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh
index f7547d40b4..37bf3433c0 100644
--- a/.github/workflows/cluster-test/postgresql/deploy.sh
+++
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh
@@ -32,6 +32,7 @@ service ssh start
#Init schema
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
+/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/initialize-jdbc-registry.sh
#Start Cluster
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
diff --git a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml
similarity index 73%
copy from .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
copy to
.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml
index 7343c8eee7..1793d94f39 100644
--- a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
+++
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml
@@ -18,12 +18,18 @@
version: "3"
services:
- ds:
- container_name: ds
- image: jdk8:ds_mysql_cluster
+ postgres:
+ container_name: postgres
+ image: postgres:14.1
restart: always
+ environment:
+ POSTGRES_PASSWORD: postgres
+ POSTGRES_DB: dolphinscheduler
ports:
- - "12345:12345"
- - "5679:5679"
- - "1235:1235"
- - "50053:50053"
+ - "5432:5432"
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U postgres"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+
diff --git
a/.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml
similarity index 100%
copy from .github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml
copy to
.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml
diff --git a/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh
similarity index 93%
copy from .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh
copy to
.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh
index 1dbd63254e..e7fd1b7204 100644
--- a/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh
+++
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh
@@ -31,9 +31,10 @@ export
SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
# Registry center configuration, determines the type and link of the registry
center
-export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
-export
REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183}
-export REGISTRY_ZOOKEEPER_BLOCK_UNTIL_CONNECTED=30000ms
+export REGISTRY_TYPE=jdbc
+export
REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:postgresql://postgres:5432/dolphinscheduler"
+export REGISTRY_HIKARI_CONFIG_USERNAME=postgres
+export REGISTRY_HIKARI_CONFIG_PASSWORD=postgres
# Tasks related configurations, need to change the configuration if you use
the related tasks.
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
diff --git a/.github/workflows/cluster-test/postgresql/install_env.sh
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
similarity index 96%
copy from .github/workflows/cluster-test/postgresql/install_env.sh
copy to
.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
index e43813479b..cd660febf8 100644
--- a/.github/workflows/cluster-test/postgresql/install_env.sh
+++
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
@@ -56,6 +56,3 @@
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
# script. The user needs to have sudo privileges and permissions to operate
hdfs. If hdfs is enabled than the root directory needs
# to be created by this user
deployUser=${deployUser:-"dolphinscheduler"}
-
-# The root of zookeeper, for now DolphinScheduler default registry server is
zookeeper.
-zkRoot=${zkRoot:-"/dolphinscheduler"}
diff --git a/.github/workflows/cluster-test/postgresql/running_test.sh
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
similarity index 100%
copy from .github/workflows/cluster-test/postgresql/running_test.sh
copy to
.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
diff --git a/.github/workflows/cluster-test/postgresql/start-job.sh
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
similarity index 72%
copy from .github/workflows/cluster-test/postgresql/start-job.sh
copy to
.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
index ba0878e3ec..e2b6b630e8 100644
--- a/.github/workflows/cluster-test/postgresql/start-job.sh
+++
b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
@@ -18,16 +18,16 @@
set -euox pipefail
#Start base service containers
-docker-compose -f
.github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d
+docker-compose -f
.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml
up -d
#Build ds postgresql cluster image
-docker build -t jdk8:ds_postgresql_cluster -f
.github/workflows/cluster-test/postgresql/Dockerfile .
+docker build -t jdk8:ds_postgresql_cluster -f
.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile .
#Start ds postgresql cluster container
-docker-compose -f
.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d
+docker-compose -f
.github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml
up -d
#Running tests
-/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh
+/bin/bash
.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
#Cleanup
docker rm -f $(docker ps -aq)
diff --git a/.github/workflows/cluster-test/postgresql/Dockerfile
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile
similarity index 77%
rename from .github/workflows/cluster-test/postgresql/Dockerfile
rename to
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile
index 38234ee7b3..077b5c97b8 100644
--- a/.github/workflows/cluster-test/postgresql/Dockerfile
+++
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile
@@ -28,12 +28,12 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin
/root/apache-dolphinschedule
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
#Setting install.sh
-COPY .github/workflows/cluster-test/postgresql/install_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
+COPY
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
#Setting dolphinscheduler_env.sh
-COPY .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
+COPY
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh
$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
#Deploy
-COPY .github/workflows/cluster-test/postgresql/deploy.sh /root/deploy.sh
+COPY
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh
/root/deploy.sh
CMD [ "/bin/bash", "/root/deploy.sh" ]
diff --git a/.github/workflows/cluster-test/postgresql/deploy.sh
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh
similarity index 100%
rename from .github/workflows/cluster-test/postgresql/deploy.sh
rename to
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh
diff --git a/.github/workflows/cluster-test/postgresql/docker-compose-base.yaml
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml
similarity index 100%
rename from .github/workflows/cluster-test/postgresql/docker-compose-base.yaml
rename to
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml
diff --git
a/.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml
similarity index 100%
rename from
.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml
rename to
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml
diff --git a/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh
similarity index 100%
rename from .github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh
rename to
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh
diff --git a/.github/workflows/cluster-test/postgresql/install_env.sh
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh
similarity index 100%
rename from .github/workflows/cluster-test/postgresql/install_env.sh
rename to
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh
diff --git a/.github/workflows/cluster-test/postgresql/running_test.sh
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
similarity index 100%
rename from .github/workflows/cluster-test/postgresql/running_test.sh
rename to
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
diff --git a/.github/workflows/cluster-test/postgresql/start-job.sh
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh
similarity index 73%
copy from .github/workflows/cluster-test/postgresql/start-job.sh
copy to
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh
index ba0878e3ec..fe755c97f1 100644
--- a/.github/workflows/cluster-test/postgresql/start-job.sh
+++
b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh
@@ -18,16 +18,16 @@
set -euox pipefail
#Start base service containers
-docker-compose -f
.github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d
+docker-compose -f
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml
up -d
#Build ds postgresql cluster image
-docker build -t jdk8:ds_postgresql_cluster -f
.github/workflows/cluster-test/postgresql/Dockerfile .
+docker build -t jdk8:ds_postgresql_cluster -f
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile .
#Start ds postgresql cluster container
-docker-compose -f
.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d
+docker-compose -f
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml
up -d
#Running tests
-/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh
+/bin/bash
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
#Cleanup
docker rm -f $(docker ps -aq)
diff --git
a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
index fd3d4b02e3..ff0088ea8d 100644
---
a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
+++
b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
@@ -27,21 +27,24 @@ import
org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
import
org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
+import
org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration;
+import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.event.EventListener;
+import org.springframework.context.annotation.FilterType;
-@SpringBootApplication
-@ComponentScan("org.apache.dolphinscheduler")
@Slf4j
+@SpringBootApplication
+@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = {
+ @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value =
JdbcRegistryAutoConfiguration.class)
+})
public class AlertServer {
@Autowired
@@ -59,11 +62,11 @@ public class AlertServer {
AlertServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount);
Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance());
Thread.currentThread().setName(Constants.THREAD_NAME_ALERT_SERVER);
- new SpringApplicationBuilder(AlertServer.class).run(args);
+ SpringApplication.run(AlertServer.class, args);
}
- @EventListener
- public void run(ApplicationReadyEvent readyEvent) {
+ @PostConstruct
+ public void run() {
log.info("Alert server is staring ...");
alertPluginManager.start();
alertRegistryClient.start();
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
index c7e6d9778f..20a6412076 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
@@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.PluginType;
import
org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
+import
org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
@@ -38,11 +39,14 @@ import
org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.boot.web.servlet.ServletComponentScan;
import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
import org.springframework.context.event.EventListener;
@ServletComponentScan
@SpringBootApplication
-@ComponentScan("org.apache.dolphinscheduler")
+@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = {
+ @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value =
JdbcRegistryAutoConfiguration.class)
+})
@Slf4j
public class ApiApplicationServer {
diff --git
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java
index 71e3be70c4..24cb022881 100644
---
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java
+++
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java
@@ -29,10 +29,10 @@ import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
+import org.springframework.stereotype.Repository;
@Slf4j
-@Component
+@Repository
public class PluginDao {
@Autowired
diff --git
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java
similarity index 95%
rename from
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java
rename to
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java
index f2cb503d9b..c0a841c1a0 100644
---
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java
+++
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java
@@ -15,10 +15,12 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao;
+package org.apache.dolphinscheduler.dao.repository.impl;
import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.enums.ProfileType;
+import org.apache.dolphinscheduler.dao.AlertDao;
+import org.apache.dolphinscheduler.dao.DaoConfiguration;
import org.apache.dolphinscheduler.dao.entity.Alert;
import java.util.List;
diff --git
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 752479e600..fd262a8c6e 100644
---
a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++
b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -24,6 +24,7 @@ import
org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.meter.metrics.MetricsProvider;
import org.apache.dolphinscheduler.meter.metrics.SystemMetrics;
+import
org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.scheduler.api.SchedulerApi;
import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
@@ -46,10 +47,13 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
import org.springframework.transaction.annotation.EnableTransactionManagement;
@SpringBootApplication
-@ComponentScan("org.apache.dolphinscheduler")
+@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = {
+ @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value =
JdbcRegistryAutoConfiguration.class)
+})
@EnableTransactionManagement
@EnableCaching
@Slf4j
diff --git
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md
index 3b1a2cb24f..554c375218 100644
---
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md
+++
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md
@@ -1,6 +1,7 @@
# Introduction
-This module is the jdbc registry plugin module, this plugin will use jdbc as
the registry center. Will use the database configuration same as
DolphinScheduler in api'yaml default.
+This module is the jdbc registry plugin module, this plugin will use jdbc as
the registry center. Will use the database
+configuration same as DolphinScheduler in api'yaml default.
# How to use
@@ -22,8 +23,11 @@ registry:
After do this two steps, you can start your DolphinScheduler cluster, your
cluster will use mysql as registry center to
store server metadata.
-NOTE: You need to add `mysql-connector-java.jar` into DS classpath if you use
mysql database, since this plugin will not bundle this driver in distribution.
-You can get the detail about <a
href="https://dolphinscheduler.apache.org/en-us/docs/3.1.2/guide/installation/pseudo-cluster";>Initialize
the Database</a>.
+NOTE: You need to add `mysql-connector-java.jar` into DS classpath if you use
mysql database, since this plugin will not
+bundle this driver in distribution.
+You can get the detail
+about <a
href="https://dolphinscheduler.apache.org/en-us/docs/3.1.2/guide/installation/pseudo-cluster";>Initialize
the
+Database</a>.
## Optional configuration
diff --git
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml
index 47b6449293..d4285edfbd 100644
---
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml
+++
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/pom.xml
@@ -62,6 +62,16 @@
<artifactId>mybatis-plus</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.baomidou</groupId>
+ <artifactId>mybatis-plus-boot-starter</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
</project>
diff --git
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConfiguration.java
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java
similarity index 65%
rename from
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConfiguration.java
rename to
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java
index 7b37749ab7..09211f99fb 100644
---
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryConfiguration.java
+++
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java
@@ -22,40 +22,56 @@ import
org.apache.dolphinscheduler.plugin.registry.jdbc.mapper.JdbcRegistryLockM
import org.apache.ibatis.session.SqlSessionFactory;
+import lombok.extern.slf4j.Slf4j;
+
import org.mybatis.spring.SqlSessionTemplate;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.autoconfigure.AutoConfigureAfter;
+import
org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import com.baomidou.mybatisplus.autoconfigure.MybatisPlusAutoConfiguration;
import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
import com.zaxxer.hikari.HikariDataSource;
-@Configuration
+@Slf4j
+@Configuration(proxyBeanMethods = false)
+@MapperScan("org.apache.dolphinscheduler.plugin.registry.jdbc.mapper")
@ConditionalOnProperty(prefix = "registry", name = "type", havingValue =
"jdbc")
-public class JdbcRegistryConfiguration {
+@AutoConfigureAfter(MybatisPlusAutoConfiguration.class)
+public class JdbcRegistryAutoConfiguration {
+
+ public JdbcRegistryAutoConfiguration() {
+ log.info("Load JdbcRegistryAutoConfiguration");
+ }
@Bean
- @ConditionalOnProperty(prefix = "registry.hikari-config", name =
"jdbc-url")
- public SqlSessionFactory
jdbcRegistrySqlSessionFactory(JdbcRegistryProperties jdbcRegistryProperties)
throws Exception {
+ @ConditionalOnMissingBean
+ public SqlSessionFactory sqlSessionFactory(JdbcRegistryProperties
jdbcRegistryProperties) throws Exception {
+ log.info("Initialize jdbcRegistrySqlSessionFactory");
MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new
MybatisSqlSessionFactoryBean();
sqlSessionFactoryBean.setDataSource(new
HikariDataSource(jdbcRegistryProperties.getHikariConfig()));
return sqlSessionFactoryBean.getObject();
}
@Bean
- public SqlSessionTemplate jdbcRegistrySqlSessionTemplate(SqlSessionFactory
jdbcRegistrySqlSessionFactory) {
-
jdbcRegistrySqlSessionFactory.getConfiguration().addMapper(JdbcRegistryDataMapper.class);
-
jdbcRegistrySqlSessionFactory.getConfiguration().addMapper(JdbcRegistryLockMapper.class);
+ @ConditionalOnMissingBean
+ public SqlSessionTemplate sqlSessionTemplate(SqlSessionFactory
jdbcRegistrySqlSessionFactory) {
+ log.info("Initialize jdbcRegistrySqlSessionTemplate");
return new SqlSessionTemplate(jdbcRegistrySqlSessionFactory);
}
@Bean
public JdbcRegistryDataMapper jdbcRegistryDataMapper(SqlSessionTemplate
jdbcRegistrySqlSessionTemplate) {
+
jdbcRegistrySqlSessionTemplate.getConfiguration().addMapper(JdbcRegistryDataMapper.class);
return
jdbcRegistrySqlSessionTemplate.getMapper(JdbcRegistryDataMapper.class);
}
@Bean
public JdbcRegistryLockMapper jdbcRegistryLockMapper(SqlSessionTemplate
jdbcRegistrySqlSessionTemplate) {
+
jdbcRegistrySqlSessionTemplate.getConfiguration().addMapper(JdbcRegistryLockMapper.class);
return
jdbcRegistrySqlSessionTemplate.getMapper(JdbcRegistryLockMapper.class);
}
diff --git a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories
similarity index 79%
rename from .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
rename to
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories
index 7343c8eee7..aabe7e3252 100644
--- a/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml
+++
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/META-INF/spring.factories
@@ -1,29 +1,19 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-version: "3"
-
-services:
- ds:
- container_name: ds
- image: jdk8:ds_mysql_cluster
- restart: always
- ports:
- - "12345:12345"
- - "5679:5679"
- - "1235:1235"
- - "50053:50053"
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
+
org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration
diff --git
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql
index 30af3066ff..6df206b391 100644
---
a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql
+++
b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/resources/mysql_registry_init.sql
@@ -15,7 +15,6 @@
* limitations under the License.
*/
-SET FOREIGN_KEY_CHECKS = 0;
DROP TABLE IF EXISTS `t_ds_jdbc_registry_data`;
CREATE TABLE `t_ds_jdbc_registry_data`
diff --git a/.github/workflows/cluster-test/postgresql/start-job.sh
b/dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh
similarity index 55%
rename from .github/workflows/cluster-test/postgresql/start-job.sh
rename to dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh
index ba0878e3ec..895d62a0bb 100644
--- a/.github/workflows/cluster-test/postgresql/start-job.sh
+++ b/dolphinscheduler-tools/src/main/bin/initialize-jdbc-registry.sh
@@ -15,19 +15,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -euox pipefail
-#Start base service containers
-docker-compose -f
.github/workflows/cluster-test/postgresql/docker-compose-base.yaml up -d
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)}
-#Build ds postgresql cluster image
-docker build -t jdk8:ds_postgresql_cluster -f
.github/workflows/cluster-test/postgresql/Dockerfile .
+if [ "$DOCKER" != "true" ]; then
+ source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh"
+fi
-#Start ds postgresql cluster container
-docker-compose -f
.github/workflows/cluster-test/postgresql/docker-compose-cluster.yaml up -d
+JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE}
-Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
-#Running tests
-/bin/bash .github/workflows/cluster-test/postgresql/running_test.sh
-
-#Cleanup
-docker rm -f $(docker ps -aq)
+$JAVA_HOME/bin/java $JAVA_OPTS \
+ -cp
"$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql"
\
+ -Dspring.profiles.active=${DATABASE} \
+ org.apache.dolphinscheduler.tools.command.CommandApplication
diff --git
a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java
b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java
new file mode 100644
index 0000000000..1e419f4d19
--- /dev/null
+++
b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/command/CommandApplication.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.tools.command;
+
+import org.apache.dolphinscheduler.dao.DaoConfiguration;
+import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import javax.sql.DataSource;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.stereotype.Component;
+
+import com.baomidou.mybatisplus.annotation.DbType;
+
+// todo: use spring-shell to manage the command
+@SpringBootApplication
+@ImportAutoConfiguration(DaoConfiguration.class)
+public class CommandApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(CommandApplication.class, args);
+ }
+
+ @Component
+ @Slf4j
+ static class JdbcRegistrySchemaInitializeCommand implements
CommandLineRunner {
+
+ @Autowired
+ private DatabaseDialect databaseDialect;
+
+ @Autowired
+ private DbType dbType;
+
+ @Autowired
+ private DataSource dataSource;
+
+ JdbcRegistrySchemaInitializeCommand() {
+ }
+
+ @Override
+ public void run(String... args) throws Exception {
+ if (databaseDialect.tableExists("t_ds_jdbc_registry_data")
+ || databaseDialect.tableExists("t_ds_jdbc_registry_lock"))
{
+ log.warn("t_ds_jdbc_registry_data/t_ds_jdbc_registry_lock
already exists");
+ return;
+ }
+ if (dbType == DbType.MYSQL) {
+ jdbcRegistrySchemaInitializeInMysql();
+ } else if (dbType == DbType.POSTGRE_SQL) {
+ jdbcRegistrySchemaInitializeInPG();
+ } else {
+ log.error("Unsupported database type: {}", dbType);
+ }
+ }
+
+ private void jdbcRegistrySchemaInitializeInMysql() throws SQLException
{
+ try (
+ Connection connection = dataSource.getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TABLE `t_ds_jdbc_registry_data`\n" +
+ "(\n" +
+ " `id` bigint(11) NOT NULL
AUTO_INCREMENT COMMENT 'primary key',\n" +
+ " `data_key` varchar(256) NOT NULL COMMENT
'key, like zookeeper node path',\n" +
+ " `data_value` text NOT NULL COMMENT
'data, like zookeeper node value',\n" +
+ " `data_type` tinyint(4) NOT NULL COMMENT
'1: ephemeral node, 2: persistent node',\n"
+ +
+ " `last_term` bigint NOT NULL COMMENT
'last term time',\n" +
+ " `last_update_time` timestamp NOT NULL DEFAULT
CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',\n"
+ +
+ " `create_time` timestamp NOT NULL DEFAULT
CURRENT_TIMESTAMP COMMENT 'create time',\n"
+ +
+ " PRIMARY KEY (`id`),\n" +
+ " unique (`data_key`)\n" +
+ ") ENGINE = InnoDB\n" +
+ " DEFAULT CHARSET = utf8;");
+
+ statement.execute("CREATE TABLE `t_ds_jdbc_registry_lock`\n" +
+ "(\n" +
+ " `id` bigint(11) NOT NULL
AUTO_INCREMENT COMMENT 'primary key',\n" +
+ " `lock_key` varchar(256) NOT NULL COMMENT
'lock path',\n" +
+ " `lock_owner` varchar(256) NOT NULL COMMENT
'the lock owner, ip_processId',\n" +
+ " `last_term` bigint NOT NULL COMMENT
'last term time',\n" +
+ " `last_update_time` timestamp NOT NULL DEFAULT
CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',\n"
+ +
+ " `create_time` timestamp NOT NULL DEFAULT
CURRENT_TIMESTAMP COMMENT 'create time',\n"
+ +
+ " PRIMARY KEY (`id`),\n" +
+ " unique (`lock_key`)\n" +
+ ") ENGINE = InnoDB\n" +
+ " DEFAULT CHARSET = utf8;");
+ }
+ }
+
+ private void jdbcRegistrySchemaInitializeInPG() throws SQLException {
+ try (
+ Connection connection = dataSource.getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("create table t_ds_jdbc_registry_data\n" +
+ "(\n" +
+ " id serial\n" +
+ " constraint t_ds_jdbc_registry_data_pk primary
key,\n" +
+ " data_key varchar
not null,\n" +
+ " data_value text
not null,\n" +
+ " data_type int4
not null,\n" +
+ " last_term bigint
not null,\n" +
+ " last_update_time timestamp default
current_timestamp not null,\n" +
+ " create_time timestamp default
current_timestamp not null\n" +
+ ");");
+ statement.execute(
+ "create unique index
t_ds_jdbc_registry_data_key_uindex on t_ds_jdbc_registry_data (data_key);");
+ statement.execute("create table t_ds_jdbc_registry_lock\n" +
+ "(\n" +
+ " id serial\n" +
+ " constraint t_ds_jdbc_registry_lock_pk primary
key,\n" +
+ " lock_key varchar
not null,\n" +
+ " lock_owner varchar
not null,\n" +
+ " last_term bigint
not null,\n" +
+ " last_update_time timestamp default
current_timestamp not null,\n" +
+ " create_time timestamp default
current_timestamp not null\n" +
+ ");");
+ statement.execute(
+ "create unique index
t_ds_jdbc_registry_lock_key_uindex on t_ds_jdbc_registry_lock (lock_key);");
+ }
+ }
+
+ }
+}
diff --git a/dolphinscheduler-tools/src/main/resources/application.yaml
b/dolphinscheduler-tools/src/main/resources/application.yaml
index 38752021dc..136a4c5fd4 100644
--- a/dolphinscheduler-tools/src/main/resources/application.yaml
+++ b/dolphinscheduler-tools/src/main/resources/application.yaml
@@ -63,6 +63,8 @@ spring:
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url:
jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+ username: root
+ password: root
---
spring:
diff --git
a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
index 86e755fc8a..b0866f7fd8 100644
---
a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
+++
b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
@@ -24,6 +24,7 @@ import
org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.meter.metrics.MetricsProvider;
import org.apache.dolphinscheduler.meter.metrics.SystemMetrics;
+import
org.apache.dolphinscheduler.plugin.registry.jdbc.JdbcRegistryAutoConfiguration;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
@@ -47,11 +48,14 @@ import
org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.FilterType;
import org.springframework.transaction.annotation.EnableTransactionManagement;
@SpringBootApplication
@EnableTransactionManagement
-@ComponentScan(basePackages = "org.apache.dolphinscheduler")
+@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = {
+ @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value =
JdbcRegistryAutoConfiguration.class)
+})
@Slf4j
public class WorkerServer implements IStoppable {
