This is an automated email from the ASF dual-hosted git repository.
zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new abe1185d28 [Improvement-16635] Using Docker Hub in Schema Check CI
(#16636)
abe1185d28 is described below
commit abe1185d2895edf474591ca2c638bbfbebccd4e7
Author: xiangzihao <[email protected]>
AuthorDate: Sun Sep 22 10:45:05 2024 +0800
[Improvement-16635] Using Docker Hub in Schema Check CI (#16636)
---
.github/workflows/backend.yml | 38 +---------------------
.../schema-check/mysql/docker-compose-base.yaml | 6 ++++
.../workflows/schema-check/mysql/running-test.sh | 9 ++++-
.github/workflows/schema-check/mysql/start-job.sh | 21 +++---------
.../postgresql/docker-compose-base.yaml | 6 ++++
.../schema-check/postgresql/running-test.sh | 8 ++++-
.../workflows/schema-check/postgresql/start-job.sh | 14 ++------
7 files changed, 34 insertions(+), 68 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index ab336eefea..e7b282a29c 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -124,39 +124,9 @@ jobs:
- name: Running cluster test
run: |
/bin/bash ${{ matrix.case.script }}
- schema-check-prepare:
- name: schema-check-prepare-${{ matrix.version }}
- needs: paths-filter
- runs-on: ubuntu-latest
- timeout-minutes: 30
- strategy:
- fail-fast: false
- matrix:
- version: [ "3.1.9", "3.2.0" ]
- steps:
- - uses: actions/cache/restore@v4
- name: Download Binary Package Cache
- with:
- path: /tmp/${{ matrix.version }}
- key: apache-dolphinscheduler-${{ matrix.version }}-bin
- restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
- - name: Download Binary Package if not cached
- run: |
- if [ ! -f /tmp/${{ matrix.version }}/apache-dolphinscheduler-${{
matrix.version }}-bin.tar.gz ]; then
- echo "Binary package not found in cache, downloading..."
- mkdir -p /tmp/${{ matrix.version }}
- wget
https://mirrors.tuna.tsinghua.edu.cn/apache/dolphinscheduler/${{ matrix.version
}}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P /tmp/${{
matrix.version }}
- else
- echo "Binary package found in cache, skipping download."
- fi
- - uses: actions/cache/save@v4
- name: Upload Binary Package Cache
- with:
- path: /tmp/${{ matrix.version }}
- key: apache-dolphinscheduler-${{ matrix.version }}-bin
schema-check:
name: ${{ matrix.case.name }}-${{ matrix.version }}
- needs: [ build, schema-check-prepare]
+ needs: [ build ]
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
@@ -181,12 +151,6 @@ jobs:
with:
name: binary-package-8
path: ds_schema_check_test/dev
- - uses: actions/cache/restore@v4
- name: Download Binary Package Cache
- with:
- path: /tmp/${{ matrix.version }}
- key: apache-dolphinscheduler-${{ matrix.version }}-bin
- restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
- name: Running Schema Check
run: |
/bin/bash ${{ matrix.case.script }} ${{ matrix.version }}
diff --git a/.github/workflows/schema-check/mysql/docker-compose-base.yaml
b/.github/workflows/schema-check/mysql/docker-compose-base.yaml
index d515a91762..4223f99fda 100644
--- a/.github/workflows/schema-check/mysql/docker-compose-base.yaml
+++ b/.github/workflows/schema-check/mysql/docker-compose-base.yaml
@@ -28,8 +28,14 @@ services:
MYSQL_DATABASE: dolphinscheduler_dev
ports:
- "3306:3306"
+ networks:
+ - schema-test
healthcheck:
test: mysqladmin ping -h 127.0.0.1 -u root
--password=$$MYSQL_ROOT_PASSWORD
interval: 5s
timeout: 60s
retries: 120
+
+networks:
+ schema-test:
+ name: schema-test
diff --git a/.github/workflows/schema-check/mysql/running-test.sh
b/.github/workflows/schema-check/mysql/running-test.sh
index 72e2fb3b6a..cc0197b9f0 100644
--- a/.github/workflows/schema-check/mysql/running-test.sh
+++ b/.github/workflows/schema-check/mysql/running-test.sh
@@ -29,8 +29,15 @@ export SPRING_DATASOURCE_PASSWORD="mysql"
bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
# Install the target version schema and upgrade it
+docker run -v
"./ds_schema_check_test/mysql-connector-java-8.0.16.jar:/opt/dolphinscheduler/tools/libs/mysql-connector-java-8.0.16.jar"
\
+--network schema-test apache/dolphinscheduler-tools:${DS_VERSION} -c \
+'export DATABASE="mysql"; \
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"; \
+export SPRING_DATASOURCE_USERNAME="root"; \
+export SPRING_DATASOURCE_PASSWORD="mysql"; \
+export
SPRING_DATASOURCE_URL='jdbc:mysql://mysql:3306/dolphinscheduler_${DATABASE_VERSION}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false';
\
+bash tools/bin/upgrade-schema.sh'
export
SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${DATABASE_VERSION}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
-bash ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
# Compare the schema
diff --git a/.github/workflows/schema-check/mysql/start-job.sh
b/.github/workflows/schema-check/mysql/start-job.sh
index 852c7f8a0e..79f2b28c83 100644
--- a/.github/workflows/schema-check/mysql/start-job.sh
+++ b/.github/workflows/schema-check/mysql/start-job.sh
@@ -21,35 +21,22 @@ DS_VERSION=$1
DATABASE_VERSION=${DS_VERSION//\./}
# Install Atlas and Create Dir
-mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
+mkdir -p ds_schema_check_test/dev
curl -sSf https://atlasgo.sh | sh
# Preparing the environment
-if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz
]; then
- wget
https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz
-P ds_schema_check_test/${DS_VERSION}
-else
- mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz
ds_schema_check_test/${DS_VERSION}
-fi
-tar -xzf
ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz
-C ds_schema_check_test/${DS_VERSION} --strip-components 1
+docker pull apache/dolphinscheduler-tools:${DS_VERSION}
tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C
ds_schema_check_test/dev --strip-components 1
-if [[ $DATABASE_VERSION -lt 300 ]]; then
- chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
ds_schema_check_test/${DS_VERSION}/script/create-dolphinscheduler.sh
-else
- chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
-fi
+chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar";
MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
wget ${MYSQL_JDBC_URL} -O ds_schema_check_test/${MYSQL_JDBC_JAR}
-for base_dir in ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}; do
- if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
- cp ds_schema_check_test/${MYSQL_JDBC_JAR} ${base_dir}/lib
- else
+for base_dir in ds_schema_check_test/dev; do
for d in alert-server api-server master-server worker-server tools; do
cp ds_schema_check_test/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
done
- fi
done
docker compose -f
.github/workflows/schema-check/mysql/docker-compose-base.yaml up -d --wait
docker exec -i mysql mysql -uroot -pmysql -e "create database
dolphinscheduler_${DATABASE_VERSION}";
diff --git a/.github/workflows/schema-check/postgresql/docker-compose-base.yaml
b/.github/workflows/schema-check/postgresql/docker-compose-base.yaml
index 9f09f0d326..1bcae2fbaf 100644
--- a/.github/workflows/schema-check/postgresql/docker-compose-base.yaml
+++ b/.github/workflows/schema-check/postgresql/docker-compose-base.yaml
@@ -27,8 +27,14 @@ services:
POSTGRES_DB: dolphinscheduler_dev
ports:
- "5432:5432"
+ networks:
+ - schema-test
healthcheck:
test: [ "CMD-SHELL", "pg_isready -U postgres" ]
interval: 5s
timeout: 60s
retries: 120
+
+networks:
+ schema-test:
+ name: schema-test
diff --git a/.github/workflows/schema-check/postgresql/running-test.sh
b/.github/workflows/schema-check/postgresql/running-test.sh
index 0118ca2477..889097f3fe 100644
--- a/.github/workflows/schema-check/postgresql/running-test.sh
+++ b/.github/workflows/schema-check/postgresql/running-test.sh
@@ -29,8 +29,14 @@ export
SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_
bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
# Install the target version schema and upgrade it
+docker run --network schema-test apache/dolphinscheduler-tools:${DS_VERSION}
-c \
+'export DATABASE="postgresql"; \
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"; \
+export SPRING_DATASOURCE_USERNAME="postgres"; \
+export SPRING_DATASOURCE_PASSWORD="postgres"; \
+export
SPRING_DATASOURCE_URL='jdbc:postgresql://postgres:5432/dolphinscheduler_${DATABASE_VERSION}';
\
+bash tools/bin/upgrade-schema.sh'
export
SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${DATABASE_VERSION}"
-bash ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
# Compare the schema
diff --git a/.github/workflows/schema-check/postgresql/start-job.sh
b/.github/workflows/schema-check/postgresql/start-job.sh
index 26308923e2..cf46dcd6f9 100644
--- a/.github/workflows/schema-check/postgresql/start-job.sh
+++ b/.github/workflows/schema-check/postgresql/start-job.sh
@@ -21,23 +21,13 @@ DS_VERSION=$1
DATABASE_VERSION=${DS_VERSION//\./}
# Install Atlas and Create Dir
-mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
+mkdir -p ds_schema_check_test/dev
curl -sSf https://atlasgo.sh | sh
# Preparing the environment
-if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz
]; then
- wget
https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz
-P ds_schema_check_test/${DS_VERSION}
-else
- mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz
ds_schema_check_test/${DS_VERSION}
-fi
-tar -xzf
ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz
-C ds_schema_check_test/${DS_VERSION} --strip-components 1
tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C
ds_schema_check_test/dev --strip-components 1
-if [[ $DATABASE_VERSION -lt 300 ]]; then
- chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
ds_schema_check_test/${DS_VERSION}/script/create-dolphinscheduler.sh
-else
- chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
-fi
+chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
docker compose -f
.github/workflows/schema-check/postgresql/docker-compose-base.yaml up -d --wait
docker exec -i postgres psql -U postgres -c "create database
dolphinscheduler_${DATABASE_VERSION}";
