This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new a89b86cef2 [Improve][CI] Optimizing CI time cost (#7997)
a89b86cef2 is described below
commit a89b86cef22a4deb1d4395323a58ae56915247de
Author: Jia Fan <[email protected]>
AuthorDate: Mon Nov 11 20:41:40 2024 +0800
[Improve][CI] Optimizing CI time cost (#7997)
---
.github/workflows/backend.yml | 123 ++++++++++++--------
.github/workflows/notify_test_workflow.yml | 2 +-
.github/workflows/schedule_backend.yml | 127 ++-------------------
.../e2e/common/container/TestContainerId.java | 19 +--
.../seatunnel/e2e/common/util/ContainerUtil.java | 34 +++++-
.../seatunnel/engine/e2e/joblog/JobLogIT.java | 7 +-
.../engine/core/job/ExecutionAddress.java | 4 +-
.../seatunnel/engine/core/job/JobDAGInfo.java | 1 +
.../seatunnel/engine/server/dag/DAGUtils.java | 3 +
.../engine/server/master/JobHistoryService.java | 33 +++---
.../seatunnel/engine/server/master/JobMaster.java | 3 +
.../engine/server/rest/service/BaseService.java | 4 +
tools/update_modules_check/update_modules_check.py | 4 +
13 files changed, 172 insertions(+), 192 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 67b2511f10..6f51b0a9e9 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -19,11 +19,19 @@ name: Backend
on:
workflow_call:
+ inputs:
+ TEST_IN_PR:
+ required: false
+ type: string
+ default: 'true'
concurrency:
group: backend-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
+env:
+ TEST_IN_PR: ${{ inputs.TEST_IN_PR }}
+
jobs:
license-header:
name: License header
@@ -137,7 +145,7 @@ jobs:
echo "engine-e2e=$true_or_false" >> $GITHUB_OUTPUT
echo "engine-e2e_files=$file_list" >> $GITHUB_OUTPUT
- api_files=`python tools/update_modules_check/check_file_updates.py
ua $workspace apache/dev origin/$current_branch "seatunnel-api/**"
"seatunnel-common/**" "seatunnel-config/**" "seatunnel-engine/**"
"seatunnel-core/**" "seatunnel-e2e/seatunnel-e2e-common/**"
"seatunnel-formats/**" "seatunnel-plugin-discovery/**"
"seatunnel-transforms-v2/**" "seatunnel-translation/**"
"seatunnel-e2e/seatunnel-transforms-v2-e2e/**" "pom.xml" "**/workflows/**"
"tools/**" "seatunnel-dist/**"`
+ api_files=`python tools/update_modules_check/check_file_updates.py
ua $workspace apache/dev origin/$current_branch "seatunnel-api/**"
"seatunnel-common/**" "seatunnel-config/**" "seatunnel-core/**"
"seatunnel-e2e/seatunnel-e2e-common/**" "seatunnel-formats/**"
"seatunnel-plugin-discovery/**" "seatunnel-transforms-v2/**"
"seatunnel-translation/**" "seatunnel-e2e/seatunnel-transforms-v2-e2e/**"
"pom.xml" "**/workflows/**" "tools/**" "seatunnel-dist/**"`
true_or_false=${api_files%%$'\n'*}
file_list=${api_files#*$'\n'}
if [[ $repository_owner == 'apache' ]];then
@@ -224,7 +232,7 @@ jobs:
- name: Make integration test modules
id: it-modules
timeout-minutes: 60
- if: ${{ steps.filter.outputs.api == 'false' &&
(steps.engine-modules.outputs.modules != '' ||
steps.cv2-modules.outputs.modules != '' ||
steps.cv2-e2e-modules.outputs.modules != '' ||
steps.cv2-flink-e2e-modules.outputs.modules != '' ||
steps.cv2-spark-e2e-modules.outputs.modules != '' ||
steps.engine-e2e-modules.outputs.modules != '') }}
+ if: ${{ steps.filter.outputs.api == 'false' &&
(steps.engine-modules.outputs.modules != '' ||
steps.cv2-modules.outputs.modules != '' ||
steps.cv2-e2e-modules.outputs.modules != '' ||
steps.cv2-flink-e2e-modules.outputs.modules != '' ||
steps.cv2-spark-e2e-modules.outputs.modules != '') }}
run: |
modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{
steps.cv2-flink-e2e-modules.outputs.modules }}${{
steps.cv2-spark-e2e-modules.outputs.modules }}${{
steps.engine-e2e-modules.outputs.modules }}${{
steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
modules=${modules: 1}
@@ -270,7 +278,7 @@ jobs:
max_attempts: 3
retry_on: error
command: |
- ./mvnw -B -q install -DskipTests -D"maven.test.skip"=true
-D"maven.javadoc.skip"=true -D"license.skipAddThirdParty"
+ ./mvnw -B install -DskipTests -D"maven.test.skip"=true
-D"maven.javadoc.skip"=true -D"license.skipAddThirdParty"
- name: Check Dependencies Licenses
run: tools/dependencies/checkLicense.sh
@@ -376,7 +384,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-1)
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 0`
if [ ! -z $sub_modules ]; then
@@ -407,7 +414,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-2)
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 1`
if [ ! -z $sub_modules ]; then
@@ -438,7 +444,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-3)
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 2`
if [ ! -z $sub_modules ]; then
@@ -469,7 +474,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-4)
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 3`
if [ ! -z $sub_modules ]; then
@@ -499,7 +503,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-5)
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 4`
if [ ! -z $sub_modules ]; then
@@ -529,7 +532,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-6)
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 5`
if [ ! -z $sub_modules ]; then
@@ -559,7 +561,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-7)
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 6`
if [ ! -z $sub_modules ]; then
@@ -590,7 +591,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-8)
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 7`
if [ ! -z $sub_modules ]; then
@@ -603,7 +603,7 @@ jobs:
engine-v2-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || needs.changes.outputs.engine-e2e == 'true'
runs-on: ${{ matrix.os }}
strategy:
matrix:
@@ -621,7 +621,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run seatunnel zeta integration test
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-seatunnel-e2e-base,:connector-console-seatunnel-e2e -am -Pci
env:
@@ -661,8 +660,11 @@ jobs:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
transform-v2-it-part-1:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -679,7 +681,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run transform-v2 integration test (part-1)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:seatunnel-transforms-v2-e2e-part-1 -am -Pci
env:
@@ -687,8 +688,11 @@ jobs:
transform-v2-it-part-2:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -705,7 +709,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run transform-v2 integration test (part-2)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:seatunnel-transforms-v2-e2e-part-2 -am -Pci
env:
@@ -713,8 +716,11 @@ jobs:
all-connectors-it-1:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -731,7 +737,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run connector-v2 integration test (part-1)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
@@ -742,8 +747,11 @@ jobs:
all-connectors-it-2:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -760,7 +768,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run connector-v2 integration test (part-2)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
@@ -771,8 +778,11 @@ jobs:
all-connectors-it-3:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -789,7 +799,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run connector-v2 integration test (part-3)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
@@ -800,8 +809,11 @@ jobs:
all-connectors-it-4:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -818,7 +830,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run connector-v2 integration test (part-4)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
@@ -829,8 +840,11 @@ jobs:
all-connectors-it-5:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -847,7 +861,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run connector-v2 integration test (part-5)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
@@ -858,8 +871,11 @@ jobs:
all-connectors-it-6:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -876,7 +892,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run connector-v2 integration test (part-6)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
@@ -887,8 +902,11 @@ jobs:
all-connectors-it-7:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -905,7 +923,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run connector-v2 integration test (part-7)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
@@ -916,8 +933,11 @@ jobs:
jdbc-connectors-it-part-1:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -934,7 +954,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-1)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-1 -am -Pci
env:
@@ -942,8 +961,11 @@ jobs:
jdbc-connectors-it-part-2:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -960,7 +982,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-2)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-2 -am -Pci
env:
@@ -968,8 +989,11 @@ jobs:
jdbc-connectors-it-part-3:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -986,7 +1010,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-3)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-3 -am -Pci
env:
@@ -994,8 +1017,11 @@ jobs:
jdbc-connectors-it-part-4:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -1012,7 +1038,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-4)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-4 -am -Pci
env:
@@ -1020,8 +1045,11 @@ jobs:
jdbc-connectors-it-part-5:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -1038,7 +1066,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-5)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-5 -am -Pci
env:
@@ -1046,8 +1073,11 @@ jobs:
jdbc-connectors-it-part-6:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -1064,7 +1094,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-6)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-6 -am -Pci
env:
@@ -1072,8 +1101,11 @@ jobs:
jdbc-connectors-it-part-7:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
strategy:
matrix:
java: [ '8', '11' ]
@@ -1090,7 +1122,6 @@ jobs:
- name: free disk space
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-7)
- if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-7 -am -Pci
env:
@@ -1104,7 +1135,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 120
+ timeout-minutes: 30
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
diff --git a/.github/workflows/notify_test_workflow.yml
b/.github/workflows/notify_test_workflow.yml
index 58d1b5f2be..797d067257 100644
--- a/.github/workflows/notify_test_workflow.yml
+++ b/.github/workflows/notify_test_workflow.yml
@@ -80,7 +80,7 @@ jobs:
status = 'completed'
const conclusion = 'action_required'
- github.rest.checks.create({
+ await github.rest.checks.create({
owner: context.repo.owner,
repo: context.repo.repo,
name: name,
diff --git a/.github/workflows/schedule_backend.yml
b/.github/workflows/schedule_backend.yml
index 503a3866df..1d2bc3fe84 100644
--- a/.github/workflows/schedule_backend.yml
+++ b/.github/workflows/schedule_backend.yml
@@ -18,128 +18,17 @@
name: Schedule Backend
on:
schedule:
- - cron: '0 0 03 * *'
+ - cron: '0 16 * * *'
concurrency:
group: backend-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
- license-header:
- if: github.repository == 'apache/seatunnel'
- name: License header
- runs-on: ubuntu-latest
- timeout-minutes: 10
- steps:
- - uses: actions/checkout@v3
- with:
- submodules: true
- - name: Check license header
- uses: apache/skywalking-eyes@985866ce7e324454f61e22eb2db2e998db09d6f3
-
- code-style:
- if: github.repository == 'apache/seatunnel'
- name: Code style
- runs-on: ubuntu-latest
- timeout-minutes: 10
- steps:
- - uses: actions/checkout@v3
- with:
- submodules: true
- - name: Check code style
- run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean
spotless:check
-
- dead-link:
- if: github.repository == 'apache/seatunnel'
- name: Dead links
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v2
- - run: sudo npm install -g [email protected]
- - run: |
- for file in $(find . -name "*.md"); do
- markdown-link-check -c .dlc.json -q "$file"
- done
-
- sanity-check:
- if: github.repository == 'apache/seatunnel'
- name: Sanity check results
- needs: [ license-header, code-style, dead-link ]
- runs-on: ubuntu-latest
- timeout-minutes: 10
- steps:
- - name: Check results
- run: |
- [[ ${{ needs.license-header.result }} == 'success' ]] || exit 1;
- [[ ${{ needs.code-style.result }} == 'success' ]] || exit 1;
- [[ ${{ needs.dead-link.result }} == 'success' ]] || exit 1;
-
- dependency-license:
- name: Dependency licenses
- needs: [ sanity-check ]
- runs-on: ubuntu-latest
- timeout-minutes: 40
- steps:
- - uses: actions/checkout@v3
- with:
- submodules: true
- - uses: actions/setup-java@v3
- with:
- distribution: 'temurin'
- java-version: '8'
- cache: 'maven'
- - name: Install
- run: >-
- ./mvnw -B -q install -DskipTests
- -D"maven.test.skip"=true
- -D"maven.javadoc.skip"=true
- -D"scalastyle.skip"=true
- -D"license.skipAddThirdParty"
- - name: Check Dependencies Licenses
- run: tools/dependencies/checkLicense.sh
-
- unit-test:
- needs: [ sanity-check ]
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- java: [ '8', '11' ]
- os: [ 'ubuntu-latest', 'windows-latest' ]
- timeout-minutes: 90
- steps:
- - uses: actions/checkout@v2
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v3
- with:
- java-version: ${{ matrix.java }}
- distribution: 'temurin'
- cache: 'maven'
- - name: run all modules unit test
- run: |
- ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false
-D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates
- env:
- MAVEN_OPTS: -Xmx2048m
-
- integration-test:
- needs: [ sanity-check ]
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- java: [ '8', '11' ]
- os: [ 'ubuntu-latest' ]
- timeout-minutes: 90
- steps:
- - uses: actions/checkout@v2
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v3
- with:
- java-version: ${{ matrix.java }}
- distribution: 'temurin'
- cache: 'maven'
- - name: run all modules integration test
- run: |
- ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates
- env:
- MAVEN_OPTS: -Xmx2048m
-
+ call-build-and-test:
+ permissions:
+ packages: write
+ name: Run
+ uses: ./.github/workflows/backend.yml
+ with:
+ TEST_IN_PR: false
diff --git
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainerId.java
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainerId.java
index db8306bfa2..bc886bf59d 100644
---
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainerId.java
+++
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainerId.java
@@ -26,18 +26,19 @@ import static
org.apache.seatunnel.e2e.common.container.EngineType.SPARK;
@AllArgsConstructor
@Getter
public enum TestContainerId {
- FLINK_1_13(FLINK, "1.13.6"),
- FLINK_1_14(FLINK, "1.14.6"),
- FLINK_1_15(FLINK, "1.15.3"),
- FLINK_1_16(FLINK, "1.16.0"),
- FLINK_1_17(FLINK, "1.17.2"),
- FLINK_1_18(FLINK, "1.18.0"),
- SPARK_2_4(SPARK, "2.4.6"),
- SPARK_3_3(SPARK, "3.3.0"),
- SEATUNNEL(EngineType.SEATUNNEL, "dev");
+ FLINK_1_13(FLINK, "1.13.6", true),
+ FLINK_1_14(FLINK, "1.14.6", false),
+ FLINK_1_15(FLINK, "1.15.3", true),
+ FLINK_1_16(FLINK, "1.16.0", false),
+ FLINK_1_17(FLINK, "1.17.2", false),
+ FLINK_1_18(FLINK, "1.18.0", true),
+ SPARK_2_4(SPARK, "2.4.6", true),
+ SPARK_3_3(SPARK, "3.3.0", true),
+ SEATUNNEL(EngineType.SEATUNNEL, "dev", true);
private final EngineType engineType;
private final String version;
+ private final boolean testInPR;
@Override
public String toString() {
diff --git
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/ContainerUtil.java
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/ContainerUtil.java
index 6c6a8e5cdd..faf744df38 100644
---
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/ContainerUtil.java
+++
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/util/ContainerUtil.java
@@ -24,6 +24,7 @@ import
org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions;
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
import org.apache.seatunnel.api.table.factory.FactoryException;
import org.apache.seatunnel.common.constants.PluginType;
+import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.commons.lang3.StringUtils;
@@ -299,7 +300,38 @@ public final class ContainerUtil {
ServiceLoader.load(TestContainer.class,
Thread.currentThread().getContextClassLoader())
.iterator()
.forEachRemaining(result::add);
- return result;
+ boolean isTestInPR =
+
Boolean.parseBoolean(System.getenv().getOrDefault("TEST_IN_PR", "true"));
+ boolean testAllContainer =
+
Boolean.parseBoolean(System.getenv().getOrDefault("RUN_ALL_CONTAINER", "true"));
+ boolean testZetaContainer =
+ Boolean.parseBoolean(
+ System.getenv().getOrDefault("RUN_ZETA_CONTAINER",
"true"));
+ log.info(
+ "Test in PR: {}, Run all container: {}, Run zeta
container: {}",
+ isTestInPR,
+ testAllContainer,
+ testZetaContainer);
+ if (isTestInPR) {
+ return result.stream()
+ .filter(container ->
container.identifier().isTestInPR())
+ .filter(
+ container -> {
+ if (testAllContainer) {
+ return true;
+ }
+ if (testZetaContainer) {
+ return container
+ .identifier()
+ .getEngineType()
+ .equals(EngineType.SEATUNNEL);
+ }
+ return true;
+ })
+ .collect(Collectors.toList());
+ } else {
+ return result;
+ }
} catch (ServiceConfigurationError e) {
log.error("Could not load service provider for containers.", e);
throw new FactoryException("Could not load service provider for
containers.", e);
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java
index 4afbfacb71..7d2b8ff006 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java
@@ -207,7 +207,12 @@ public class JobLogIT extends SeaTunnelContainer {
"sh", "-c", "find /tmp/seatunnel/logs -name " +
tuple2.f1() + "\n");
String file1 = execResult.getStdout();
Assertions.assertEquals(
- tuple2.f0(), StringUtils.isBlank(file) &&
StringUtils.isBlank(file1));
+ tuple2.f0(),
+ StringUtils.isBlank(file) && StringUtils.isBlank(file1),
+ "Server Logs: \n"
+ + server.getLogs()
+ + "\n SecondServer Logs: \n"
+ + secondServer.getLogs());
}
}
diff --git
a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/ExecutionAddress.java
b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/ExecutionAddress.java
index ef445b50cb..605890c518 100644
---
a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/ExecutionAddress.java
+++
b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/ExecutionAddress.java
@@ -22,6 +22,6 @@ import java.io.Serializable;
@AllArgsConstructor
@NoArgsConstructor
public class ExecutionAddress implements Serializable {
- String hostname;
- int port;
+ private String hostname;
+ private int port;
}
diff --git
a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/JobDAGInfo.java
b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/JobDAGInfo.java
index f0ef24bad6..9790f48e39 100644
---
a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/JobDAGInfo.java
+++
b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/JobDAGInfo.java
@@ -39,6 +39,7 @@ public class JobDAGInfo implements Serializable {
Map<String, Object> envOptions;
Map<Integer, List<Edge>> pipelineEdges;
Map<Long, VertexInfo> vertexInfoMap;
+ ExecutionAddress master;
Set<ExecutionAddress> historyExecutionPlan;
public JsonObject toJsonObject() {
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/dag/DAGUtils.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/dag/DAGUtils.java
index 62df65c917..e8e05aa0e8 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/dag/DAGUtils.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/dag/DAGUtils.java
@@ -55,6 +55,7 @@ public class DAGUtils {
JobImmutableInformation jobImmutableInformation,
EngineConfig engineConfig,
boolean isPhysicalDAGIInfo,
+ ExecutionAddress master,
Set<ExecutionAddress> historyExecutionAddress) {
List<Pipeline> pipelines =
new ExecutionPlanGenerator(logicalDag,
jobImmutableInformation, engineConfig)
@@ -93,6 +94,7 @@ public class DAGUtils {
logicalDag.getJobConfig().getEnvOptions(),
pipelineWithEdges,
vertexInfoMap,
+ master,
historyExecutionAddress);
} else {
// Generate LogicalPlan DAG
@@ -141,6 +143,7 @@ public class DAGUtils {
logicalDag.getJobConfig().getEnvOptions(),
pipelineWithEdges,
vertexInfoMap,
+ master,
historyExecutionAddress);
}
}
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobHistoryService.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobHistoryService.java
index 7b0d83ef03..ed09e3ee9d 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobHistoryService.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobHistoryService.java
@@ -309,19 +309,26 @@ public class JobHistoryService {
JobDAGInfo jobDagInfo = event.getOldValue();
try {
Set<ExecutionAddress> historyExecutionPlan =
jobDagInfo.getHistoryExecutionPlan();
-
- historyExecutionPlan.forEach(
- address -> {
- logger.info("clean job log, jobId: " + jobId + ",
address: " + address);
- try {
- NodeEngineUtil.sendOperationToMemberNode(
- nodeEngine,
- new CleanLogOperation(jobId),
- new Address(address.getHostname(),
address.getPort()));
- } catch (UnknownHostException e) {
- throw new RuntimeException(e);
- }
- });
+ Stream.concat(historyExecutionPlan.stream(),
Stream.of(jobDagInfo.getMaster()))
+ .forEach(
+ address -> {
+ logger.info(
+ "clean job log, jobId: "
+ + jobId
+ + ", address: "
+ + address);
+ try {
+
NodeEngineUtil.sendOperationToMemberNode(
+ nodeEngine,
+ new
CleanLogOperation(jobId),
+ new Address(
+
address.getHostname(),
+
address.getPort()))
+ .join();
+ } catch (UnknownHostException e) {
+ throw new RuntimeException(e);
+ }
+ });
} catch (Exception e) {
logger.warning("clean job log err", e);
}
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java
index 49b676c97e..db3d7b1d21 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/master/JobMaster.java
@@ -589,6 +589,9 @@ public class JobMaster {
jobImmutableInformation,
engineConfig,
isPhysicalDAGIInfo,
+ new ExecutionAddress(
+ this.nodeEngine.getThisAddress().getHost(),
+
this.nodeEngine.getThisAddress().getPort()),
historyExecutionAddress);
}
return jobDAGInfo;
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/service/BaseService.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/service/BaseService.java
index f342aaba45..fc99feb264 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/service/BaseService.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/service/BaseService.java
@@ -31,6 +31,7 @@ import org.apache.seatunnel.engine.common.config.JobConfig;
import org.apache.seatunnel.engine.common.utils.PassiveCompletableFuture;
import org.apache.seatunnel.engine.core.classloader.ClassLoaderService;
import org.apache.seatunnel.engine.core.dag.logical.LogicalDag;
+import org.apache.seatunnel.engine.core.job.ExecutionAddress;
import org.apache.seatunnel.engine.core.job.JobDAGInfo;
import org.apache.seatunnel.engine.core.job.JobImmutableInformation;
import org.apache.seatunnel.engine.core.job.JobInfo;
@@ -166,6 +167,9 @@ public abstract class BaseService {
jobImmutableInformation,
getSeaTunnelServer(false).getSeaTunnelConfig().getEngineConfig(),
true,
+ new ExecutionAddress(
+ this.nodeEngine.getMasterAddress().getHost(),
+ this.nodeEngine.getMasterAddress().getPort()),
new HashSet<>());
jobInfoJson
diff --git a/tools/update_modules_check/update_modules_check.py
b/tools/update_modules_check/update_modules_check.py
index 8359bc7d97..f2f33abd10 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -176,6 +176,10 @@ def get_sub_update_it_modules(modules, total_num,
current_num):
module_list.remove("connector-rocketmq-e2e")
if "seatunnel-engine-k8s-e2e" in module_list:
module_list.remove("seatunnel-engine-k8s-e2e")
+ if "connector-seatunnel-e2e-base" in module_list:
+ module_list.remove("connector-seatunnel-e2e-base")
+ if "connector-console-seatunnel-e2e" in module_list:
+ module_list.remove("connector-console-seatunnel-e2e")
if "connector-doris-e2e" in module_list:
module_list.remove("connector-doris-e2e")
if "connector-paimon-e2e" in module_list:
