This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 81de0a69cc [Improve][CI]skip ui module, improve module dependent
(#8225)
81de0a69cc is described below
commit 81de0a69cc907f8ee638210c2332be5328c0ccab
Author: zhangdonghao <[email protected]>
AuthorDate: Tue Dec 17 18:45:53 2024 +0800
[Improve][CI]skip ui module, improve module dependent (#8225)
---
.github/workflows/backend.yml | 81 ++++++-----
.../seatunnel/hudi/catalog/HudiCatalog.java | 4 +-
.../cdc/oracle/OracleCDCWithSchemaChangeIT.java | 13 +-
.../connector-hudi-e2e/pom.xml | 11 +-
.../hudi/HudiSeatunnelS3MultiTableIT.java | 148 ++++++++++-----------
.../connector/hudi/HudiSparkS3MultiTableIT.java | 145 +++++++++-----------
.../seatunnel/e2e/connector/hudi/MinIoUtils.java | 100 ++++++++++++++
.../connector-paimon-e2e/pom.xml | 8 --
.../e2e/connector/paimon/PaimonWithS3IT.java | 8 +-
.../connector-starrocks-e2e/pom.xml | 6 -
.../engine/e2e/console/FakeSourceToConsoleIT.java | 4 +-
.../FakeSourceToConsoleWithEventReportIT.java | 4 +-
...r.java => ClusterSeaTunnelEngineContainer.java} | 2 +-
.../seatunnel/engine/e2e/JobClientJobProxyIT.java | 2 +-
.../apache/seatunnel/engine/e2e/JobRestoreIT.java | 2 +-
...ontainer.java => SeaTunnelEngineContainer.java} | 5 +-
.../seatunnel/engine/e2e/SinkPlaceholderIT.java | 2 +-
.../engine/e2e/classloader/ClassLoaderITBase.java | 4 +-
.../seatunnel/engine/e2e/joblog/JobLogIT.java | 4 +-
seatunnel-engine/seatunnel-engine-ui/pom.xml | 2 +
tools/update_modules_check/update_modules_check.py | 8 ++
21 files changed, 326 insertions(+), 237 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 7c07d3dab6..8f720d9503 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -284,7 +284,7 @@ jobs:
max_attempts: 3
retry_on: error
command: |
- ./mvnw -B install -DskipTests -D"maven.test.skip"=true
-D"maven.javadoc.skip"=true -D"license.skipAddThirdParty"
+ ./mvnw -B install -DskipTests -D"maven.test.skip"=true
-D"maven.javadoc.skip"=true -D"license.skipAddThirdParty" -D"skip.ui"=true
- name: Check Dependencies Licenses
run: tools/dependencies/checkLicense.sh
@@ -326,22 +326,18 @@ jobs:
steps:
- name: Checkout PR
uses: actions/checkout@v3
-
- uses: actions/setup-node@v2
with:
node-version: 20.x
-
- name: Install Dependencies and Check Code Style
run: |
cd seatunnel-engine/seatunnel-engine-ui/
npm install
npm run lint
-
- name: Run unit tests
run: |
cd seatunnel-engine/seatunnel-engine-ui/
npm run test:unit
-
- name: Build SeaTunnel UI
run: |
cd seatunnel-engine/seatunnel-engine-ui/
@@ -366,7 +362,7 @@ jobs:
cache: 'maven'
- name: run all modules unit test
run: |
- ./mvnw -B -T 1 clean verify -DskipUT=false -DskipIT=true
-D"license.skipAddThirdParty"=true --no-snapshot-updates
+ ./mvnw -B -T 1 clean verify -DskipUT=false -DskipIT=true
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx4096m
@@ -393,8 +389,8 @@ jobs:
run: |
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 0`
if [ ! -z $sub_modules ]; then
- echo $sub_modules
- ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am
-Pci
+ echo $sub_modules
+ ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
else
echo "sub modules is empty, skipping"
fi
@@ -425,7 +421,7 @@ jobs:
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 1`
if [ ! -z $sub_modules ]; then
echo $sub_modules
- ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am
-Pci
+ ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
else
echo "sub modules is empty, skipping"
fi
@@ -456,7 +452,7 @@ jobs:
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 2`
if [ ! -z $sub_modules ]; then
echo $sub_modules
- ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am
-Pci
+ ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
else
echo "sub modules is empty, skipping"
fi
@@ -487,7 +483,7 @@ jobs:
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 3`
if [ ! -z $sub_modules ]; then
echo $sub_modules
- ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am
-Pci
+ ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
else
echo "sub modules is empty, skipping"
fi
@@ -517,7 +513,7 @@ jobs:
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 4`
if [ ! -z $sub_modules ]; then
echo $sub_modules
- ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am
-Pci
+ ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
else
echo "sub modules is empty, skipping"
fi
@@ -547,7 +543,7 @@ jobs:
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 5`
if [ ! -z $sub_modules ]; then
echo $sub_modules
- ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am
-Pci
+ ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
else
echo "sub modules is empty, skipping"
fi
@@ -577,7 +573,7 @@ jobs:
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 6`
if [ ! -z $sub_modules ]; then
echo $sub_modules
- ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am
-Pci
+ ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
else
echo "sub modules is empty, skipping"
fi
@@ -608,7 +604,7 @@ jobs:
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 7`
if [ ! -z $sub_modules ]; then
echo $sub_modules
- ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am
-Pci
+ ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
else
echo "sub modules is empty, skipping"
fi
@@ -639,6 +635,7 @@ jobs:
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-seatunnel-e2e-base,:connector-console-seatunnel-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
+
engine-k8s-it:
needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' ||
contains(needs.changes.outputs.it-modules, 'seatunnel-engine-k8s-e2e')
@@ -668,10 +665,11 @@ jobs:
cache: 'maven'
- name: run seatunnel zeta on k8s test
run: |
- ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:seatunnel-engine-k8s-e2e -am -Pci
+ ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:seatunnel-engine-k8s-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+
transform-v2-it-part-1:
needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
@@ -696,7 +694,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run transform-v2 integration test (part-1)
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:seatunnel-transforms-v2-e2e-part-1 -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:seatunnel-transforms-v2-e2e-part-1 -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -724,7 +722,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run transform-v2 integration test (part-2)
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:seatunnel-transforms-v2-e2e-part-2 -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:seatunnel-transforms-v2-e2e-part-2 -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -755,7 +753,7 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 0`
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules
-am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -786,7 +784,7 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 1`
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules
-am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -817,7 +815,7 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 2`
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules
-am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -848,7 +846,7 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 3`
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules
-am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -879,7 +877,7 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 4`
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules
-am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -910,7 +908,7 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 5`
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules
-am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -941,7 +939,7 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 6`
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $run_it_modules
-am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -969,7 +967,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-1)
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-1 -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-1 -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -997,7 +995,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-2)
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-2 -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-2 -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1025,7 +1023,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-3)
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-3 -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-3 -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1053,7 +1051,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-4)
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-4 -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-4 -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1081,7 +1079,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-5)
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-5 -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-5 -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1109,7 +1107,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-6)
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-6 -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-6 -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1137,7 +1135,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run jdbc connectors integration test (part-7)
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-7 -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-jdbc-e2e-part-7 -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1162,7 +1160,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run kudu connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-kudu-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-kudu-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1187,7 +1185,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run amazonsqs connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-amazonsqs-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-amazonsqs-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1212,7 +1210,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run kafka connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-kafka-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-kafka-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1237,7 +1235,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run rocket connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-rocketmq-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-rocketmq-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1263,11 +1261,10 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run doris connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-doris-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-doris-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
-
paimon-connector-it:
needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' ||
contains(needs.changes.outputs.it-modules, 'connector-paimon-e2e')
@@ -1289,7 +1286,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run paimon connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-paimon-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-paimon-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1344,7 +1341,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run file local connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-file-local-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-file-local-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1369,7 +1366,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run file sftp connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-file-sftp-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-file-sftp-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1394,6 +1391,6 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run redis connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true --no-snapshot-updates -pl
:connector-redis-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-redis-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
diff --git
a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/catalog/HudiCatalog.java
b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/catalog/HudiCatalog.java
index 0d238c193d..63a02ff7fb 100644
---
a/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/catalog/HudiCatalog.java
+++
b/seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/catalog/HudiCatalog.java
@@ -100,7 +100,9 @@ public class HudiCatalog implements Catalog {
@Override
public void close() throws CatalogException {
try {
- fs.close();
+ if (fs != null) {
+ fs.close();
+ }
} catch (Exception e) {
log.info("Hudi catalog close error.", e);
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-oracle-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/OracleCDCWithSchemaChangeIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-oracle-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/OracleCDCWithSchemaChangeIT.java
index 3502b54757..a6a1ed920d 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-oracle-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/OracleCDCWithSchemaChangeIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-oracle-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/OracleCDCWithSchemaChangeIT.java
@@ -37,6 +37,7 @@ import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
+import org.apache.seatunnel.e2e.common.util.JobIdGenerator;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
@@ -65,6 +66,7 @@ import java.util.stream.Stream;
import static org.awaitility.Awaitility.await;
import static org.awaitility.Awaitility.with;
import static org.awaitility.Durations.TWO_SECONDS;
+import static org.testcontainers.shaded.org.awaitility.Awaitility.given;
@Slf4j
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@@ -192,17 +194,24 @@ public class OracleCDCWithSchemaChangeIT extends
AbstractOracleCDCIT implements
dropTable(ORACLE_CONTAINER.getJdbcUrl(), SCEHMA_NAME, SOURCE_TABLE1);
dropTable(ORACLE_CONTAINER.getJdbcUrl(), SCEHMA_NAME, SOURCE_TABLE1 +
"_SINK");
createAndInitialize("full_types", ADMIN_USER, ADMIN_PWD);
+ String jobId = String.valueOf(JobIdGenerator.newJobId());
CompletableFuture.runAsync(
() -> {
try {
-
container.executeJob("/oraclecdc_to_mysql_with_schema_change.conf");
+
container.executeJob("/oraclecdc_to_mysql_with_schema_change.conf", jobId);
} catch (Exception e) {
log.error("Commit task exception :" + e.getMessage());
throw new RuntimeException(e);
}
});
- Thread.sleep(10000L);
+ given().pollDelay(10, TimeUnit.SECONDS)
+ .await()
+ .pollDelay(5000L, TimeUnit.MILLISECONDS)
+ .untilAsserted(
+ () -> {
+ Assertions.assertEquals("RUNNING",
container.getJobStatus(jobId));
+ });
assertSchemaEvolution(
ORACLE_CONTAINER.getJdbcUrl(),
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/pom.xml
index 7fe8cc8523..ac49a5bec2 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/pom.xml
@@ -59,11 +59,16 @@
<dependency>
<groupId>org.apache.seatunnel</groupId>
- <artifactId>connector-seatunnel-e2e-base</artifactId>
+ <artifactId>seatunnel-hadoop3-3.1.4-uber</artifactId>
<version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
+ <classifier>optional</classifier>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java
index 237fd100d2..64451bb278 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java
@@ -18,8 +18,8 @@
package org.apache.seatunnel.e2e.connector.hudi;
import org.apache.seatunnel.common.utils.FileUtils;
+import org.apache.seatunnel.e2e.common.container.seatunnel.SeaTunnelContainer;
import org.apache.seatunnel.e2e.common.util.ContainerUtil;
-import org.apache.seatunnel.engine.e2e.SeaTunnelContainer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalFileSystem;
@@ -32,16 +32,13 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.MinIOContainer;
import io.minio.BucketExistsArgs;
-import io.minio.DownloadObjectArgs;
-import io.minio.ListObjectsArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
-import io.minio.Result;
-import io.minio.messages.Item;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
@@ -51,6 +48,7 @@ import java.util.concurrent.TimeUnit;
import static org.awaitility.Awaitility.given;
@Slf4j
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class HudiSeatunnelS3MultiTableIT extends SeaTunnelContainer {
private static final String MINIO_DOCKER_IMAGE =
"minio/minio:RELEASE.2024-06-13T22-53-53Z";
@@ -88,7 +86,6 @@ public class HudiSeatunnelS3MultiTableIT extends
SeaTunnelContainer {
String s3URL = container.getS3URL();
- // configuringClient
minioClient =
MinioClient.builder()
.endpoint(s3URL)
@@ -111,11 +108,13 @@ public class HudiSeatunnelS3MultiTableIT extends
SeaTunnelContainer {
"wget -P "
+ SEATUNNEL_HOME
+ "lib "
+ + " --timeout=180 "
+ AWS_SDK_DOWNLOAD
+ " &&"
+ "wget -P "
+ SEATUNNEL_HOME
+ "lib "
+ + " --timeout=180 "
+ HADOOP_AWS_DOWNLOAD
+ " &&"
+ ContainerUtil.adaptPathForWin(
@@ -132,90 +131,81 @@ public class HudiSeatunnelS3MultiTableIT extends
SeaTunnelContainer {
}
}
+ @Override
+ protected boolean isIssueWeAlreadyKnow(String threadName) {
+ return super.isIssueWeAlreadyKnow(threadName)
+ // hudi with s3
+ || threadName.startsWith("s3a-transfer");
+ }
+
@Test
public void testS3MultiWrite() throws IOException, InterruptedException {
copyFileToContainer("/hudi/core-site.xml",
"/tmp/seatunnel/config/core-site.xml");
- Container.ExecResult textWriteResult =
executeSeaTunnelJob("/hudi/s3_fake_to_hudi.conf");
+ Container.ExecResult textWriteResult =
executeJob("/hudi/s3_fake_to_hudi.conf");
Assertions.assertEquals(0, textWriteResult.getExitCode());
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", LocalFileSystem.DEFAULT_FS);
- given().ignoreExceptions()
+ given().pollDelay(10, TimeUnit.SECONDS)
+ .pollInterval(1, TimeUnit.SECONDS)
.await()
- .atMost(60000, TimeUnit.MILLISECONDS)
+ .atMost(300, TimeUnit.SECONDS)
.untilAsserted(
() -> {
// copy hudi to local
- Path inputPath1 =
- downloadNewestCommitFile(DATABASE_1 + "/"
+ TABLE_NAME_1 + "/");
- Path inputPath2 =
- downloadNewestCommitFile(DATABASE_2 + "/"
+ TABLE_NAME_2 + "/");
- ParquetReader<Group> reader1 =
- ParquetReader.builder(new
GroupReadSupport(), inputPath1)
- .withConf(configuration)
- .build();
- ParquetReader<Group> reader2 =
- ParquetReader.builder(new
GroupReadSupport(), inputPath2)
- .withConf(configuration)
- .build();
-
- long rowCount1 = 0;
- long rowCount2 = 0;
- // Read data and count rows
- while (reader1.read() != null) {
- rowCount1++;
- }
- // Read data and count rows
- while (reader2.read() != null) {
- rowCount2++;
+ Path inputPath1 = null;
+ Path inputPath2 = null;
+ try {
+ inputPath1 =
+ new Path(
+
MinIoUtils.downloadNewestCommitFile(
+ minioClient,
+ BUCKET,
+ String.format(
+ "%s/%s/",
DATABASE_1, TABLE_NAME_1),
+ DOWNLOAD_PATH));
+ log.info(
+ "download from s3 success, the parquet
file is at: {}",
+ inputPath1);
+ inputPath2 =
+ new Path(
+
MinIoUtils.downloadNewestCommitFile(
+ minioClient,
+ BUCKET,
+ String.format(
+ "%s/%s/",
DATABASE_2, TABLE_NAME_2),
+ DOWNLOAD_PATH));
+ log.info(
+ "download from s3 success, the parquet
file is at: {}",
+ inputPath2);
+ ParquetReader<Group> reader1 =
+ ParquetReader.builder(new
GroupReadSupport(), inputPath1)
+ .withConf(configuration)
+ .build();
+ ParquetReader<Group> reader2 =
+ ParquetReader.builder(new
GroupReadSupport(), inputPath2)
+ .withConf(configuration)
+ .build();
+
+ long rowCount1 = 0;
+ long rowCount2 = 0;
+ // Read data and count rows
+ while (reader1.read() != null) {
+ rowCount1++;
+ }
+ // Read data and count rows
+ while (reader2.read() != null) {
+ rowCount2++;
+ }
+ Assertions.assertEquals(100, rowCount1);
+ Assertions.assertEquals(240, rowCount2);
+ } finally {
+ if (inputPath1 != null) {
+
FileUtils.deleteFile(inputPath1.toUri().getPath());
+ }
+ if (inputPath2 != null) {
+
FileUtils.deleteFile(inputPath2.toUri().getPath());
+ }
}
- FileUtils.deleteFile(inputPath1.toUri().getPath());
- FileUtils.deleteFile(inputPath2.toUri().getPath());
- Assertions.assertEquals(100, rowCount1);
- Assertions.assertEquals(240, rowCount2);
});
}
-
- public Path downloadNewestCommitFile(String pathPrefix) throws IOException
{
- Iterable<Result<Item>> listObjects =
- minioClient.listObjects(
-
ListObjectsArgs.builder().bucket(BUCKET).prefix(pathPrefix).build());
- String newestCommitFileabsolutePath = "";
- String newestCommitFileName = "";
- long newestCommitTime = 0L;
- for (Result<Item> listObject : listObjects) {
- Item item;
- try {
- item = listObject.get();
- } catch (Exception e) {
- throw new IOException("List minio file error.", e);
- }
- if (item.isDir() || !item.objectName().endsWith(".parquet")) {
- continue;
- }
- long fileCommitTime =
- Long.parseLong(
- item.objectName()
- .substring(
- item.objectName().lastIndexOf("_")
+ 1,
-
item.objectName().lastIndexOf(".parquet")));
- if (fileCommitTime > newestCommitTime) {
- newestCommitFileabsolutePath = item.objectName();
- newestCommitFileName =
- newestCommitFileabsolutePath.substring(
- item.objectName().lastIndexOf("/") + 1);
- newestCommitTime = fileCommitTime;
- }
- }
- try {
- minioClient.downloadObject(
- DownloadObjectArgs.builder()
- .bucket(BUCKET)
- .object(newestCommitFileabsolutePath)
- .filename(DOWNLOAD_PATH + newestCommitFileName)
- .build());
- } catch (Exception e) {
- log.error("Download file from minio error.");
- }
- return new Path(DOWNLOAD_PATH + newestCommitFileName);
- }
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSparkS3MultiTableIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSparkS3MultiTableIT.java
index f91f340f3c..8b598d443b 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSparkS3MultiTableIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSparkS3MultiTableIT.java
@@ -32,21 +32,18 @@ import org.apache.parquet.example.data.Group;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.example.GroupReadSupport;
-import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestTemplate;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.MinIOContainer;
import io.minio.BucketExistsArgs;
-import io.minio.DownloadObjectArgs;
-import io.minio.ListObjectsArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
-import io.minio.Result;
-import io.minio.messages.Item;
import lombok.extern.slf4j.Slf4j;
+import okhttp3.OkHttpClient;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
@@ -72,7 +69,7 @@ public class HudiSparkS3MultiTableIT extends TestSuiteBase
implements TestResour
private static final String TABLE_NAME_2 = "st_test_2";
private static final String DOWNLOAD_PATH = "/tmp/seatunnel/";
- @BeforeEach
+ @BeforeAll
@Override
public void startUp() throws Exception {
container =
@@ -86,11 +83,15 @@ public class HudiSparkS3MultiTableIT extends TestSuiteBase
implements TestResour
String s3URL = container.getS3URL();
+ OkHttpClient.Builder builder = new OkHttpClient.Builder();
+ builder.connectTimeout(10, TimeUnit.SECONDS);
+ builder.readTimeout(10, TimeUnit.SECONDS);
// configuringClient
minioClient =
MinioClient.builder()
.endpoint(s3URL)
.credentials(container.getUserName(),
container.getPassword())
+ .httpClient(builder.build())
.build();
// create bucket
@@ -100,7 +101,7 @@ public class HudiSparkS3MultiTableIT extends TestSuiteBase
implements TestResour
Assertions.assertTrue(minioClient.bucketExists(existsArgs));
}
- @AfterEach
+ @AfterAll
@Override
public void tearDown() throws Exception {
if (container != null) {
@@ -121,82 +122,68 @@ public class HudiSparkS3MultiTableIT extends
TestSuiteBase implements TestResour
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", LocalFileSystem.DEFAULT_FS);
given().ignoreExceptions()
+ .pollDelay(5, TimeUnit.SECONDS)
+ .pollInterval(2, TimeUnit.SECONDS)
.await()
- .atMost(60000, TimeUnit.MILLISECONDS)
+ .atMost(300, TimeUnit.SECONDS)
.untilAsserted(
() -> {
// copy hudi to local
- Path inputPath1 =
- downloadNewestCommitFile(DATABASE_1 + "/"
+ TABLE_NAME_1 + "/");
- Path inputPath2 =
- downloadNewestCommitFile(DATABASE_2 + "/"
+ TABLE_NAME_2 + "/");
- ParquetReader<Group> reader1 =
- ParquetReader.builder(new
GroupReadSupport(), inputPath1)
- .withConf(configuration)
- .build();
- ParquetReader<Group> reader2 =
- ParquetReader.builder(new
GroupReadSupport(), inputPath2)
- .withConf(configuration)
- .build();
-
- long rowCount1 = 0;
- long rowCount2 = 0;
- // Read data and count rows
- while (reader1.read() != null) {
- rowCount1++;
- }
- // Read data and count rows
- while (reader2.read() != null) {
- rowCount2++;
+ // copy hudi to local
+ Path inputPath1 = null;
+ Path inputPath2 = null;
+ try {
+ inputPath1 =
+ new Path(
+
MinIoUtils.downloadNewestCommitFile(
+ minioClient,
+ BUCKET,
+ String.format(
+ "%s/%s/",
DATABASE_1, TABLE_NAME_1),
+ DOWNLOAD_PATH));
+ log.info(
+ "download from s3 success, the parquet
file is at: {}",
+ inputPath1);
+ inputPath2 =
+ new Path(
+
MinIoUtils.downloadNewestCommitFile(
+ minioClient,
+ BUCKET,
+ String.format(
+ "%s/%s/",
DATABASE_2, TABLE_NAME_2),
+ DOWNLOAD_PATH));
+ log.info(
+ "download from s3 success, the parquet
file is at: {}",
+ inputPath2);
+ ParquetReader<Group> reader1 =
+ ParquetReader.builder(new
GroupReadSupport(), inputPath1)
+ .withConf(configuration)
+ .build();
+ ParquetReader<Group> reader2 =
+ ParquetReader.builder(new
GroupReadSupport(), inputPath2)
+ .withConf(configuration)
+ .build();
+
+ long rowCount1 = 0;
+ long rowCount2 = 0;
+ // Read data and count rows
+ while (reader1.read() != null) {
+ rowCount1++;
+ }
+ // Read data and count rows
+ while (reader2.read() != null) {
+ rowCount2++;
+ }
+ Assertions.assertEquals(100, rowCount1);
+ Assertions.assertEquals(240, rowCount2);
+ } finally {
+ if (inputPath1 != null) {
+
FileUtils.deleteFile(inputPath1.toUri().getPath());
+ }
+ if (inputPath2 != null) {
+
FileUtils.deleteFile(inputPath2.toUri().getPath());
+ }
}
- FileUtils.deleteFile(inputPath1.toUri().getPath());
- FileUtils.deleteFile(inputPath2.toUri().getPath());
- Assertions.assertEquals(100, rowCount1);
- Assertions.assertEquals(240, rowCount2);
});
}
-
- public Path downloadNewestCommitFile(String pathPrefix) throws IOException
{
- Iterable<Result<Item>> listObjects =
- minioClient.listObjects(
-
ListObjectsArgs.builder().bucket(BUCKET).prefix(pathPrefix).build());
- String newestCommitFileabsolutePath = "";
- String newestCommitFileName = "";
- long newestCommitTime = 0L;
- for (Result<Item> listObject : listObjects) {
- Item item;
- try {
- item = listObject.get();
- } catch (Exception e) {
- throw new IOException("List minio file error.", e);
- }
- if (item.isDir() || !item.objectName().endsWith(".parquet")) {
- continue;
- }
- long fileCommitTime =
- Long.parseLong(
- item.objectName()
- .substring(
- item.objectName().lastIndexOf("_")
+ 1,
-
item.objectName().lastIndexOf(".parquet")));
- if (fileCommitTime > newestCommitTime) {
- newestCommitFileabsolutePath = item.objectName();
- newestCommitFileName =
- newestCommitFileabsolutePath.substring(
- item.objectName().lastIndexOf("/") + 1);
- newestCommitTime = fileCommitTime;
- }
- }
- try {
- minioClient.downloadObject(
- DownloadObjectArgs.builder()
- .bucket(BUCKET)
- .object(newestCommitFileabsolutePath)
- .filename(DOWNLOAD_PATH + newestCommitFileName)
- .build());
- } catch (Exception e) {
- log.error("Download file from minio error.");
- }
- return new Path(DOWNLOAD_PATH + newestCommitFileName);
- }
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/MinIoUtils.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/MinIoUtils.java
new file mode 100644
index 0000000000..9cdfdc531e
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/MinIoUtils.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.e2e.connector.hudi;
+
+import io.minio.GetObjectArgs;
+import io.minio.ListObjectsArgs;
+import io.minio.MinioClient;
+import io.minio.Result;
+import io.minio.messages.Item;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.FilterInputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+@Slf4j
+public class MinIoUtils {
+
+ public static String downloadNewestCommitFile(
+ MinioClient minioClient, String bucketName, String pathPrefix,
String downloadPath) {
+ Iterable<Result<Item>> listObjects =
+ minioClient.listObjects(
+
ListObjectsArgs.builder().bucket(bucketName).prefix(pathPrefix).build());
+ long newestCommitTime = 0L;
+ String objectPath = null;
+ for (Result<Item> listObject : listObjects) {
+ Item item = null;
+ try {
+ item = listObject.get();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ if (item.isDir() || !item.objectName().endsWith(".parquet")) {
+ continue;
+ }
+ long fileCommitTime =
+ Long.parseLong(
+ item.objectName()
+ .substring(
+ item.objectName().lastIndexOf("_")
+ 1,
+
item.objectName().lastIndexOf(".parquet")));
+ if (fileCommitTime > newestCommitTime) {
+ objectPath = item.objectName();
+ }
+ }
+ log.info("download object path: {}", objectPath);
+ assert objectPath != null;
+ Path path =
+ Paths.get(
+ createDir(downloadPath)
+ +
objectPath.substring(objectPath.lastIndexOf("/") + 1));
+ try (FilterInputStream inputStream =
+ minioClient.getObject(
+ GetObjectArgs.builder()
+ .bucket(bucketName)
+ .object(objectPath)
+ .build());
+ OutputStream outputStream = Files.newOutputStream(path)) {
+ byte[] buffer = new byte[1024];
+ int bytesRead;
+ while ((bytesRead = inputStream.read(buffer)) != -1) {
+ outputStream.write(buffer, 0, bytesRead);
+ }
+ } catch (Exception e) {
+ log.error("download error \n", e);
+ throw new RuntimeException(e);
+ }
+ log.info("download success path: {}", path);
+ return path.toFile().getAbsolutePath();
+ }
+
+ private static String createDir(String downloadPath) {
+ Path path = Paths.get(downloadPath);
+ if (!Files.exists(path)) {
+ try {
+ Files.createDirectories(path);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return downloadPath;
+ }
+}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml
index c1ce438be3..3fc619d225 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml
@@ -57,14 +57,6 @@
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>connector-seatunnel-e2e-base</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-hadoop3-3.1.4-uber</artifactId>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonWithS3IT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonWithS3IT.java
index 2df1a5e49b..a618aad8b3 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonWithS3IT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/src/test/java/org/apache/seatunnel/e2e/connector/paimon/PaimonWithS3IT.java
@@ -17,13 +17,14 @@
package org.apache.seatunnel.e2e.connector.paimon;
+import org.apache.seatunnel.e2e.common.container.seatunnel.SeaTunnelContainer;
import org.apache.seatunnel.e2e.common.util.ContainerUtil;
-import org.apache.seatunnel.engine.e2e.SeaTunnelContainer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.MinIOContainer;
@@ -34,6 +35,7 @@ import io.minio.MinioClient;
import java.nio.file.Paths;
import java.util.Map;
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class PaimonWithS3IT extends SeaTunnelContainer {
private static final String MINIO_DOCKER_IMAGE =
"minio/minio:RELEASE.2024-06-13T22-53-53Z";
@@ -121,10 +123,10 @@ public class PaimonWithS3IT extends SeaTunnelContainer {
@Test
public void testFaceCDCSinkPaimonWithS3Filesystem() throws Exception {
- Container.ExecResult execResult =
executeSeaTunnelJob("/fake_to_paimon_with_s3.conf");
+ Container.ExecResult execResult =
executeJob("/fake_to_paimon_with_s3.conf");
Assertions.assertEquals(0, execResult.getExitCode());
- Container.ExecResult readResult =
executeSeaTunnelJob("/paimon_with_s3_to_assert.conf");
+ Container.ExecResult readResult =
executeJob("/paimon_with_s3_to_assert.conf");
Assertions.assertEquals(0, readResult.getExitCode());
}
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml
index 05829ea893..7f11b08d8b 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml
@@ -55,12 +55,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.seatunnel</groupId>
- <artifactId>connector-cdc-mysql-e2e</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>connector-assert</artifactId>
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java
index 832be6fc18..c63982cbbf 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java
@@ -17,7 +17,7 @@
package org.apache.seatunnel.engine.e2e.console;
-import org.apache.seatunnel.engine.e2e.SeaTunnelContainer;
+import org.apache.seatunnel.engine.e2e.SeaTunnelEngineContainer;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -25,7 +25,7 @@ import org.testcontainers.containers.Container;
import java.io.IOException;
-public class FakeSourceToConsoleIT extends SeaTunnelContainer {
+public class FakeSourceToConsoleIT extends SeaTunnelEngineContainer {
@Test
public void testFakeSourceToConsoleSink() throws IOException,
InterruptedException {
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleWithEventReportIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleWithEventReportIT.java
index 79a4dbe1f8..e23cadfe41 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleWithEventReportIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleWithEventReportIT.java
@@ -21,7 +21,7 @@ import
org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode;
import
org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ArrayNode;
import org.apache.seatunnel.api.event.EventType;
-import org.apache.seatunnel.engine.e2e.SeaTunnelContainer;
+import org.apache.seatunnel.engine.e2e.SeaTunnelEngineContainer;
import org.apache.seatunnel.engine.server.event.JobEventHttpReportHandler;
import org.junit.jupiter.api.AfterAll;
@@ -52,7 +52,7 @@ import static
org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PA
import static org.awaitility.Awaitility.given;
@Slf4j
-public class FakeSourceToConsoleWithEventReportIT extends SeaTunnelContainer {
+public class FakeSourceToConsoleWithEventReportIT extends
SeaTunnelEngineContainer {
private static final int MOCK_SERVER_PORT = 1024;
private MockWebServer mockWebServer;
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelEngineContainer.java
similarity index 99%
rename from
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java
rename to
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelEngineContainer.java
index 6bdf1c2415..28b8adb41f 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelEngineContainer.java
@@ -53,7 +53,7 @@ import static
org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PA
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.in;
-public class ClusterSeaTunnelContainer extends SeaTunnelContainer {
+public class ClusterSeaTunnelEngineContainer extends SeaTunnelEngineContainer {
private GenericContainer<?> secondServer;
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobClientJobProxyIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobClientJobProxyIT.java
index c44161f2a4..febdaa332d 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobClientJobProxyIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobClientJobProxyIT.java
@@ -28,7 +28,7 @@ import java.io.IOException;
import static
org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH;
-public class JobClientJobProxyIT extends SeaTunnelContainer {
+public class JobClientJobProxyIT extends SeaTunnelEngineContainer {
@Override
@BeforeAll
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobRestoreIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobRestoreIT.java
index f251d9019f..9a0e94f89e 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobRestoreIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobRestoreIT.java
@@ -26,7 +26,7 @@ import java.io.IOException;
import static
org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH;
-public class JobRestoreIT extends SeaTunnelContainer {
+public class JobRestoreIT extends SeaTunnelEngineContainer {
@Override
@BeforeAll
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelEngineContainer.java
similarity index 91%
rename from
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java
rename to
seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelEngineContainer.java
index 0b96193f0c..1d73690493 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelEngineContainer.java
@@ -17,6 +17,8 @@
package org.apache.seatunnel.engine.e2e;
+import org.apache.seatunnel.e2e.common.container.seatunnel.SeaTunnelContainer;
+
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestInstance;
@@ -28,8 +30,7 @@ import java.io.IOException;
@Slf4j
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public abstract class SeaTunnelContainer
- extends
org.apache.seatunnel.e2e.common.container.seatunnel.SeaTunnelContainer {
+public abstract class SeaTunnelEngineContainer extends SeaTunnelContainer {
@Override
@BeforeAll
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SinkPlaceholderIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SinkPlaceholderIT.java
index eee3705452..a577b2b6bd 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SinkPlaceholderIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SinkPlaceholderIT.java
@@ -23,7 +23,7 @@ import org.testcontainers.containers.Container;
import java.io.IOException;
-public class SinkPlaceholderIT extends SeaTunnelContainer {
+public class SinkPlaceholderIT extends SeaTunnelEngineContainer {
@Test
public void testSinkPlaceholder() throws IOException, InterruptedException
{
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java
index 0a4e090598..a8234bd59e 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/classloader/ClassLoaderITBase.java
@@ -18,7 +18,7 @@
package org.apache.seatunnel.engine.e2e.classloader;
import org.apache.seatunnel.e2e.common.util.ContainerUtil;
-import org.apache.seatunnel.engine.e2e.SeaTunnelContainer;
+import org.apache.seatunnel.engine.e2e.SeaTunnelEngineContainer;
import org.apache.seatunnel.engine.server.rest.RestConstant;
import org.awaitility.Awaitility;
@@ -41,7 +41,7 @@ import static io.restassured.RestAssured.given;
import static
org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH;
import static org.hamcrest.Matchers.equalTo;
-public abstract class ClassLoaderITBase extends SeaTunnelContainer {
+public abstract class ClassLoaderITBase extends SeaTunnelEngineContainer {
private static final String CONF_FILE =
"/classloader/fake_to_inmemory.conf";
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java
index 18d08b7506..37310b9fe0 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/joblog/JobLogIT.java
@@ -19,7 +19,7 @@ package org.apache.seatunnel.engine.e2e.joblog;
import org.apache.seatunnel.common.constants.JobMode;
import org.apache.seatunnel.e2e.common.util.ContainerUtil;
-import org.apache.seatunnel.engine.e2e.SeaTunnelContainer;
+import org.apache.seatunnel.engine.e2e.SeaTunnelEngineContainer;
import org.apache.seatunnel.engine.server.rest.RestConstant;
import org.awaitility.Awaitility;
@@ -53,7 +53,7 @@ import static io.restassured.RestAssured.given;
import static
org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH;
import static org.hamcrest.Matchers.equalTo;
-public class JobLogIT extends SeaTunnelContainer {
+public class JobLogIT extends SeaTunnelEngineContainer {
private static final String CUSTOM_JOB_NAME = "test-job-log-file";
private static final String CUSTOM_JOB_NAME2 = "test-job-log-file2";
diff --git a/seatunnel-engine/seatunnel-engine-ui/pom.xml
b/seatunnel-engine/seatunnel-engine-ui/pom.xml
index 5244e73c5b..0ac5f9954b 100644
--- a/seatunnel-engine/seatunnel-engine-ui/pom.xml
+++ b/seatunnel-engine/seatunnel-engine-ui/pom.xml
@@ -35,6 +35,7 @@
<nodemodules.dir>node_modules</nodemodules.dir>
<dist.dir>../seatunnel-engine-server/src/main/resources/ui</dist.dir>
<deployed.dir>.deployed</deployed.dir>
+ <skip.ui>false</skip.ui>
</properties>
<build>
@@ -70,6 +71,7 @@
<configuration>
<nodeVersion>${build.node.version}</nodeVersion>
<npmVersion>${build.npm.version}</npmVersion>
+ <skip>${skip.ui}</skip>
</configuration>
<executions>
<execution>
diff --git a/tools/update_modules_check/update_modules_check.py
b/tools/update_modules_check/update_modules_check.py
index cef49ad4c3..7b3fddfab1 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -154,6 +154,10 @@ def get_sub_it_modules(modules, total_num, current_num):
modules_arr.remove("connector-file-local-e2e")
modules_arr.remove("connector-file-sftp-e2e")
modules_arr.remove("connector-redis-e2e")
+ if "connector-seatunnel-e2e-base" in modules_arr:
+ modules_arr.remove("connector-seatunnel-e2e-base")
+ if "connector-console-seatunnel-e2e" in modules_arr:
+ modules_arr.remove("connector-console-seatunnel-e2e")
output = ""
for i, module in enumerate(modules_arr):
if len(module) > 0 and i % int(total_num) == int(current_num):
@@ -195,6 +199,10 @@ def get_sub_update_it_modules(modules, total_num,
current_num):
module_list.remove("connector-file-sftp-e2e")
if "connector-redis-e2e" in module_list:
module_list.remove("connector-redis-e2e")
+ if "connector-seatunnel-e2e-base" in module_list:
+ module_list.remove("connector-seatunnel-e2e-base")
+ if "connector-console-seatunnel-e2e" in module_list:
+ module_list.remove("connector-console-seatunnel-e2e")
for i, module in enumerate(module_list):
if len(module) > 0 and i % int(total_num) == int(current_num):
final_modules.append(":" + module)
