This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new bbc871908 fix ci problem (#3279)
bbc871908 is described below
commit bbc871908b9a9385d260437d4cbfb9f446389990
Author: Eric <[email protected]>
AuthorDate: Thu Nov 3 13:31:07 2022 +0800
fix ci problem (#3279)
---
.github/workflows/backend.yml | 25 ++++++++--------
.github/workflows/schedule_backend.yml | 7 +----
tools/update_modules_check/update_modules_check.py | 34 ++++++++++++++++++++--
3 files changed, 45 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index b1dd0dc94..df7d7bd15 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -200,11 +200,11 @@ jobs:
run: |
modules='${{ steps.engine-modules.outputs.modules }}${{
steps.cv2-modules.outputs.modules }}'
modules=${modules: 1}
- includes=`python tools/update_modules_check/update_modules_check.py
tree $modules`
- ./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree
$includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
- modules=`python tools/update_modules_check/update_modules_check.py
final /tmp/tree_out.txt`
- echo $modules
- echo "modules=$modules" >> $GITHUB_OUTPUT
+ pl_modules=`python
tools/update_modules_check/update_modules_check.py replace $modules`
+ ./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree
-DoutputType=text -DoutputFile=/tmp/tree_out.txt -amd -pl $pl_modules
+ build_modules=`python
tools/update_modules_check/update_modules_check.py final_ut /tmp/tree_out.txt`
+ echo $build_modules
+ echo "modules=$build_modules" >> $GITHUB_OUTPUT
- name: Make integration test modules
id: it-modules
@@ -212,11 +212,11 @@ jobs:
run: |
modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{
steps.cv2-flink-e2e-modules.outputs.modules }}${{
steps.cv2-spark-e2e-modules.outputs.modules }}${{
steps.engine-e2e-modules.outputs.modules }}${{
steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
modules=${modules: 1}
- includes=`python tools/update_modules_check/update_modules_check.py
tree $modules`
- ./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree
$includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
- modules=`python tools/update_modules_check/update_modules_check.py
final /tmp/tree_out.txt`
- echo $modules
- echo "modules=$modules" >> $GITHUB_OUTPUT
+ pl_modules=`python
tools/update_modules_check/update_modules_check.py replace $modules`
+ ./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree
-DoutputType=text -DoutputFile=/tmp/tree_out.txt -amd -pl $pl_modules
+ build_modules=`python
tools/update_modules_check/update_modules_check.py final_it /tmp/tree_out.txt`
+ echo $build_modules
+ echo "modules=$build_modules" >> $GITHUB_OUTPUT
dependency-license:
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
@@ -270,12 +270,13 @@ jobs:
- name: run updated modules unit test
if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.ut-modules != ''
run: |
- ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -amd
-Pno_dist
+ ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -Pno_dist
env:
MAVEN_OPTS: -Xmx2048m
integration-test:
needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api ==
'false' && needs.changes.outputs.it-modules != '')
runs-on: ${{ matrix.os }}
strategy:
matrix:
@@ -300,7 +301,7 @@ jobs:
- name: run updated modules integration test
if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
- ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -amd
-Pno_dist
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -Pno_dist
env:
MAVEN_OPTS: -Xmx2048m
diff --git a/.github/workflows/schedule_backend.yml
b/.github/workflows/schedule_backend.yml
index 518904195..8b33e76c3 100644
--- a/.github/workflows/schedule_backend.yml
+++ b/.github/workflows/schedule_backend.yml
@@ -143,9 +143,4 @@ jobs:
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"scalastyle.skip"=true
-D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m
- - name: run updated modules integration test
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
- run: |
- ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"scalastyle.skip"=true
-D"license.skipAddThirdParty"=true --no-snapshot-updates
- env:
- MAVEN_OPTS: -Xmx2048m
+
diff --git a/tools/update_modules_check/update_modules_check.py
b/tools/update_modules_check/update_modules_check.py
index 0e8b4f24d..8c104a2c4 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -44,6 +44,10 @@ def get_modules(files, index, start_pre, root_module):
if module_name.startswith(start_pre):
modules_name_set.add(module_name)
+ sub_module_name = file.split('/')[index + 1]
+ if sub_module_name.startswith(start_pre):
+ modules_name_set.add(sub_module_name)
+
output_module = ""
if len(modules_name_set) > 0:
for module in modules_name_set:
@@ -54,6 +58,11 @@ def get_modules(files, index, start_pre, root_module):
print(output_module)
+def replace_comma_to_commacolon(modules_str):
+ modules_str = modules_str.replace(",", ",:")
+ modules_str = ":" + modules_str
+ print(modules_str)
+
def get_dependency_tree_includes(modules_str):
modules = modules_str.split(',')
output = ""
@@ -64,14 +73,29 @@ def get_dependency_tree_includes(modules_str):
output = "-Dincludes=" + output
print(output)
-def get_final_modules(file):
+def get_final_it_modules(file):
f = open(file, 'rb')
output = ""
for line in f.readlines():
if line.startswith("org.apache.seatunnel"):
con = line.split(":")
+ # find all e2e modules
+ if con[2] == "jar" and "-e2e" in con[1] and "transform" not in
con[1]:
+ output = output + "," + ":" + con[1]
+
+ output = output[1:len(output)]
+ print(output)
+
+def get_final_ut_modules(file):
+ f = open(file, 'rb')
+ output = ""
+ for line in f.readlines():
+ if line.startswith("org.apache.seatunnel"):
+ con = line.split(":")
+ # find all e2e modules
if con[2] == "jar":
output = output + "," + ":" + con[1]
+
output = output[1:len(output)]
print(output)
@@ -90,8 +114,12 @@ def main(argv):
get_engine_e2e_modules(argv[2])
elif argv[1] == "tree":
get_dependency_tree_includes(argv[2])
- elif argv[1] == "final":
- get_final_modules(argv[2])
+ elif argv[1] == "final_it":
+ get_final_it_modules(argv[2])
+ elif argv[1] == "final_ut":
+ get_final_ut_modules(argv[2])
+ elif argv[1] == "replace":
+ replace_comma_to_commacolon(argv[2])
if __name__ == "__main__":
main(sys.argv)
\ No newline at end of file
