This is an automated email from the ASF dual-hosted git repository. gaojun2048 pushed a commit to branch revert-3279-improve_ci_it in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
commit 1c23357a0d1f8bf180167858f99848da1814f2b7 Author: Eric <[email protected]> AuthorDate: Thu Nov 3 13:48:35 2022 +0800 Revert "fix ci problem (#3279)" This reverts commit bbc871908b9a9385d260437d4cbfb9f446389990. --- .github/workflows/backend.yml | 25 ++++++++-------- .github/workflows/schedule_backend.yml | 7 ++++- tools/update_modules_check/update_modules_check.py | 34 ++-------------------- 3 files changed, 21 insertions(+), 45 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index df7d7bd15..b1dd0dc94 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -200,11 +200,11 @@ jobs: run: | modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' modules=${modules: 1} - pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules` - ./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree -DoutputType=text -DoutputFile=/tmp/tree_out.txt -amd -pl $pl_modules - build_modules=`python tools/update_modules_check/update_modules_check.py final_ut /tmp/tree_out.txt` - echo $build_modules - echo "modules=$build_modules" >> $GITHUB_OUTPUT + includes=`python tools/update_modules_check/update_modules_check.py tree $modules` + ./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt + modules=`python tools/update_modules_check/update_modules_check.py final /tmp/tree_out.txt` + echo $modules + echo "modules=$modules" >> $GITHUB_OUTPUT - name: Make integration test modules id: it-modules @@ -212,11 +212,11 @@ jobs: run: | modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' modules=${modules: 1} - pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules` - ./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree -DoutputType=text -DoutputFile=/tmp/tree_out.txt -amd -pl $pl_modules - build_modules=`python tools/update_modules_check/update_modules_check.py final_it /tmp/tree_out.txt` - echo $build_modules - echo "modules=$build_modules" >> $GITHUB_OUTPUT + includes=`python tools/update_modules_check/update_modules_check.py tree $modules` + ./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt + modules=`python tools/update_modules_check/update_modules_check.py final /tmp/tree_out.txt` + echo $modules + echo "modules=$modules" >> $GITHUB_OUTPUT dependency-license: if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' @@ -270,13 +270,12 @@ jobs: - name: run updated modules unit test if: needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != '' run: | - ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -Pno_dist + ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -amd -Pno_dist env: MAVEN_OPTS: -Xmx2048m integration-test: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '') runs-on: ${{ matrix.os }} strategy: matrix: @@ -301,7 +300,7 @@ jobs: - name: run updated modules integration test if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' run: | - ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -Pno_dist + ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -amd -Pno_dist env: MAVEN_OPTS: -Xmx2048m diff --git a/.github/workflows/schedule_backend.yml b/.github/workflows/schedule_backend.yml index 8b33e76c3..518904195 100644 --- a/.github/workflows/schedule_backend.yml +++ b/.github/workflows/schedule_backend.yml @@ -143,4 +143,9 @@ jobs: ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates env: MAVEN_OPTS: -Xmx2048m - + - name: run updated modules integration test + if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != '' + run: | + ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates + env: + MAVEN_OPTS: -Xmx2048m diff --git a/tools/update_modules_check/update_modules_check.py b/tools/update_modules_check/update_modules_check.py index 8c104a2c4..0e8b4f24d 100644 --- a/tools/update_modules_check/update_modules_check.py +++ b/tools/update_modules_check/update_modules_check.py @@ -44,10 +44,6 @@ def get_modules(files, index, start_pre, root_module): if module_name.startswith(start_pre): modules_name_set.add(module_name) - sub_module_name = file.split('/')[index + 1] - if sub_module_name.startswith(start_pre): - modules_name_set.add(sub_module_name) - output_module = "" if len(modules_name_set) > 0: for module in modules_name_set: @@ -58,11 +54,6 @@ def get_modules(files, index, start_pre, root_module): print(output_module) -def replace_comma_to_commacolon(modules_str): - modules_str = modules_str.replace(",", ",:") - modules_str = ":" + modules_str - print(modules_str) - def get_dependency_tree_includes(modules_str): modules = modules_str.split(',') output = "" @@ -73,29 +64,14 @@ def get_dependency_tree_includes(modules_str): output = "-Dincludes=" + output print(output) -def get_final_it_modules(file): +def get_final_modules(file): f = open(file, 'rb') output = "" for line in f.readlines(): if line.startswith("org.apache.seatunnel"): con = line.split(":") - # find all e2e modules - if con[2] == "jar" and "-e2e" in con[1] and "transform" not in con[1]: - output = output + "," + ":" + con[1] - - output = output[1:len(output)] - print(output) - -def get_final_ut_modules(file): - f = open(file, 'rb') - output = "" - for line in f.readlines(): - if line.startswith("org.apache.seatunnel"): - con = line.split(":") - # find all e2e modules if con[2] == "jar": output = output + "," + ":" + con[1] - output = output[1:len(output)] print(output) @@ -114,12 +90,8 @@ def main(argv): get_engine_e2e_modules(argv[2]) elif argv[1] == "tree": get_dependency_tree_includes(argv[2]) - elif argv[1] == "final_it": - get_final_it_modules(argv[2]) - elif argv[1] == "final_ut": - get_final_ut_modules(argv[2]) - elif argv[1] == "replace": - replace_comma_to_commacolon(argv[2]) + elif argv[1] == "final": + get_final_modules(argv[2]) if __name__ == "__main__": main(sys.argv) \ No newline at end of file
