https://github.com/llvmbot created https://github.com/llvm/llvm-project/pull/150947
Backport 871d65bfdd580cec349e63d687e6829e0ef62824 afff28e4cb4b56dc5c77ecdb5aad9ec10e170999 921287e126465d6850954855ded640f0f78d72fd 796f5512442782c7d37b8008740b3afcac1feb8b bcfd29418f8851b93309746f987c22644bac6307 Requested by: @boomanaiden154 >From 8391eafd7632367acab3003e077d610e6873f7f9 Mon Sep 17 00:00:00 2001 From: Aiden Grossman <aidengross...@google.com> Date: Thu, 17 Jul 2025 10:13:43 -0700 Subject: [PATCH 1/5] [CI] Migrate monolithic-linux script to sccache This is in preparation for migrating to Google Cloud Storage (GCS) based caching soon which is only supported by sccache. Reviewers: Keenuts, gburgessiv, dschuff, lnihlen, cmtice Reviewed By: cmtice Pull Request: https://github.com/llvm/llvm-project/pull/149195 (cherry picked from commit 871d65bfdd580cec349e63d687e6829e0ef62824) --- .ci/monolithic-linux.sh | 7 ++++--- .github/workflows/premerge.yaml | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index 8d1faab13986c..c6ece4e45a186 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -21,7 +21,7 @@ BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}" INSTALL_DIR="${BUILD_DIR}/install" rm -rf "${BUILD_DIR}" -ccache --zero-stats +sccache --zero-stats if [[ -n "${CLEAR_CACHE:-}" ]]; then echo "clearing cache" @@ -36,7 +36,7 @@ export CLANG_CRASH_DIAGNOSTICS_DIR=`realpath artifacts/reproducers` function at-exit { retcode=$? - ccache --print-stats > artifacts/ccache_stats.txt + sccache --show-stats > artifacts/sccache_stats.txt cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log cp "${BUILD_DIR}"/test-results.*.xml artifacts/ || : @@ -78,7 +78,8 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_LIT_ARGS="${lit_args}" \ -D LLVM_ENABLE_LLD=ON \ -D CMAKE_CXX_FLAGS=-gmlt \ - -D LLVM_CCACHE_BUILD=ON \ + -D CMAKE_C_COMPILER_LAUNCHER=sccache \ + -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \ -D LIBCXX_CXX_ABI=libcxxabi \ -D MLIR_ENABLE_BINDINGS_PYTHON=ON \ -D LLDB_ENABLE_PYTHON=ON \ diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml index ff63355222065..7b5ecd62080f3 100644 --- a/.github/workflows/premerge.yaml +++ b/.github/workflows/premerge.yaml @@ -37,6 +37,7 @@ jobs: - name: Setup ccache uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17 with: + variant: "sccache" max-size: "2000M" - name: Build and Test # Mark the job as a success even if the step fails so that people do >From da4877236d22e34c7ce38e1835ad6eae5a31776d Mon Sep 17 00:00:00 2001 From: Andy Kaylor <akay...@nvidia.com> Date: Thu, 17 Jul 2025 11:17:52 -0700 Subject: [PATCH 2/5] [CI][Github] Enable CIR CI build and test (#147430) This change modifies CI scripts to add a pseudo-project for CIR and detect when CIR-specific files are modified. It also enables building clang with CIR enabled whenever both the clang and mlir projects are being built. Building and testing CIR is only enabled on Linux at this time, as CIR doesn't properly support Windows or MacOS yet. (cherry picked from commit afff28e4cb4b56dc5c77ecdb5aad9ec10e170999) --- .ci/compute_projects.py | 19 ++++++++++++++++++ .ci/compute_projects_test.py | 34 ++++++++++++++++++++++++++++++++- .ci/monolithic-linux.sh | 2 ++ .github/workflows/premerge.yaml | 2 +- 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/.ci/compute_projects.py b/.ci/compute_projects.py index c3cf714ce6c10..8e25fd61d6b32 100644 --- a/.ci/compute_projects.py +++ b/.ci/compute_projects.py @@ -19,6 +19,7 @@ PROJECT_DEPENDENCIES = { "llvm": set(), "clang": {"llvm"}, + "CIR": {"clang", "mlir"}, "bolt": {"clang", "lld", "llvm"}, "clang-tools-extra": {"clang", "llvm"}, "compiler-rt": {"clang", "lld"}, @@ -55,6 +56,7 @@ ".ci": { "llvm", "clang", + "CIR", "lld", "lldb", "bolt", @@ -128,6 +130,7 @@ "lldb": "check-lldb", "llvm": "check-llvm", "clang": "check-clang", + "CIR": "check-clang-cir", "bolt": "check-bolt", "lld": "check-lld", "flang": "check-flang", @@ -247,6 +250,14 @@ def _get_modified_projects(modified_files: list[str]) -> Set[str]: # capacity. if len(path_parts) > 3 and path_parts[:3] == ("llvm", "utils", "gn"): continue + # If the file is in the clang/lib/CIR directory, add the CIR project. + if len(path_parts) > 3 and ( + path_parts[:3] == ("clang", "lib", "CIR") + or path_parts[:3] == ("clang", "test", "CIR") + or path_parts[:4] == ("clang", "include", "clang", "CIR") + ): + modified_projects.add("CIR") + # Fall through to add clang. modified_projects.add(pathlib.Path(modified_file).parts[0]) return modified_projects @@ -267,6 +278,13 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]: runtimes_check_targets_needs_reconfig = _compute_project_check_targets( runtimes_to_test_needs_reconfig ) + + # CIR is used as a pseudo-project in this script. It is built as part of the + # clang build, but it requires an explicit option to enable. We set that + # option here, and remove it from the projects_to_build list. + enable_cir = "ON" if "CIR" in projects_to_build else "OFF" + projects_to_build.discard("CIR") + # We use a semicolon to separate the projects/runtimes as they get passed # to the CMake invocation and thus we need to use the CMake list separator # (;). We use spaces to separate the check targets as they end up getting @@ -279,6 +297,7 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]: "runtimes_check_targets_needs_reconfig": " ".join( sorted(runtimes_check_targets_needs_reconfig) ), + "enable_cir": enable_cir, } diff --git a/.ci/compute_projects_test.py b/.ci/compute_projects_test.py index 6299931e1ec34..732514c96f5a6 100644 --- a/.ci/compute_projects_test.py +++ b/.ci/compute_projects_test.py @@ -104,6 +104,10 @@ def test_clang(self): env_variables["runtimes_check_targets_needs_reconfig"], "check-cxx check-cxxabi check-unwind", ) + self.assertEqual( + env_variables["enable_cir"], + "OFF", + ) def test_clang_windows(self): env_variables = compute_projects.get_env_variables( @@ -126,6 +130,32 @@ def test_clang_windows(self): env_variables["runtimes_check_targets_needs_reconfig"], "check-cxx check-cxxabi check-unwind", ) + self.assertEqual(env_variables["enable_cir"], "OFF") + + def test_cir(self): + env_variables = compute_projects.get_env_variables( + ["clang/lib/CIR/CMakeLists.txt"], "Linux" + ) + self.assertEqual( + env_variables["projects_to_build"], + "clang;clang-tools-extra;lld;llvm;mlir", + ) + self.assertEqual( + env_variables["project_check_targets"], + "check-clang check-clang-cir check-clang-tools", + ) + self.assertEqual( + env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind" + ) + self.assertEqual( + env_variables["runtimes_check_targets"], + "check-compiler-rt", + ) + self.assertEqual( + env_variables["runtimes_check_targets_needs_reconfig"], + "check-cxx check-cxxabi check-unwind", + ) + self.assertEqual(env_variables["enable_cir"], "ON") def test_bolt(self): env_variables = compute_projects.get_env_variables( @@ -158,6 +188,7 @@ def test_mlir(self): self.assertEqual(env_variables["runtimes_to_build"], "") self.assertEqual(env_variables["runtimes_check_targets"], "") self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "") + self.assertEqual(env_variables["enable_cir"], "OFF") def test_flang(self): env_variables = compute_projects.get_env_variables( @@ -168,6 +199,7 @@ def test_flang(self): self.assertEqual(env_variables["runtimes_to_build"], "") self.assertEqual(env_variables["runtimes_check_targets"], "") self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "") + self.assertEqual(env_variables["enable_cir"], "OFF") def test_invalid_subproject(self): env_variables = compute_projects.get_env_variables( @@ -237,7 +269,7 @@ def test_ci(self): ) self.assertEqual( env_variables["project_check_targets"], - "check-bolt check-clang check-clang-tools check-flang check-lld check-lldb check-llvm check-mlir check-polly", + "check-bolt check-clang check-clang-cir check-clang-tools check-flang check-lld check-lldb check-llvm check-mlir check-polly", ) self.assertEqual( env_variables["runtimes_to_build"], diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index c6ece4e45a186..e6d34cf52e192 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -53,6 +53,7 @@ targets="${2}" runtimes="${3}" runtime_targets="${4}" runtime_targets_needs_reconfig="${5}" +enable_cir="${6}" lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests" @@ -72,6 +73,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -G Ninja \ -D CMAKE_PREFIX_PATH="${HOME}/.local" \ -D CMAKE_BUILD_TYPE=Release \ + -D CLANG_ENABLE_CIR=${enable_cir} \ -D LLVM_ENABLE_ASSERTIONS=ON \ -D LLVM_BUILD_EXAMPLES=ON \ -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml index 7b5ecd62080f3..73943bc86eadd 100644 --- a/.github/workflows/premerge.yaml +++ b/.github/workflows/premerge.yaml @@ -62,7 +62,7 @@ jobs: export CC=/opt/llvm/bin/clang export CXX=/opt/llvm/bin/clang++ - ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}" + ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}" "${enable_cir}" - name: Upload Artifacts if: '!cancelled()' uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 >From 39174d16afa3531b4114f769895151ee0e3656de Mon Sep 17 00:00:00 2001 From: Aiden Grossman <aidengross...@google.com> Date: Tue, 22 Jul 2025 13:31:25 -0700 Subject: [PATCH 3/5] [CI] Enable sccache GCS on Linux premerge (#149923) This patch enables sccache using GCS for Linux premerge. (cherry picked from commit 921287e126465d6850954855ded640f0f78d72fd) --- .github/workflows/premerge.yaml | 12 +++++++----- polly/CMakeLists.txt | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml index 73943bc86eadd..80c0a97d03160 100644 --- a/.github/workflows/premerge.yaml +++ b/.github/workflows/premerge.yaml @@ -34,11 +34,6 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 2 - - name: Setup ccache - uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17 - with: - variant: "sccache" - max-size: "2000M" - name: Build and Test # Mark the job as a success even if the step fails so that people do # not get notified while the new premerge pipeline is in an @@ -62,6 +57,13 @@ jobs: export CC=/opt/llvm/bin/clang export CXX=/opt/llvm/bin/clang++ + # This environment variable is passes into the container through the + # runner pod definition. This differs between our two clusters which + # why we do not hardcode it. + export SCCACHE_GCS_BUCKET=$CACHE_GCS_BUCKET + export SCCACHE_GCS_RW_MODE=READ_WRITE + sccache --start-server + ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}" "${enable_cir}" - name: Upload Artifacts if: '!cancelled()' diff --git a/polly/CMakeLists.txt b/polly/CMakeLists.txt index 52d1be6fe295a..770fddca86b22 100644 --- a/polly/CMakeLists.txt +++ b/polly/CMakeLists.txt @@ -1,3 +1,4 @@ +# test # Check if this is a in tree build. if (NOT DEFINED LLVM_MAIN_SRC_DIR) project(Polly) >From dd5358a974f449a49f3a247c056ca4b807c13aa7 Mon Sep 17 00:00:00 2001 From: Aiden Grossman <aidengross...@google.com> Date: Tue, 22 Jul 2025 15:53:52 -0700 Subject: [PATCH 4/5] [CI] Enable Windows Caching Through Sccache GCS (#150114) Same as 921287e126465d6850954855ded640f0f78d72fd but for Windows. (cherry picked from commit 796f5512442782c7d37b8008740b3afcac1feb8b) --- .github/workflows/premerge.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml index 80c0a97d03160..c51325e2f0d45 100644 --- a/.github/workflows/premerge.yaml +++ b/.github/workflows/premerge.yaml @@ -88,11 +88,6 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 2 - - name: Setup ccache - uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17 - with: - variant: "sccache" - max-size: "2000M" - name: Compute Projects id: vars run: | @@ -115,7 +110,7 @@ jobs: shell: cmd run: | call C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64 - bash .ci/monolithic-windows.sh "${{ steps.vars.outputs.windows-projects }}" "${{ steps.vars.outputs.windows-check-targets }}" + bash -c "export SCCACHE_GCS_BUCKET=$CACHE_GCS_BUCKET; export SCCACHE_GCS_RW_MODE=READ_WRITE; sccache --start-server; .ci/monolithic-windows.sh \"${{ steps.vars.outputs.windows-projects }}\" \"${{ steps.vars.outputs.windows-check-targets }}\"" - name: Upload Artifacts if: '!cancelled()' uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 >From e948e082f1c3c821646db9a95576f367c4298693 Mon Sep 17 00:00:00 2001 From: Aiden Grossman <aidengross...@google.com> Date: Thu, 24 Jul 2025 15:11:24 -0700 Subject: [PATCH 5/5] [CI][Github] Set SCCACHE_IDLE_TIMEOUT to 0 (#150508) This patch sets the sccache idle timeout to zero which ensures that it will run indefinitely. Without this, sccache will sometimes stop when several long test suites get run consequitively. This resets the statistics which we save at the end. Ensuring the process stays running means we have accurate statistics when we collect them at the end. (cherry picked from commit bcfd29418f8851b93309746f987c22644bac6307) --- .github/workflows/premerge.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml index c51325e2f0d45..d0518fa6879e2 100644 --- a/.github/workflows/premerge.yaml +++ b/.github/workflows/premerge.yaml @@ -62,6 +62,12 @@ jobs: # why we do not hardcode it. export SCCACHE_GCS_BUCKET=$CACHE_GCS_BUCKET export SCCACHE_GCS_RW_MODE=READ_WRITE + + # Set the idle timeout to zero to ensure sccache runs for the + # entire duration of the job. Otherwise it might stop if we run + # several test suites in a row and discard statistics that we want + # to save in the end. + export SCCACHE_IDLE_TIMEOUT=0 sccache --start-server ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}" "${enable_cir}" @@ -110,7 +116,9 @@ jobs: shell: cmd run: | call C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64 - bash -c "export SCCACHE_GCS_BUCKET=$CACHE_GCS_BUCKET; export SCCACHE_GCS_RW_MODE=READ_WRITE; sccache --start-server; .ci/monolithic-windows.sh \"${{ steps.vars.outputs.windows-projects }}\" \"${{ steps.vars.outputs.windows-check-targets }}\"" + # See the comments above in the Linux job for why we define each of + # these environment variables. + bash -c "export SCCACHE_GCS_BUCKET=$CACHE_GCS_BUCKET; export SCCACHE_GCS_RW_MODE=READ_WRITE; export SCCACHE_IDLE_TIMEOUT=0; sccache --start-server; .ci/monolithic-windows.sh \"${{ steps.vars.outputs.windows-projects }}\" \"${{ steps.vars.outputs.windows-check-targets }}\"" - name: Upload Artifacts if: '!cancelled()' uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
