This is an automated email from the ASF dual-hosted git repository. hgruszecki pushed a commit to branch improve-cache in repository https://gitbox.apache.org/repos/asf/iggy.git
commit e7f49e0a9db45b8024c007b2650816e3f588a121 Author: Hubert Gruszecki <[email protected]> AuthorDate: Thu Jan 15 15:00:50 2026 +0100 chore(ci): improve sccache cache effectiveness --- .github/actions/csharp-dotnet/pre-merge/action.yml | 1 + .github/actions/go/pre-merge/action.yml | 1 + .github/actions/java-gradle/pre-merge/action.yml | 1 + .github/actions/node-npm/pre-merge/action.yml | 1 + .../actions/python-maturin/pre-merge/action.yml | 1 + .github/actions/rust/post-merge/action.yml | 3 +- .github/actions/rust/pre-merge/action.yml | 22 ++++++---- .github/actions/utils/docker-buildx/action.yml | 10 +++++ .../actions/utils/setup-node-with-cache/action.yml | 12 +----- .../actions/utils/setup-rust-with-cache/action.yml | 49 ++++++++++++++-------- .github/workflows/_build_python_wheels.yml | 1 + .github/workflows/_build_rust_artifacts.yml | 2 + .github/workflows/_publish_rust_crates.yml | 2 +- .github/workflows/_test_bdd.yml | 1 + .github/workflows/_test_examples.yml | 1 + .github/workflows/post-merge.yml | 1 + .github/workflows/publish.yml | 3 +- 17 files changed, 70 insertions(+), 42 deletions(-) diff --git a/.github/actions/csharp-dotnet/pre-merge/action.yml b/.github/actions/csharp-dotnet/pre-merge/action.yml index ac4d404e9..b8422b071 100644 --- a/.github/actions/csharp-dotnet/pre-merge/action.yml +++ b/.github/actions/csharp-dotnet/pre-merge/action.yml @@ -37,6 +37,7 @@ runs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + save-cache: "false" # Only builds server, let Rust test job save comprehensive cache - name: Restore dependencies run: | diff --git a/.github/actions/go/pre-merge/action.yml b/.github/actions/go/pre-merge/action.yml index 6bcff99b5..dd4c2bc3c 100644 --- a/.github/actions/go/pre-merge/action.yml +++ b/.github/actions/go/pre-merge/action.yml @@ -124,6 +124,7 @@ runs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false + save-cache: "false" # Only builds server, let Rust test job save comprehensive cache - name: Setup server for e2e tests if: inputs.task == 'e2e' diff --git a/.github/actions/java-gradle/pre-merge/action.yml b/.github/actions/java-gradle/pre-merge/action.yml index a8ddec1db..b362a8729 100644 --- a/.github/actions/java-gradle/pre-merge/action.yml +++ b/.github/actions/java-gradle/pre-merge/action.yml @@ -66,6 +66,7 @@ runs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + save-cache: "false" # Only builds server, let Rust test job save comprehensive cache - name: Start Iggy server if: inputs.task == 'test' diff --git a/.github/actions/node-npm/pre-merge/action.yml b/.github/actions/node-npm/pre-merge/action.yml index bb4ff3972..8a6672441 100644 --- a/.github/actions/node-npm/pre-merge/action.yml +++ b/.github/actions/node-npm/pre-merge/action.yml @@ -39,6 +39,7 @@ runs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + save-cache: "false" # Only builds server, let Rust test job save comprehensive cache - name: Install netcat if: inputs.task == 'e2e' diff --git a/.github/actions/python-maturin/pre-merge/action.yml b/.github/actions/python-maturin/pre-merge/action.yml index 7b5c0d611..dcd1fe89b 100644 --- a/.github/actions/python-maturin/pre-merge/action.yml +++ b/.github/actions/python-maturin/pre-merge/action.yml @@ -35,6 +35,7 @@ runs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + save-cache: "false" # Only builds wheel, let Rust test job save comprehensive cache - name: Cache pip uses: actions/cache@v4 diff --git a/.github/actions/rust/post-merge/action.yml b/.github/actions/rust/post-merge/action.yml index 1255ae469..909facf8e 100644 --- a/.github/actions/rust/post-merge/action.yml +++ b/.github/actions/rust/post-merge/action.yml @@ -36,8 +36,7 @@ runs: - name: Setup Rust with cache uses: ./.github/actions/utils/setup-rust-with-cache with: - cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) - show-stats: false # Don't need stats for publishing + enabled: "false" # Post-merge runs infrequently, no need to cache - name: Validate package run: | diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index a857f4739..dc08d8d78 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -45,7 +45,8 @@ runs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) - show-stats: true + # Only test (most comprehensive) and build-* tasks should save cache + save-cache: ${{ inputs.task == 'test' || startsWith(inputs.task, 'build-') }} - name: Install tools for specific tasks run: | @@ -125,14 +126,6 @@ runs: echo "Test time: ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" echo "Total time: $((build_duration + test_duration))s ($(date -ud @$((build_duration + test_duration)) +'%M:%S'))" echo "=========================================" - - if command -v sccache &> /dev/null; then - echo "" - echo "=========================================" - echo "sccache statistics:" - echo "=========================================" - sccache --show-stats - fi shell: bash - name: Backwards compatibility check @@ -173,3 +166,14 @@ runs: if: inputs.task == 'build-macos-aarch64' run: cargo build --locked --all-targets shell: bash + + - name: Show sccache statistics + if: always() + run: | + if command -v sccache &> /dev/null; then + echo "=========================================" + echo "sccache statistics:" + echo "=========================================" + sccache --show-stats + fi + shell: bash diff --git a/.github/actions/utils/docker-buildx/action.yml b/.github/actions/utils/docker-buildx/action.yml index 7f06d4826..a140117e5 100644 --- a/.github/actions/utils/docker-buildx/action.yml +++ b/.github/actions/utils/docker-buildx/action.yml @@ -46,6 +46,10 @@ inputs: description: "Single platform to build (e.g., linux/amd64). If set, builds only this platform without QEMU. Leave empty for multi-arch build." required: false default: "" + gha-cache: + description: "Whether to use GitHub Actions cache for Docker layers (disable to save cache space)" + required: false + default: "true" outputs: digest: @@ -271,6 +275,12 @@ runs: fi fi + # Filter out GHA cache entries if gha-cache is disabled + if [ "${{ inputs.gha-cache }}" = "false" ]; then + CACHE_TO=$(echo "$CACHE_TO" | grep -v "type=gha" || true) + CACHE_FROM=$(echo "$CACHE_FROM" | grep -v "type=gha" || true) + fi + { echo 'CACHE_TO<<EOF' echo "$CACHE_TO" diff --git a/.github/actions/utils/setup-node-with-cache/action.yml b/.github/actions/utils/setup-node-with-cache/action.yml index e82bf581e..3c5cfe59c 100644 --- a/.github/actions/utils/setup-node-with-cache/action.yml +++ b/.github/actions/utils/setup-node-with-cache/action.yml @@ -44,19 +44,9 @@ runs: uses: actions/setup-node@v4 with: node-version: ${{ inputs.node-version }} - cache: "npm" + cache: ${{ inputs.enabled == 'true' && 'npm' || '' }} cache-dependency-path: ${{ inputs.cache-dependency-path }} - - name: Setup npm cache - if: inputs.enabled == 'true' - uses: actions/cache@v4 - with: - path: ~/.npm - key: npm-${{ runner.os }}-${{ hashFiles(inputs.cache-dependency-path) }} - restore-keys: | - npm-${{ runner.os }}- - - - name: Install dependencies if: inputs.enabled == 'true' run: | diff --git a/.github/actions/utils/setup-rust-with-cache/action.yml b/.github/actions/utils/setup-rust-with-cache/action.yml index 17a5e857a..d761716fb 100644 --- a/.github/actions/utils/setup-rust-with-cache/action.yml +++ b/.github/actions/utils/setup-rust-with-cache/action.yml @@ -25,14 +25,18 @@ inputs: description: "Whether to enable caching" required: false default: "true" - show-stats: - description: "Whether to show sccache statistics at the end" - required: false - default: "false" cache-targets: description: "Whether to cache target directory (passed to rust-cache)" required: false default: "false" + profile: + description: "Build profile for cache separation (dev or release)" + required: false + default: "dev" + save-cache: + description: "Whether to save cache at end of job (set false for jobs that don't compile much)" + required: false + default: "true" runs: using: "composite" @@ -60,18 +64,36 @@ runs: uses: Swatinem/rust-cache@v2 with: cache-targets: ${{ inputs.cache-targets }} + shared-key: rust-deps # All jobs share same dependency cache (profile doesn't affect downloads) - - name: Setup sccache cache - if: inputs.enabled == 'true' + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + shell: bash + + - name: Setup sccache cache (restore and save) + if: inputs.enabled == 'true' && inputs.save-cache == 'true' uses: actions/cache@v4 with: path: | ~/.cache/sccache ~/Library/Caches/sccache ~/.local/share/sccache - key: sccache-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} + key: sccache-${{ runner.os }}-${{ runner.arch }}-${{ inputs.profile }}-${{ steps.date.outputs.date }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} + restore-keys: | + sccache-${{ runner.os }}-${{ runner.arch }}-${{ inputs.profile }}- + + - name: Setup sccache cache (restore only) + if: inputs.enabled == 'true' && inputs.save-cache != 'true' + uses: actions/cache/restore@v4 + with: + path: | + ~/.cache/sccache + ~/Library/Caches/sccache + ~/.local/share/sccache + key: sccache-${{ runner.os }}-${{ runner.arch }}-${{ inputs.profile }}-${{ steps.date.outputs.date }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} restore-keys: | - sccache-${{ runner.os }}-${{ runner.arch }}- + sccache-${{ runner.os }}-${{ runner.arch }}-${{ inputs.profile }}- - name: Install sccache if: inputs.enabled == 'true' @@ -160,7 +182,7 @@ runs: mkdir -p "$SCCACHE_DIR" echo "SCCACHE_DIR=$SCCACHE_DIR" >> $GITHUB_ENV - echo "SCCACHE_CACHE_SIZE=5G" >> $GITHUB_ENV + echo "SCCACHE_CACHE_SIZE=2G" >> $GITHUB_ENV echo "SCCACHE_ERROR_BEHAVIOR=warn" >> $GITHUB_ENV echo "SCCACHE_IGNORE_SERVER_IO_ERROR=1" >> $GITHUB_ENV @@ -196,12 +218,3 @@ runs: echo "⚠️ sccache not available, continuing without cache" fi shell: bash - - - name: Show sccache stats on completion - if: always() && inputs.enabled == 'true' && inputs.show-stats == 'true' - run: | - if command -v sccache &> /dev/null; then - echo "### Final sccache Statistics ###" - sccache --show-stats || true - fi - shell: bash diff --git a/.github/workflows/_build_python_wheels.yml b/.github/workflows/_build_python_wheels.yml index 7b2b674c7..3b5e79080 100644 --- a/.github/workflows/_build_python_wheels.yml +++ b/.github/workflows/_build_python_wheels.yml @@ -88,6 +88,7 @@ jobs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false + profile: release - name: Build wheels uses: PyO3/maturin-action@v1 diff --git a/.github/workflows/_build_rust_artifacts.yml b/.github/workflows/_build_rust_artifacts.yml index 14c8e7904..5afb9af1e 100644 --- a/.github/workflows/_build_rust_artifacts.yml +++ b/.github/workflows/_build_rust_artifacts.yml @@ -103,6 +103,7 @@ jobs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false + profile: release - name: Add Rust target run: rustup target add ${{ matrix.target }} @@ -205,6 +206,7 @@ jobs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false + profile: release - name: Add Rust target run: rustup target add ${{ matrix.target }} diff --git a/.github/workflows/_publish_rust_crates.yml b/.github/workflows/_publish_rust_crates.yml index d2e7db074..5b04d92ac 100644 --- a/.github/workflows/_publish_rust_crates.yml +++ b/.github/workflows/_publish_rust_crates.yml @@ -98,7 +98,7 @@ jobs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false - show-stats: false + profile: release - name: Extract versions id: versions diff --git a/.github/workflows/_test_bdd.yml b/.github/workflows/_test_bdd.yml index 907868913..dbba0b26d 100644 --- a/.github/workflows/_test_bdd.yml +++ b/.github/workflows/_test_bdd.yml @@ -59,6 +59,7 @@ jobs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + save-cache: "false" # Only builds server+cli, let Rust test job save comprehensive cache - name: Build server for BDD tests if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') diff --git a/.github/workflows/_test_examples.yml b/.github/workflows/_test_examples.yml index d30e2a068..492239cf4 100644 --- a/.github/workflows/_test_examples.yml +++ b/.github/workflows/_test_examples.yml @@ -50,6 +50,7 @@ jobs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + save-cache: "false" # Only builds server+examples, let Rust test job save comprehensive cache - name: Setup Node with cache for examples if: inputs.component == 'examples-suite' diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index 13fbc6520..3f7be680e 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -138,6 +138,7 @@ jobs: version: ${{ steps.ver.outputs.version }} platform: ${{ matrix.platform }} dry_run: ${{ github.event.repository.fork }} + gha-cache: "false" # Use registry cache only to save GHA cache space - name: Export digest if: ${{ !github.event.repository.fork }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 646955d19..dbbc4ced4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -580,6 +580,7 @@ jobs: version: ${{ steps.ver.outputs.version }} platform: ${{ matrix.platform }} dry_run: ${{ inputs.dry_run }} + gha-cache: "false" # Use registry cache only to save GHA cache space - name: Export digest if: ${{ !inputs.dry_run }} @@ -768,7 +769,7 @@ jobs: uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false - show-stats: false + profile: release - name: Debug matrix run: echo '${{ toJson(matrix) }}'
