This is an automated email from the ASF dual-hosted git repository. hubcio pushed a commit to branch perf/rust-pre-merge-coverage in repository https://gitbox.apache.org/repos/asf/iggy.git
commit affe6e098bbb39ee6333e61e259a81e396abab33 Author: Hubert Gruszecki <[email protected]> AuthorDate: Wed Oct 14 16:57:50 2026 +0200 perf(ci): avoid redundant Rust coverage builds Rust shards and BDD jobs rebuilt tests, while coverage settings and Python's native-library environment prevented dependency reuse. Own Rust checks, the shared test build, and parallel shards in one reusable workflow. Reuse it for PRs and full master coverage while preserving affected-task selection and test failure propagation. Instrument BDD tests during image builds and reuse Python's prebuilt server. Collect Rust coverage only from tests and BDD, with matching master reports and two-hour post-merge timeouts. Align master cache warmers with the SDK jobs. Keep compiler errors visible and avoid hashing raw coverage files. Remove npm and Bazel caching and shorten shared setup action names. --- .config/nextest.toml | 3 +- .github/actions/cpp-bazel/pre-merge/action.yml | 8 +- .github/actions/csharp-dotnet/pre-merge/action.yml | 6 +- .github/actions/go/pre-merge/action.yml | 6 +- .github/actions/java-gradle/post-merge/action.yml | 4 +- .github/actions/java-gradle/pre-merge/action.yml | 8 +- .github/actions/node-npm/pre-merge/action.yml | 7 +- .github/actions/php/pre-merge/action.yml | 4 +- .../actions/python-maturin/pre-merge/action.yml | 50 ++-- .github/actions/rust/post-merge/action.yml | 4 +- .github/actions/rust/pre-merge/action.yml | 255 +++++++++++---------- .github/actions/swift/pre-merge/action.yml | 2 +- .github/actions/utils/server-start/action.yml | 13 +- .../{setup-cpp-with-cache => setup-cpp}/action.yml | 9 +- .../{setup-go-with-cache => setup-go}/action.yml | 4 +- .../action.yml | 2 +- .../action.yml | 29 +-- .../action.yml | 86 ++++--- .../utils/validate-third-party-licenses/action.yml | 5 +- .github/config/components.yml | 6 +- .github/workflows/_build_python_wheels.yml | 12 +- .github/workflows/_build_rust_artifacts.yml | 9 +- .github/workflows/_common.yml | 1 + .github/workflows/_detect.yml | 19 +- .github/workflows/_publish_rust_crates.yml | 4 +- .github/workflows/_rust.yml | 196 ++++++++++++++++ .github/workflows/_test.yml | 76 +----- .github/workflows/_test_bdd.yml | 43 +++- .github/workflows/_test_examples.yml | 63 +++-- .github/workflows/coverage-baseline.yml | 191 ++++++--------- .github/workflows/pr-skill-review-run.yml | 2 + .github/workflows/pre-merge.yml | 32 +-- .github/workflows/publish.yml | 2 +- bdd/docker-compose.cluster.yml | 3 + bdd/docker-compose.coverage.yml | 21 +- bdd/docker-compose.server.yml | 2 + bdd/docker-compose.yml | 2 + bdd/rust/Dockerfile | 40 +++- codecov.yml | 3 + scripts/ci/coverage-baseline-affected.sh | 2 +- scripts/ci/storage-compat.sh | 4 + scripts/run-bdd-tests.sh | 3 +- 42 files changed, 698 insertions(+), 543 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index bd0f105e5..1acb8e17f 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -17,8 +17,7 @@ # Floor for the flags CI and scripts/ci/storage-compat.sh pass: --no-tests # (0.9.75), --run-ignored only (0.9.76) and --ignore-default-filter (0.9.77). -# nextest refuses to run below it, and setup-rust-with-cache replaces a -# cache-restored binary that fails `cargo nextest show-config version`. +# nextest refuses to run below it; setup-rust installs a pinned version above this floor. nextest-version = { required = "0.9.77" } [[profile.default.overrides]] diff --git a/.github/actions/cpp-bazel/pre-merge/action.yml b/.github/actions/cpp-bazel/pre-merge/action.yml index b21f3eb2e..eaa15c9c6 100644 --- a/.github/actions/cpp-bazel/pre-merge/action.yml +++ b/.github/actions/cpp-bazel/pre-merge/action.yml @@ -26,12 +26,12 @@ inputs: runs: using: "composite" steps: - - name: Setup Bazel with cache + - name: Setup Bazel if: inputs.task != 'lint' - uses: ./.github/actions/utils/setup-cpp-with-cache + uses: ./.github/actions/utils/setup-cpp - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust - name: Install clang-format if: inputs.task == 'lint' diff --git a/.github/actions/csharp-dotnet/pre-merge/action.yml b/.github/actions/csharp-dotnet/pre-merge/action.yml index d68f930c5..d0471cd79 100644 --- a/.github/actions/csharp-dotnet/pre-merge/action.yml +++ b/.github/actions/csharp-dotnet/pre-merge/action.yml @@ -33,9 +33,9 @@ runs: 8.0.x 10.0.x - - name: Setup Rust with cache - if: inputs.task == 'test' || startsWith(inputs.task, 'e2e') - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + if: startsWith(inputs.task, 'e2e') + uses: ./.github/actions/utils/setup-rust - name: Restore dependencies run: | diff --git a/.github/actions/go/pre-merge/action.yml b/.github/actions/go/pre-merge/action.yml index b06f680a5..30a4a0010 100644 --- a/.github/actions/go/pre-merge/action.yml +++ b/.github/actions/go/pre-merge/action.yml @@ -27,7 +27,7 @@ runs: using: "composite" steps: - name: Setup Go - uses: ./.github/actions/utils/setup-go-with-cache + uses: ./.github/actions/utils/setup-go - name: Format Check shell: bash @@ -117,9 +117,9 @@ runs: # Build the module go build -v ./... - - name: Setup Rust with cache for e2e + - name: Setup Rust for e2e if: inputs.task == 'e2e' - uses: ./.github/actions/utils/setup-rust-with-cache + uses: ./.github/actions/utils/setup-rust # The Go SDK speaks the VSR wire protocol, so every e2e lane runs against # a locally built VSR server rather than a container of the legacy one. diff --git a/.github/actions/java-gradle/post-merge/action.yml b/.github/actions/java-gradle/post-merge/action.yml index e4e1927ea..79b7cf73e 100644 --- a/.github/actions/java-gradle/post-merge/action.yml +++ b/.github/actions/java-gradle/post-merge/action.yml @@ -30,8 +30,8 @@ inputs: runs: using: "composite" steps: - - name: Setup Java with cache - uses: ./.github/actions/utils/setup-java-with-cache + - name: Setup Java + uses: ./.github/actions/utils/setup-java - name: Build for publishing shell: bash diff --git a/.github/actions/java-gradle/pre-merge/action.yml b/.github/actions/java-gradle/pre-merge/action.yml index db377bd43..a6047a619 100644 --- a/.github/actions/java-gradle/pre-merge/action.yml +++ b/.github/actions/java-gradle/pre-merge/action.yml @@ -26,8 +26,8 @@ inputs: runs: using: "composite" steps: - - name: Setup Java with cache - uses: ./.github/actions/utils/setup-java-with-cache + - name: Setup Java + uses: ./.github/actions/utils/setup-java with: gradle-cache-disabled: "true" @@ -76,9 +76,9 @@ runs: # Exit with build exit code exit $BUILD_EXIT_CODE - - name: Setup Rust with cache + - name: Setup Rust if: inputs.task == 'test' - uses: ./.github/actions/utils/setup-rust-with-cache + uses: ./.github/actions/utils/setup-rust - name: Start Iggy server if: inputs.task == 'test' diff --git a/.github/actions/node-npm/pre-merge/action.yml b/.github/actions/node-npm/pre-merge/action.yml index 97771a05b..4214810f1 100644 --- a/.github/actions/node-npm/pre-merge/action.yml +++ b/.github/actions/node-npm/pre-merge/action.yml @@ -31,16 +31,15 @@ runs: with: node-version: "23" registry-url: "https://registry.npmjs.org"; - cache: "npm" - cache-dependency-path: foreign/node/package-lock.json + package-manager-cache: false - name: Pin npm to 11 to match Dependabot lockfile format run: npm install -g npm@11 shell: bash - - name: Setup Rust with cache + - name: Setup Rust if: inputs.task == 'e2e' - uses: ./.github/actions/utils/setup-rust-with-cache + uses: ./.github/actions/utils/setup-rust - name: Install netcat if: inputs.task == 'e2e' diff --git a/.github/actions/php/pre-merge/action.yml b/.github/actions/php/pre-merge/action.yml index f69b57657..6386086a0 100644 --- a/.github/actions/php/pre-merge/action.yml +++ b/.github/actions/php/pre-merge/action.yml @@ -58,8 +58,8 @@ runs: "${php_config}" --version "${php_bin}" "${composer_bin}" --version - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust with: shared-key: dev save-cache: "false" diff --git a/.github/actions/python-maturin/pre-merge/action.yml b/.github/actions/python-maturin/pre-merge/action.yml index dcb69b146..3233c3a75 100644 --- a/.github/actions/python-maturin/pre-merge/action.yml +++ b/.github/actions/python-maturin/pre-merge/action.yml @@ -26,14 +26,37 @@ inputs: runs: using: "composite" steps: + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust + + # setup-python changes PKG_CONFIG_PATH and invalidates cached native server dependencies. + - name: Build server Docker image for TLS tests + if: inputs.task == 'test' + run: | + # test_tls.py spawns the server in a container; build it from the + # same binaries the plain-TCP tests run against. + cargo build --locked --bin iggy-server --bin iggy + docker build \ + -f core/server/Dockerfile \ + --target runtime-prebuilt \ + --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ + --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ + -t iggy-server:local . + shell: bash + + - name: Start Iggy server + if: inputs.task == 'test' + id: iggy + uses: ./.github/actions/utils/server-start + with: + mode: bin + bin: target/debug/iggy-server + - name: Setup Python uses: actions/setup-python@v6 with: python-version: "3.11" - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache - - name: Install cargo-llvm-cov if: inputs.task == 'test' uses: taiki-e/install-action@v2 @@ -147,27 +170,6 @@ runs: uv run --no-sync maturin develop shell: bash - - name: Build server Docker image for TLS tests - if: inputs.task == 'test' - run: | - # test_tls.py spawns the server in a container; build it from the - # same binaries the plain-TCP tests run against. - cargo build --locked --bin iggy-server --bin iggy - docker build \ - -f core/server/Dockerfile \ - --target runtime-prebuilt \ - --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ - --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ - -t iggy-server:local . - shell: bash - - - name: Start Iggy server - if: inputs.task == 'test' - id: iggy - uses: ./.github/actions/utils/server-start - with: - cargo-bin: iggy-server - - name: Run Python integration tests if: inputs.task == 'test' run: | diff --git a/.github/actions/rust/post-merge/action.yml b/.github/actions/rust/post-merge/action.yml index caec167ee..13868253d 100644 --- a/.github/actions/rust/post-merge/action.yml +++ b/.github/actions/rust/post-merge/action.yml @@ -34,8 +34,8 @@ inputs: runs: using: "composite" steps: - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust - name: Validate package env: diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index de49148fd..34d6c5d3c 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -20,22 +20,41 @@ description: Rust pre-merge testing and linting github iggy actions inputs: task: - description: "Task to run (check, check-msrv, fmt, clippy, clippy-macos-aarch64, sort, machete, doctest, verify-publish, test-1, test-2, test-3, test-storage-compat, miri)" + description: "Task to run (check, check-msrv, fmt, clippy, clippy-macos-aarch64, sort, machete, doctest, verify-publish, test-build, test, test-storage-compat, miri)" required: true component: description: "Component name (for context)" required: false default: "" + test-scope: + description: "Test selection: affected or workspace" + required: false + default: affected + partition: + description: "One-based nextest partition index for the test task" + required: false + partitions: + description: "Total nextest partitions for the test task" + required: false + save-cache: + description: "Save build dependencies for subsequent PRs" + required: false + default: "false" + free-disk-space-aggressive: + description: "Use deep disk cleanup for full workspace coverage builds" + required: false + default: "false" runs: using: "composite" steps: - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust with: # Only test jobs execute nextest. Avoid downloading it in lint, metadata, # and cross-build jobs. - install-nextest: ${{ startsWith(inputs.task, 'test-') }} + install-nextest: ${{ inputs.task == 'test' || startsWith(inputs.task, 'test-') }} + coverage: ${{ inputs.task == 'test-build' || inputs.task == 'test' }} # These tasks do not compile workspace crates and need no native # libraries. All compiling tasks keep the existing dependency setup. install-system-dependencies: ${{ inputs.task != 'fmt' && inputs.task != 'sort' && inputs.task != 'machete' }} @@ -48,14 +67,14 @@ runs: # the warm `~/.cargo` restore is a free win, and its 1.95 target # artifacts rebuild regardless since the rustc hash differs. shared-key: ${{ inputs.task == 'miri' && 'miri' || 'dev' }} - # fmt/sort/machete never build into target/, so the multi-GB cache - # restore is pure overhead. Compiling legs (check/clippy/doctest/test-*) - # keep it; a cold cache there recompiles the whole dep tree. - read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete') && 'false' || 'true' }} + # Shards download the build artifact. Metadata-only tasks need no target cache. + read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete' || inputs.task == 'test') && 'false' || 'true' }} + save-cache: ${{ inputs.save-cache }} # Light legs fit in the runner's ~89 GiB default headroom; skip the # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers - # images on test-*, cross-builds, miri, verify-publish) keep it. + # images on test jobs, cross-builds, miri, verify-publish) keep it. free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }} + free-disk-space-aggressive: ${{ inputs.free-disk-space-aggressive }} - name: Install cargo-sort if: inputs.task == 'sort' @@ -84,7 +103,7 @@ runs: # Cargo-Rail release it is qualified with (0.30.1 for v10.1.1). - name: Plan affected Rust work id: rail - if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat' + if: inputs.task == 'test-build' && inputs.test-scope == 'affected' continue-on-error: true uses: loadingalias/cargo-rail-action@409962f05d90a78aae995ef6b1c8466010c0a460 # v10.1.1 @@ -149,45 +168,22 @@ runs: shell: bash - name: Install dependencies for Rust tests - if: startsWith(inputs.task, 'test-') && runner.os == 'Linux' + if: (inputs.task == 'test' || inputs.task == 'test-storage-compat') && runner.os == 'Linux' run: | ./scripts/ci/apt-install.sh musl-tools gnome-keyring keyutils dbus-x11 libsecret-tools rm -f "$HOME"/.local/share/keyrings/* shell: bash - - name: Install cargo-llvm-cov - if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat' - uses: taiki-e/install-action@v2 - with: - tool: cargo-llvm-cov - - - name: Build and test with coverage - # test-storage-compat drives its own nextest invocation from - # scripts/ci/storage-compat.sh. Without this exclusion it would also run - # here without partition arguments, i.e. a fourth *unpartitioned* copy - # of the whole suite on top of test-1/2/3. - if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat' + - name: Build coverage test archive + if: inputs.task == 'test-build' env: PLAN_FILE: ${{ steps.rail.outputs.plan-file }} PLAN_STATUS: ${{ steps.rail.outcome }} - TASK: ${{ inputs.task }} + TEST_SELECTION: ${{ inputs.test-scope }} run: | set -euo pipefail - - # Parse partition index from task name (test-1 -> hash:1/3, test-2 -> hash:2/3, ...). - # TEST_PARTITIONS must match the number of test-N tasks in - # .github/config/components.yml. Cluster bootstrap makes each test - # CPU-heavy, so partitions stay small. - # The regex is numeric-only on purpose: non-numeric test-* tasks - # (test-storage-compat) opt out of partitioning rather than producing - # an out-of-range hash:N/3. - TEST_PARTITIONS=3 - PARTITION_ARGS=() - if [[ "$TASK" =~ ^test-([0-9]+)$ ]]; then - PARTITION_INDEX="${BASH_REMATCH[1]}" - PARTITION_ARGS=(--partition "hash:${PARTITION_INDEX}/${TEST_PARTITIONS}") - echo "::notice::Running test partition ${PARTITION_INDEX}/${TEST_PARTITIONS}" - fi + ARTIFACT_DIR="$RUNNER_TEMP/rust-test-build" + mkdir -p "$ARTIFACT_DIR" # Read only validated selectors. Argument and package selectors are # NUL-delimited, hence `mapfile -d ''`. Any planner or selector failure @@ -195,12 +191,18 @@ runs: # target-args emits `--test` pairs only when every selected target of # every selected package is an integration test, so it never drops a # package or its unit tests from a mixed selection. - SELECTOR_DIR="${RUNNER_TEMP:?RUNNER_TEMP is required}/cargo-rail-selectors-${TASK}" + SELECTOR_DIR="${RUNNER_TEMP:?RUNNER_TEMP is required}/cargo-rail-selectors" mkdir -p "$SELECTOR_DIR" TEST_CARGO_ARGS=() TEST_TARGET_ARGS=() TEST_PACKAGES=() - if [[ "$PLAN_STATUS" == "success" ]] && + if [[ "$TEST_SELECTION" == "workspace" ]]; then + TEST_SCOPE=workspace + RUNTIME_SCOPE=workspace + elif [[ "$TEST_SELECTION" != "affected" ]]; then + echo "::error::Invalid test scope: $TEST_SELECTION" + exit 1 + elif [[ "$PLAN_STATUS" == "success" ]] && TEST_SCOPE=$(cargo-rail-action plan cargo-scope "$PLAN_FILE" cargo.test) && RUNTIME_SCOPE=$(cargo-rail-action plan cargo-scope "$PLAN_FILE" runtime-artifacts) && cargo-rail-action plan cargo-args "$PLAN_FILE" cargo.test > "$SELECTOR_DIR/test-args" && @@ -216,61 +218,44 @@ runs: echo "::warning::Cargo-Rail plan unavailable or invalid; running the full workspace" fi + printf '%s\n' "$TEST_SCOPE" > "$ARTIFACT_DIR/scope" + git rev-parse HEAD > "$ARTIFACT_DIR/revision" if [[ "$TEST_SCOPE" == "skipped" ]]; then echo "::notice::Cargo-Rail reports no affected Rust tests" exit 0 fi - # Runtime prerequisites (spawned binaries, dlopened connector plugins) - # are built alongside the tested packages. A package in both lists is - # built once. + # Only spawned binaries and dlopened plugins need a separate dev build. + # nextest archive compiles the selected tests with their dev-dependency features. BUILD_ARGS=() - if [[ "$TEST_SCOPE" == "packages" && "$RUNTIME_SCOPE" != "workspace" ]]; then - mapfile -d '' -t BUILD_PACKAGES < <(sort -zu "$SELECTOR_DIR/test-packages" "$SELECTOR_DIR/runtime-packages") - for package in "${BUILD_PACKAGES[@]}"; do + if [[ "$RUNTIME_SCOPE" == "packages" ]]; then + mapfile -d '' -t RUNTIME_PACKAGES < "$SELECTOR_DIR/runtime-packages" + for package in "${RUNTIME_PACKAGES[@]}"; do BUILD_ARGS+=(-p "$package") done - BUILD_SUMMARY="${#BUILD_PACKAGES[@]} packages" - else - BUILD_SUMMARY="full workspace" fi - if [[ "$TEST_SCOPE" == "packages" ]]; then - TEST_SUMMARY="${#TEST_PACKAGES[@]} packages" - else - TEST_SUMMARY="full workspace" - fi - echo "::notice::Cargo-Rail: testing ${TEST_SUMMARY}, building ${BUILD_SUMMARY}" - - source <(cargo llvm-cov show-env --export-prefix) - - # Doris 4.0.3's start_be.sh hard-`exit 1`s unless vm.max_map_count >= 2000000. - # This kernel param can only be set on the host (no container can raise it), - # so we raise it here. Everything else about booting Doris — image, heap/mem - # caps, port mappings, BE-alive wait — lives in the testcontainers fixture - # at core/integration/tests/connectors/fixtures/doris/container.rs, so - # `cargo test` and CI follow exactly the same path. - if [[ "$RUNNER_OS" == "Linux" ]]; then - sudo sysctl -w vm.max_map_count=2000000 || true + : > "$ARTIFACT_DIR/build.jsonl" + : > "$ARTIFACT_DIR/runtime-files" + if [[ "$RUNTIME_SCOPE" != "skipped" ]]; then + cargo build --locked "${BUILD_ARGS[@]}" --message-format=json-render-diagnostics > "$ARTIFACT_DIR/build.jsonl" + jq -r 'select(.reason == "compiler-artifact") | + (select(.target.kind | index("bin")) | .executable // empty), + (select(.target.crate_types | index("cdylib")) | .filenames[] | select(endswith(".so")))' \ + "$ARTIFACT_DIR/build.jsonl" | sort -u | \ + while IFS= read -r artifact; do + realpath --relative-to="$GITHUB_WORKSPACE" "$artifact" + done > "$ARTIFACT_DIR/runtime-files" fi - - bins_start=$(date +%s) - cargo build --locked "${BUILD_ARGS[@]}" - bins_end=$(date +%s) - bins_duration=$((bins_end - bins_start)) - echo "::notice::Binaries and libraries built in ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" - - compile_start=$(date +%s) - cargo test --locked --no-run "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}" - compile_end=$(date +%s) - compile_duration=$((compile_end - compile_start)) - echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" + tar --zstd -cf "$ARTIFACT_DIR/runtime.tar.zst" -T "$ARTIFACT_DIR/runtime-files" + rm "$ARTIFACT_DIR/build.jsonl" "$ARTIFACT_DIR/runtime-files" # api_handler_tests, version_firewall_tests, and server_e2e_tests need gitignored - # wire fixtures. Generate when iggy-gateway-kafka is in the Cargo-Rail test scope, or (on - # a full-workspace run) when gateways/** changed vs the PR base — avoid building - # kafka-message-gen / kafka-protocol when the gateway was not touched. + # wire fixtures. Master always generates them. On PRs, use the selected + # packages or the gateway diff to avoid building fixture tools unnecessarily. NEEDS_KAFKA_FIXTURES=false - if [[ "$TEST_SCOPE" == "packages" ]]; then + if [[ "$TEST_SELECTION" == "workspace" ]]; then + NEEDS_KAFKA_FIXTURES=true + elif [[ "$TEST_SCOPE" == "packages" ]]; then for package in "${TEST_PACKAGES[@]}"; do if [[ "$package" == "iggy-gateway-kafka" ]]; then NEEDS_KAFKA_FIXTURES=true @@ -313,9 +298,44 @@ runs: echo "::warning::git diff against the PR base failed (base: ${BASE_SHA:-none, HEAD is not a merge commit}; ${DIFF_OUTPUT:-}); generating Kafka fixtures unconditionally as a fail-safe" fi fi + printf '%s\n' "$NEEDS_KAFKA_FIXTURES" > "$ARTIFACT_DIR/kafka-fixtures" if [[ "$NEEDS_KAFKA_FIXTURES" == true ]]; then ./gateways/kafka/scripts/ci-wire-fixtures.sh generate - trap './gateways/kafka/scripts/ci-wire-fixtures.sh cleanup' EXIT + tar --zstd -cf "$ARTIFACT_DIR/kafka-fixtures.tar.zst" gateways/kafka/tools/kafka-tool/kafka_messages + fi + + cargo nextest archive --locked --profile ci \ + "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}" \ + --archive-file "$ARTIFACT_DIR/nextest.tar.zst" + shell: bash + + - name: Run coverage test partition + if: inputs.task == 'test' + env: + TEST_PARTITION: ${{ inputs.partition }} + TEST_PARTITIONS: ${{ inputs.partitions }} + run: | + set -euo pipefail + if [[ ! "$TEST_PARTITION" =~ ^[1-9][0-9]*$ || ! "$TEST_PARTITIONS" =~ ^[1-9][0-9]*$ ]] || + (( TEST_PARTITION > TEST_PARTITIONS )); then + echo "::error::Invalid test partition: $TEST_PARTITION/$TEST_PARTITIONS" + exit 1 + fi + ARTIFACT_DIR="$RUNNER_TEMP/rust-test-build" + artifact_revision=$(cat "$ARTIFACT_DIR/revision") + checkout_revision=$(git rev-parse HEAD) + if [[ "$artifact_revision" != "$checkout_revision" ]]; then + echo "::error::Coverage archive revision $artifact_revision does not match checkout $checkout_revision" + exit 1 + fi + TEST_SCOPE=$(cat "$ARTIFACT_DIR/scope") + if [[ "$TEST_SCOPE" == "skipped" ]]; then + echo "::notice::Cargo-Rail reports no affected Rust tests" + exit 0 + fi + tar --zstd -xf "$ARTIFACT_DIR/runtime.tar.zst" + if [[ "$(cat "$ARTIFACT_DIR/kafka-fixtures")" == true ]]; then + tar --zstd -xf "$ARTIFACT_DIR/kafka-fixtures.tar.zst" # Fixtures are expected to exist from here on; a missing one now means # generation silently failed, not a legitimate local-dev skip. export KAFKA_FIXTURES_REQUIRED=1 @@ -349,52 +369,23 @@ runs: # Previously this ran before `cargo build`, leaving a 7+ minute idle # window that triggered org.freedesktop.Secret.Error.IsLocked ~10% of runs. if [[ "$RUNNER_OS" == "Linux" ]]; then - eval $(dbus-launch --sh-syntax) + eval "$(dbus-launch --sh-syntax)" export DBUS_SESSION_BUS_ADDRESS - eval $(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets) + eval "$(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets)" echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup fi - test_start=$(date +%s) - if command -v cargo-nextest &> /dev/null; then - if [[ "$TEST_SCOPE" == "packages" ]]; then - # Every test target in the `bdd` crate is `required-features = - # ["bdd"]`, so the `cargo test --no-run` above builds no binaries - # for it. A PR touching only that crate then produces a valid - # package scope that matches zero tests, and nextest's default - # `--no-tests=fail` turns that into exit 4 on a green PR. - cargo nextest run --locked --no-fail-fast --no-tests=warn --profile ci \ - "${PARTITION_ARGS[@]}" "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}" - else - cargo nextest run --locked --no-fail-fast --profile ci "${PARTITION_ARGS[@]}" - fi - else - if [[ ${#PARTITION_ARGS[@]} -gt 0 ]]; then - echo "::error::cargo-nextest not found, falling back to cargo test without partitioning (all tests will run on every partition)" - fi - cargo test --locked --no-fail-fast "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}" + # Doris requires this host setting before its testcontainers fixture starts. + sudo sysctl -w vm.max_map_count=2000000 || true + NO_TESTS=fail + if [[ "$TEST_SCOPE" == "packages" ]]; then + # BDD targets require features, so a BDD-only scope can contain no runnable tests. + NO_TESTS=warn fi - test_end=$(date +%s) - test_duration=$((test_end - test_start)) - echo "::notice::Tests executed in ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" - - build_duration=$((bins_duration + compile_duration)) - total_duration=$((build_duration + test_duration)) - echo "" - echo "=========================================" - echo "Cargo-Rail scope (test): ${TEST_SUMMARY}" - echo "Cargo-Rail scope (build): ${BUILD_SUMMARY}" - echo "All targets build: ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" - echo "Tests compile: ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" - echo "Tests execute: ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))" - echo "-----------------------------------------" - echo "Total build: ${build_duration}s ($(date -ud @${build_duration} +'%M:%S'))" - echo "Total time: ${total_duration}s ($(date -ud @${total_duration} +'%M:%S'))" - echo "=========================================" - - cargo llvm-cov report --codecov --output-path codecov.json - echo "Coverage report generated: codecov.json" - ls -la codecov.json + cargo nextest run --archive-file "$ARTIFACT_DIR/nextest.tar.zst" \ + --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --workspace-remap "$GITHUB_WORKSPACE" \ + --no-fail-fast --no-tests="$NO_TESTS" --profile ci \ + --partition "hash:${TEST_PARTITION}/${TEST_PARTITIONS}" shell: bash # On-disk format backwards compatibility. The script builds an iggy-server @@ -478,7 +469,19 @@ runs: run: | target/debug/iggy-server --fresh > server-boot.log 2>&1 & SERVER_PID=$! - trap 'kill "$SERVER_PID" 2>/dev/null || true' EXIT + stop_server() { + local stop_timeout=30 + kill "$SERVER_PID" 2>/dev/null || true + for ((elapsed = 0; elapsed < stop_timeout; elapsed++)); do + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + break + fi + sleep 1 + done + kill -KILL "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" || true + } + trap stop_server EXIT for _ in $(seq 1 30); do if ! kill -0 "$SERVER_PID" 2>/dev/null; then diff --git a/.github/actions/swift/pre-merge/action.yml b/.github/actions/swift/pre-merge/action.yml index f21a11903..8623a6a53 100644 --- a/.github/actions/swift/pre-merge/action.yml +++ b/.github/actions/swift/pre-merge/action.yml @@ -80,7 +80,7 @@ runs: # the lane skips the disk cleanup, the apt install, and the cache restore. - name: Setup Rust for the golden vectors if: inputs.task == 'golden' - uses: ./.github/actions/utils/setup-rust-with-cache + uses: ./.github/actions/utils/setup-rust with: free-disk-space: "false" read-cache: "false" diff --git a/.github/actions/utils/server-start/action.yml b/.github/actions/utils/server-start/action.yml index 664582f7e..6b76e9cc5 100644 --- a/.github/actions/utils/server-start/action.yml +++ b/.github/actions/utils/server-start/action.yml @@ -112,14 +112,13 @@ runs: else OUT="target/debug/$NAME" fi - if [[ ! -x "$OUT" ]]; then - echo "Building $NAME with cargo ($PROFILE)…" - CARGO_ARGS=(build --locked --bin "$NAME") - if [[ "$PROFILE" == "release" ]]; then - CARGO_ARGS+=(--release) - fi - cargo "${CARGO_ARGS[@]}" + # An existing executable may belong to a different checkout or feature set. + echo "Building $NAME with cargo ($PROFILE)..." + CARGO_ARGS=(build --locked --bin "$NAME") + if [[ "$PROFILE" == "release" ]]; then + CARGO_ARGS+=(--release) fi + cargo "${CARGO_ARGS[@]}" BIN_PATH="$OUT" else BIN_PATH="${{ inputs.bin }}" diff --git a/.github/actions/utils/setup-cpp-with-cache/action.yml b/.github/actions/utils/setup-cpp/action.yml similarity index 82% rename from .github/actions/utils/setup-cpp-with-cache/action.yml rename to .github/actions/utils/setup-cpp/action.yml index 1ffb584bf..9f3a43bfd 100644 --- a/.github/actions/utils/setup-cpp-with-cache/action.yml +++ b/.github/actions/utils/setup-cpp/action.yml @@ -15,18 +15,15 @@ # specific language governing permissions and limitations # under the License. -name: setup-cpp-with-cache -description: Setup Bazel toolchain and caching (bazelisk + repository + disk cache) +name: setup-cpp +description: Setup Bazel toolchain runs: using: "composite" steps: - - name: Setup Bazel with cache + - name: Setup Bazel uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # v0.19.0 with: - bazelisk-cache: true - disk-cache: true - repository-cache: true module-root: foreign/cpp - name: Verify Bazel installation diff --git a/.github/actions/utils/setup-go-with-cache/action.yml b/.github/actions/utils/setup-go/action.yml similarity index 98% rename from .github/actions/utils/setup-go-with-cache/action.yml rename to .github/actions/utils/setup-go/action.yml index 10f9c2a7e..ae896bbf9 100644 --- a/.github/actions/utils/setup-go-with-cache/action.yml +++ b/.github/actions/utils/setup-go/action.yml @@ -16,10 +16,10 @@ # under the License. # NOTE: Go builds are fast, so caching is disabled by default. -# The read-cache/save-cache inputs exist for parity with setup-rust-with-cache +# The read-cache/save-cache inputs exist for parity with setup-rust # but both default to "false". Enable only if Go build times become a bottleneck. -name: setup-go-with-cache +name: setup-go description: Setup Go toolchain and optional caching (go mod cache + golangci-lint cache) inputs: go-version: diff --git a/.github/actions/utils/setup-java-with-cache/action.yml b/.github/actions/utils/setup-java/action.yml similarity index 98% rename from .github/actions/utils/setup-java-with-cache/action.yml rename to .github/actions/utils/setup-java/action.yml index 2a757b24e..e9f60cd28 100644 --- a/.github/actions/utils/setup-java-with-cache/action.yml +++ b/.github/actions/utils/setup-java/action.yml @@ -18,7 +18,7 @@ # NOTE: this action sets up the Java toolchain with optional Gradle caching. # Gradle itself is provided via the wrapper (gradlew) in each project. -name: setup-java-with-cache +name: setup-java description: Setup Java toolchain with optional Gradle caching (uses gradlew wrapper) inputs: java-version: diff --git a/.github/actions/utils/setup-node-with-cache/action.yml b/.github/actions/utils/setup-node/action.yml similarity index 68% rename from .github/actions/utils/setup-node-with-cache/action.yml rename to .github/actions/utils/setup-node/action.yml index a46b199e4..4c4767694 100644 --- a/.github/actions/utils/setup-node-with-cache/action.yml +++ b/.github/actions/utils/setup-node/action.yml @@ -15,24 +15,13 @@ # specific language governing permissions and limitations # under the License. -# NOTE: this action sets up the Node.js toolchain + npm cache for dependencies, -# it is a convenience wrapper, so that we can use it in all workflows. - -name: setup-node-with-cache -description: Setup Node.js toolchain and comprehensive caching (npm cache for dependencies) +name: setup-node +description: Setup Node.js toolchain and install dependencies inputs: node-version: description: "Node.js version to use" required: false default: "23" - enabled: - description: "Whether to enable caching" - required: false - default: "true" - cache-dependency-path: - description: "Path to lock file for npm caching" - required: false - default: "package-lock.json" workspace: description: "Workspace directory" required: false @@ -44,25 +33,13 @@ runs: uses: actions/setup-node@v6 with: node-version: ${{ inputs.node-version }} - cache: "npm" - cache-dependency-path: ${{ inputs.cache-dependency-path }} + package-manager-cache: false - name: Pin npm to 11 to match Dependabot lockfile format run: npm install -g npm@11 shell: bash - - name: Setup npm cache - if: inputs.enabled == 'true' - uses: actions/cache@v5 - with: - path: ~/.npm - key: npm-${{ runner.os }}-${{ hashFiles(inputs.cache-dependency-path) }} - restore-keys: | - npm-${{ runner.os }}- - - - name: Install dependencies - if: inputs.enabled == 'true' run: | echo "📦 Installing Node.js dependencies..." if [ "${{ inputs.workspace }}" != "." ]; then diff --git a/.github/actions/utils/setup-rust-with-cache/action.yml b/.github/actions/utils/setup-rust/action.yml similarity index 74% rename from .github/actions/utils/setup-rust-with-cache/action.yml rename to .github/actions/utils/setup-rust/action.yml index 023483300..c4f0a2026 100644 --- a/.github/actions/utils/setup-rust-with-cache/action.yml +++ b/.github/actions/utils/setup-rust/action.yml @@ -15,10 +15,14 @@ # specific language governing permissions and limitations # under the License. -name: setup-rust-with-cache +name: setup-rust description: Setup Rust toolchain with Swatinem/rust-cache inputs: + coverage: + description: "Instrument workspace crates for coverage before compiling" + required: false + default: "false" install-nextest: description: "Whether to install cargo-nextest" required: false @@ -88,15 +92,35 @@ runs: rustup-init -y --no-modify-path --default-toolchain none fi rustup show + # rust-cache hashes every installed toolchain, including unused runner defaults. + active_toolchain=$(rustup show active-toolchain) + active_toolchain=${active_toolchain%% *} + installed_toolchains=$(rustup toolchain list --quiet) + while read -r toolchain; do + if [[ "$toolchain" != "$active_toolchain" ]]; then + rustup toolchain uninstall "$toolchain" + fi + done <<< "$installed_toolchains" echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" shell: bash + - name: Configure Cargo for CI + run: | + echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" + echo "CARGO_PROFILE_DEV_DEBUG=0" >> "$GITHUB_ENV" + echo "CARGO_PROFILE_TEST_DEBUG=0" >> "$GITHUB_ENV" + echo "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse" >> "$GITHUB_ENV" + shell: bash + + # llvm-cov instruments workspace crates, which rust-cache removes before saving. + # External dependencies can share the cache with check, lint and SDK server builds. - name: Setup Rust cache id: rust-cache if: inputs.read-cache == 'true' uses: Swatinem/rust-cache@v2 with: shared-key: ${{ inputs.shared-key }} + cache-workspace-crates: "false" save-if: ${{ inputs.save-cache == 'true' }} - name: Cache status @@ -123,42 +147,32 @@ runs: shell: bash - name: Install cargo-nextest - if: runner.os == 'Linux' && inputs.install-nextest == 'true' - run: | - # A binary restored from the cargo cache can predate the floor in - # .config/nextest.toml. `show-config version` exits non-zero below it, - # and on a nextest too old to know the subcommand at all, so both fall - # through to a fresh install instead of aborting the lane later. - if command -v cargo-nextest &> /dev/null && cargo nextest show-config version; then - echo "cargo-nextest already installed" - exit 0 - fi - - ARCH="$(uname -m)" - case "$ARCH" in - x86_64) NEXTEST_PLATFORM="linux" ;; - aarch64) NEXTEST_PLATFORM="linux-arm" ;; - *) - echo "Unsupported architecture: $ARCH, skipping nextest" - exit 0 - ;; - esac + if: inputs.install-nextest == 'true' + uses: taiki-e/install-action@v2 + with: + tool: [email protected] - CARGO_BIN="${CARGO_HOME:-$HOME/.cargo}/bin" - tarball="$(mktemp)" - # Staged to a file because curl --retry cannot rewind a pipe. - curl -LsSf --retry 5 --retry-all-errors -o "$tarball" "https://get.nexte.st/latest/${NEXTEST_PLATFORM}"; - tar xzf "$tarball" -C "$CARGO_BIN" - rm -f "$tarball" - cargo nextest --version - shell: bash - continue-on-error: true + - name: Install cargo-llvm-cov + if: inputs.coverage == 'true' + uses: taiki-e/install-action@v2 + with: + tool: [email protected] - - name: Configure Cargo for CI - if: inputs.read-cache == 'true' + - name: Enable coverage for subsequent builds and tests + if: inputs.coverage == 'true' run: | - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - echo "CARGO_PROFILE_DEV_DEBUG=0" >> "$GITHUB_ENV" - echo "CARGO_PROFILE_TEST_DEBUG=0" >> "$GITHUB_ENV" - echo "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse" >> "$GITHUB_ENV" + set -euo pipefail + rustup component add llvm-tools-preview + coverage_env=$(cargo llvm-cov show-env --sh) + eval "$coverage_env" + if [[ "${{ inputs.read-cache }}" == true ]]; then + cargo llvm-cov clean --workspace + fi + while IFS= read -r line; do + if [[ "$line" =~ ^export\ ([A-Za-z_][A-Za-z0-9_]*)= ]]; then + name=${BASH_REMATCH[1]} + delimiter="COVERAGE_ENV_${RANDOM}_${RANDOM}" + printf '%s<<%s\n%s\n%s\n' "$name" "$delimiter" "${!name}" "$delimiter" >> "$GITHUB_ENV" + fi + done <<< "$coverage_env" shell: bash diff --git a/.github/actions/utils/validate-third-party-licenses/action.yml b/.github/actions/utils/validate-third-party-licenses/action.yml index 772da695d..ca853e53b 100644 --- a/.github/actions/utils/validate-third-party-licenses/action.yml +++ b/.github/actions/utils/validate-third-party-licenses/action.yml @@ -37,9 +37,10 @@ runs: uses: actions/setup-node@v6 with: node-version: ${{ inputs.node-version }} + package-manager-cache: false - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust with: read-cache: "false" diff --git a/.github/config/components.yml b/.github/config/components.yml index b8c5dd036..4714969fe 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -144,7 +144,7 @@ components: # existing deployments, which no other check covers. Matching is literal # glob with no crate-graph expansion, so every crate has to be spelled out: # `core/server/**` does not cover `core/server_common/**`. - # `breaking:storage` on the PR skips it (see .github/workflows/_test.yml). + # `breaking:storage` on the PR skips it (see .github/workflows/_rust.yml). rust-storage-compat: depends_on: - "rust-workspace" @@ -212,9 +212,7 @@ components: - "sort" - "doctest" - "machete" - - "test-1" - - "test-2" - - "test-3" + - "test" - "build-aarch64-gnu" - "build-aarch64-musl" - "build-macos-aarch64" diff --git a/.github/workflows/_build_python_wheels.yml b/.github/workflows/_build_python_wheels.yml index 1942a1ba1..af8e43c57 100644 --- a/.github/workflows/_build_python_wheels.yml +++ b/.github/workflows/_build_python_wheels.yml @@ -91,8 +91,8 @@ jobs: id: interpreters run: ./scripts/ci/wheel-interpreters.sh >> "$GITHUB_OUTPUT" - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust - name: Generate third-party license manifest run: | @@ -166,8 +166,8 @@ jobs: with: python-version: ${{ steps.interpreters.outputs.python_versions }} - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust - name: Generate third-party license manifest run: | @@ -240,8 +240,8 @@ jobs: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust with: shared-key: python-wheels-windows diff --git a/.github/workflows/_build_rust_artifacts.yml b/.github/workflows/_build_rust_artifacts.yml index 5d8715f63..df77bba9c 100644 --- a/.github/workflows/_build_rust_artifacts.yml +++ b/.github/workflows/_build_rust_artifacts.yml @@ -102,8 +102,8 @@ jobs: if: matrix.libc == 'musl' run: ./scripts/ci/apt-install.sh musl-tools - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust - name: Add Rust target run: rustup target add ${{ matrix.target }} @@ -112,6 +112,7 @@ jobs: uses: actions/setup-node@v7 with: node-version: "22" + package-manager-cache: false - name: Build Web UI static files run: npm --prefix web ci && npm --prefix web run build:static @@ -213,8 +214,8 @@ jobs: /tmp/copy-latest-from-master.sh save .github scripts /tmp/copy-latest-from-master.sh apply - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust - name: Add Rust target run: rustup target add ${{ matrix.target }} diff --git a/.github/workflows/_common.yml b/.github/workflows/_common.yml index 56459044a..e4e247cdc 100644 --- a/.github/workflows/_common.yml +++ b/.github/workflows/_common.yml @@ -151,6 +151,7 @@ jobs: uses: actions/setup-node@v7 with: node-version: "23" + package-manager-cache: false - name: Install markdownlint-cli run: npm install -g markdownlint-cli diff --git a/.github/workflows/_detect.yml b/.github/workflows/_detect.yml index 38902b859..8cd417c1c 100644 --- a/.github/workflows/_detect.yml +++ b/.github/workflows/_detect.yml @@ -23,9 +23,9 @@ on: third_party_licenses: description: "Whether third-party license inputs changed" value: ${{ jobs.detect.outputs.third_party_licenses }} - rust_matrix: - description: "Matrix for Rust components" - value: ${{ jobs.detect.outputs.rust_matrix }} + rust_plan: + description: "Selected Rust checks and tests; empty when Rust is unaffected" + value: ${{ jobs.detect.outputs.rust_plan }} python_matrix: description: "Matrix for Python SDK" value: ${{ jobs.detect.outputs.python_matrix }} @@ -69,7 +69,7 @@ jobs: timeout-minutes: 10 outputs: third_party_licenses: ${{ github.event_name != 'pull_request' || steps.changed.outputs.third_party_licenses == 'true' }} - rust_matrix: ${{ steps.mk.outputs.rust_matrix }} + rust_plan: ${{ steps.mk.outputs.rust_plan }} python_matrix: ${{ steps.mk.outputs.python_matrix }} php_matrix: ${{ steps.mk.outputs.php_matrix }} node_matrix: ${{ steps.mk.outputs.node_matrix }} @@ -111,7 +111,7 @@ jobs: - 'scripts/ci/render-node-licenses.mjs' - 'scripts/ci/apt-install.sh' - '.github/actions/utils/validate-third-party-licenses/**' - - '.github/actions/utils/setup-rust-with-cache/**' + - '.github/actions/utils/setup-rust/**' - '.github/actions/utils/free-disk-space/**' - '.github/workflows/_detect.yml' - '.github/workflows/_common.yml' @@ -290,7 +290,7 @@ jobs: const entries = cfg.tasks.map(task => ({ component: name, task })); taskedComponents.push({ name, tasks: cfg.tasks, trigger }); - if (name === 'rust') groups.rust.push(...entries); + if (name === 'rust' || name.startsWith('rust-')) groups.rust.push(...entries); else if (name === 'sdk-python') groups.python.push(...entries); else if (name === 'sdk-php') groups.php.push(...entries); else if (name === 'sdk-node') groups.node.push(...entries); @@ -354,7 +354,7 @@ jobs: for (const [name, cfg] of Object.entries(components)) { if (!cfg.tasks || cfg.tasks.length === 0) continue; const entries = cfg.tasks.map(task => ({ component: name, task })); - if (name === 'rust') groups.rust.push(...entries); + if (name === 'rust' || name.startsWith('rust-')) groups.rust.push(...entries); else if (name === 'sdk-python') groups.python.push(...entries); else if (name === 'sdk-php') groups.php.push(...entries); else if (name === 'sdk-node') groups.node.push(...entries); @@ -426,7 +426,10 @@ jobs: require('fs').appendFileSync(process.env.GITHUB_OUTPUT, `${output}\n`); }; - setOutput('rust_matrix', JSON.stringify(matrix(groups.rust))); + setOutput('rust_plan', groups.rust.length ? JSON.stringify({ + checks: groups.rust.filter(entry => entry.task !== 'test'), + tests: groups.rust.some(entry => entry.task === 'test'), + }) : ''); setOutput('python_matrix', JSON.stringify(matrix(groups.python))); setOutput('php_matrix', JSON.stringify(matrix(groups.php))); setOutput('node_matrix', JSON.stringify(matrix(groups.node))); diff --git a/.github/workflows/_publish_rust_crates.yml b/.github/workflows/_publish_rust_crates.yml index ae2a8d739..2f268231d 100644 --- a/.github/workflows/_publish_rust_crates.yml +++ b/.github/workflows/_publish_rust_crates.yml @@ -112,8 +112,8 @@ jobs: /tmp/copy-latest-from-master.sh apply - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust # Normalize the inbound commit to a canonical full SHA. Caller (publish.yml) # already does this; a direct workflow_call caller must pass an explicit diff --git a/.github/workflows/_rust.yml b/.github/workflows/_rust.yml new file mode 100644 index 000000000..7d5458b72 --- /dev/null +++ b/.github/workflows/_rust.yml @@ -0,0 +1,196 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: _rust +on: + workflow_call: + inputs: + plan: + description: "JSON with checks (component/task entries) and tests (boolean)" + type: string + required: true + test-scope: + description: "Test selection: affected or workspace" + type: string + default: affected + timeout-minutes: + type: number + default: 60 + save-cache: + description: "Save dependencies from the test build for subsequent PRs" + type: boolean + default: false + build-metadata: + description: "Embed CI build metadata in the server binaries" + type: boolean + default: false + secrets: + CODECOV_TOKEN: + required: false + +permissions: + contents: read + issues: read # live breaking:storage label lookup + +env: + IGGY_CI_BUILD: ${{ inputs.build-metadata }} + +jobs: + checks: + name: ${{ matrix.component }}/${{ matrix.task }} + if: ${{ fromJSON(inputs.plan).checks[0] != null }} + strategy: + fail-fast: true + matrix: + include: ${{ fromJSON(inputs.plan).checks }} + runs-on: ${{ + (matrix.task == 'build-aarch64-gnu' || matrix.task == 'build-aarch64-musl') && 'ubuntu-26.04-arm' || + (matrix.task == 'build-macos-aarch64' || matrix.task == 'clippy-macos-aarch64') && 'macos-14' || + matrix.task == 'build-windows-sdk' && 'windows-latest' || + 'ubuntu-26.04' + }} + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - uses: actions/[email protected] + + # Re-runs replay the original event labels. Read live labels so applying + # breaking:storage followed by a re-run can skip an intentional format break. + - name: Resolve breaking:storage escape hatch + id: storage_hatch + if: matrix.task == 'test-storage-compat' && github.event.pull_request.number + uses: actions/github-script@v9 + with: + script: | + let skip = false; + try { + const labels = await github.paginate( + github.rest.issues.listLabelsOnIssue, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + per_page: 100, + }, + ); + skip = labels.some(l => l.name === 'breaking:storage'); + } catch (e) { + core.warning(`listLabelsOnIssue failed (${e.status ?? e.code ?? 'network'}), running the check`); + } + core.info(`breaking:storage escape hatch: skip=${skip}`); + core.setOutput('skip', String(skip)); + + - name: Run Rust check + if: steps.storage_hatch.outputs.skip != 'true' + uses: ./.github/actions/rust/pre-merge + with: + task: ${{ matrix.task }} + component: ${{ matrix.component }} + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: ${{ matrix.component }}-${{ matrix.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }} + path: reports/** + if-no-files-found: ignore + retention-days: 7 + + test-build: + name: Build test artifacts + if: ${{ fromJSON(inputs.plan).tests }} + runs-on: ubuntu-26.04 + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - uses: actions/[email protected] + + - name: Build coverage test archive + uses: ./.github/actions/rust/pre-merge + with: + task: test-build + test-scope: ${{ inputs.test-scope }} + save-cache: ${{ inputs.save-cache }} + free-disk-space-aggressive: ${{ inputs.test-scope == 'workspace' }} + + - name: Upload test artifacts + uses: actions/upload-artifact@v7 + with: + name: rust-test-build-${{ github.run_id }} + path: ${{ runner.temp }}/rust-test-build/ + compression-level: 0 + if-no-files-found: error + overwrite: true + retention-days: 7 + + tests: + name: Test shard ${{ matrix.partition }}/${{ strategy.job-total }} + needs: test-build + strategy: + fail-fast: false + matrix: + partition: [1, 2, 3] + runs-on: ubuntu-26.04 + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - uses: actions/[email protected] + + - name: Download test artifacts + uses: actions/download-artifact@v8 + with: + name: rust-test-build-${{ github.run_id }} + path: ${{ runner.temp }}/rust-test-build + + - name: Run test shard + uses: ./.github/actions/rust/pre-merge + with: + task: test + partition: ${{ matrix.partition }} + partitions: ${{ strategy.job-total }} + + - name: Generate coverage report + if: always() && !cancelled() + run: | + if compgen -G 'target/*.profraw' > /dev/null; then + cargo llvm-cov report --codecov --output-path codecov.json + else + echo "::notice::No coverage profiles found; skipping report" + fi + shell: bash + + - name: Upload coverage to Codecov + if: always() && !cancelled() && hashFiles('codecov.json') != '' + uses: codecov/[email protected] + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: codecov.json + disable_search: true + flags: rust + fail_ci_if_error: false + verbose: true + override_pr: ${{ github.event.pull_request.number }} + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: rust-test-${{ matrix.partition }}-reports-${{ github.run_id }}-${{ github.run_attempt }} + path: | + reports/** + target/llvm-cov/** + coverage.lcov + codecov.json + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 7fb5ff909..0d24d4da2 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -33,7 +33,6 @@ on: permissions: contents: read - issues: read # listLabelsOnIssue, for the breaking:storage escape hatch security-events: write pull-requests: write @@ -41,10 +40,7 @@ jobs: run: # Select runner based on build target runs-on: ${{ - (inputs.task == 'build-aarch64-gnu' || inputs.task == 'build-aarch64-musl') && 'ubuntu-26.04-arm' || - (inputs.task == 'build-macos-aarch64' || inputs.task == 'clippy-macos-aarch64') && 'macos-14' || inputs.task == 'build-macos' && 'macos-15' || - inputs.task == 'build-windows-sdk' && 'windows-latest' || 'ubuntu-26.04' }} timeout-minutes: 60 @@ -56,75 +52,6 @@ jobs: if: inputs.component == 'noop' run: echo "No changes detected, skipping tests" - # Rust - # `breaking:storage` is the documented escape hatch for a deliberate - # on-disk format change, which the compat check cannot pass by - # construction. The label has to be read live: pre-merge.yml does not - # subscribe to `labeled` (pr-triage-apply.yml writes S-* labels, so every - # triage write would re-run the whole gate), and a re-run replays the - # webhook-frozen `pull_request.labels` from the last push. The operator - # flow is "check fails, apply label, re-run", which the frozen payload - # cannot see. Paginated so a PR with more than 100 labels still resolves. - - name: Resolve breaking:storage escape hatch - id: storage_hatch - if: >- - startsWith(inputs.component, 'rust') && - inputs.task == 'test-storage-compat' && - github.event.pull_request.number - uses: actions/github-script@v9 - with: - script: | - let skip = false; - try { - const labels = await github.paginate( - github.rest.issues.listLabelsOnIssue, - { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - per_page: 100, - }, - ); - skip = labels.some(l => l.name === 'breaking:storage'); - } catch (e) { - // Fail toward running. A rate limit or a narrowed token must not - // wave an on-disk format break through as a green check. - core.warning(`listLabelsOnIssue failed (${e.status ?? e.code ?? 'network'}), running the check`); - } - core.info(`breaking:storage escape hatch: skip=${skip}`); - core.setOutput('skip', String(skip)); - - - name: Run Rust task - # Skipping the step (not the job) keeps the leg green so the pre-merge - # status roll-up stays reportable instead of pending. - if: >- - startsWith(inputs.component, 'rust') && - steps.storage_hatch.outputs.skip != 'true' - uses: ./.github/actions/rust/pre-merge - with: - task: ${{ inputs.task }} - component: ${{ inputs.component }} - - - name: Upload coverage to Codecov - # test-storage-compat runs its own nextest invocation without llvm-cov, - # so it emits no codecov.json. Uploading an empty report under the - # `rust` flag would read as a coverage drop on the PR. A test task that - # Cargo-Rail found no affected tests for writes no report either. - if: >- - startsWith(inputs.component, 'rust') && - startsWith(inputs.task, 'test-') && - inputs.task != 'test-storage-compat' && - hashFiles('codecov.json') != '' - uses: codecov/[email protected] - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: codecov.json - disable_search: true - flags: rust - fail_ci_if_error: false - verbose: true - override_pr: ${{ github.event.pull_request.number }} - # Python SDK - name: Set up Docker Buildx for Python if: inputs.component == 'sdk-python' && inputs.task == 'test' @@ -283,8 +210,7 @@ jobs: uses: actions/setup-node@v7 with: node-version: "24" - cache: npm - cache-dependency-path: web/package-lock.json + package-manager-cache: false - name: Run Web UI task if: inputs.component == 'web-ui' diff --git a/.github/workflows/_test_bdd.yml b/.github/workflows/_test_bdd.yml index 945f58054..0e713c05d 100644 --- a/.github/workflows/_test_bdd.yml +++ b/.github/workflows/_test_bdd.yml @@ -27,6 +27,14 @@ on: type: string required: true description: "Task to run (e.g., 'bdd-rust', 'bdd-python', etc.)" + timeout-minutes: + type: number + default: 60 + description: "Job timeout in minutes" + + secrets: + CODECOV_TOKEN: + required: false permissions: contents: read @@ -35,7 +43,7 @@ permissions: jobs: run: runs-on: ubuntu-26.04 - timeout-minutes: 60 + timeout-minutes: ${{ inputs.timeout-minutes }} steps: - name: Checkout code uses: actions/[email protected] @@ -44,9 +52,11 @@ jobs: if: inputs.component == 'noop' run: echo "No changes detected, skipping tests" - - name: Setup Rust with cache for BDD + - name: Setup Rust for BDD if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') - uses: ./.github/actions/utils/setup-rust-with-cache + uses: ./.github/actions/utils/setup-rust + with: + coverage: ${{ inputs.task == 'bdd-rust' }} - name: Build server for BDD tests if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') @@ -88,7 +98,11 @@ jobs: export IGGY_CLI_PATH="target/debug/iggy" export IGGY_ROOT_USERNAME="iggy" export IGGY_ROOT_PASSWORD="iggy" - ./scripts/run-bdd-tests.sh "$SDK_NAME" + COVERAGE_ARGS=() + if [[ "$SDK_NAME" == rust ]]; then + COVERAGE_ARGS=(--coverage) + fi + ./scripts/run-bdd-tests.sh "${COVERAGE_ARGS[@]}" "$SDK_NAME" - name: Clean up Docker resources (BDD) if: always() && startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') @@ -96,6 +110,26 @@ jobs: ./scripts/run-bdd-tests.sh clean || true docker system prune -f || true + - name: Generate Rust coverage report + if: always() && !cancelled() && inputs.task == 'bdd-rust' + run: | + if compgen -G 'target/*.profraw' > /dev/null; then + cargo llvm-cov report --codecov --output-path codecov.json + else + echo "::notice::No coverage profiles found; skipping report" + fi + + - name: Upload Rust coverage to Codecov + if: always() && !cancelled() && inputs.task == 'bdd-rust' && hashFiles('codecov.json', 'reports/rust-bdd-coverage.lcov') != '' + uses: codecov/[email protected] + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: codecov.json,reports/rust-bdd-coverage.lcov + disable_search: true + flags: rust-bdd + fail_ci_if_error: false + override_pr: ${{ github.event.pull_request.number }} + - name: Upload reports if: always() uses: actions/upload-artifact@v7 @@ -105,5 +139,6 @@ jobs: reports/** target/llvm-cov/** coverage.lcov + codecov.json if-no-files-found: ignore retention-days: 7 diff --git a/.github/workflows/_test_examples.yml b/.github/workflows/_test_examples.yml index 024961914..6b3de5130 100644 --- a/.github/workflows/_test_examples.yml +++ b/.github/workflows/_test_examples.yml @@ -27,6 +27,7 @@ on: type: string required: true description: "Task to run (e.g., 'examples-rust', 'examples-go', 'examples-node')" + permissions: contents: read security-events: write @@ -43,28 +44,17 @@ jobs: if: inputs.component == 'noop' run: echo "No changes detected, skipping tests" - - name: Setup Rust with cache for examples + - name: Setup Rust for examples if: startsWith(inputs.component, 'examples-') - uses: ./.github/actions/utils/setup-rust-with-cache + uses: ./.github/actions/utils/setup-rust - - name: Setup Node with cache for examples + - name: Setup Node for examples if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-node' - uses: ./.github/actions/utils/setup-node-with-cache + uses: ./.github/actions/utils/setup-node with: node-version: "23" - cache-dependency-path: examples/node/package-lock.json workspace: examples/node - - name: Setup Python - if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python' - uses: actions/[email protected] - with: - python-version: "3.11" - - - name: Setup uv - if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python' - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - - name: Install PHP build dependencies if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-php' shell: bash @@ -91,22 +81,15 @@ jobs: "${php_bin}" --version "${php_config}" --version - - name: Cache uv - if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python' - uses: actions/[email protected] - with: - path: ~/.cache/uv - key: uv-${{ runner.os }}-${{ hashFiles('examples/python/uv.lock', 'foreign/python/uv.lock', 'bdd/python/uv.lock') }} - - - name: Setup Java with cache for examples + - name: Setup Java for examples if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-java' - uses: ./.github/actions/utils/setup-java-with-cache + uses: ./.github/actions/utils/setup-java with: gradle-cache-disabled: "true" - name: Setup Go for examples if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-go' - uses: ./.github/actions/utils/setup-go-with-cache + uses: ./.github/actions/utils/setup-go with: download-deps: "false" @@ -130,12 +113,11 @@ jobs: # against the VSR server. SERVER_BIN="iggy-server" echo "Building ${SERVER_BIN}..." - cargo build --locked --bin "${SERVER_BIN}" - + BUILD_ARGS=(--bin "$SERVER_BIN") if [[ "${{ inputs.task }}" == "examples-rust" ]]; then - echo "Building additional binaries for Rust examples..." - cargo build --locked --bin iggy --examples + BUILD_ARGS+=(--bin iggy --examples) fi + cargo build --locked "${BUILD_ARGS[@]}" # Verify server binary was built (needed by all examples) echo "Verifying server binary:" @@ -160,6 +142,24 @@ jobs: fi fi + # Keep Python's PKG_CONFIG_PATH out of the server build above. + - name: Setup Python + if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python' + uses: actions/[email protected] + with: + python-version: "3.11" + + - name: Setup uv + if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python' + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + + - name: Cache uv + if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python' + uses: actions/[email protected] + with: + path: ~/.cache/uv + key: uv-${{ runner.os }}-${{ hashFiles('examples/python/uv.lock', 'foreign/python/uv.lock', 'bdd/python/uv.lock') }} + - name: Build PHP extension for examples if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-php' shell: bash @@ -218,9 +218,6 @@ jobs: uses: actions/upload-artifact@v7 with: name: ${{ inputs.component }}-${{ inputs.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }} - path: | - reports/** - target/llvm-cov/** - coverage.lcov + path: reports/** if-no-files-found: ignore retention-days: 7 diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index 5cd48bef4..78962fc16 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -83,87 +83,48 @@ jobs: name: Rust coverage baseline needs: detect if: needs.detect.outputs.rust == 'true' - runs-on: ubuntu-26.04 - timeout-minutes: 45 - steps: - - uses: actions/[email protected] - - - name: Install system dependencies - run: | - ./scripts/ci/apt-install.sh gnome-keyring keyutils dbus-x11 libsecret-tools - rm -f "$HOME"/.local/share/keyrings/* - - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache - with: - install-nextest: "true" - # Also warms the GitHub Actions build cache for subsequent PR builds - save-cache: "true" - # llvm-cov instrumentation roughly doubles object sizes; the light - # cleanup leaves too little headroom (build hit "No space left on - # device" at ~34 MB free). - free-disk-space-aggressive: "true" - - - name: Install cargo-llvm-cov - uses: taiki-e/[email protected] - with: - tool: cargo-llvm-cov - - # Doris 4.0.3's start_be.sh hard-`exit 1`s unless vm.max_map_count >= 2000000. - # Host-only kernel param (no container can raise it). Mirrors the same step in - # .github/actions/rust/pre-merge/action.yml; without it every Doris connector - # integration test panics at fixture setup. - - name: Raise vm.max_map_count for Doris - run: sudo sysctl -w vm.max_map_count=2000000 || true - shell: bash - - - name: Build and test with coverage - run: | - source <(cargo llvm-cov show-env --export-prefix) - cargo build --locked - cargo test --locked --no-run - - # Start D-Bus and unlock keyring right before test execution to avoid - # gnome-keyring auto-locking the collection during the build phase. - # Previously this ran before `cargo build`, leaving a 7+ minute idle - # window that triggered org.freedesktop.Secret.Error.IsLocked ~10% of runs. - eval $(dbus-launch --sh-syntax) - export DBUS_SESSION_BUS_ADDRESS - eval $(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets) - echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup - - cargo nextest run --locked --no-fail-fast --profile ci - shell: bash - - - name: Generate coverage report - run: | - source <(cargo llvm-cov show-env --export-prefix) - cargo llvm-cov report --codecov --output-path codecov.json - shell: bash - - - name: Upload to Codecov - uses: codecov/[email protected] - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: codecov.json - disable_search: true - flags: rust - fail_ci_if_error: false + permissions: + contents: read + issues: read + uses: ./.github/workflows/_rust.yml + with: + plan: '{"checks":[],"tests":true}' + test-scope: workspace + timeout-minutes: 120 + save-cache: true + build-metadata: true + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + rust-bdd-coverage: + name: Rust BDD coverage baseline + needs: detect + if: needs.detect.outputs.rust == 'true' + permissions: + contents: read + security-events: write + uses: ./.github/workflows/_test_bdd.yml + with: + component: bdd-rust + task: bdd-rust + timeout-minutes: 120 + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} java-coverage: name: Java coverage baseline needs: detect if: needs.detect.outputs.java == 'true' runs-on: ubuntu-26.04 - timeout-minutes: 30 + timeout-minutes: 120 steps: - uses: actions/[email protected] - - name: Setup Java with cache - uses: ./.github/actions/utils/setup-java-with-cache + - name: Setup Java + uses: ./.github/actions/utils/setup-java - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust with: save-cache: "false" @@ -200,7 +161,7 @@ jobs: needs: detect if: needs.detect.outputs.csharp == 'true' runs-on: ubuntu-26.04 - timeout-minutes: 30 + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -215,8 +176,8 @@ jobs: 8.0.x 10.0.x - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust with: save-cache: "false" @@ -285,20 +246,41 @@ jobs: needs: detect if: needs.detect.outputs.python == 'true' runs-on: ubuntu-26.04 - timeout-minutes: 30 + timeout-minutes: 120 steps: - uses: actions/[email protected] + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust + with: + save-cache: "false" + + # setup-python changes PKG_CONFIG_PATH and invalidates cached native server dependencies. + - name: Build server Docker image for TLS tests + run: | + # test_tls.py spawns the server in a container; build it from the + # same binaries the plain-TCP tests run against. + cargo build --locked --bin iggy-server --bin iggy + docker build \ + -f core/server/Dockerfile \ + --target runtime-prebuilt \ + --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ + --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ + -t iggy-server:local . + shell: bash + + - name: Start Iggy server + id: iggy + uses: ./.github/actions/utils/server-start + with: + mode: bin + bin: target/debug/iggy-server + - name: Setup Python uses: actions/[email protected] with: python-version: "3.11" - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache - with: - save-cache: "false" - - name: Install cargo-llvm-cov uses: taiki-e/[email protected] with: @@ -326,25 +308,6 @@ jobs: uv run --no-sync maturin develop shell: bash - - name: Build server Docker image for TLS tests - run: | - # test_tls.py spawns the server in a container; build it from the - # same binaries the plain-TCP tests run against. - cargo build --locked --bin iggy-server --bin iggy - docker build \ - -f core/server/Dockerfile \ - --target runtime-prebuilt \ - --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ - --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ - -t iggy-server:local . - shell: bash - - - name: Start Iggy server - id: iggy - uses: ./.github/actions/utils/server-start - with: - cargo-bin: iggy-server - - name: Run tests run: | cd foreign/python @@ -409,7 +372,7 @@ jobs: needs: detect if: needs.detect.outputs.php == 'true' runs-on: ubuntu-26.04 - timeout-minutes: 30 + timeout-minutes: 120 steps: - uses: actions/[email protected] @@ -432,7 +395,7 @@ jobs: needs: detect if: needs.detect.outputs.node == 'true' runs-on: ubuntu-26.04 - timeout-minutes: 30 + timeout-minutes: 120 steps: - uses: actions/[email protected] @@ -440,14 +403,13 @@ jobs: uses: actions/setup-node@v7 with: node-version: "23" - cache: "npm" - cache-dependency-path: foreign/node/package-lock.json + package-manager-cache: false - name: Pin npm to 11 to match Dependabot lockfile format run: npm install -g npm@11 - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust with: save-cache: "false" @@ -499,15 +461,15 @@ jobs: needs: detect if: needs.detect.outputs.go == 'true' runs-on: ubuntu-26.04 - timeout-minutes: 30 + timeout-minutes: 120 steps: - uses: actions/[email protected] - name: Setup Go - uses: ./.github/actions/utils/setup-go-with-cache + uses: ./.github/actions/utils/setup-go - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust with: save-cache: "false" @@ -587,7 +549,7 @@ jobs: needs: detect if: needs.detect.outputs.cpp == 'true' runs-on: ubuntu-26.04 - timeout-minutes: 45 + timeout-minutes: 120 steps: - uses: actions/[email protected] @@ -608,17 +570,14 @@ jobs: warm-cache-macos: name: Warm macOS cache needs: detect - # Gated on the rust key: the cache goes stale only when Rust sources - # change, and the Swatinem key is Cargo.lock-based, so skipping the save - # on non-Rust pushes leaves the previous cache valid. if: needs.detect.outputs.rust == 'true' runs-on: macos-14 - timeout-minutes: 30 + timeout-minutes: 120 steps: - uses: actions/[email protected] - - name: Setup Rust with cache - uses: ./.github/actions/utils/setup-rust-with-cache + - name: Setup Rust + uses: ./.github/actions/utils/setup-rust with: save-cache: "true" diff --git a/.github/workflows/pr-skill-review-run.yml b/.github/workflows/pr-skill-review-run.yml index 28d48c327..96286c04a 100644 --- a/.github/workflows/pr-skill-review-run.yml +++ b/.github/workflows/pr-skill-review-run.yml @@ -231,6 +231,8 @@ jobs: - name: Setup Node if: steps.parse.outputs.status == 'ok' uses: actions/setup-node@v7 + with: + package-manager-cache: false - name: Install the agent if: steps.parse.outputs.status == 'ok' diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 24c0017f1..bcb96cf60 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -55,16 +55,12 @@ jobs: # Rust components test-rust: - name: Rust • ${{ matrix.component }}/${{ matrix.task }} + name: Rust needs: detect - if: ${{ fromJson(needs.detect.outputs.rust_matrix).include[0].component != 'noop' }} - strategy: - fail-fast: true - matrix: ${{ fromJson(needs.detect.outputs.rust_matrix) }} - uses: ./.github/workflows/_test.yml + if: needs.detect.outputs.rust_plan != '' + uses: ./.github/workflows/_rust.yml with: - component: ${{ matrix.component }} - task: ${{ matrix.task }} + plan: ${{ needs.detect.outputs.rust_plan }} secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -211,6 +207,8 @@ jobs: with: component: ${{ matrix.component }} task: ${{ matrix.task }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} # Examples Tests test-examples: @@ -237,10 +235,11 @@ jobs: uses: actions/github-script@v9 with: script: | - const jobs = await github.rest.actions.listJobsForWorkflowRun({ + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { owner: context.repo.owner, repo: context.repo.repo, - run_id: context.runId + run_id: context.runId, + per_page: 100 }); const jobTimes = {}; @@ -263,7 +262,7 @@ jobs: // Log job names for debugging console.log('Job names found:'); - for (const job of jobs.data.jobs) { + for (const job of jobs) { console.log(` - ${job.name}: ${job.status} (conclusion: ${job.conclusion || 'N/A'})`); jobStatuses[job.name] = job.conclusion || job.status; @@ -305,7 +304,6 @@ jobs: }; // Set outputs for each component - const rust = findJobInfo('Rust •'); const python = findJobInfo('Python •'); const php = findJobInfo('PHP •'); const node = findJobInfo('Node •'); @@ -327,7 +325,13 @@ jobs: : { time: null, status: 'skipped' }; // Output formatted durations - core.setOutput('rust_time', formatJobDuration(rust)); + const rustJobs = jobs.filter(job => job.name.startsWith('Rust /') && + job.conclusion !== 'skipped' && job.started_at && job.completed_at); + const rustDuration = rustJobs.length ? formatDuration( + Math.max(...rustJobs.map(job => new Date(job.completed_at).getTime())) - + Math.min(...rustJobs.map(job => new Date(job.started_at).getTime())) + ) : '-'; + core.setOutput('rust_time', rustDuration); core.setOutput('python_time', formatJobDuration(python)); core.setOutput('php_time', formatJobDuration(php)); core.setOutput('node_time', formatJobDuration(node)); @@ -345,7 +349,7 @@ jobs: // Calculate total time - find the earliest job start time let earliestStart = null; let latestEnd = null; - for (const job of jobs.data.jobs) { + for (const job of jobs) { if (job.started_at) { const start = new Date(job.started_at); if (!earliestStart || start < earliestStart) { diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 425f2b91e..15705dfaa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1040,7 +1040,7 @@ jobs: # need the toolchain. - name: Setup Rust toolchain (if needed) if: matrix.type == 'python' - uses: ./.github/actions/utils/setup-rust-with-cache + uses: ./.github/actions/utils/setup-rust - name: Debug matrix run: echo '${{ toJson(matrix) }}' diff --git a/bdd/docker-compose.cluster.yml b/bdd/docker-compose.cluster.yml index fb9b02954..0ccc3a676 100644 --- a/bdd/docker-compose.cluster.yml +++ b/bdd/docker-compose.cluster.yml @@ -52,6 +52,7 @@ x-cluster-node: &cluster-node hard: -1 x-cluster-topology: &cluster-topology + LLVM_PROFILE_FILE: ${BDD_SERVER_PROFILE_FILE:-} IGGY_ROOT_USERNAME: iggy IGGY_ROOT_PASSWORD: iggy IGGY_CLUSTER_ENABLED: "true" @@ -116,6 +117,7 @@ services: ipv4_address: 172.28.0.101 volumes: - iggy_leader_data:/app/local_data_leader + - ../target:/profiles iggy-follower: <<: *cluster-node @@ -142,6 +144,7 @@ services: ipv4_address: 172.28.0.102 volumes: - iggy_follower_data:/app/local_data_follower + - ../target:/profiles rust-bdd: <<: *cluster-bdd-deps diff --git a/bdd/docker-compose.coverage.yml b/bdd/docker-compose.coverage.yml index 79ea67d3b..e1c58fea8 100644 --- a/bdd/docker-compose.coverage.yml +++ b/bdd/docker-compose.coverage.yml @@ -23,20 +23,25 @@ services: rust-bdd: + build: + args: + COVERAGE: "true" volumes: - ../reports:/reports command: - - sh + - bash + - -euo + - pipefail - -c - | - rustup component add llvm-tools-preview \ - && cargo install cargo-llvm-cov \ - && case "$$BDD_FEATURE" in - basic_messaging) cargo llvm-cov test -p bdd --features "$$BDD_RUST_FEATURES" --test basic_messaging --lcov --output-path /reports/rust-bdd-coverage.lcov ;; - leader_redirection) cargo llvm-cov test -p bdd --features "$$BDD_RUST_FEATURES" --test leader_redirection --lcov --output-path /reports/rust-bdd-coverage.lcov ;; - stream_crud) cargo llvm-cov test -p bdd --features "$$BDD_RUST_FEATURES" --test stream_crud --lcov --output-path /reports/rust-bdd-coverage.lcov ;; - *) cargo llvm-cov test -p bdd --features "$$BDD_RUST_FEATURES" --lcov --output-path /reports/rust-bdd-coverage.lcov ;; + coverage_env=$$(cargo llvm-cov show-env --sh --remap-path-prefix) + eval "$$coverage_env" + trap 'cargo llvm-cov report --lcov --output-path /reports/rust-bdd-coverage.lcov' EXIT + TEST_ARGS=() + case "$$BDD_FEATURE" in + basic_messaging|leader_redirection|raw_command|stream_crud) TEST_ARGS=(--test "$$BDD_FEATURE") ;; esac + cargo test --locked -p bdd --features "$$BDD_RUST_FEATURES" "$${TEST_ARGS[@]}" python-bdd: volumes: diff --git a/bdd/docker-compose.server.yml b/bdd/docker-compose.server.yml index 88868ace3..dcda9539e 100644 --- a/bdd/docker-compose.server.yml +++ b/bdd/docker-compose.server.yml @@ -60,6 +60,7 @@ services: retries: 30 start_period: 2s environment: + - LLVM_PROFILE_FILE=${BDD_SERVER_PROFILE_FILE:-} - RUST_LOG=info - IGGY_ROOT_USERNAME=iggy - IGGY_ROOT_PASSWORD=iggy @@ -71,6 +72,7 @@ services: - IGGY_WEBSOCKET_ADDRESS=0.0.0.0:8070 volumes: - iggy_data:/app/local_data + - ../target:/profiles networks: - iggy-bdd-network diff --git a/bdd/docker-compose.yml b/bdd/docker-compose.yml index 01dee9292..37a460ca8 100644 --- a/bdd/docker-compose.yml +++ b/bdd/docker-compose.yml @@ -21,6 +21,8 @@ services: build: context: .. dockerfile: bdd/rust/Dockerfile + args: + BDD_RUST_FEATURES: ${BDD_RUST_FEATURES:-bdd} environment: - IGGY_ROOT_USERNAME=iggy - IGGY_ROOT_PASSWORD=iggy diff --git a/bdd/rust/Dockerfile b/bdd/rust/Dockerfile index ff967433d..1ffa83ea2 100644 --- a/bdd/rust/Dockerfile +++ b/bdd/rust/Dockerfile @@ -18,22 +18,44 @@ ARG RUST_VERSION=1.99 FROM rust:${RUST_VERSION}-slim-trixie +ARG COVERAGE=false +ARG BDD_RUST_FEATURES=bdd +ARG LLVM_COV_VERSION=0.9.1 + +ENV RUSTFLAGS="-C linker=gcc" \ + CARGO_INCREMENTAL=0 \ + CARGO_PROFILE_DEV_DEBUG=0 \ + CARGO_PROFILE_TEST_DEBUG=0 + +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] WORKDIR /app RUN apt-get update && apt-get install -y \ + ca-certificates \ + curl \ libhwloc-dev \ libudev-dev \ pkg-config \ && rm -rf /var/lib/apt/lists/* -# Copy everything -COPY . . +RUN if [[ "$COVERAGE" == true ]]; then \ + rustup component add llvm-tools-preview; \ + host=$(rustc -vV | sed -n 's/^host: //p'); \ + curl -fsSL --retry 5 --retry-all-errors \ + "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${LLVM_COV_VERSION}/cargo-llvm-cov-${host}.tar.gz"; \ + -o /tmp/llvm-cov.tar.gz; \ + tar -xzf /tmp/llvm-cov.tar.gz -C /usr/local/cargo/bin; \ + rm /tmp/llvm-cov.tar.gz; \ + fi -# Create features directory for BDD feature files -RUN mkdir -p /app/features - -# Build tests -RUN cargo test --no-run -p bdd +COPY . . +# The shared Docker context includes host server binaries for other images. +# Their coverage mappings belong in the host report, not the BDD report. +RUN rm -rf target && mkdir -p /app/features +RUN if [[ "$COVERAGE" == true ]]; then \ + coverage_env=$(cargo llvm-cov show-env --sh --remap-path-prefix); \ + eval "$coverage_env"; \ + fi; \ + cargo test --locked --no-run -p bdd --features "$BDD_RUST_FEATURES" -# Default command will be overridden by docker-compose -CMD ["cargo", "test", "-p", "bdd", "--features", "bdd"] +CMD ["cargo", "test", "--locked", "-p", "bdd", "--features", "bdd"] diff --git a/codecov.yml b/codecov.yml index 2d927b241..f33f0de4e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -55,6 +55,9 @@ flag_management: - name: rust paths: - core/ + - name: rust-bdd + paths: + - core/ - name: java paths: - foreign/java/ diff --git a/scripts/ci/coverage-baseline-affected.sh b/scripts/ci/coverage-baseline-affected.sh index aa1187d8f..3b0f96af8 100755 --- a/scripts/ci/coverage-baseline-affected.sh +++ b/scripts/ci/coverage-baseline-affected.sh @@ -75,7 +75,7 @@ changed() { for gate in "${GATES[@]}"; do case "$gate" in # .config holds the nextest and Cargo-Rail policy every Rust test run reads. - rust) paths=(core gateways Cargo.toml Cargo.lock rust-toolchain.toml .cargo .config) ;; + rust) paths=(core gateways bdd/rust bdd/scenarios ':(glob)bdd/docker-compose*.yml' examples/rust Cargo.toml Cargo.lock rust-toolchain.toml .cargo .config) ;; java) paths=(foreign/java) ;; csharp) paths=(foreign/csharp) ;; python) paths=(foreign/python) ;; diff --git a/scripts/ci/storage-compat.sh b/scripts/ci/storage-compat.sh index edfabe02c..296893268 100755 --- a/scripts/ci/storage-compat.sh +++ b/scripts/ci/storage-compat.sh @@ -243,6 +243,10 @@ else BASELINE_TARGET_DIR="${WORKTREE_DIR}/target" ( cd "${WORKTREE_DIR}" + # The reference revision must not contribute counters to HEAD's coverage report. + if [[ "${__CARGO_LLVM_COV_RUSTC_WRAPPER:-}" == 1 ]]; then + export RUSTC_WRAPPER="${__CARGO_LLVM_COV_RUSTC_WRAPPER_PRE_EXISTING:-}" + fi CARGO_TARGET_DIR="${BASELINE_TARGET_DIR}" cargo build --locked -p server --bin iggy-server ) diff --git a/scripts/run-bdd-tests.sh b/scripts/run-bdd-tests.sh index acc6f913b..aadd6e2e9 100755 --- a/scripts/run-bdd-tests.sh +++ b/scripts/run-bdd-tests.sh @@ -78,7 +78,8 @@ case "$FEATURE" in esac if [ "$COVERAGE" = "1" ]; then COMPOSE_FILES+=(-f docker-compose.coverage.yml) - mkdir -p ../reports + mkdir -p ../reports ../target + export BDD_SERVER_PROFILE_FILE="/profiles/bdd-server-%h-%p-%m.profraw" fi cleanup(){
