This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new f139f0e5b ci: gate Docker :edge refresh by cargo-rail DAG (#3369)
f139f0e5b is described below
commit f139f0e5b8bde5dde5e7a507b82f26db8fbbeb2e
Author: Hubert Gruszecki <[email protected]>
AuthorDate: Fri May 29 13:51:44 2026 +0200
ci: gate Docker :edge refresh by cargo-rail DAG (#3369)
---
.github/actions/utils/docker-buildx/action.yml | 2 +-
.github/actions/utils/docker-login/action.yml | 113 +++++++++++++++++++++
.github/config/publish.yml | 24 +++++
.github/workflows/post-merge.yml | 20 +++-
.github/workflows/publish.yml | 2 +-
scripts/ci/edge-affected-images.sh | 135 +++++++++++++++++++++++++
6 files changed, 291 insertions(+), 5 deletions(-)
diff --git a/.github/actions/utils/docker-buildx/action.yml
b/.github/actions/utils/docker-buildx/action.yml
index b76b5c8ec..803c8832e 100644
--- a/.github/actions/utils/docker-buildx/action.yml
+++ b/.github/actions/utils/docker-buildx/action.yml
@@ -143,7 +143,7 @@ runs:
- name: Login to Docker Hub
if: steps.config.outputs.should_push == 'true'
- uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 #
v4.0.0
+ uses: ./.github/actions/utils/docker-login
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_TOKEN }}
diff --git a/.github/actions/utils/docker-login/action.yml
b/.github/actions/utils/docker-login/action.yml
new file mode 100644
index 000000000..7de493770
--- /dev/null
+++ b/.github/actions/utils/docker-login/action.yml
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: docker-login
+description: >
+ Log in to a Docker registry with exponential-backoff retry. The
+ post-merge publish occasionally hits a transient registry/auth flake
+ (e.g. `Error response from daemon: Get https://registry-1.docker.io/v2/:
+ unknown:`) that a single attempt cannot survive; docker/login-action has
+ no built-in retry. Mirrors the wait-for-url / wait-for-crate backoff idiom.
+
+inputs:
+ username:
+ description: "Registry username."
+ required: true
+ password:
+ description: "Registry password or access token. Passed via stdin, never
argv."
+ required: true
+ registry:
+ description: "Registry host. Empty means Docker Hub (docker.io)."
+ required: false
+ default: ""
+ max_attempts:
+ description: "Maximum login attempts before giving up."
+ required: false
+ default: "5"
+ initial_sleep_seconds:
+ description: "Sleep between the first two attempts in seconds. Doubles
each attempt, capped at max_sleep_seconds."
+ required: false
+ default: "2"
+ max_sleep_seconds:
+ description: "Upper bound on per-attempt sleep, in seconds."
+ required: false
+ default: "30"
+
+runs:
+ using: composite
+ steps:
+ - name: Docker login with retry
+ shell: bash
+ env:
+ USERNAME: ${{ inputs.username }}
+ PASSWORD: ${{ inputs.password }}
+ REGISTRY: ${{ inputs.registry }}
+ MAX_ATTEMPTS: ${{ inputs.max_attempts }}
+ INITIAL_SLEEP_SECONDS: ${{ inputs.initial_sleep_seconds }}
+ MAX_SLEEP_SECONDS: ${{ inputs.max_sleep_seconds }}
+ run: |
+ set -euo pipefail
+
+ if [ -z "${USERNAME}" ] || [ -z "${PASSWORD}" ]; then
+ echo "❌ docker-login: username and password are required"
+ exit 1
+ fi
+ if ! [[ "${MAX_ATTEMPTS}" =~ ^[0-9]+$ ]] || [ "${MAX_ATTEMPTS}" -lt 1
]; then
+ echo "❌ docker-login: max_attempts '${MAX_ATTEMPTS}' must be a
positive integer"
+ exit 1
+ fi
+ if ! [[ "${INITIAL_SLEEP_SECONDS}" =~ ^[0-9]+$ ]]; then
+ echo "❌ docker-login: initial_sleep_seconds
'${INITIAL_SLEEP_SECONDS}' must be a non-negative integer"
+ exit 1
+ fi
+ if ! [[ "${MAX_SLEEP_SECONDS}" =~ ^[0-9]+$ ]] || [
"${MAX_SLEEP_SECONDS}" -lt 1 ]; then
+ echo "❌ docker-login: max_sleep_seconds '${MAX_SLEEP_SECONDS}' must
be a positive integer"
+ exit 1
+ fi
+
+ target="${REGISTRY:-docker.io}"
+ echo "🔐 Logging in to ${target} (up to ${MAX_ATTEMPTS} attempts)"
+
+ # Empty REGISTRY means Docker Hub: omit the host arg entirely.
+ login_args=(-u "${USERNAME}" --password-stdin)
+ if [ -n "${REGISTRY}" ]; then
+ login_args=("${REGISTRY}" "${login_args[@]}")
+ fi
+
+ sleep_s="${INITIAL_SLEEP_SECONDS}"
+ for attempt in $(seq 1 "${MAX_ATTEMPTS}"); do
+ # Token on stdin only: never argv or a logged env echo. GitHub masks
+ # the secret regardless; this keeps it out of process listings too.
+ if printf '%s' "${PASSWORD}" | docker login "${login_args[@]}"; then
+ echo "✅ Logged in to ${target} (attempt
${attempt}/${MAX_ATTEMPTS})"
+ exit 0
+ fi
+
+ if [ "${attempt}" -eq "${MAX_ATTEMPTS}" ]; then
+ break
+ fi
+ echo "⏳ Login failed (attempt ${attempt}/${MAX_ATTEMPTS}), retrying
in ${sleep_s}s"
+ sleep "${sleep_s}"
+ sleep_s=$(( sleep_s * 2 ))
+ if [ "${sleep_s}" -gt "${MAX_SLEEP_SECONDS}" ]; then
+ sleep_s="${MAX_SLEEP_SECONDS}"
+ fi
+ done
+
+ echo "❌ docker login to ${target} failed after ${MAX_ATTEMPTS}
attempts"
+ echo " Likely a transient registry outage or auth rate-limit; rerun
the job."
+ exit 1
diff --git a/.github/config/publish.yml b/.github/config/publish.yml
index 4ac68180b..36376d34c 100644
--- a/.github/config/publish.yml
+++ b/.github/config/publish.yml
@@ -55,6 +55,13 @@ components:
platforms: ["linux/amd64", "linux/arm64"]
version_file: "core/server/Cargo.toml"
version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"'
+ # :edge refresh gate (scripts/ci/edge-affected-images.sh). Ships server +
CLI
+ # with the web UI embedded at compile time, so web-only changes must
rebuild
+ # it. crates: matched against `cargo rail plan`; paths: extra git pathspecs
+ # (the dockerfile: above is gated automatically).
+ gate:
+ crates: [server, iggy-cli]
+ paths: [web]
rust-mcp:
tag_pattern:
"^mcp-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$"
@@ -64,6 +71,8 @@ components:
platforms: ["linux/amd64", "linux/arm64"]
version_file: "core/ai/mcp/Cargo.toml"
version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"'
+ gate:
+ crates: [iggy-mcp]
rust-bench-dashboard:
tag_pattern:
"^bench-dashboard-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$"
@@ -73,6 +82,11 @@ components:
platforms: ["linux/amd64", "linux/arm64"]
version_file: "core/bench/dashboard/server/Cargo.toml"
version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"'
+ # Image bundles the server binary plus the built WASM frontend.
+ gate:
+ crates: [iggy-bench-dashboard-server, bench-dashboard-frontend]
+ paths:
+ - core/bench/dashboard/server/docker-entrypoint.sh
rust-connectors:
tag_pattern:
"^connectors-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$"
@@ -82,6 +96,9 @@ components:
platforms: ["linux/amd64", "linux/arm64"]
version_file: "core/connectors/runtime/Cargo.toml"
version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"'
+ # Image ships only the runtime binary; plugin .so files are not bundled.
+ gate:
+ crates: [iggy-connectors]
web-ui:
tag_pattern:
"^web-ui-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$"
@@ -91,6 +108,13 @@ components:
platforms: ["linux/amd64", "linux/arm64"]
version_file: "web/package.json"
version_regex: '"version"\s*:\s*"([^"]+)"'
+ # Not a cargo crate: path-gated only. The Dockerfile also COPYs two
+ # license scripts from scripts/ci/ into the build.
+ gate:
+ paths:
+ - web
+ - scripts/ci/third-party-licenses.sh
+ - scripts/ci/render-node-licenses.mjs
# ── Other SDKs ─────────────────────────────────────────────────────────────
sdk-python:
diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml
index c40fb5773..6e45fa57e 100644
--- a/.github/workflows/post-merge.yml
+++ b/.github/workflows/post-merge.yml
@@ -61,15 +61,25 @@ jobs:
chmod +x /usr/local/bin/yq
fi
+ # cargo-rail computes the affected crate set for the Docker :edge gate.
+ # Metadata-only `cargo rail plan` (no compile), so it runs on the
runner's
+ # preinstalled cargo (rust-toolchain.toml pins 1.95.0) and skips the
+ # heavyweight build-cache restore.
+ - name: Install cargo-rail
+ uses: taiki-e/install-action@v2
+ with:
+ tool: cargo-rail
+
- name: Check all components
id: check
run: |
chmod +x scripts/extract-version.sh
- # Get all Docker components (always publish :edge)
- DOCKER_COMPONENTS=$(yq -r '.components | to_entries | .[] |
select(.value.registry == "dockerhub") | .key' .github/config/publish.yml | tr
'\n' ',' | sed 's/,$//')
+ # Refresh :edge only for Docker images whose crate closure or build
+ # context changed in this push (DAG gate). Fail-open to all images.
+ DOCKER_COMPONENTS=$(scripts/ci/edge-affected-images.sh "${{
github.event.before }}" "${{ github.sha }}")
echo "docker_components=$DOCKER_COMPONENTS" >> "$GITHUB_OUTPUT"
- echo "Docker components: $DOCKER_COMPONENTS"
+ echo "Docker components to refresh: ${DOCKER_COMPONENTS:-<none>}"
# Check Rust crates for pre-release versions without tags
CRATES_TO_PUBLISH=""
@@ -138,6 +148,10 @@ jobs:
call-publish:
name: Publish components
needs: check-auto-publish
+ if: >-
+ needs.check-auto-publish.outputs.crates_to_publish != '' ||
+ needs.check-auto-publish.outputs.docker_components != '' ||
+ needs.check-auto-publish.outputs.sdks_to_publish != ''
uses: ./.github/workflows/publish.yml
with:
commit: ${{ github.sha }}
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 8371a70e2..12b6a261e 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -944,7 +944,7 @@ jobs:
uses:
docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
- name: Login to Docker Hub
- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 #
v4.1.0
+ uses: ./.github/actions/utils/docker-login
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_TOKEN }}
diff --git a/scripts/ci/edge-affected-images.sh
b/scripts/ci/edge-affected-images.sh
new file mode 100755
index 000000000..8559aed4b
--- /dev/null
+++ b/scripts/ci/edge-affected-images.sh
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+# Decide which DockerHub :edge images a master push actually changed, so
+# post-merge only refreshes the affected ones instead of all of them.
+#
+# Each Rust image builds via cargo-chef over the whole workspace (COPY . .),
+# so the shipped binary changes only when its crate dependency closure does.
+# That closure is exactly what `cargo rail plan` reports, the same DAG that
+# scopes test runs in .github/actions/rust/pre-merge. web-ui is not a crate,
+# and a Dockerfile-only edit touches no crate source, so each image also
+# declares fallback `gate.paths`.
+#
+# Fail-open: on any uncertainty (force push, workspace-global change,
+# cargo-rail failure, unconfigured image) emit the full image list so a needed
+# edge refresh is never skipped.
+#
+# Usage: edge-affected-images.sh <base-sha> <head-sha>
+# Output: comma-separated publish.yml component keys (e.g.
"rust-server,web-ui")
+
+BASE="${1:-}"
+HEAD="${2:-}"
+CONFIG=".github/config/publish.yml"
+ZERO="0000000000000000000000000000000000000000"
+
+if [[ -z "$BASE" || -z "$HEAD" ]]; then
+ echo "usage: $0 <base-sha> <head-sha>" >&2
+ exit 2
+fi
+
+CFG_JSON="$(yq -o=json -I=0 '.components' "$CONFIG")"
+mapfile -t ALL_IMAGES < <(jq -r 'to_entries[] | select(.value.registry ==
"dockerhub") | .key' <<<"$CFG_JSON")
+
+emit() { (IFS=,; echo "$*"); }
+emit_all() { emit "${ALL_IMAGES[@]}"; }
+
+# Unusable base: initial push, force-push, or a commit not in history.
+if [[ "$BASE" == "$ZERO" ]] || ! git cat-file -e "${BASE}^{commit}"
2>/dev/null; then
+ echo "::notice::edge-gate: unusable base '$BASE', refreshing all images" >&2
+ emit_all
+ exit 0
+fi
+
+# Workspace-global edits rebuild every crate. cargo-rail does not flag
+# toolchain/lockfile changes (they are not crate sources), so escalate here.
+if ! git diff --quiet "$BASE" "$HEAD" -- Cargo.toml Cargo.lock
rust-toolchain.toml .cargo; then
+ echo "::notice::edge-gate: workspace-global change, refreshing all images"
>&2
+ emit_all
+ exit 0
+fi
+
+RAIL_ERR="$(mktemp)"
+trap 'rm -f "$RAIL_ERR"' EXIT
+
+# Capture cargo-rail's exit status explicitly: a nonzero exit means the plan is
+# untrustworthy, so fall open. `|| true` would hide the failure and let
+# malformed stdout reach the parsing below.
+if ! PLAN="$(cargo rail plan --since "$BASE" -f json 2>"$RAIL_ERR")"; then
+ echo "::warning::edge-gate: cargo-rail exited nonzero, refreshing all
images. $(cat "$RAIL_ERR" 2>/dev/null || true)" >&2
+ emit_all
+ exit 0
+fi
+
+# Parse defensively: malformed output must fall OPEN, never abort under set -e
+# (that fails closed and skips the publish). jq prints nothing on a parse
error,
+# leaving MODE empty. A valid {mode:crates, crates:[]} is the normal
+# docs/SDK/config-only push and must publish nothing, so only an unparsable
+# MODE or a non-"crates" mode escalates to emit_all.
+MODE="$(jq -r '.scope.mode // "full"' <<<"$PLAN" 2>/dev/null || true)"
+if [[ -z "$MODE" ]]; then
+ echo "::warning::edge-gate: unparsable cargo-rail output, refreshing all
images" >&2
+ emit_all
+ exit 0
+fi
+if [[ "$MODE" != "crates" ]]; then
+ echo "::notice::edge-gate: cargo-rail reports full workspace, refreshing all
images" >&2
+ emit_all
+ exit 0
+fi
+
+mapfile -t AFFECTED < <(jq -r '.scope.crates // [] | .[]' <<<"$PLAN")
+declare -A AFFECTED_SET=()
+for crate in ${AFFECTED[@]+"${AFFECTED[@]}"}; do
+ AFFECTED_SET["$crate"]=1
+done
+
+SELECTED=()
+for img in "${ALL_IMAGES[@]}"; do
+ # No gate block fails open (always refreshed). An incomplete gate fails
closed
+ # (under-publishes): enumerate every embedded/COPYed source.
+ if [[ "$(jq -r --arg k "$img" '.[$k] | has("gate")' <<<"$CFG_JSON")" !=
"true" ]]; then
+ SELECTED+=("$img")
+ continue
+ fi
+
+ keep=false
+
+ while IFS= read -r crate; do
+ if [[ -n "${AFFECTED_SET[$crate]:-}" ]]; then
+ keep=true
+ break
+ fi
+ done < <(jq -r --arg k "$img" '.[$k].gate.crates // [] | .[]' <<<"$CFG_JSON")
+
+ if [[ "$keep" == false ]]; then
+ # dockerfile: is the build context's primary input; gate.paths lists any
+ # extra sources COPYed or embedded beyond the crate closure.
+ mapfile -t gate_paths < <(jq -r --arg k "$img" \
+ '[.[$k].dockerfile] + (.[$k].gate.paths // []) | .[] | select(. !=
null)' <<<"$CFG_JSON")
+ if [[ ${#gate_paths[@]} -gt 0 ]] && ! git diff --quiet "$BASE" "$HEAD" --
"${gate_paths[@]}"; then
+ keep=true
+ fi
+ fi
+
+ [[ "$keep" == true ]] && SELECTED+=("$img")
+done
+
+emit ${SELECTED[@]+"${SELECTED[@]}"}
