This is an automated email from the ASF dual-hosted git repository. hgruszecki pushed a commit to branch refactor-post-merge-publish in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 8df752a670943c82af8b6926c6d724af5a27f667 Author: Hubert Gruszecki <[email protected]> AuthorDate: Mon Dec 15 13:13:03 2025 +0100 feat(ci): use native runners for Docker builds instead of QEMU emulation --- .github/actions/utils/docker-buildx/action.yml | 40 ++++- .github/workflows/post-merge.yml | 90 +++++++++- .github/workflows/publish.yml | 235 ++++++++++++++++++++++--- 3 files changed, 331 insertions(+), 34 deletions(-) diff --git a/.github/actions/utils/docker-buildx/action.yml b/.github/actions/utils/docker-buildx/action.yml index 276cb6e0b..247fcd516 100644 --- a/.github/actions/utils/docker-buildx/action.yml +++ b/.github/actions/utils/docker-buildx/action.yml @@ -42,6 +42,10 @@ inputs: description: "Libc to use (glibc/musl)" required: false default: "musl" + platform: + description: "Single platform to build (e.g., linux/amd64). If set, builds only this platform without QEMU. Leave empty for multi-arch build." + required: false + default: "" runs: using: "composite" @@ -112,6 +116,8 @@ runs: fi - name: Set up QEMU + # Skip QEMU when building single platform on native runner (no emulation needed) + if: inputs.platform == '' uses: docker/setup-qemu-action@v3 with: platforms: all @@ -130,30 +136,56 @@ runs: username: ${{ env.DOCKERHUB_USER }} password: ${{ env.DOCKERHUB_TOKEN }} + - name: Compute platform suffix + id: suffix + shell: bash + run: | + platform="${{ inputs.platform }}" + if [ -n "$platform" ]; then + # Extract arch from platform (e.g., linux/amd64 -> amd64) + suffix="${platform##*/}" + echo "suffix=-${suffix}" >> "$GITHUB_OUTPUT" + echo "is_single=true" >> "$GITHUB_OUTPUT" + echo "📦 Platform suffix: -${suffix}" + else + echo "suffix=" >> "$GITHUB_OUTPUT" + echo "is_single=false" >> "$GITHUB_OUTPUT" + echo "📦 No platform suffix (multi-arch)" + fi + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: ${{ steps.config.outputs.image }} tags: | - type=raw,value=${{ inputs.version }} - type=raw,value=latest,enable=${{ steps.config.outputs.should_push == 'true' && inputs.version != 'test' && !contains(inputs.version, 'edge') && !contains(inputs.version, 'rc') && !contains(inputs.version, 'alpha') && !contains(inputs.version, 'beta') }} + type=raw,value=${{ inputs.version }}${{ steps.suffix.outputs.suffix }} + type=raw,value=latest,enable=${{ steps.suffix.outputs.is_single == 'false' && steps.config.outputs.should_push == 'true' && inputs.version != 'test' && !contains(inputs.version, 'edge') && !contains(inputs.version, 'rc') && !contains(inputs.version, 'alpha') && !contains(inputs.version, 'beta') }} type=sha,enable=${{ inputs.version == 'test' }} - name: Determine platforms id: platforms shell: bash run: | + p_input="${{ inputs.platform }}" p_cfg="${{ steps.config.outputs.cfg_platforms }}" - if [ -n "$p_cfg" ]; then + + # Priority: input platform > config platforms > defaults + if [ -n "$p_input" ]; then + # Single platform specified - native build on correct runner + platforms="$p_input" + echo "🚀 Single platform build (native): $platforms" + elif [ -n "$p_cfg" ]; then platforms="$p_cfg" + echo "🖥️ Platforms from config: $platforms" elif [ "${{ steps.config.outputs.should_push }}" = "true" ]; then platforms="linux/amd64,linux/arm64" + echo "🖥️ Multi-arch build: $platforms" else platforms="linux/amd64" + echo "🖥️ Default platform: $platforms" fi echo "platforms=$platforms" >> "$GITHUB_OUTPUT" - echo "🖥️ Platforms: $platforms" - name: Compose cache config (no registry on dry-run) id: cachecfg diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index 84a2bc9ed..c4725bd21 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -39,6 +39,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.mk.outputs.matrix }} + components: ${{ steps.mk.outputs.components }} steps: - uses: actions/checkout@v4 @@ -63,21 +64,44 @@ jobs: const b64 = `${{ steps.cfg.outputs.components_b64 }}` || ''; if (!b64) { core.setOutput('matrix', JSON.stringify({ include: [{ component: 'noop' }] })); + core.setOutput('components', JSON.stringify({ include: [{ component: 'noop' }] })); return; } const comps = JSON.parse(Buffer.from(b64, 'base64').toString('utf8')); - const include = Object.entries(comps) + const components = Object.entries(comps) .filter(([_, v]) => v && v.registry === 'dockerhub') - .map(([k]) => ({ component: k })); - const uniq = Array.from(new Map(include.map(i => [i.component, i])).values()); - core.setOutput('matrix', JSON.stringify(uniq.length ? { include: uniq } : { include: [{ component: 'noop' }] })); + .map(([k]) => k); + const uniqComponents = [...new Set(components)]; + + // Output just the component list for manifest creation + const componentMatrix = uniqComponents.length + ? { include: uniqComponents.map(c => ({ component: c })) } + : { include: [{ component: 'noop' }] }; + core.setOutput('components', JSON.stringify(componentMatrix)); + + // Build cross-product matrix: components × platforms + const platforms = [ + { platform: 'linux/amd64', arch: 'amd64', runner: 'ubuntu-latest' }, + { platform: 'linux/arm64', arch: 'arm64', runner: 'ubuntu-24.04-arm' } + ]; + + const matrix = []; + for (const comp of uniqComponents) { + for (const p of platforms) { + matrix.push({ component: comp, ...p }); + } + } + + core.setOutput('matrix', JSON.stringify(matrix.length ? { include: matrix } : { include: [{ component: 'noop' }] })); + console.log(`Components: ${uniqComponents.join(', ')}`); + console.log(`Matrix size: ${matrix.length} jobs (${uniqComponents.length} components × 2 platforms)`); docker-edge: - name: ${{ matrix.component }} + name: ${{ matrix.component }} (${{ matrix.arch }}) needs: plan if: ${{ fromJson(needs.plan.outputs.matrix).include[0].component != 'noop' }} - runs-on: ubuntu-latest - timeout-minutes: 120 + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 strategy: fail-fast: false matrix: ${{ fromJson(needs.plan.outputs.matrix) }} @@ -93,7 +117,57 @@ jobs: libc: musl component: ${{ matrix.component }} version: edge - dry_run: ${{ github.event.repository.fork }} # forks: always dry-run + platform: ${{ matrix.platform }} + dry_run: ${{ github.event.repository.fork }} + + docker-manifests: + name: Create manifests + needs: [plan, docker-edge] + if: ${{ !github.event.repository.fork && fromJson(needs.plan.outputs.components).include[0].component != 'noop' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan.outputs.components) }} + env: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - name: Resolve image from config + id: config + shell: bash + run: | + if ! command -v yq >/dev/null 2>&1; then + YQ_VERSION="v4.47.1" + YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" + curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1 + chmod +x /usr/local/bin/yq + fi + image=$(yq ".components.${{ matrix.component }}.image" .github/config/publish.yml) + echo "image=$image" >> "$GITHUB_OUTPUT" + echo "📦 Image: $image" + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKERHUB_USER }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Create and push manifest + run: | + IMAGE="${{ steps.config.outputs.image }}" + VERSION="edge" + + echo "Creating manifest for $IMAGE:$VERSION" + + docker manifest create "${IMAGE}:${VERSION}" \ + "${IMAGE}:${VERSION}-amd64" \ + "${IMAGE}:${VERSION}-arm64" + + docker manifest push "${IMAGE}:${VERSION}" + echo "✅ Pushed manifest: ${IMAGE}:${VERSION}" build-artifacts: name: Build artifacts diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 77470d3c5..f55eb3149 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -136,10 +136,14 @@ jobs: outputs: targets: ${{ steps.mk.outputs.targets }} non_rust_targets: ${{ steps.mk.outputs.non_rust_targets }} + non_docker_targets: ${{ steps.mk.outputs.non_docker_targets }} + docker_matrix: ${{ steps.mk.outputs.docker_matrix }} + docker_components: ${{ steps.mk.outputs.docker_components }} count: ${{ steps.mk.outputs.count }} go_sdk_version: ${{ steps.mk.outputs.go_sdk_version }} has_python: ${{ steps.mk.outputs.has_python }} has_rust_crates: ${{ steps.mk.outputs.has_rust_crates }} + has_docker: ${{ steps.mk.outputs.has_docker }} steps: - name: Download latest copy script from master if: inputs.use_latest_ci @@ -263,9 +267,14 @@ jobs: } } + // Separate Docker targets from other non-Rust targets + const dockerTargets = nonRustTargets.filter(t => t.type === 'docker'); + const nonDockerTargets = nonRustTargets.filter(t => t.type !== 'docker'); + console.log(`Publishing ${targets.length} components:`); targets.forEach(t => console.log(` - ${t.name} (${t.type}) -> ${t.registry || 'N/A'}`)); console.log(` (${nonRustTargets.length} non-Rust, ${targets.length - nonRustTargets.length} Rust crates)`); + console.log(` (${dockerTargets.length} Docker, ${nonDockerTargets.length} other SDKs)`); // Output all targets for reference and tag creation core.setOutput('targets', JSON.stringify(targets.length ? { include: targets } : { include: [{ key: 'noop', type: 'noop' }] })); @@ -273,6 +282,26 @@ jobs: // Output only non-Rust targets for the parallel publish job core.setOutput('non_rust_targets', JSON.stringify(nonRustTargets.length ? { include: nonRustTargets } : { include: [{ key: 'noop', type: 'noop' }] })); + // Output non-Docker, non-Rust targets (SDKs only) + core.setOutput('non_docker_targets', JSON.stringify(nonDockerTargets.length ? { include: nonDockerTargets } : { include: [{ key: 'noop', type: 'noop' }] })); + + // Build Docker matrix: components × platforms for native runner builds + const platforms = [ + { platform: 'linux/amd64', arch: 'amd64', runner: 'ubuntu-latest' }, + { platform: 'linux/arm64', arch: 'arm64', runner: 'ubuntu-24.04-arm' } + ]; + + const dockerMatrix = []; + for (const t of dockerTargets) { + for (const p of platforms) { + dockerMatrix.push({ ...t, ...p }); + } + } + + core.setOutput('docker_matrix', JSON.stringify(dockerMatrix.length ? { include: dockerMatrix } : { include: [{ key: 'noop', type: 'noop' }] })); + core.setOutput('docker_components', JSON.stringify(dockerTargets.length ? { include: dockerTargets } : { include: [{ key: 'noop', type: 'noop' }] })); + core.setOutput('has_docker', String(dockerTargets.length > 0)); + core.setOutput('count', String(targets.length)); core.setOutput('go_sdk_version', goVersion); core.setOutput('has_rust_crates', String(hasRustCrates)); @@ -603,20 +632,168 @@ jobs: if: always() run: echo "status=${{ job.status }}" >> "$GITHUB_OUTPUT" + # Docker publishing on native runners (no QEMU emulation) + publish-docker: + name: Docker ${{ matrix.name }} (${{ matrix.arch }}) + needs: [validate, plan, check-tags, build-python-wheels, publish-rust-crates] + if: | + always() && + needs.validate.outputs.has_targets == 'true' && + needs.plan.outputs.has_docker == 'true' && + fromJson(needs.plan.outputs.docker_matrix).include[0].key != 'noop' && + (needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped') && + (needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped') + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan.outputs.docker_matrix) }} + env: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + steps: + - name: Download latest copy script from master + if: inputs.use_latest_ci + run: | + curl -sSL "https://raw.githubusercontent.com/${{ github.repository }}/master/scripts/copy-latest-from-master.sh" \ + -o /tmp/copy-latest-from-master.sh + chmod +x /tmp/copy-latest-from-master.sh + echo "✅ Downloaded latest copy script from master" + + - name: Checkout at commit + uses: actions/checkout@v4 + with: + ref: ${{ needs.validate.outputs.commit }} + fetch-depth: 0 + + - name: Save and apply latest CI from master + if: inputs.use_latest_ci + run: | + /tmp/copy-latest-from-master.sh save \ + .github \ + scripts \ + web/Dockerfile \ + core/server/Dockerfile \ + core/ai/mcp/Dockerfile \ + core/connectors/runtime/Dockerfile \ + core/bench/dashboard/server/Dockerfile + + /tmp/copy-latest-from-master.sh apply + + - name: Ensure version extractor is executable + run: | + test -x scripts/extract-version.sh || chmod +x scripts/extract-version.sh + + - name: Extract version + id: ver + run: | + VERSION=$(scripts/extract-version.sh "${{ matrix.key }}") + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "✅ Resolved ${{ matrix.key }} -> version=$VERSION" + + - name: Publish Docker image + uses: ./.github/actions/utils/docker-buildx + with: + task: publish + component: ${{ matrix.key }} + version: ${{ steps.ver.outputs.version }} + platform: ${{ matrix.platform }} + dry_run: ${{ inputs.dry_run }} + + # Create multi-arch Docker manifests after platform-specific images are pushed + docker-manifests: + name: Docker manifests + needs: [validate, plan, publish-docker] + if: | + always() && + needs.validate.outputs.has_targets == 'true' && + needs.plan.outputs.has_docker == 'true' && + fromJson(needs.plan.outputs.docker_components).include[0].key != 'noop' && + needs.publish-docker.result == 'success' && + inputs.dry_run == false + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan.outputs.docker_components) }} + env: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.validate.outputs.commit }} + + - name: Ensure version extractor is executable + run: | + test -x scripts/extract-version.sh || chmod +x scripts/extract-version.sh + + - name: Extract version + id: ver + run: | + VERSION=$(scripts/extract-version.sh "${{ matrix.key }}") + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Resolve image from config + id: config + run: | + if ! command -v yq >/dev/null 2>&1; then + YQ_VERSION="v4.47.1" + YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" + curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1 + chmod +x /usr/local/bin/yq + fi + image=$(yq ".components.${{ matrix.key }}.image" .github/config/publish.yml) + echo "image=$image" >> "$GITHUB_OUTPUT" + echo "📦 Image: $image" + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKERHUB_USER }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Create and push manifest + run: | + IMAGE="${{ steps.config.outputs.image }}" + VERSION="${{ steps.ver.outputs.version }}" + + echo "Creating manifest for $IMAGE:$VERSION" + + docker manifest create "${IMAGE}:${VERSION}" \ + "${IMAGE}:${VERSION}-amd64" \ + "${IMAGE}:${VERSION}-arm64" + + docker manifest push "${IMAGE}:${VERSION}" + echo "✅ Pushed manifest: ${IMAGE}:${VERSION}" + + # Also create 'latest' tag for stable releases (not edge/rc/alpha/beta) + if [[ ! "$VERSION" =~ (edge|rc|alpha|beta) ]]; then + echo "Creating 'latest' manifest" + docker manifest create "${IMAGE}:latest" \ + "${IMAGE}:${VERSION}-amd64" \ + "${IMAGE}:${VERSION}-arm64" + docker manifest push "${IMAGE}:latest" + echo "✅ Pushed manifest: ${IMAGE}:latest" + fi + + # Non-Docker, non-Rust publishing (Python, Node, Java, C#, Go SDKs) publish: name: ${{ matrix.name }} needs: - [validate, plan, check-tags, build-python-wheels, publish-rust-crates] + [validate, plan, check-tags, build-python-wheels, publish-rust-crates, publish-docker, docker-manifests] if: | always() && needs.validate.outputs.has_targets == 'true' && - fromJson(needs.plan.outputs.non_rust_targets).include[0].key != 'noop' && + fromJson(needs.plan.outputs.non_docker_targets).include[0].key != 'noop' && (needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped') && - (needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped') + (needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped') && + (needs.publish-docker.result == 'success' || needs.publish-docker.result == 'skipped') && + (needs.docker-manifests.result == 'success' || needs.docker-manifests.result == 'skipped') runs-on: ubuntu-latest strategy: fail-fast: false - matrix: ${{ fromJson(needs.plan.outputs.non_rust_targets) }} + matrix: ${{ fromJson(needs.plan.outputs.non_docker_targets) }} env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} @@ -696,18 +873,6 @@ jobs: echo "tag=$TAG" >> "$GITHUB_OUTPUT" echo "✅ Resolved ${{ matrix.key }} -> version=$VERSION tag=${TAG:-<none>}" - # ───────────────────────────────────────── - # Docker Publishing - # ───────────────────────────────────────── - - name: Publish Docker image - if: matrix.type == 'docker' - uses: ./.github/actions/utils/docker-buildx - with: - task: publish - component: ${{ matrix.key }} - version: ${{ steps.ver.outputs.version }} - dry_run: ${{ inputs.dry_run }} - # ───────────────────────────────────────── # Python SDK Publishing # ───────────────────────────────────────── @@ -774,6 +939,8 @@ jobs: check-tags, build-python-wheels, publish-rust-crates, + publish-docker, + docker-manifests, publish, ] if: | @@ -783,6 +950,8 @@ jobs: inputs.skip_tag_creation == false && (needs.publish.result == 'success' || needs.publish.result == 'skipped') && (needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped') && + (needs.publish-docker.result == 'success' || needs.publish-docker.result == 'skipped') && + (needs.docker-manifests.result == 'success' || needs.docker-manifests.result == 'skipped') && (needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped') runs-on: ubuntu-latest permissions: @@ -873,6 +1042,8 @@ jobs: check-tags, build-python-wheels, publish-rust-crates, + publish-docker, + docker-manifests, publish, create-tags, ] @@ -1021,13 +1192,31 @@ jobs: echo fi - # Other publishing status - echo "### Other Publishing" + # Docker publishing status + if [ "${{ needs.plan.outputs.has_docker }}" = "true" ]; then + echo "### Docker Publishing (Native Runners)" + case "${{ needs.publish-docker.result }}" in + success) echo "✅ **Docker images built and pushed successfully on native runners**" ;; + failure) echo "❌ **Docker image publishing failed - check logs for details**" ;; + skipped) echo "⏭️ **Docker publishing was skipped**" ;; + esac + echo + echo "### Docker Manifests" + case "${{ needs.docker-manifests.result }}" in + success) echo "✅ **Multi-arch manifests created successfully**" ;; + failure) echo "❌ **Manifest creation failed - check logs for details**" ;; + skipped) echo "⏭️ **Manifest creation was skipped**" ;; + esac + echo + fi + + # SDK publishing status + echo "### SDK Publishing" case "${{ needs.publish.result }}" in - success) echo "✅ **Publishing completed successfully**" ;; - failure) echo "❌ **Publishing failed - check logs for details**" ;; - cancelled) echo "🚫 **Publishing was cancelled**" ;; - *) echo "⏭️ **Publishing was skipped**" ;; + success) echo "✅ **SDK publishing completed successfully**" ;; + failure) echo "❌ **SDK publishing failed - check logs for details**" ;; + cancelled) echo "🚫 **SDK publishing was cancelled**" ;; + *) echo "⏭️ **SDK publishing was skipped**" ;; esac if [ "${{ inputs.dry_run }}" = "true" ]; then echo @@ -1055,6 +1244,8 @@ jobs: plan, build-python-wheels, publish-rust-crates, + publish-docker, + docker-manifests, publish, create-tags, summary,
