This is an automated email from the ASF dual-hosted git repository. hgruszecki pushed a commit to branch refactor-post-merge-publish in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 9de452c10b75d27fbd9b074fb9c6749f3ffdfaaf Author: Hubert Gruszecki <[email protected]> AuthorDate: Mon Dec 15 12:58:48 2025 +0100 feat(ci): replace Docker buildx with native runner builds No more QEMU emulation for Rust images. Changes: - Add _build_docker_images.yml workflow using native ARM/x86 runners - Add prebuilt stages to mcp, connectors, bench-dashboard Dockerfiles - Extend _build_rust_artifacts.yml with iggy-mcp and bench-dashboard - Update post-merge.yml and publish.yml to use new workflow - Deprecate docker-buildx action (kept only for web-ui) --- .github/actions/utils/docker-buildx/action.yml | 16 +- .github/workflows/_build_docker_images.yml | 409 +++++++++++++++++++++++++ .github/workflows/_build_rust_artifacts.yml | 98 +++++- .github/workflows/post-merge.yml | 80 ++--- .github/workflows/publish.yml | 127 +++++++- core/ai/mcp/Dockerfile | 21 +- core/bench/dashboard/server/Dockerfile | 62 +++- core/connectors/runtime/Dockerfile | 21 +- 8 files changed, 749 insertions(+), 85 deletions(-) diff --git a/.github/actions/utils/docker-buildx/action.yml b/.github/actions/utils/docker-buildx/action.yml index 276cb6e0b..ba34b5f39 100644 --- a/.github/actions/utils/docker-buildx/action.yml +++ b/.github/actions/utils/docker-buildx/action.yml @@ -15,8 +15,22 @@ # specific language governing permissions and limitations # under the License. +# ================================================================================== +# DEPRECATION NOTICE +# ================================================================================== +# This action uses Docker buildx with QEMU emulation for multi-arch builds. +# For Rust components (server, mcp, connectors, bench-dashboard), use the new +# _build_docker_images.yml workflow instead, which uses native runners for each +# architecture, resulting in ~4x faster builds. +# +# This action is ONLY used for: +# - web-ui (Node.js) - QEMU overhead is minimal for Node.js builds +# +# Do NOT use this action for new Rust Docker images. +# ================================================================================== + name: docker-buildx -description: Multi-arch Docker build and push +description: Multi-arch Docker build and push (DEPRECATED for Rust images - use _build_docker_images.yml instead) inputs: task: description: "Task to run (build/publish/docker)" diff --git a/.github/workflows/_build_docker_images.yml b/.github/workflows/_build_docker_images.yml new file mode 100644 index 000000000..3846a5748 --- /dev/null +++ b/.github/workflows/_build_docker_images.yml @@ -0,0 +1,409 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: _build_docker_images +on: + workflow_call: + inputs: + version: + type: string + required: false + default: "edge" + description: "Version tag for Docker images" + dry_run: + type: boolean + required: false + default: false + description: "If true, build images but don't push" + components: + type: string + required: false + default: "rust-server,rust-mcp,rust-connectors,rust-bench-dashboard" + description: "Comma-separated list of components to build" + +env: + IGGY_CI_BUILD: true + +jobs: + build-images: + name: ${{ matrix.component }} (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + # Note: web-ui is NOT included here - it's a Node.js project that doesn't need + # pre-built Rust artifacts. It continues to use docker-buildx action directly. + component: [rust-server, rust-mcp, rust-connectors, rust-bench-dashboard] + include: + - arch: amd64 + runner: ubuntu-latest + artifact_target: x86_64-unknown-linux-gnu + - arch: arm64 + runner: ubuntu-24.04-arm + artifact_target: aarch64-unknown-linux-gnu + steps: + - name: Check if component should be built + id: check + run: | + components="${{ inputs.components }}" + component="${{ matrix.component }}" + # Use word boundary matching to avoid partial matches + if echo "$components" | grep -wq "$component"; then + echo "should_build=true" >> "$GITHUB_OUTPUT" + else + echo "should_build=false" >> "$GITHUB_OUTPUT" + echo "Skipping $component (not in component list)" + fi + + - name: Checkout code + if: steps.check.outputs.should_build == 'true' + uses: actions/checkout@v4 + + - name: Download binaries artifact + if: steps.check.outputs.should_build == 'true' && matrix.component != 'rust-bench-dashboard' + uses: actions/download-artifact@v4 + with: + name: binaries-${{ matrix.artifact_target }} + path: artifacts + + - name: Download bench-dashboard artifact + if: steps.check.outputs.should_build == 'true' && matrix.component == 'rust-bench-dashboard' + uses: actions/download-artifact@v4 + with: + name: bench-dashboard-${{ matrix.artifact_target }} + path: artifacts + + - name: Extract artifacts + if: steps.check.outputs.should_build == 'true' + run: | + cd artifacts + # Extract all tarballs (handles single or multiple files safely) + for tarball in *.tar.gz; do + [ -f "$tarball" ] || continue + echo "Extracting $tarball..." + tar xzf "$tarball" + rm -f "$tarball" + done + echo "Extracted artifacts:" + ls -la + + - name: Set up Docker + if: steps.check.outputs.should_build == 'true' + run: | + # Ensure docker is available + docker version + + - name: Login to Docker Hub + if: steps.check.outputs.should_build == 'true' && inputs.dry_run == false + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Resolve image config + if: steps.check.outputs.should_build == 'true' + id: config + run: | + component="${{ matrix.component }}" + case "$component" in + rust-server) + echo "image=apache/iggy" >> "$GITHUB_OUTPUT" + echo "dockerfile=core/server/Dockerfile" >> "$GITHUB_OUTPUT" + echo "binary=iggy-server" >> "$GITHUB_OUTPUT" + echo "prebuilt_arg=PREBUILT_IGGY_SERVER" >> "$GITHUB_OUTPUT" + echo "extra_args=--build-arg PREBUILT_IGGY_CLI=artifacts/iggy" >> "$GITHUB_OUTPUT" + ;; + rust-mcp) + echo "image=apache/iggy-mcp" >> "$GITHUB_OUTPUT" + echo "dockerfile=core/ai/mcp/Dockerfile" >> "$GITHUB_OUTPUT" + echo "binary=iggy-mcp" >> "$GITHUB_OUTPUT" + echo "prebuilt_arg=PREBUILT_IGGY_MCP" >> "$GITHUB_OUTPUT" + echo "extra_args=" >> "$GITHUB_OUTPUT" + ;; + rust-connectors) + echo "image=apache/iggy-connect" >> "$GITHUB_OUTPUT" + echo "dockerfile=core/connectors/runtime/Dockerfile" >> "$GITHUB_OUTPUT" + echo "binary=iggy-connectors" >> "$GITHUB_OUTPUT" + echo "prebuilt_arg=PREBUILT_IGGY_CONNECTORS" >> "$GITHUB_OUTPUT" + echo "extra_args=" >> "$GITHUB_OUTPUT" + ;; + rust-bench-dashboard) + echo "image=apache/iggy-bench-dashboard" >> "$GITHUB_OUTPUT" + echo "dockerfile=core/bench/dashboard/server/Dockerfile" >> "$GITHUB_OUTPUT" + echo "binary=iggy-bench-dashboard-server" >> "$GITHUB_OUTPUT" + echo "prebuilt_arg=PREBUILT_IGGY_BENCH_DASHBOARD_SERVER" >> "$GITHUB_OUTPUT" + echo "extra_args=--build-arg PREBUILT_FRONTEND_DIST=artifacts/dist" >> "$GITHUB_OUTPUT" + ;; + *) + echo "Unknown component: $component" + exit 1 + ;; + esac + + - name: Resolve version + if: steps.check.outputs.should_build == 'true' + id: version + run: | + input_version="${{ inputs.version }}" + component="${{ matrix.component }}" + + # If version is "auto", extract actual version from component + if [ "$input_version" = "auto" ]; then + chmod +x scripts/extract-version.sh 2>/dev/null || true + VERSION=$(scripts/extract-version.sh "$component" 2>/dev/null || echo "") + if [ -z "$VERSION" ]; then + echo "❌ Failed to extract version for $component" + exit 1 + fi + echo "📦 Extracted version for $component: $VERSION" + else + # Use provided version (e.g., "edge", "0.8.0") + VERSION="$input_version" + echo "📦 Using provided version: $VERSION" + fi + + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Build Docker image + if: steps.check.outputs.should_build == 'true' + run: | + IMAGE="${{ steps.config.outputs.image }}" + VERSION="${{ steps.version.outputs.version }}" + ARCH="${{ matrix.arch }}" + DOCKERFILE="${{ steps.config.outputs.dockerfile }}" + BINARY="${{ steps.config.outputs.binary }}" + PREBUILT_ARG="${{ steps.config.outputs.prebuilt_arg }}" + EXTRA_ARGS="${{ steps.config.outputs.extra_args }}" + + TAG="${IMAGE}:${VERSION}-${ARCH}" + + echo "Building $TAG" + echo " Dockerfile: $DOCKERFILE" + echo " Binary: artifacts/$BINARY" + echo " Prebuilt arg: $PREBUILT_ARG" + + docker build \ + --file "$DOCKERFILE" \ + --target runtime-prebuilt \ + --tag "$TAG" \ + --build-arg "${PREBUILT_ARG}=artifacts/${BINARY}" \ + $EXTRA_ARGS \ + . + + echo "Built image: $TAG" + docker images "$IMAGE" + + - name: Push Docker image + if: steps.check.outputs.should_build == 'true' && inputs.dry_run == false + run: | + IMAGE="${{ steps.config.outputs.image }}" + VERSION="${{ steps.version.outputs.version }}" + ARCH="${{ matrix.arch }}" + TAG="${IMAGE}:${VERSION}-${ARCH}" + + echo "Pushing $TAG" + docker push "$TAG" + + - name: Save image digest + if: steps.check.outputs.should_build == 'true' + id: digest + run: | + IMAGE="${{ steps.config.outputs.image }}" + VERSION="${{ steps.version.outputs.version }}" + ARCH="${{ matrix.arch }}" + TAG="${IMAGE}:${VERSION}-${ARCH}" + + if [ "${{ inputs.dry_run }}" = "false" ]; then + DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$TAG" 2>/dev/null || echo "") + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" + else + echo "digest=dry-run" >> "$GITHUB_OUTPUT" + fi + + create-manifests: + name: Create manifests + needs: build-images + if: inputs.dry_run == false + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + component: [rust-server, rust-mcp, rust-connectors, rust-bench-dashboard] + steps: + - name: Check if component should be built + id: check + run: | + components="${{ inputs.components }}" + component="${{ matrix.component }}" + # Use word boundary matching to avoid partial matches + if echo "$components" | grep -wq "$component"; then + echo "should_build=true" >> "$GITHUB_OUTPUT" + else + echo "should_build=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout code + if: steps.check.outputs.should_build == 'true' + uses: actions/checkout@v4 + + - name: Login to Docker Hub + if: steps.check.outputs.should_build == 'true' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Resolve image name + if: steps.check.outputs.should_build == 'true' + id: config + run: | + component="${{ matrix.component }}" + case "$component" in + rust-server) echo "image=apache/iggy" >> "$GITHUB_OUTPUT" ;; + rust-mcp) echo "image=apache/iggy-mcp" >> "$GITHUB_OUTPUT" ;; + rust-connectors) echo "image=apache/iggy-connect" >> "$GITHUB_OUTPUT" ;; + rust-bench-dashboard) echo "image=apache/iggy-bench-dashboard" >> "$GITHUB_OUTPUT" ;; + esac + + - name: Resolve version + if: steps.check.outputs.should_build == 'true' + id: version + run: | + input_version="${{ inputs.version }}" + component="${{ matrix.component }}" + + # If version is "auto", extract actual version from component + if [ "$input_version" = "auto" ]; then + chmod +x scripts/extract-version.sh 2>/dev/null || true + VERSION=$(scripts/extract-version.sh "$component" 2>/dev/null || echo "") + if [ -z "$VERSION" ]; then + echo "❌ Failed to extract version for $component" + exit 1 + fi + echo "📦 Extracted version for $component: $VERSION" + else + VERSION="$input_version" + echo "📦 Using provided version: $VERSION" + fi + + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Create and push manifest + if: steps.check.outputs.should_build == 'true' + run: | + IMAGE="${{ steps.config.outputs.image }}" + VERSION="${{ steps.version.outputs.version }}" + + echo "Creating manifest for $IMAGE:$VERSION" + + # Create manifest from platform-specific images + docker manifest create "${IMAGE}:${VERSION}" \ + "${IMAGE}:${VERSION}-amd64" \ + "${IMAGE}:${VERSION}-arm64" + + # Push the manifest + docker manifest push "${IMAGE}:${VERSION}" + + echo "Pushed manifest: ${IMAGE}:${VERSION}" + + - name: Create and push latest tag + if: steps.check.outputs.should_build == 'true' + run: | + IMAGE="${{ steps.config.outputs.image }}" + VERSION="${{ steps.version.outputs.version }}" + + # Only create 'latest' tag for stable releases (not edge, rc, alpha, beta) + if [ "$VERSION" = "edge" ]; then + echo "Skipping latest tag for edge version" + exit 0 + fi + + if echo "$VERSION" | grep -qE '(rc|alpha|beta)'; then + echo "Skipping latest tag for pre-release version: $VERSION" + exit 0 + fi + + echo "Creating latest manifest for $IMAGE (version: $VERSION)" + + # Create manifest for latest + docker manifest create "${IMAGE}:latest" \ + "${IMAGE}:${VERSION}-amd64" \ + "${IMAGE}:${VERSION}-arm64" + + # Push the latest manifest + docker manifest push "${IMAGE}:latest" + + echo "Pushed manifest: ${IMAGE}:latest" + + summary: + name: Summary + needs: [build-images, create-manifests] + if: always() + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Generate summary + run: | + { + echo "## Docker Images Build Summary" + echo "" + echo "| Component | Version | Status |" + echo "|-----------|---------|--------|" + + components="${{ inputs.components }}" + input_version="${{ inputs.version }}" + dry_run="${{ inputs.dry_run }}" + + for component in rust-server rust-mcp rust-connectors rust-bench-dashboard; do + # Use word boundary matching to avoid partial matches + if echo "$components" | grep -wq "$component"; then + case "$component" in + rust-server) image="apache/iggy" ;; + rust-mcp) image="apache/iggy-mcp" ;; + rust-connectors) image="apache/iggy-connect" ;; + rust-bench-dashboard) image="apache/iggy-bench-dashboard" ;; + esac + + # Resolve version for display + if [ "$input_version" = "auto" ]; then + chmod +x scripts/extract-version.sh 2>/dev/null || true + version=$(scripts/extract-version.sh "$component" 2>/dev/null || echo "unknown") + else + version="$input_version" + fi + + if [ "$dry_run" = "true" ]; then + echo "| $component | $version | Built (dry-run) |" + else + echo "| $component | $version | Published to \`$image:$version\` |" + fi + else + echo "| $component | - | Skipped |" + fi + done + + echo "" + if [ "$dry_run" = "true" ]; then + echo "**Mode:** Dry run (images not pushed)" + else + echo "**Mode:** Production (images pushed to Docker Hub)" + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/_build_rust_artifacts.yml b/.github/workflows/_build_rust_artifacts.yml index f5b3c5006..8ecfe440a 100644 --- a/.github/workflows/_build_rust_artifacts.yml +++ b/.github/workflows/_build_rust_artifacts.yml @@ -41,7 +41,7 @@ on: binaries: type: string required: false - default: "iggy-server,iggy,iggy-bench,iggy-connectors" + default: "iggy-server,iggy,iggy-bench,iggy-connectors,iggy-mcp" description: "Comma-separated list of binaries to build" connector_plugins: type: string @@ -245,9 +245,102 @@ jobs: retention-days: 7 if-no-files-found: error + build-bench-dashboard: + name: Bench dashboard ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + runner: ubuntu-latest + - target: aarch64-unknown-linux-gnu + runner: ubuntu-24.04-arm + steps: + - name: Download latest copy script from master + if: inputs.use_latest_ci + run: | + curl -sSL "https://raw.githubusercontent.com/${{ github.repository }}/master/scripts/copy-latest-from-master.sh" \ + -o /tmp/copy-latest-from-master.sh + chmod +x /tmp/copy-latest-from-master.sh + + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit || github.sha }} + + - name: Apply latest CI from master + if: inputs.use_latest_ci + run: | + /tmp/copy-latest-from-master.sh save .github scripts + /tmp/copy-latest-from-master.sh apply + + - name: Setup Rust with cache + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false + + - name: Add Rust targets + run: | + rustup target add ${{ matrix.target }} + rustup target add wasm32-unknown-unknown + + - name: Install trunk + run: | + # Install trunk for WASM frontend build + TRUNK_VERSION="0.21.15" + ARCH="$(uname -m)" + case "$ARCH" in + x86_64) TRUNK_ARCH="x86_64-unknown-linux-gnu" ;; + aarch64) TRUNK_ARCH="aarch64-unknown-linux-gnu" ;; + *) + echo "Unsupported architecture: $ARCH" + exit 1 + ;; + esac + curl -L "https://github.com/trunk-rs/trunk/releases/download/v${TRUNK_VERSION}/trunk-${TRUNK_ARCH}.tar.gz"; | tar xz + sudo mv trunk /usr/local/bin/ + trunk --version + + - name: Build frontend (WASM) + run: | + cd core/bench/dashboard/frontend + trunk build --release + + - name: Build backend binary + run: | + cargo build --locked --release --target ${{ matrix.target }} --bin iggy-bench-dashboard-server + + - name: Package bench-dashboard + id: pkg + run: | + target="${{ matrix.target }}" + version="${{ inputs.version }}" + outdir="dist/${target}" + mkdir -p "${outdir}/dist" + + # Copy backend binary + cp "target/${target}/release/iggy-bench-dashboard-server" "${outdir}/" + + # Copy frontend dist folder + cp -r core/bench/dashboard/frontend/dist/* "${outdir}/dist/" + + tarball="iggy-bench-dashboard-${target}-${version}.tar.gz" + tar czf "${tarball}" -C "${outdir}" . + echo "tarball=${tarball}" >> "$GITHUB_OUTPUT" + + - name: Upload artifact + if: inputs.upload_artifacts + uses: actions/upload-artifact@v4 + with: + name: bench-dashboard-${{ matrix.target }} + path: ${{ steps.pkg.outputs.tarball }} + retention-days: 7 + if-no-files-found: error + collect: name: Collect artifacts - needs: [build-binaries, build-connector-plugins] + needs: [build-binaries, build-connector-plugins, build-bench-dashboard] if: inputs.upload_artifacts runs-on: ubuntu-latest outputs: @@ -288,6 +381,7 @@ jobs: size=$(ls -lh "$tarball" | awk '{print $5}') if [[ "$filename" == iggy-connectors-* ]]; then type="plugins" + elif [[ "$filename" == iggy-bench-dashboard-* ]]; then type="dashboard" else type="binaries"; fi if [[ "$filename" == *"x86_64"* ]]; then target="x86_64" diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index 84a2bc9ed..50300cdb0 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -34,67 +34,6 @@ env: IGGY_CI_BUILD: true jobs: - plan: - name: Plan dockerhub components - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.mk.outputs.matrix }} - steps: - - uses: actions/checkout@v4 - - - name: Load publish config (base64) - id: cfg - shell: bash - run: | - if ! command -v yq >/dev/null 2>&1; then - YQ_VERSION="v4.47.1" - YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" - curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 - echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1 - chmod +x /usr/local/bin/yq - fi - echo "components_b64=$(yq -o=json -I=0 '.components' .github/config/publish.yml | base64 -w0)" >> "$GITHUB_OUTPUT" - - - name: Build matrix - id: mk - uses: actions/github-script@v7 - with: - script: | - const b64 = `${{ steps.cfg.outputs.components_b64 }}` || ''; - if (!b64) { - core.setOutput('matrix', JSON.stringify({ include: [{ component: 'noop' }] })); - return; - } - const comps = JSON.parse(Buffer.from(b64, 'base64').toString('utf8')); - const include = Object.entries(comps) - .filter(([_, v]) => v && v.registry === 'dockerhub') - .map(([k]) => ({ component: k })); - const uniq = Array.from(new Map(include.map(i => [i.component, i])).values()); - core.setOutput('matrix', JSON.stringify(uniq.length ? { include: uniq } : { include: [{ component: 'noop' }] })); - - docker-edge: - name: ${{ matrix.component }} - needs: plan - if: ${{ fromJson(needs.plan.outputs.matrix).include[0].component != 'noop' }} - runs-on: ubuntu-latest - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.plan.outputs.matrix) }} - env: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - steps: - - uses: actions/checkout@v4 - - - uses: ./.github/actions/utils/docker-buildx - with: - task: publish - libc: musl - component: ${{ matrix.component }} - version: edge - dry_run: ${{ github.event.repository.fork }} # forks: always dry-run - build-artifacts: name: Build artifacts uses: ./.github/workflows/_build_rust_artifacts.yml @@ -102,6 +41,17 @@ jobs: version: edge upload_artifacts: true + docker-images: + name: Docker images + needs: build-artifacts + if: needs.build-artifacts.result == 'success' + uses: ./.github/workflows/_build_docker_images.yml + with: + version: edge + dry_run: ${{ github.event.repository.fork }} + components: "rust-server,rust-mcp,rust-connectors,rust-bench-dashboard" + secrets: inherit + create-prerelease: name: Create edge pre-release runs-on: ubuntu-latest @@ -153,6 +103,10 @@ jobs: - `iggy` - The command-line interface - `iggy-bench` - The benchmarking tool - `iggy-connectors` - The connectors runtime + - `iggy-mcp` - Model Context Protocol server for LLM integration + + ## Bench Dashboard + - `iggy-bench-dashboard-server` - Benchmarking web dashboard (with frontend) ## Connector plugins included (.so) - `iggy_connector_elasticsearch_sink` @@ -176,6 +130,10 @@ jobs: - `iggy-connectors-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64 (glibc) - `iggy-connectors-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64 (glibc) + ### Bench Dashboard + - `iggy-bench-dashboard-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64 (glibc) + - `iggy-bench-dashboard-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64 (glibc) + ## Build info - Server version: ${{ steps.meta.outputs.server_version }} - Commit: ${{ github.sha }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 77470d3c5..29856b00b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -136,10 +136,13 @@ jobs: outputs: targets: ${{ steps.mk.outputs.targets }} non_rust_targets: ${{ steps.mk.outputs.non_rust_targets }} + non_docker_targets: ${{ steps.mk.outputs.non_docker_targets }} count: ${{ steps.mk.outputs.count }} go_sdk_version: ${{ steps.mk.outputs.go_sdk_version }} has_python: ${{ steps.mk.outputs.has_python }} has_rust_crates: ${{ steps.mk.outputs.has_rust_crates }} + rust_docker_components: ${{ steps.mk.outputs.rust_docker_components }} + has_web_ui: ${{ steps.mk.outputs.has_web_ui }} steps: - name: Download latest copy script from master if: inputs.use_latest_ci @@ -263,9 +266,13 @@ jobs: } } + // Filter non-Docker targets (for SDK publishing, not handled by _build_docker_images.yml) + const nonDockerTargets = nonRustTargets.filter(t => t.type !== 'docker'); + console.log(`Publishing ${targets.length} components:`); targets.forEach(t => console.log(` - ${t.name} (${t.type}) -> ${t.registry || 'N/A'}`)); console.log(` (${nonRustTargets.length} non-Rust, ${targets.length - nonRustTargets.length} Rust crates)`); + console.log(` (${nonDockerTargets.length} non-Docker SDKs for matrix job)`); // Output all targets for reference and tag creation core.setOutput('targets', JSON.stringify(targets.length ? { include: targets } : { include: [{ key: 'noop', type: 'noop' }] })); @@ -273,10 +280,24 @@ jobs: // Output only non-Rust targets for the parallel publish job core.setOutput('non_rust_targets', JSON.stringify(nonRustTargets.length ? { include: nonRustTargets } : { include: [{ key: 'noop', type: 'noop' }] })); + // Output only non-Docker, non-Rust targets (SDKs only) + core.setOutput('non_docker_targets', JSON.stringify(nonDockerTargets.length ? { include: nonDockerTargets } : { include: [{ key: 'noop', type: 'noop' }] })); + core.setOutput('count', String(targets.length)); core.setOutput('go_sdk_version', goVersion); core.setOutput('has_rust_crates', String(hasRustCrates)); + // Separate Rust Docker components from web-ui + // web-ui uses docker-buildx action directly (Node.js, doesn't need pre-built artifacts) + const rustDockerComponents = targets + .filter(t => t.type === 'docker' && t.key !== 'web-ui') + .map(t => t.key) + .join(','); + const hasWebUi = targets.some(t => t.key === 'web-ui'); + + core.setOutput('rust_docker_components', rustDockerComponents); + core.setOutput('has_web_ui', String(hasWebUi)); + // Check if Python SDK is in targets and extract version const pythonTarget = targets.find(t => t.key === 'sdk-python'); if (pythonTarget) { @@ -455,6 +476,84 @@ jobs: use_latest_ci: ${{ inputs.use_latest_ci }} commit: ${{ needs.validate.outputs.commit }} + build-docker-artifacts: + name: Build Docker artifacts + needs: [validate, plan, check-tags] + if: | + needs.validate.outputs.has_targets == 'true' && + inputs.publish_dockerhub != '' + uses: ./.github/workflows/_build_rust_artifacts.yml + with: + version: "publish" + upload_artifacts: true + use_latest_ci: ${{ inputs.use_latest_ci }} + commit: ${{ needs.validate.outputs.commit }} + + publish-docker-images: + name: Publish Rust Docker images + needs: [validate, plan, check-tags, build-docker-artifacts] + if: | + needs.validate.outputs.has_targets == 'true' && + needs.plan.outputs.rust_docker_components != '' && + needs.build-docker-artifacts.result == 'success' + uses: ./.github/workflows/_build_docker_images.yml + with: + version: "auto" + dry_run: ${{ inputs.dry_run }} + components: ${{ needs.plan.outputs.rust_docker_components }} + secrets: inherit + + # web-ui uses docker-buildx directly (Node.js, doesn't benefit from pre-built artifacts) + publish-web-ui: + name: Publish web-ui Docker image + needs: [validate, plan, check-tags] + if: | + needs.validate.outputs.has_targets == 'true' && + needs.plan.outputs.has_web_ui == 'true' + runs-on: ubuntu-latest + steps: + - name: Download latest copy script from master + if: inputs.use_latest_ci + run: | + curl -sSL "https://raw.githubusercontent.com/${{ github.repository }}/master/scripts/copy-latest-from-master.sh" \ + -o /tmp/copy-latest-from-master.sh + chmod +x /tmp/copy-latest-from-master.sh + echo "✅ Downloaded latest copy script from master" + + - name: Checkout at commit + uses: actions/checkout@v4 + with: + ref: ${{ needs.validate.outputs.commit }} + + - name: Save and apply latest CI from master + if: inputs.use_latest_ci + run: | + /tmp/copy-latest-from-master.sh save \ + .github \ + scripts \ + web/Dockerfile + + /tmp/copy-latest-from-master.sh apply + + - name: Extract web-ui version + id: version + run: | + chmod +x scripts/extract-version.sh + VERSION=$(scripts/extract-version.sh web-ui) + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "📦 web-ui version: $VERSION" + + - name: Build and push web-ui image + uses: ./.github/actions/utils/docker-buildx + env: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + task: publish + component: web-ui + version: ${{ steps.version.outputs.version }} + dry_run: ${{ inputs.dry_run }} + # Sequential Rust crate publishing to handle dependencies properly publish-rust-crates: name: Publish Rust Crates @@ -610,17 +709,15 @@ jobs: if: | always() && needs.validate.outputs.has_targets == 'true' && - fromJson(needs.plan.outputs.non_rust_targets).include[0].key != 'noop' && + fromJson(needs.plan.outputs.non_docker_targets).include[0].key != 'noop' && (needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped') && (needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped') runs-on: ubuntu-latest strategy: fail-fast: false - matrix: ${{ fromJson(needs.plan.outputs.non_rust_targets) }} + matrix: ${{ fromJson(needs.plan.outputs.non_docker_targets) }} env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} NEXUS_USER: ${{ secrets.NEXUS_USER }} @@ -667,7 +764,7 @@ jobs: test -x scripts/extract-version.sh || chmod +x scripts/extract-version.sh - name: Setup Rust toolchain (if needed) - if: matrix.type == 'rust' || matrix.type == 'docker' || matrix.type == 'python' + if: matrix.type == 'rust' || matrix.type == 'python' uses: ./.github/actions/utils/setup-rust-with-cache with: cache-targets: false @@ -696,18 +793,6 @@ jobs: echo "tag=$TAG" >> "$GITHUB_OUTPUT" echo "✅ Resolved ${{ matrix.key }} -> version=$VERSION tag=${TAG:-<none>}" - # ───────────────────────────────────────── - # Docker Publishing - # ───────────────────────────────────────── - - name: Publish Docker image - if: matrix.type == 'docker' - uses: ./.github/actions/utils/docker-buildx - with: - task: publish - component: ${{ matrix.key }} - version: ${{ steps.ver.outputs.version }} - dry_run: ${{ inputs.dry_run }} - # ───────────────────────────────────────── # Python SDK Publishing # ───────────────────────────────────────── @@ -774,6 +859,8 @@ jobs: check-tags, build-python-wheels, publish-rust-crates, + publish-docker-images, + publish-web-ui, publish, ] if: | @@ -782,6 +869,8 @@ jobs: inputs.dry_run == false && inputs.skip_tag_creation == false && (needs.publish.result == 'success' || needs.publish.result == 'skipped') && + (needs.publish-docker-images.result == 'success' || needs.publish-docker-images.result == 'skipped') && + (needs.publish-web-ui.result == 'success' || needs.publish-web-ui.result == 'skipped') && (needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped') && (needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped') runs-on: ubuntu-latest @@ -873,6 +962,8 @@ jobs: check-tags, build-python-wheels, publish-rust-crates, + publish-docker-images, + publish-web-ui, publish, create-tags, ] @@ -1055,6 +1146,8 @@ jobs: plan, build-python-wheels, publish-rust-crates, + publish-docker-images, + publish-web-ui, publish, create-tags, summary, diff --git a/core/ai/mcp/Dockerfile b/core/ai/mcp/Dockerfile index f5fa7e977..6566d347d 100644 --- a/core/ai/mcp/Dockerfile +++ b/core/ai/mcp/Dockerfile @@ -85,7 +85,26 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG fi # -# Final runtime - Debian trixie Slim +# Prebuilt path (FAST) - for CI with pre-compiled binaries +# +FROM debian:trixie-slim AS prebuilt +ARG PREBUILT_IGGY_MCP +WORKDIR /out +COPY ${PREBUILT_IGGY_MCP} /out/iggy-mcp +RUN chmod +x /out/iggy-mcp + +# +# Final runtime using prebuilt binary +# +FROM debian:trixie-slim AS runtime-prebuilt +WORKDIR /app + +COPY --from=prebuilt /out/iggy-mcp /usr/local/bin/iggy-mcp + +ENTRYPOINT ["iggy-mcp"] + +# +# Final runtime - Debian trixie Slim (from source build) # FROM debian:trixie-slim AS runtime WORKDIR /app diff --git a/core/bench/dashboard/server/Dockerfile b/core/bench/dashboard/server/Dockerfile index 29e103824..417e1ff48 100644 --- a/core/bench/dashboard/server/Dockerfile +++ b/core/bench/dashboard/server/Dockerfile @@ -53,8 +53,66 @@ RUN cd core/bench/dashboard/frontend && trunk build --release # Build the server with release profile RUN cargo build --release --bin iggy-bench-dashboard-server -# Runtime stage -FROM debian:trixie-slim +# +# Prebuilt path (FAST) - for CI with pre-compiled binaries and frontend +# +FROM debian:trixie-slim AS prebuilt +ARG PREBUILT_IGGY_BENCH_DASHBOARD_SERVER +ARG PREBUILT_FRONTEND_DIST +WORKDIR /out +COPY ${PREBUILT_IGGY_BENCH_DASHBOARD_SERVER} /out/iggy-bench-dashboard-server +COPY ${PREBUILT_FRONTEND_DIST} /out/frontend/dist +RUN chmod +x /out/iggy-bench-dashboard-server + +# +# Runtime stage using prebuilt binaries +# +FROM debian:trixie-slim AS runtime-prebuilt + +WORKDIR /app + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + openssl \ + curl \ + libssl3 \ + && rm -rf /var/lib/apt/lists/* + +# Copy prebuilt binary and frontend files +COPY --from=prebuilt /out/iggy-bench-dashboard-server /app/ +COPY --from=prebuilt /out/frontend/dist /app/frontend/dist + +# Create data directory and non-root user +RUN groupadd -r iggy && \ + useradd -r -g iggy -s /bin/false iggy && \ + mkdir -p /data/performance_results && \ + chown -R iggy:iggy /app /data && \ + chmod -R 755 /data/performance_results + +# Copy the entrypoint script +COPY core/bench/dashboard/server/docker-entrypoint.sh /app/ +RUN chmod +x /app/docker-entrypoint.sh && \ + chown iggy:iggy /app/docker-entrypoint.sh + +# Set default environment variables for configuration +ENV HOST=0.0.0.0 \ + PORT=80 \ + RESULTS_DIR=/data/performance_results + +# Set volume for results with proper permissions +VOLUME ["/data/performance_results"] + +# Switch to non-root user +USER iggy + +# Set the entrypoint script +ENTRYPOINT ["/app/docker-entrypoint.sh"] + +# +# Runtime stage (from source build) +# +FROM debian:trixie-slim AS runtime WORKDIR /app diff --git a/core/connectors/runtime/Dockerfile b/core/connectors/runtime/Dockerfile index 7e78fe1a5..c65159133 100644 --- a/core/connectors/runtime/Dockerfile +++ b/core/connectors/runtime/Dockerfile @@ -85,7 +85,26 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG fi # -# Final runtime - Debian trixie Slim +# Prebuilt path (FAST) - for CI with pre-compiled binaries +# +FROM debian:trixie-slim AS prebuilt +ARG PREBUILT_IGGY_CONNECTORS +WORKDIR /out +COPY ${PREBUILT_IGGY_CONNECTORS} /out/iggy-connectors +RUN chmod +x /out/iggy-connectors + +# +# Final runtime using prebuilt binary +# +FROM debian:trixie-slim AS runtime-prebuilt +WORKDIR /app + +COPY --from=prebuilt /out/iggy-connectors /usr/local/bin/iggy-connectors + +ENTRYPOINT ["iggy-connectors"] + +# +# Final runtime - Debian trixie Slim (from source build) # FROM debian:trixie-slim AS runtime WORKDIR /app
