This is an automated email from the ASF dual-hosted git repository.
piotr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 64d4e8ccf feat(ci): add ARM64 builds and connector plugins to edge
release (#2472)
64d4e8ccf is described below
commit 64d4e8ccf8ecf3d0a5337052b019fe1df1793344
Author: Hubert Gruszecki <[email protected]>
AuthorDate: Fri Dec 12 09:11:30 2025 +0100
feat(ci): add ARM64 builds and connector plugins to edge release (#2472)
- Fix edge tag not updating by using --cleanup-tag on release delete
- Add ARM64 binary builds using native ARM runners
- Add iggy-connectors binary to release artifacts
- Add connector plugin shared libraries (.so) for x86_64 and ARM64
- Refactor build logic into reusable _build_rust_artifacts.yml
---
...rust_binaries.yml => _build_rust_artifacts.yml} | 177 +++++++++++++++------
.github/workflows/_test_bdd.yml | 8 +
.github/workflows/post-merge.yml | 57 ++++---
3 files changed, 170 insertions(+), 72 deletions(-)
diff --git a/.github/workflows/_build_rust_binaries.yml
b/.github/workflows/_build_rust_artifacts.yml
similarity index 52%
rename from .github/workflows/_build_rust_binaries.yml
rename to .github/workflows/_build_rust_artifacts.yml
index 195d00df7..e087136a5 100644
--- a/.github/workflows/_build_rust_binaries.yml
+++ b/.github/workflows/_build_rust_artifacts.yml
@@ -15,8 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-# .github/workflows/_build_rust_binaries.yml
-name: _build_rust_binaries
+name: _build_rust_artifacts
on:
workflow_call:
inputs:
@@ -38,23 +37,28 @@ on:
type: string
required: false
default: ""
- description: "Specific commit to checkout for building binaries"
+ description: "Specific commit to checkout for building"
binaries:
type: string
required: false
- default: "iggy-server,iggy,iggy-bench"
+ default: "iggy-server,iggy,iggy-bench,iggy-connectors"
description: "Comma-separated list of binaries to build"
+ connector_plugins:
+ type: string
+ required: false
+ default:
"iggy_connector_elasticsearch_sink,iggy_connector_elasticsearch_source,iggy_connector_iceberg_sink,iggy_connector_postgres_sink,iggy_connector_postgres_source,iggy_connector_quickwit_sink,iggy_connector_random_source,iggy_connector_stdout_sink"
+ description: "Comma-separated list of connector plugin crates to build
as shared libraries"
outputs:
artifact_name:
- description: "Name of the uploaded artifact containing all binaries"
+ description: "Name of the uploaded artifact containing all artifacts"
value: ${{ jobs.collect.outputs.artifact_name }}
env:
IGGY_CI_BUILD: true
jobs:
- linux:
- name: Linux ${{ matrix.target }}
+ build-binaries:
+ name: Binaries ${{ matrix.target }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 60
strategy:
@@ -67,12 +71,16 @@ jobs:
- target: x86_64-unknown-linux-musl
runner: ubuntu-latest
libc: musl
+ - target: aarch64-unknown-linux-gnu
+ runner: ubuntu-24.04-arm
+ libc: gnu
+ - target: aarch64-unknown-linux-musl
+ runner: ubuntu-24.04-arm
+ libc: musl
steps:
- name: Download latest copy script from master
if: inputs.use_latest_ci
- shell: bash
run: |
- set -euo pipefail
curl -sSL "https://raw.githubusercontent.com/${{ github.repository
}}/master/scripts/copy-latest-from-master.sh" \
-o /tmp/copy-latest-from-master.sh
chmod +x /tmp/copy-latest-from-master.sh
@@ -81,11 +89,9 @@ jobs:
with:
ref: ${{ inputs.commit || github.sha }}
- - name: Save and apply latest CI from master
+ - name: Apply latest CI from master
if: inputs.use_latest_ci
- shell: bash
run: |
- set -euo pipefail
/tmp/copy-latest-from-master.sh save .github scripts
/tmp/copy-latest-from-master.sh apply
@@ -102,10 +108,7 @@ jobs:
run: rustup target add ${{ matrix.target }}
- name: Build binaries
- shell: bash
run: |
- set -euo pipefail
-
binaries="${{ inputs.binaries }}"
bin_flags=()
@@ -120,10 +123,7 @@ jobs:
- name: Package binaries
id: pkg
- shell: bash
run: |
- set -euo pipefail
-
target="${{ matrix.target }}"
version="${{ inputs.version }}"
outdir="dist/${target}"
@@ -139,23 +139,102 @@ jobs:
tarball="iggy-${target}-${version}.tar.gz"
tar czf "${tarball}" -C "${outdir}" .
+ echo "tarball=${tarball}" >> "$GITHUB_OUTPUT"
+
+ - name: Upload artifact
+ if: inputs.upload_artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: binaries-${{ matrix.target }}
+ path: ${{ steps.pkg.outputs.tarball }}
+ retention-days: 7
+ if-no-files-found: error
+
+ build-connector-plugins:
+ name: Connector plugins ${{ matrix.target }}
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - target: x86_64-unknown-linux-gnu
+ runner: ubuntu-latest
+ - target: aarch64-unknown-linux-gnu
+ runner: ubuntu-24.04-arm
+ steps:
+ - name: Download latest copy script from master
+ if: inputs.use_latest_ci
+ run: |
+ curl -sSL "https://raw.githubusercontent.com/${{ github.repository
}}/master/scripts/copy-latest-from-master.sh" \
+ -o /tmp/copy-latest-from-master.sh
+ chmod +x /tmp/copy-latest-from-master.sh
+
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.commit || github.sha }}
+
+ - name: Apply latest CI from master
+ if: inputs.use_latest_ci
+ run: |
+ /tmp/copy-latest-from-master.sh save .github scripts
+ /tmp/copy-latest-from-master.sh apply
+
+ - name: Setup Rust with cache
+ uses: ./.github/actions/utils/setup-rust-with-cache
+ with:
+ cache-targets: false
+
+ - name: Add Rust target
+ run: rustup target add ${{ matrix.target }}
+
+ - name: Build connector plugins
+ run: |
+ plugins="${{ inputs.connector_plugins }}"
+ pkg_flags=()
+
+ IFS=',' read -ra pkgs <<< "$plugins"
+ for pkg in "${pkgs[@]}"; do
+ name="$(echo "$pkg" | xargs)"
+ [[ -z "$name" ]] && continue
+ pkg_flags+=(--package "$name")
+ done
+
+ cargo build --locked --release --target ${{ matrix.target }}
"${pkg_flags[@]}"
+ - name: Package connector plugins
+ id: pkg
+ run: |
+ target="${{ matrix.target }}"
+ version="${{ inputs.version }}"
+ outdir="dist/${target}"
+ mkdir -p "${outdir}"
+
+ plugins="${{ inputs.connector_plugins }}"
+ IFS=',' read -ra pkgs <<< "$plugins"
+ for pkg in "${pkgs[@]}"; do
+ lib_name="$(echo "$pkg" | xargs)"
+ [[ -z "$lib_name" ]] && continue
+ so_file="target/${target}/release/lib${lib_name}.so"
+ [[ -f "$so_file" ]] && cp "$so_file" "${outdir}/"
+ done
+
+ tarball="iggy-connectors-${target}-${version}.tar.gz"
+ tar czf "${tarball}" -C "${outdir}" .
echo "tarball=${tarball}" >> "$GITHUB_OUTPUT"
- echo "Built: ${tarball}"
- ls -lh "${tarball}"
- name: Upload artifact
if: inputs.upload_artifacts
uses: actions/upload-artifact@v4
with:
- name: rust-binaries-${{ matrix.target }}
+ name: connector-plugins-${{ matrix.target }}
path: ${{ steps.pkg.outputs.tarball }}
retention-days: 7
if-no-files-found: error
collect:
- name: Collect all binaries
- needs: [linux]
+ name: Collect artifacts
+ needs: [build-binaries, build-connector-plugins]
if: inputs.upload_artifacts
runs-on: ubuntu-latest
outputs:
@@ -169,59 +248,57 @@ jobs:
id: meta
run: |
chmod +x scripts/extract-version.sh
- server_version=$(scripts/extract-version.sh rust-server)
- echo "server_version=${server_version}" >> "$GITHUB_OUTPUT"
+ echo "server_version=$(scripts/extract-version.sh rust-server)" >>
"$GITHUB_OUTPUT"
- - name: Download all binaries
+ - name: Download all artifacts
uses: actions/download-artifact@v4
with:
- pattern: rust-binaries-*
- merge-multiple: true
path: dist
+ merge-multiple: true
- - name: List binaries
- shell: bash
+ - name: Generate summary
run: |
- set -euo pipefail
shopt -s nullglob
-
tarballs=(dist/*.tar.gz)
- if ((${#tarballs[@]} == 0)); then
- echo "⚠️ No tarballs found in dist/, skipping summary." >>
"$GITHUB_STEP_SUMMARY"
- exit 0
- fi
+ (( ${#tarballs[@]} == 0 )) && exit 0
- echo "## 📦 Built Rust Binaries" >> "$GITHUB_STEP_SUMMARY"
- echo "" >> "$GITHUB_STEP_SUMMARY"
- echo "| Target | Libc | File | Size |" >> "$GITHUB_STEP_SUMMARY"
- echo "|--------|------|------|------|" >> "$GITHUB_STEP_SUMMARY"
+ {
+ echo "## Built Rust Artifacts"
+ echo ""
+ echo "| Type | Target | File | Size |"
+ echo "|------|--------|------|------|"
+ } >> "$GITHUB_STEP_SUMMARY"
for tarball in "${tarballs[@]}"; do
filename=$(basename "$tarball")
size=$(ls -lh "$tarball" | awk '{print $5}')
- if [[ "$filename" == *"gnu"* ]]; then libc="glibc"
- elif [[ "$filename" == *"musl"* ]]; then libc="musl (static)"
- else libc="unknown"; fi
+ if [[ "$filename" == iggy-connectors-* ]]; then type="plugins"
+ else type="binaries"; fi
if [[ "$filename" == *"x86_64"* ]]; then target="x86_64"
elif [[ "$filename" == *"aarch64"* ]]; then target="aarch64"
else target="unknown"; fi
- echo "| $target | $libc | \`$filename\` | $size |" >>
"$GITHUB_STEP_SUMMARY"
+ if [[ "$filename" == *"musl"* ]]; then target="$target (musl)"
+ elif [[ "$filename" == *"gnu"* ]]; then target="$target (glibc)"
+ fi
+
+ echo "| $type | $target | \`$filename\` | $size |" >>
"$GITHUB_STEP_SUMMARY"
done
- echo "" >> "$GITHUB_STEP_SUMMARY"
- echo "**Server version:** ${{ steps.meta.outputs.server_version }}"
>> "$GITHUB_STEP_SUMMARY"
- echo "" >> "$GITHUB_STEP_SUMMARY"
- echo "**Binaries included:** ${{ inputs.binaries }}" >>
"$GITHUB_STEP_SUMMARY"
+ {
+ echo ""
+ echo "**Server version:** ${{ steps.meta.outputs.server_version }}"
+ echo "**Binaries:** ${{ inputs.binaries }}"
+ } >> "$GITHUB_STEP_SUMMARY"
- name: Upload combined artifact
uses: actions/upload-artifact@v4
with:
- name: rust-binaries-all
+ name: rust-artifacts-all
path: dist
retention-days: 30
- id: output
- run: echo "artifact_name=rust-binaries-all" >> $GITHUB_OUTPUT
+ run: echo "artifact_name=rust-artifacts-all" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/_test_bdd.yml b/.github/workflows/_test_bdd.yml
index 06f93ad63..907868913 100644
--- a/.github/workflows/_test_bdd.yml
+++ b/.github/workflows/_test_bdd.yml
@@ -39,6 +39,14 @@ jobs:
env:
IGGY_CI_BUILD: true
steps:
+ - name: Cleanup disk space
+ run: |
+ echo "Disk space before cleanup:"
+ df -h
+ sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
+ echo "Disk space after cleanup:"
+ df -h
+
- name: Checkout code
uses: actions/checkout@v4
diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml
index 0bdc2f508..84a2bc9ed 100644
--- a/.github/workflows/post-merge.yml
+++ b/.github/workflows/post-merge.yml
@@ -95,9 +95,9 @@ jobs:
version: edge
dry_run: ${{ github.event.repository.fork }} # forks: always dry-run
- build-rust-binaries:
- name: Build Rust binaries
- uses: ./.github/workflows/_build_rust_binaries.yml
+ build-artifacts:
+ name: Build artifacts
+ uses: ./.github/workflows/_build_rust_artifacts.yml
with:
version: edge
upload_artifacts: true
@@ -105,8 +105,8 @@ jobs:
create-prerelease:
name: Create edge pre-release
runs-on: ubuntu-latest
- needs: build-rust-binaries
- if: needs.build-rust-binaries.result == 'success' &&
!github.event.repository.fork
+ needs: build-artifacts
+ if: needs.build-artifacts.result == 'success' &&
!github.event.repository.fork
permissions:
contents: write
steps:
@@ -119,38 +119,32 @@ jobs:
server_version=$(scripts/extract-version.sh rust-server)
echo "server_version=${server_version}" >> "$GITHUB_OUTPUT"
- - name: Download edge artifacts
+ - name: Download all artifacts
uses: actions/download-artifact@v4
with:
- name: rust-binaries-all
+ name: rust-artifacts-all
path: ./artifacts
- - name: Delete existing edge release
+ - name: Delete existing edge release and tag
env:
GH_TOKEN: ${{ github.token }}
- shell: bash
run: |
- set -euo pipefail
if gh release view edge &>/dev/null; then
- echo "Deleting existing edge release..."
- if ! gh release delete edge --yes; then
- echo "Release delete failed (likely already deleted by
concurrent run), continuing."
- fi
- else
- echo "No existing edge release to delete"
+ echo "Deleting existing edge release and tag..."
+ gh release delete edge --cleanup-tag --yes || echo "Delete failed,
continuing..."
fi
- name: Create edge pre-release
uses: softprops/action-gh-release@v2
with:
tag_name: edge
- name: Edge (latest)
+ name: edge
draft: false
prerelease: true
make_latest: false
- files: artifacts/**/*.tar.gz
+ files: artifacts/*.tar.gz
body: |
- Rolling edge build of Apache Iggy binaries.
+ Rolling edge build of Apache Iggy binaries and connector plugins.
**This release is automatically updated on every push to master.**
@@ -158,13 +152,32 @@ jobs:
- `iggy-server` - The server binary
- `iggy` - The command-line interface
- `iggy-bench` - The benchmarking tool
+ - `iggy-connectors` - The connectors runtime
+
+ ## Connector plugins included (.so)
+ - `iggy_connector_elasticsearch_sink`
+ - `iggy_connector_elasticsearch_source`
+ - `iggy_connector_iceberg_sink`
+ - `iggy_connector_postgres_sink`
+ - `iggy_connector_postgres_source`
+ - `iggy_connector_quickwit_sink`
+ - `iggy_connector_random_source`
+ - `iggy_connector_stdout_sink`
## Downloads
- - `iggy-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux (glibc)
- - `iggy-x86_64-unknown-linux-musl-edge.tar.gz` - Linux (musl,
static)
+
+ ### Binaries
+ - `iggy-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64
(glibc)
+ - `iggy-x86_64-unknown-linux-musl-edge.tar.gz` - Linux x86_64
(musl, static)
+ - `iggy-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64
(glibc)
+ - `iggy-aarch64-unknown-linux-musl-edge.tar.gz` - Linux ARM64
(musl, static)
+
+ ### Connector plugins
+ - `iggy-connectors-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux
x86_64 (glibc)
+ - `iggy-connectors-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux
ARM64 (glibc)
## Build info
- Server version: ${{ steps.meta.outputs.server_version }}
- Commit: ${{ github.sha }}
- ⚠️ **Not an official ASF release** - for development/testing only.
+ **Not an official ASF release** - for development/testing only.
