This is an automated email from the ASF dual-hosted git repository.
piotr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new d7afbb60e feat(ci): add automatic crates.io publishing for edge/rc
versions (#2508)
d7afbb60e is described below
commit d7afbb60eb911f6b040ae0d84c2d6988002ba019
Author: Hubert Gruszecki <[email protected]>
AuthorDate: Sun Dec 21 10:35:44 2025 +0100
feat(ci): add automatic crates.io publishing for edge/rc versions (#2508)
Extract Rust crate publishing into reusable workflow and add auto-publish
to post-merge for versions containing `-edge` or `-rc` suffix.
- Add _publish_rust_crates.yml reusable workflow
- Add check-auto-publish job to detect edge/rc versions
- Skip publishing if git tag already exists
- Refactor publish.yml to use the new reusable workflow
---
.github/workflows/_publish_rust_crates.yml | 214 +++++++++++++++++++++++++++++
.github/workflows/post-merge.yml | 74 ++++++++++
.github/workflows/publish.yml | 149 ++------------------
3 files changed, 297 insertions(+), 140 deletions(-)
diff --git a/.github/workflows/_publish_rust_crates.yml
b/.github/workflows/_publish_rust_crates.yml
new file mode 100644
index 000000000..d2e7db074
--- /dev/null
+++ b/.github/workflows/_publish_rust_crates.yml
@@ -0,0 +1,214 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: _publish_rust_crates
+on:
+ workflow_call:
+ inputs:
+ crates:
+ type: string
+ required: true
+ description: "Comma-separated list of crates to publish:
rust-common,rust-binary-protocol,rust-sdk,rust-cli"
+ dry_run:
+ type: boolean
+ required: false
+ default: false
+ description: "Dry run mode - validate without publishing"
+ create_tags:
+ type: boolean
+ required: false
+ default: true
+ description: "Create git tags after successful publishing"
+ commit:
+ type: string
+ required: false
+ default: ""
+ description: "Specific commit to checkout (defaults to github.sha)"
+ use_latest_ci:
+ type: boolean
+ required: false
+ default: false
+ description: "Use latest CI configuration from master branch"
+ secrets:
+ CARGO_REGISTRY_TOKEN:
+ required: true
+ outputs:
+ status:
+ description: "Publishing status"
+ value: ${{ jobs.publish.outputs.status }}
+
+permissions:
+ contents: write
+
+env:
+ IGGY_CI_BUILD: true
+
+jobs:
+ publish:
+ name: Publish Rust crates
+ runs-on: ubuntu-latest
+ env:
+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+ outputs:
+ status: ${{ steps.final-status.outputs.status }}
+ steps:
+ - name: Download latest copy script from master
+ if: inputs.use_latest_ci
+ run: |
+ curl -sSL "https://raw.githubusercontent.com/${{ github.repository
}}/master/scripts/copy-latest-from-master.sh" \
+ -o /tmp/copy-latest-from-master.sh
+ chmod +x /tmp/copy-latest-from-master.sh
+ echo "✅ Downloaded latest copy script from master"
+
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.commit || github.sha }}
+ fetch-depth: 0
+
+ - name: Save and apply latest CI from master
+ if: inputs.use_latest_ci
+ run: |
+ /tmp/copy-latest-from-master.sh save \
+ .github \
+ scripts \
+ web/Dockerfile \
+ core/server/Dockerfile \
+ core/ai/mcp/Dockerfile \
+ core/connectors/runtime/Dockerfile \
+ core/bench/dashboard/server/Dockerfile
+
+ /tmp/copy-latest-from-master.sh apply
+
+ - name: Setup Rust with cache
+ uses: ./.github/actions/utils/setup-rust-with-cache
+ with:
+ cache-targets: false
+ show-stats: false
+
+ - name: Extract versions
+ id: versions
+ run: |
+ chmod +x scripts/extract-version.sh
+ echo "common=$(scripts/extract-version.sh rust-common)" >>
"$GITHUB_OUTPUT"
+ echo "protocol=$(scripts/extract-version.sh rust-binary-protocol)"
>> "$GITHUB_OUTPUT"
+ echo "sdk=$(scripts/extract-version.sh rust-sdk)" >> "$GITHUB_OUTPUT"
+ echo "cli=$(scripts/extract-version.sh rust-cli)" >> "$GITHUB_OUTPUT"
+
+ echo "📦 Versions to publish:"
+ echo " iggy_common: $(scripts/extract-version.sh rust-common)"
+ echo " iggy_binary_protocol: $(scripts/extract-version.sh
rust-binary-protocol)"
+ echo " iggy: $(scripts/extract-version.sh rust-sdk)"
+ echo " iggy-cli: $(scripts/extract-version.sh rust-cli)"
+
+ # Step 1: Publish iggy_common
+ - name: Publish iggy_common
+ if: contains(inputs.crates, 'rust-common')
+ uses: ./.github/actions/rust/post-merge
+ with:
+ package: iggy_common
+ version: ${{ steps.versions.outputs.common }}
+ dry_run: ${{ inputs.dry_run }}
+
+ - name: Wait for iggy_common to be available
+ if: contains(inputs.crates, 'rust-common') && inputs.dry_run == false
+ run: |
+ echo "⏳ Waiting for iggy_common to be available on crates.io..."
+ for i in {1..30}; do
+ if cargo search iggy_common --limit 1 | grep -q "^iggy_common =
\"${{ steps.versions.outputs.common }}\""; then
+ echo "✅ iggy_common is now available"
+ break
+ fi
+ echo "Waiting... (attempt $i/30)"
+ sleep 10
+ done
+
+ # Step 2: Publish iggy_binary_protocol (depends on common)
+ - name: Publish iggy_binary_protocol
+ if: contains(inputs.crates, 'rust-binary-protocol')
+ uses: ./.github/actions/rust/post-merge
+ with:
+ package: iggy_binary_protocol
+ version: ${{ steps.versions.outputs.protocol }}
+ dry_run: ${{ inputs.dry_run }}
+
+ - name: Wait for iggy_binary_protocol to be available
+ if: contains(inputs.crates, 'rust-binary-protocol') && inputs.dry_run
== false
+ run: |
+ echo "⏳ Waiting for iggy_binary_protocol to be available on
crates.io..."
+ for i in {1..30}; do
+ if cargo search iggy_binary_protocol --limit 1 | grep -q
"^iggy_binary_protocol = \"${{ steps.versions.outputs.protocol }}\""; then
+ echo "✅ iggy_binary_protocol is now available"
+ break
+ fi
+ echo "Waiting... (attempt $i/30)"
+ sleep 10
+ done
+
+ # Step 3: Publish iggy SDK (depends on common and protocol)
+ - name: Publish iggy SDK
+ if: contains(inputs.crates, 'rust-sdk')
+ uses: ./.github/actions/rust/post-merge
+ with:
+ package: iggy
+ version: ${{ steps.versions.outputs.sdk }}
+ dry_run: ${{ inputs.dry_run }}
+
+ - name: Wait for iggy SDK to be available
+ if: contains(inputs.crates, 'rust-sdk') && inputs.dry_run == false
+ run: |
+ echo "⏳ Waiting for iggy to be available on crates.io..."
+ for i in {1..30}; do
+ if cargo search iggy --limit 1 | grep -q "^iggy = \"${{
steps.versions.outputs.sdk }}\""; then
+ echo "✅ iggy SDK is now available"
+ break
+ fi
+ echo "Waiting... (attempt $i/30)"
+ sleep 10
+ done
+
+ # Step 4: Publish iggy-cli (depends on SDK and protocol)
+ - name: Publish iggy-cli
+ if: contains(inputs.crates, 'rust-cli')
+ uses: ./.github/actions/rust/post-merge
+ with:
+ package: iggy-cli
+ version: ${{ steps.versions.outputs.cli }}
+ dry_run: ${{ inputs.dry_run }}
+
+ # Create git tags
+ - name: Create git tags
+ if: inputs.create_tags && inputs.dry_run == false
+ run: |
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+
+ for crate in $(echo "${{ inputs.crates }}" | tr ',' ' '); do
+ TAG=$(scripts/extract-version.sh "$crate" --tag)
+ if ! git rev-parse "$TAG" >/dev/null 2>&1; then
+ git tag -a "$TAG" -m "Release $TAG"
+ git push origin "$TAG"
+ echo "✅ Created tag: $TAG"
+ else
+ echo "⏭️ Tag $TAG already exists"
+ fi
+ done
+
+ - name: Set final status output
+ id: final-status
+ if: always()
+ run: echo "status=${{ job.status }}" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml
index c47091e41..9d38472e7 100644
--- a/.github/workflows/post-merge.yml
+++ b/.github/workflows/post-merge.yml
@@ -309,3 +309,77 @@ jobs:
- Commit: ${{ github.sha }}
**Not an official ASF release** - for development/testing only.
+
+ # Check if auto-publish should run for edge/rc versions
+ check-auto-publish:
+ name: Check auto-publish
+ runs-on: ubuntu-latest
+ if: ${{ !github.event.repository.fork }}
+ outputs:
+ should_publish: ${{ steps.check.outputs.should_publish }}
+ crates_to_publish: ${{ steps.check.outputs.crates_to_publish }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Check versions and tags for each crate
+ id: check
+ run: |
+ chmod +x scripts/extract-version.sh
+
+ CRATES_TO_PUBLISH=""
+
+ # Check each crate individually
+ for crate in rust-common rust-binary-protocol rust-sdk rust-cli; do
+ VERSION=$(scripts/extract-version.sh "$crate")
+ TAG=$(scripts/extract-version.sh "$crate" --tag)
+
+ echo "Checking $crate: version=$VERSION, tag=$TAG"
+
+ # Skip if version doesn't contain edge or rc
+ if [[ ! "$VERSION" =~ -(edge|rc) ]]; then
+ echo " ⏭️ Stable version - skipping"
+ continue
+ fi
+
+ # Skip if tag already exists
+ if git rev-parse "$TAG" >/dev/null 2>&1; then
+ echo " ⏭️ Tag exists - skipping"
+ continue
+ fi
+
+ echo " ✅ Will publish"
+ if [ -n "$CRATES_TO_PUBLISH" ]; then
+ CRATES_TO_PUBLISH="$CRATES_TO_PUBLISH,$crate"
+ else
+ CRATES_TO_PUBLISH="$crate"
+ fi
+ done
+
+ if [ -z "$CRATES_TO_PUBLISH" ]; then
+ echo ""
+ echo "No crates need publishing"
+ echo "should_publish=false" >> "$GITHUB_OUTPUT"
+ echo "crates_to_publish=" >> "$GITHUB_OUTPUT"
+ else
+ echo ""
+ echo "Crates to publish: $CRATES_TO_PUBLISH"
+ echo "should_publish=true" >> "$GITHUB_OUTPUT"
+ echo "crates_to_publish=$CRATES_TO_PUBLISH" >> "$GITHUB_OUTPUT"
+ fi
+
+ # Auto-publish Rust crates for edge/rc versions
+ publish-rust-crates:
+ name: Auto-publish Rust crates
+ needs: check-auto-publish
+ if: needs.check-auto-publish.outputs.should_publish == 'true'
+ permissions:
+ contents: write # Required for git tag push
+ uses: ./.github/workflows/_publish_rust_crates.yml
+ with:
+ crates: ${{ needs.check-auto-publish.outputs.crates_to_publish }}
+ dry_run: false
+ create_tags: true
+ secrets:
+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index e5c02f49b..646955d19 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -486,151 +486,20 @@ jobs:
# Sequential Rust crate publishing to handle dependencies properly
publish-rust-crates:
- name: Publish Rust Crates
+ name: Publish Rust crates
needs: [validate, plan, check-tags]
if: |
needs.validate.outputs.has_targets == 'true' &&
contains(inputs.publish_crates, 'rust-')
- runs-on: ubuntu-latest
- env:
+ uses: ./.github/workflows/_publish_rust_crates.yml
+ with:
+ crates: ${{ inputs.publish_crates }}
+ dry_run: ${{ inputs.dry_run }}
+ create_tags: false # publish.yml handles tags separately in create-tags
job
+ commit: ${{ needs.validate.outputs.commit }}
+ use_latest_ci: ${{ inputs.use_latest_ci }}
+ secrets:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
- DRY_RUN: ${{ inputs.dry_run }}
- outputs:
- status: ${{ steps.final-status.outputs.status }}
- steps:
- - name: Download latest copy script from master
- if: inputs.use_latest_ci
- run: |
- curl -sSL "https://raw.githubusercontent.com/${{ github.repository
}}/master/scripts/copy-latest-from-master.sh" \
- -o /tmp/copy-latest-from-master.sh
- chmod +x /tmp/copy-latest-from-master.sh
- echo "✅ Downloaded latest copy script from master"
-
- - name: Checkout at commit
- uses: actions/checkout@v4
- with:
- ref: ${{ needs.validate.outputs.commit }}
- fetch-depth: 0
-
- - name: Save and apply latest CI from master
- if: inputs.use_latest_ci
- run: |
- /tmp/copy-latest-from-master.sh save \
- .github \
- scripts \
- web/Dockerfile \
- core/server/Dockerfile \
- core/ai/mcp/Dockerfile \
- core/connectors/runtime/Dockerfile \
- core/bench/dashboard/server/Dockerfile
-
- /tmp/copy-latest-from-master.sh apply
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- cache-targets: false
- show-stats: false
-
- - name: Extract versions
- id: versions
- run: |
- # Extract version for each crate
- chmod +x scripts/extract-version.sh
-
- echo "common_version=$(scripts/extract-version.sh rust-common)" >>
$GITHUB_OUTPUT
- echo "protocol_version=$(scripts/extract-version.sh
rust-binary-protocol)" >> $GITHUB_OUTPUT
- echo "sdk_version=$(scripts/extract-version.sh rust-sdk)" >>
$GITHUB_OUTPUT
- echo "cli_version=$(scripts/extract-version.sh rust-cli)" >>
$GITHUB_OUTPUT
-
- # Step 1: Publish iggy_common first
- - name: Publish iggy_common
- if: contains(inputs.publish_crates, 'rust-common')
- uses: ./.github/actions/rust/post-merge
- with:
- package: iggy_common
- version: ${{ steps.versions.outputs.common_version }}
- dry_run: ${{ inputs.dry_run }}
-
- # Wait for crates.io to index (only in non-dry-run mode)
- - name: Wait for iggy_common to be available
- if: |
- contains(inputs.publish_crates, 'rust-common') &&
- inputs.dry_run == 'false'
- run: |
- echo "⏳ Waiting for iggy_common to be available on crates.io..."
- for i in {1..30}; do
- if cargo search iggy_common --limit 1 | grep -q "^iggy_common =
\"${{ steps.versions.outputs.common_version }}\""; then
- echo "✅ iggy_common is now available"
- break
- fi
- echo "Waiting... (attempt $i/30)"
- sleep 10
- done
-
- # Step 2: Publish iggy_binary_protocol (depends on common)
- - name: Publish iggy_binary_protocol
- if: contains(inputs.publish_crates, 'rust-binary-protocol')
- uses: ./.github/actions/rust/post-merge
- with:
- package: iggy_binary_protocol
- version: ${{ steps.versions.outputs.protocol_version }}
- dry_run: ${{ inputs.dry_run }}
-
- # Wait for crates.io to index
- - name: Wait for iggy_binary_protocol to be available
- if: |
- contains(inputs.publish_crates, 'rust-binary-protocol') &&
- inputs.dry_run == 'false'
- run: |
- echo "⏳ Waiting for iggy_binary_protocol to be available on
crates.io..."
- for i in {1..30}; do
- if cargo search iggy_binary_protocol --limit 1 | grep -q
"^iggy_binary_protocol = \"${{ steps.versions.outputs.protocol_version }}\"";
then
- echo "✅ iggy_binary_protocol is now available"
- break
- fi
- echo "Waiting... (attempt $i/30)"
- sleep 10
- done
-
- # Step 3: Publish iggy SDK (depends on common and protocol)
- - name: Publish iggy SDK
- if: contains(inputs.publish_crates, 'rust-sdk')
- uses: ./.github/actions/rust/post-merge
- with:
- package: iggy
- version: ${{ steps.versions.outputs.sdk_version }}
- dry_run: ${{ inputs.dry_run }}
-
- # Wait for crates.io to index
- - name: Wait for iggy SDK to be available
- if: |
- contains(inputs.publish_crates, 'rust-sdk') &&
- inputs.dry_run == 'false'
- run: |
- echo "⏳ Waiting for iggy to be available on crates.io..."
- for i in {1..30}; do
- if cargo search iggy --limit 1 | grep -q "^iggy = \"${{
steps.versions.outputs.sdk_version }}\""; then
- echo "✅ iggy SDK is now available"
- break
- fi
- echo "Waiting... (attempt $i/30)"
- sleep 10
- done
-
- # Step 4: Publish iggy-cli (depends on SDK and protocol)
- - name: Publish iggy-cli
- if: contains(inputs.publish_crates, 'rust-cli')
- uses: ./.github/actions/rust/post-merge
- with:
- package: iggy-cli
- version: ${{ steps.versions.outputs.cli_version }}
- dry_run: ${{ inputs.dry_run }}
-
- - name: Set final status output
- id: final-status
- if: always()
- run: echo "status=${{ job.status }}" >> "$GITHUB_OUTPUT"
# Docker publishing on native runners (no QEMU emulation)
publish-docker:
