HyukjinKwon commented on code in PR #84:
URL: https://github.com/apache/spark-connect-rust/pull/84#discussion_r3886008058
##########
.github/workflows/release.yml:
##########
@@ -233,28 +303,234 @@ jobs:
name: wheels-sdist
path: dist/*.tar.gz
- publish-wheels:
- name: Publish wheel + sdist to PyPI
- needs: [build-wheels, build-sdist]
- if: github.event_name == 'push' || (github.event_name ==
'workflow_dispatch' && inputs.dry_run == false)
+ # ---- Dry run: rehearse the crates.io packaging without publishing ----
+ dryrun-crates:
+ name: Dry-run crates.io packaging
+ needs: resolve
+ if: needs.resolve.outputs.mode == 'dryrun'
runs-on: ubuntu-latest
steps:
- - uses: actions/setup-python@v5
+ - uses: actions/checkout@v4
with:
- python-version: "3.11"
+ repository: ${{ env.UPSTREAM_REPO }}
+ ref: ${{ needs.resolve.outputs.sha }}
+ - name: Install Rust toolchain
+ run: |
+ rustup toolchain install stable --profile minimal
+ rustup default stable
+ - name: Install protoc
+ run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+ - name: cargo publish --dry-run (dependency-ordered)
+ run: |
+ set -euo pipefail
+ cargo publish --dry-run \
+ -p apache-spark-connect-proto \
+ -p apache-spark-connect-core \
+ -p apache-spark-connect
+
+ # ---- Cut an RC: tag + GitHub pre-release with the artifacts (no
crates.io/PyPI) ----
+ publish-rc:
+ name: Cut RC GitHub pre-release
+ needs: [resolve, build-wheels, build-sdist]
+ if: needs.resolve.outputs.mode == 'rc'
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write # push the RC tag + create the GitHub Release on the
fork
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ repository: ${{ env.UPSTREAM_REPO }}
+ ref: ${{ needs.resolve.outputs.sha }}
+ fetch-depth: 0 # full history so the tag push carries all reachable
objects
- uses: actions/download-artifact@v4
with:
- # Matches every wheels-<target> artifact plus wheels-sdist.
pattern: wheels-*
path: dist
merge-multiple: true
- - name: Publish to PyPI
- # Fork-based release: authenticate with the committer's PyPI token via
twine
- # (avoids third-party publish actions; ASF Actions policy allowlists
actions/*).
+ - name: Create and push the RC tag to this fork
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TAG: ${{ needs.resolve.outputs.rc_tag }}
+ VERSION: ${{ needs.resolve.outputs.version }}
+ run: |
+ set -euo pipefail
+ git config user.name "${{ github.actor }}"
+ git config user.email "${{ github.actor }}@users.noreply.github.com"
+ if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then
+ echo "Local tag ${TAG} already exists; reusing."
+ else
+ git tag -a "${TAG}" -m "Release candidate ${TAG} of
pyspark-client-rust ${VERSION}"
+ fi
+ # Push the RC tag (and the objects it reaches) to THIS fork, so the
GitHub
+ # pre-release below can hang off it. Force so re-cutting the same RC
is idempotent.
+ git push --force
"https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git";
"refs/tags/${TAG}"
+ - name: Create the GitHub pre-release with the artifacts
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TAG: ${{ needs.resolve.outputs.rc_tag }}
+ VERSION: ${{ needs.resolve.outputs.version }}
+ run: |
+ set -euo pipefail
+ ls -l dist
+ NOTES="Release candidate ${TAG} of pyspark-client-rust
${VERSION}."$'\n\n'"> Not published to PyPI or crates.io. Install directly from
a wheel/sdist asset below, e.g. \`pip install <asset-url>\`. This pre-release
is deleted when the RC is finalized (or dropped)."
+ # Idempotent: a re-run updates the existing release and clobbers its
assets.
+ if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null
2>&1; then
+ gh release upload "${TAG}" dist/* --repo "${GITHUB_REPOSITORY}"
--clobber
+ else
+ gh release create "${TAG}" dist/* --repo "${GITHUB_REPOSITORY}" \
+ --prerelease --title "${TAG}" --notes "${NOTES}"
+ fi
+
+ # ---- Finalize: promote the named RC into the official release
(IRREVERSIBLE) ----
+ finalize:
+ name: Finalize release to PyPI + crates.io (IRREVERSIBLE)
+ needs: resolve
+ if: needs.resolve.outputs.mode == 'finalize'
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write # delete the RC pre-releases + tags on the fork
+ steps:
+ - name: Abort window
+ run: |
+ echo
"=============================================================================="
+ echo " CONVERTING RC ${{ needs.resolve.outputs.rc_tag }} INTO THE
OFFICIAL RELEASE"
+ echo " ${{ needs.resolve.outputs.version }} -> PyPI + crates.io.
THIS IS IRREVERSIBLE."
+ echo " Cancel this workflow now if you did not intend to finalize."
+ echo " Continuing in 60 seconds..."
+ echo
"=============================================================================="
+ sleep 60
+ - name: Verify the RC pre-release exists
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TAG: ${{ needs.resolve.outputs.rc_tag }}
+ run: |
+ set -euo pipefail
+ if ! gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}"
>/dev/null 2>&1; then
+ echo "::error::RC GitHub pre-release ${TAG} not found on
${GITHUB_REPOSITORY}. Cut the RC first."
+ exit 1
+ fi
+ - name: Check out the RC tag (the exact released source)
+ uses: actions/checkout@v4
+ with:
+ # The RC tag lives on the fork where the RC was cut (this
repository).
+ repository: ${{ github.repository }}
+ ref: ${{ needs.resolve.outputs.rc_tag }}
+ fetch-depth: 0
+ - name: Validate the RC source versions
+ run: |
+ set -euo pipefail
+ V="${{ needs.resolve.outputs.version }}"
+ CARGO_VER=$(grep -m1 '^version' Cargo.toml | awk -F'"' '{print $2}')
+ PYPROJECT_VER=$(grep -m1 '^version' pyproject.toml | awk -F'"'
'{print $2}')
+ echo "RC tag source: Cargo=${CARGO_VER} pyproject=${PYPROJECT_VER};
release-version=${V}"
+ # Cargo and pyproject must both equal the release-version (agreement
+ expected value).
+ if [ "$CARGO_VER" != "$V" ] || [ "$PYPROJECT_VER" != "$V" ]; then
+ echo "::error::RC tag ${{ needs.resolve.outputs.rc_tag }} declares
Cargo=${CARGO_VER} / pyproject=${PYPROJECT_VER}, not ${V}."
+ exit 1
+ fi
+ # Path-dep pins must match, or `cargo publish` ships a stale hard
dep (same check as resolve).
+ BAD=$(grep -rEn 'path = "\.\./spark-connect[^"]*"'
crates/*/Cargo.toml \
+ | grep -E 'version = "' \
+ | grep -v "version = \"${CARGO_VER}\"" || true)
+ if [ -n "$BAD" ]; then
+ echo "::error::internal path-dep version pin(s) in the RC source
do not match ${CARGO_VER}:"
+ echo "$BAD"
+ exit 1
+ fi
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+ - name: Download the RC artifacts (the exact tested wheels + sdist)
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TAG: ${{ needs.resolve.outputs.rc_tag }}
+ run: |
+ set -euo pipefail
+ mkdir -p dist
+ gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" --dir
dist --pattern '*'
+ ls -l dist
+ - name: Publish wheels + sdist to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
run: |
+ set -euo pipefail
python -m pip install --upgrade twine
# --skip-existing makes a re-run after a partial upload safe.
twine upload --skip-existing dist/*
+ - name: Install Rust toolchain
+ run: |
+ rustup toolchain install stable --profile minimal
+ rustup default stable
+ - name: Install protoc
+ run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+ - name: Publish crates to crates.io (from the identical RC source)
+ env:
+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+ run: |
+ set -euo pipefail
+ CRATES=(apache-spark-connect-proto apache-spark-connect-core
apache-spark-connect)
+ VER="${{ needs.resolve.outputs.version }}"
+ # Skip any crate already on crates.io at this version so a re-run is
idempotent.
+ pkgs=()
+ for c in "${CRATES[@]}"; do
+ if curl -sf -H "User-Agent: spark-connect-rust-release
([email protected])" \
+ "https://crates.io/api/v1/crates/${c}/${VER}"; >/dev/null; then
+ echo "${c} ${VER} is already on crates.io — skipping"
+ else
+ pkgs+=(-p "$c")
+ fi
+ done
+ if [ ${#pkgs[@]} -eq 0 ]; then
+ echo "All crates already published at ${VER}; nothing to do."
+ else
+ # cargo publishes in dependency order and waits for the index
between crates.
+ cargo publish "${pkgs[@]}"
+ fi
+ - name: Tag the released commit with the permanent v<version> tag
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TAG: v${{ needs.resolve.outputs.version }}
+ VERSION: ${{ needs.resolve.outputs.version }}
+ run: |
+ set -euo pipefail
+ git config user.name "${{ github.actor }}"
+ git config user.email "${{ github.actor }}@users.noreply.github.com"
+ # HEAD is the RC tag's commit (the exact released source), checked
out above.
+ if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then
+ echo "Local tag ${TAG} already exists; reusing."
+ else
+ git tag -a "${TAG}" -m "pyspark-client-rust ${VERSION}"
+ fi
+ # Push the permanent release tag to this fork. NOT forced: a release
tag is
+ # immutable, so if ${TAG} already points at this commit the push is
a no-op,
+ # and if it somehow exists elsewhere we must not silently move it.
+ if git push
"https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git";
"refs/tags/${TAG}"; then
Review Comment:
Fixed in ccbccf2. The tag push no longer uses a catch-all `else`. It now
looks the remote tag up explicitly with `git ls-remote --tags` (so a
network/auth failure aborts under `set -e` instead of being read as "already
exists"), then: absent -> push (a push failure is fatal); already at the
**same** commit -> no-op; already at a **different** commit -> hard error
(`exit 1`), so an immutable release tag is never silently moved and finalize
never reports success while the tag marks the wrong source. Verified the
annotated-tag (`^{}` peeled) vs lightweight parsing across
same/different/absent cases.
##########
.github/workflows/release.yml:
##########
@@ -57,118 +95,150 @@ concurrency:
group: release
cancel-in-progress: false
+env:
+ # The release source is always fetched from this canonical apache repository.
+ UPSTREAM_REPO: apache/spark-connect-rust
+
jobs:
- validate-version:
- name: Validate version
+ resolve:
+ name: Resolve release mode
runs-on: ubuntu-latest
+ # In the canonical apache repo, only dry runs are allowed (release-version
and
+ # rc-count empty). A fork may perform real runs. This mirrors apache/spark.
+ # NOTE: the `env` context is not available in a job-level `if:`, so the
canonical
+ # repo is spelled out literally here; it must match `env.UPSTREAM_REPO`
below.
+ if: >-
Review Comment:
Fixed in ccbccf2. Added a `reject-real-release-from-apache` job that runs
only when `github.repository == 'apache/spark-connect-rust' && (release-version
!= '' || rc-count != '')` and fails with an explanatory error pointing the
committer to dispatch from a fork. The apache dry-run path (empty inputs) and
the scheduled dry run are unaffected — they don't match the condition, so the
job is skipped.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
