steveloughran commented on code in PR #3548: URL: https://github.com/apache/parquet-java/pull/3548#discussion_r3350678830
########## .github/workflows/ci-release-scripts.yml: ########## @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Test Release Scripts + +on: + pull_request: + paths: + - 'release/**' + push: + branches: + - master + paths: + - 'release/**' + +jobs: + bats: + name: Release Script Unit Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 Review Comment: looking @ dependabot updated scripts elsewhere, the latest is ``` actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 ``` ########## .github/workflows/release-prepare-rc.yml: ########## @@ -0,0 +1,94 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Release - Prepare RC + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g., 1.18.0)' + required: true + type: string + rc_number: + description: 'RC number override (leave empty for auto-detect)' + required: false + type: string + default: '' + dry_run: + description: 'Dry run mode (no actual changes)' + required: false + type: boolean + default: true + +concurrency: + group: release-${{ inputs.version }} + cancel-in-progress: false + +jobs: + prepare-rc: + if: github.repository_owner == 'apache' + name: Prepare Release Candidate + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up JDK 11 + uses: actions/setup-java@v4 Review Comment: uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 ########## release/libs/_github.sh: ########## @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[[ -n "${_GITHUB_LOADED:-}" ]] && return 0 2>/dev/null || true +_GITHUB_LOADED=1 + +LIBS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${LIBS_DIR}/_log.sh" +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_exec.sh" + +function check_github_checks_passed() { + local commit_sha="$1" + + print_info "Checking GitHub CI status for commit ${commit_sha}..." + + if [[ ${DRY_RUN:-1} -eq 1 ]]; then + print_info "DRY_RUN is enabled, skipping GitHub check verification" + return 0 + fi + + if [[ -z "${GITHUB_TOKEN:-}" ]]; then + print_error "GITHUB_TOKEN is required to verify CI checks" + return 1 + fi + + local repo_info="${GITHUB_REPO}" + + local num_incomplete + if ! num_incomplete=$(gh api "repos/${repo_info}/commits/${commit_sha}/check-runs" \ + --jq '[.check_runs[] | select(.status != "completed")] | length'); then + print_error "Failed to fetch GitHub check runs for commit ${commit_sha}" + return 1 + fi + + if [[ ${num_incomplete} -ne 0 ]]; then + print_error "Found ${num_incomplete} still-running GitHub checks for commit ${commit_sha}" + gh api "repos/${repo_info}/commits/${commit_sha}/check-runs" \ + --jq '.check_runs[] | select(.status != "completed") | " - \(.name): \(.status)"' >&2 + return 1 + fi + + local num_failed + if ! num_failed=$(gh api "repos/${repo_info}/commits/${commit_sha}/check-runs" \ + --jq '[.check_runs[] | select(.conclusion != "success" and .conclusion != "skipped")] | length'); then + print_error "Failed to fetch GitHub check runs for commit ${commit_sha}" + return 1 + fi + + if [[ ${num_failed} -ne 0 ]]; then + print_error "Found ${num_failed} failed GitHub checks for commit ${commit_sha}" + gh api "repos/${repo_info}/commits/${commit_sha}/check-runs" \ + --jq '.check_runs[] | select(.conclusion != "success" and .conclusion != "skipped") | " - \(.name): \(.conclusion)"' >&2 + return 1 + fi + + print_info "All GitHub checks passed for commit ${commit_sha}" Review Comment: so you can't make a release from a broken build, but the checks are decoupled from the actual release process which is therefore nice and fast? slick. ########## release/libs/_maven.sh: ########## @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[[ -n "${_MAVEN_LOADED:-}" ]] && return 0 2>/dev/null || true +_MAVEN_LOADED=1 + +LIBS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_exec.sh" + +function generate_maven_settings { + local settings_file="${1:-.release-settings.xml}" + + if [[ -z "${NEXUS_USERNAME:-}" || -z "${NEXUS_PASSWORD:-}" ]]; then + print_warning "NEXUS_USERNAME or NEXUS_PASSWORD not set; Maven deploy may fail" + fi + + cat > "${settings_file}" <<'SETTINGS_EOF' +<?xml version="1.0" encoding="UTF-8"?> Review Comment: this is such a maven hack isn't it, it requiring storage of the credentials for publishing artifacts to be stored in plaintext. Hopefully the north koreans haven't added copying ~/.m2/settings.xml to their package manager worms. This release mech is *more secure* that all of us keeping it locally in the file ########## .github/workflows/ci-release-scripts.yml: ########## @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Test Release Scripts + +on: + pull_request: + paths: + - 'release/**' + push: + branches: + - master + paths: + - 'release/**' + +jobs: + bats: + name: Release Script Unit Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 Review Comment: add ``` persist-credentials: false ``` ########## .github/workflows/release-cancel-rc.yml: ########## @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Release - Cancel RC + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g., 1.18.0)' + required: true + type: string + rc_number: + description: 'RC number to cancel (e.g., 0)' + required: true + type: string + staging_repo_id: + description: 'Nexus staging repository ID to drop (e.g., orgapacheparquet-1234)' + required: true + type: string + dry_run: + description: 'Dry run mode (no actual changes)' + required: false + type: boolean + default: true + +jobs: + cancel-rc: + name: Cancel Release Candidate + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 Review Comment: ``` actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 ``` ########## .github/workflows/release-cancel-rc.yml: ########## @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Release - Cancel RC + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g., 1.18.0)' + required: true + type: string + rc_number: + description: 'RC number to cancel (e.g., 0)' + required: true + type: string + staging_repo_id: + description: 'Nexus staging repository ID to drop (e.g., orgapacheparquet-1234)' + required: true + type: string + dry_run: + description: 'Dry run mode (no actual changes)' + required: false + type: boolean + default: true + +jobs: + cancel-rc: + name: Cancel Release Candidate + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y subversion + + - name: Cancel Release Candidate + env: + DRY_RUN: ${{ inputs.dry_run && '1' || '0' }} Review Comment: + env expansion is the way to defend against malicious inputs before they are executed in scripts ########## release/bin/prepare-rc.sh: ########## @@ -0,0 +1,392 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIBS_DIR="${SCRIPT_DIR}/../libs" + +source "${LIBS_DIR}/_constants.sh" +source "${LIBS_DIR}/_log.sh" +source "${LIBS_DIR}/_exec.sh" +source "${LIBS_DIR}/_version.sh" +source "${LIBS_DIR}/_github.sh" +source "${LIBS_DIR}/_nexus.sh" +source "${LIBS_DIR}/_maven.sh" +source "${LIBS_DIR}/_svn.sh" + +trap 'rm -f .release-settings.xml' EXIT + +# --------------------------------------------------------------------------- +# Usage +# --------------------------------------------------------------------------- +function usage { + cat <<EOF +Usage: $0 <version> [OPTIONS] + +Prepare a release candidate for Apache Parquet Java. + +Arguments: + version Release version (e.g., 1.18.0) + +Options: + --rc <num> Override RC number (default: auto-detect) + --skip-branch-creation Do not create the release branch + --help Show this help + +Environment variables: + DRY_RUN Set to 0 for real execution (default: 1) + NEXUS_USERNAME Apache Nexus username + NEXUS_PASSWORD Apache Nexus password + SVN_USERNAME SVN username for dist.apache.org + SVN_PASSWORD SVN password + GITHUB_TOKEN GitHub token for CI checks and release creation + +Example: + DRY_RUN=1 $0 1.18.0 + DRY_RUN=0 $0 1.18.0 --rc 2 +EOF + exit "${1:-0}" +} + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +version="" +rc_override="" +skip_branch=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --rc) + if [[ -z "${2:-}" ]]; then + print_error "--rc requires a value" + usage 1 + fi + rc_override="$2" + if [[ ! "${rc_override}" =~ ^[0-9]+$ ]]; then + print_error "--rc value must be a non-negative integer, got: '${rc_override}'" + exit 1 + fi + shift 2 + ;; + --skip-branch-creation) + skip_branch=true + shift + ;; + --help|-h) + usage 0 + ;; + -*) + print_error "Unknown option: $1" + usage 1 + ;; + *) + if [[ -z "${version}" ]]; then + version="$1" + else + print_error "Unexpected argument: $1" + usage 1 + fi + shift + ;; + esac +done + +if [[ -z "${version}" ]]; then + print_error "Version is required" + usage 1 +fi + +# --------------------------------------------------------------------------- +# Step 0: Validate inputs +# --------------------------------------------------------------------------- +step_summary "## Release Candidate Preparation" +step_summary "" + +if [[ ${DRY_RUN:-1} -eq 1 ]]; then + step_summary "> **DRY RUN** -- no changes will be made" + step_summary "" +fi + +if ! validate_and_extract_version "${version}"; then + print_error "Invalid version format: '${version}'. Expected: X.Y.Z" + exit 1 +fi + +step_summary "| Parameter | Value |" +step_summary "| --- | --- |" +step_summary "| Version | \`${version}\` |" + +# Check prerequisites +if ! command -v gpg &>/dev/null; then + print_warning "gpg not found -- GPG signing will fail" +fi +if ! command -v svn &>/dev/null; then + print_warning "svn not found -- SVN staging will fail" +fi +if [[ ! -x ./mvnw ]]; then + print_error "mvnw not found in current directory. Run from the repo root." + exit 1 +fi + +# --------------------------------------------------------------------------- +# Step 1: Create release branch (idempotent) +# --------------------------------------------------------------------------- +release_branch="${BRANCH_PREFIX}${major}.${minor}.x" +step_summary "| Release branch | \`${release_branch}\` |" + +if [[ "${skip_branch}" == "true" ]]; then + print_info "Skipping branch creation (--skip-branch-creation)" +elif git show-ref --verify --quiet "refs/remotes/origin/${release_branch}" 2>/dev/null; then + print_info "Release branch ${release_branch} already exists, skipping creation" +else + print_info "Creating release branch ${release_branch} from master..." + exec_process git branch "${release_branch}" origin/master + exec_process git push origin "${release_branch}" --set-upstream + step_summary "" + step_summary "Created release branch \`${release_branch}\`" +fi + +# Switch to the release branch +if [[ "$(git branch --show-current)" != "${release_branch}" ]]; then + print_info "Switching to ${release_branch}..." + exec_process git checkout "${release_branch}" +fi + +# --------------------------------------------------------------------------- +# Step 2: Auto-detect RC number +# --------------------------------------------------------------------------- +if [[ -n "${rc_override}" ]]; then + rc_number="${rc_override}" + print_info "Using RC override: rc${rc_number}" +else + find_next_rc_number "${version}" + print_info "Auto-detected next RC: rc${rc_number}" +fi + +rc_tag="${TAG_PREFIX}${version}-rc${rc_number}" +step_summary "| RC number | \`${rc_number}\` |" +step_summary "| RC tag | \`${rc_tag}\` |" + +# Check if tag already exists +if git rev-parse "${rc_tag}" >/dev/null 2>&1; then + print_error "Tag ${rc_tag} already exists. Use --rc to specify a different RC number." + exit 1 +fi + +# --------------------------------------------------------------------------- +# Step 3: Verify CI checks +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### CI Verification" + +current_commit=$(git rev-parse HEAD) +step_summary "| Verified commit | \`${current_commit}\` |" + +if ! check_github_checks_passed "${current_commit}"; then + print_error "CI checks are not passing. Fix CI before creating an RC." + step_summary "CI checks: **FAILED**" + exit 1 +fi +step_summary "CI checks on \`${current_commit}\`: **PASSED**" +step_summary "" +step_summary "> **Note:** the RC tag is placed on a child commit (\"Set" +step_summary "> version to ${version} for release\") that is created in" +step_summary "> the next step. That commit is not directly CI-verified;" +step_summary "> it is a mechanical version bump on top of the verified" +step_summary "> commit above." + +# --------------------------------------------------------------------------- +# Step 4: Set POM versions (only on rc0 or if version doesn't match) +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Version Update" + +current_pom_version=$(get_current_pom_version || echo "unknown") +print_info "Current POM version: ${current_pom_version}" + +if [[ "${current_pom_version}" == "${version}" ]]; then + print_info "POM version already set to ${version}, skipping version update" + step_summary "POM version already at \`${version}\`, no update needed" +else + print_info "Setting POM version to ${version}..." + set_pom_version "${version}" + step_summary "Updated POM version: \`${current_pom_version}\` -> \`${version}\`" + + # Commit version changes + exec_process git add -A + exec_process git commit -m "Set version to ${version} for release" +fi + +# --------------------------------------------------------------------------- +# Step 5: Create RC tag and push +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Tag and Push" + +exec_process git tag -a "${rc_tag}" -m "Apache Parquet ${version} RC${rc_number}" +exec_process git push origin "${release_branch}" +exec_process git push origin "${rc_tag}" + +tag_commit=$(git rev-parse HEAD) +step_summary "Created tag \`${rc_tag}\` at \`${tag_commit}\`" + +# --------------------------------------------------------------------------- +# Step 6: Deploy to Nexus +# --------------------------------------------------------------------------- +step_summary "" +step_summary "### Nexus Deployment" + +settings_file=".release-settings.xml" +generate_maven_settings "${settings_file}" + +maven_deploy "${settings_file}" + +step_summary "Deployed artifacts to Apache Nexus staging" + +# Find and close the staging repo. Verify it actually contains the +# artifacts we just deployed before closing -- another concurrent run Review Comment: I've hit some bizarre stuff with behind-VPN releases where a single docker container release build ended up pushing up artifacts to multiple staging repos -nexus splits repos by source IP Address for security, and and if routing is inconsistent the release breaks. hopefully GHA hosting will be good, otherwise more artifact auditing is needed (i.e. everything) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
