tvalentyn commented on a change in pull request #14325: URL: https://github.com/apache/beam/pull/14325#discussion_r601823780
########## File path: release/src/main/scripts/deploy_release_candidate_pypi.sh ########## @@ -0,0 +1,181 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script will deploy a Release Candidate to pypi, includes: +# 1. Download python binary artifacts +# 2. Deploy Release Candidate to pypi + +set -e + +function usage() { + echo 'Usage: deploy_release_candidate.sh --release <version> --rc <rc> --commit <commit> --user <user> [--deploy]' Review comment: We should take RC number as parameter and find corresponding commit by looking up the commit accociated with the RC tag. ########## File path: release/src/main/scripts/deploy_release_candidate_pypi.sh ########## @@ -0,0 +1,181 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script will deploy a Release Candidate to pypi, includes: +# 1. Download python binary artifacts +# 2. Deploy Release Candidate to pypi + +set -e + +function usage() { + echo 'Usage: deploy_release_candidate.sh --release <version> --rc <rc> --commit <commit> --user <user> [--deploy]' +} + +RELEASE= +RC= +COMMIT= +USER_GITHUB_ID= +DEPLOY=no +PYTHON_ARTIFACTS_DIR=python +BEAM_ROOT_DIR=beam +GIT_REPO_BASE_URL=apache/beam + +while [[ $# -gt 0 ]] ; do + arg="$1" + + case $arg in + --release) + shift + RELEASE=$1 + shift + ;; + + --rc) + shift + RC=$1 + shift + ;; + + --commit) + shift + COMMIT=$1 + shift + ;; + + --user) + shift + USER_GITHUB_ID=$1 + shift + ;; + + --deploy) + DEPLOY=yes + shift + ;; + + *) + usage + exit 1 + ;; + esac +done + +if [[ -z "$RELEASE" ]] ; then + echo 'No release version supplied.' + usage + exit 1 +fi + +if [[ -z "$RC" ]] ; then + echo 'No RC number supplied' + usage + exit 1 +fi + +if [[ -z "$COMMIT" ]] ; then + echo 'No commit hash supplied.' + usage + exit 1 +fi + +if [[ -z "$USER_GITHUB_ID" ]] ; then + echo 'No github user supplied.' + usage + exit 1 +fi + +function clean_up(){ + echo "Do you want to clean local clone repo ${LOCAL_CLONE_DIR}? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + cd ~ + rm -rf ${LOCAL_CLONE_DIR} + echo "Cleaned up local repo." + fi +} + +RC_TAG="v${RELEASE}-RC${RC}" +LOCAL_CLONE_DIR="beam_release_${RC_TAG}" +SCRIPT_DIR=$(dirname $0) + +echo "================Checking Environment Variables==============" +echo "working on release version: ${RELEASE}" +echo "working on release branch: ${RC_TAG}" Review comment: There seems to be confusion between release branches and tags - we only create one release branch per release, and then tag some commit ids when building an RC. We don't create separate branches for RCs. ########## File path: .github/workflows/build_wheels.yml ########## @@ -71,6 +73,24 @@ jobs: run: python -m pip install -r build-requirements.txt - name: Install wheels run: python -m pip install wheel + - name: Get tag + id: get_tag + run: | + echo ::set-output name=TAG::${GITHUB_REF#refs/*/} + - name: Check for rc version Review comment: `Check whether an -RC tag was applied to the commit.` ########## File path: release/src/main/scripts/deploy_release_candidate_pypi.sh ########## @@ -0,0 +1,181 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script will deploy a Release Candidate to pypi, includes: +# 1. Download python binary artifacts +# 2. Deploy Release Candidate to pypi + +set -e + +function usage() { + echo 'Usage: deploy_release_candidate.sh --release <version> --rc <rc> --commit <commit> --user <user> [--deploy]' +} + +RELEASE= +RC= +COMMIT= +USER_GITHUB_ID= +DEPLOY=no +PYTHON_ARTIFACTS_DIR=python +BEAM_ROOT_DIR=beam +GIT_REPO_BASE_URL=apache/beam + +while [[ $# -gt 0 ]] ; do + arg="$1" + + case $arg in + --release) + shift + RELEASE=$1 + shift + ;; + + --rc) + shift + RC=$1 + shift + ;; + + --commit) + shift + COMMIT=$1 + shift + ;; + + --user) + shift + USER_GITHUB_ID=$1 + shift + ;; + + --deploy) + DEPLOY=yes + shift + ;; + + *) + usage + exit 1 + ;; + esac +done + +if [[ -z "$RELEASE" ]] ; then + echo 'No release version supplied.' + usage + exit 1 +fi + +if [[ -z "$RC" ]] ; then + echo 'No RC number supplied' + usage + exit 1 +fi + +if [[ -z "$COMMIT" ]] ; then + echo 'No commit hash supplied.' + usage + exit 1 +fi + +if [[ -z "$USER_GITHUB_ID" ]] ; then + echo 'No github user supplied.' + usage + exit 1 +fi + +function clean_up(){ + echo "Do you want to clean local clone repo ${LOCAL_CLONE_DIR}? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + cd ~ + rm -rf ${LOCAL_CLONE_DIR} + echo "Cleaned up local repo." + fi +} + +RC_TAG="v${RELEASE}-RC${RC}" +LOCAL_CLONE_DIR="beam_release_${RC_TAG}" +SCRIPT_DIR=$(dirname $0) + +echo "================Checking Environment Variables==============" +echo "working on release version: ${RELEASE}" +echo "working on release branch: ${RC_TAG}" +echo "will create release candidate: RC${RC}" +echo "Please review all environment variables and confirm: [y|N]" +read confirmation +if [[ $confirmation != "y" ]]; then + echo "Please rerun this script and make sure you have the right inputs." + exit +fi + +echo "=====================Clear folder==============================" +cd ~ +if [[ -d ${LOCAL_CLONE_DIR} ]]; then + echo "Deleting existing local clone repo ${LOCAL_CLONE_DIR}." + rm -rf "${LOCAL_CLONE_DIR}" +fi +mkdir "${LOCAL_CLONE_DIR}" +LOCAL_CLONE_DIR_ROOT=$(pwd)/${LOCAL_CLONE_DIR} + +echo "================Download python artifacts======================" +cd - +python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \ + --github-user "${USER_GITHUB_ID}" \ + --repo-url "${GIT_REPO_BASE_URL}" \ + --release-branch "${RC_TAG}" \ + --release-commit "${COMMIT}" \ + --artifacts_dir "${LOCAL_CLONE_DIR_ROOT}" \ + --is_rc_version + +cd ${LOCAL_CLONE_DIR_ROOT} + +echo "------Checking Hash Value for apache-beam-${RELEASE}rc${RC}.zip-----" +sha512sum -c "apache-beam-${RELEASE}rc${RC}.zip.sha512" + +for artifact in *.whl; do + echo "----------Checking Hash Value for ${artifact} wheel-----------" + sha512sum -c "${artifact}.sha512" +done + +echo "===================Removing sha512 files=======================" +rm $(ls | grep -i ".*.sha512$") + +echo "====================Upload rc to pypi========================" +virtualenv deploy_pypi_env +source ./deploy_pypi_env/bin/activate +pip install twine + +mkdir dist && mv $(ls | grep apache) dist && cd dist +echo "Will upload the following files to PyPI:" +ls +echo "Are the files listed correct? [y|N]" +read confirmation +if [[ $confirmation != "y" ]]; then + echo "Exiting without deploying artifacts to PyPI." + clean_up + exit +fi + +if [[ "$DEPLOY" == yes ]] ; then + twine upload * +else + echo "Not deploying to pypi with tag $RC." Review comment: ```Skipping deployment to PyPI. Run the script with --deploy to stage the artifacts.``` ########## File path: .github/workflows/build_wheels.yml ########## @@ -71,6 +73,24 @@ jobs: run: python -m pip install -r build-requirements.txt - name: Install wheels run: python -m pip install wheel + - name: Get tag + id: get_tag + run: | + echo ::set-output name=TAG::${GITHUB_REF#refs/*/} + - name: Check for rc version + id: is_rc + run: | + echo ${{ steps.get_tag.outputs.TAG }} > temp + OUTPUT=$( if grep -e '-RC.' -q temp; then echo 1; else echo 0; fi) + echo "::set-output name=is_rc::$OUTPUT" + - name: Get RC_NUM and VERSION + if: steps.is_rc.outputs.is_rc == 1 + id: get_rc_version + run: | + OUTPUT=$(sed -n "s/^.*-RC\([0-9]*\)/\1/p" temp) Review comment: s/OUTPUT/RC_NUM? ########## File path: website/www/site/content/en/contribute/release-guide.md ########## @@ -598,6 +598,36 @@ See the source of the script for more details, or to run commands manually in ca Artifact names should follow [the existing format](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) in which artifact name mirrors directory structure, e.g., `beam-sdks-java-io-kafka`. Carefully review any new artifacts. +### Upload release candidate to PyPi + +* **Script:** [deploy_release_candidate_pypi.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/deploy_release_candidate_pypi.sh) + +* **Usage** + + ./release/src/main/scripts/deploy_release_candidate_pypi.sh \ + --release "${RELEASE_VERSION}" \ + --rc "${RC_NUM}" \ + --commit "${COMMIT_REF}" \ + --user "${GITHUB_USER}" \ + --deploy + +* **The script will:** + 1. Download source distribution and wheels tagged as `rc`. + 1. Deploy release candidate to PyPI + +__Attention:__ Verify that the following files are correct at [Dowload Files](https://pypi.org/project/apache-beam/#files): + +* All wheels should be published +* Release source's zip should be published +* Signatures and hashes do not need to be uploaded +* There should be a pre-release version with the `rc` tag Review comment: Do you mean `File names version should include ``rc-#`` suffix `? ########## File path: .github/workflows/build_wheels.yml ########## @@ -71,6 +73,24 @@ jobs: run: python -m pip install -r build-requirements.txt - name: Install wheels run: python -m pip install wheel + - name: Get tag + id: get_tag + run: | + echo ::set-output name=TAG::${GITHUB_REF#refs/*/} + - name: Check for rc version + id: is_rc + run: | + echo ${{ steps.get_tag.outputs.TAG }} > temp + OUTPUT=$( if grep -e '-RC.' -q temp; then echo 1; else echo 0; fi) + echo "::set-output name=is_rc::$OUTPUT" + - name: Get RC_NUM and VERSION Review comment: Consider: Get RELEASE_VERSION and RC_NUM ########## File path: release/src/main/scripts/deploy_release_candidate_pypi.sh ########## @@ -0,0 +1,181 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script will deploy a Release Candidate to pypi, includes: +# 1. Download python binary artifacts +# 2. Deploy Release Candidate to pypi + +set -e + +function usage() { + echo 'Usage: deploy_release_candidate.sh --release <version> --rc <rc> --commit <commit> --user <user> [--deploy]' +} + +RELEASE= +RC= +COMMIT= +USER_GITHUB_ID= +DEPLOY=no +PYTHON_ARTIFACTS_DIR=python +BEAM_ROOT_DIR=beam +GIT_REPO_BASE_URL=apache/beam + +while [[ $# -gt 0 ]] ; do + arg="$1" + + case $arg in + --release) + shift + RELEASE=$1 + shift + ;; + + --rc) + shift + RC=$1 + shift + ;; + + --commit) + shift + COMMIT=$1 + shift + ;; + + --user) + shift + USER_GITHUB_ID=$1 + shift + ;; + + --deploy) + DEPLOY=yes + shift + ;; + + *) + usage + exit 1 + ;; + esac +done + +if [[ -z "$RELEASE" ]] ; then + echo 'No release version supplied.' + usage + exit 1 +fi + +if [[ -z "$RC" ]] ; then + echo 'No RC number supplied' + usage + exit 1 +fi + +if [[ -z "$COMMIT" ]] ; then + echo 'No commit hash supplied.' + usage + exit 1 +fi + +if [[ -z "$USER_GITHUB_ID" ]] ; then + echo 'No github user supplied.' + usage + exit 1 +fi + +function clean_up(){ + echo "Do you want to clean local clone repo ${LOCAL_CLONE_DIR}? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + cd ~ + rm -rf ${LOCAL_CLONE_DIR} + echo "Cleaned up local repo." + fi +} + +RC_TAG="v${RELEASE}-RC${RC}" +LOCAL_CLONE_DIR="beam_release_${RC_TAG}" +SCRIPT_DIR=$(dirname $0) + +echo "================Checking Environment Variables==============" +echo "working on release version: ${RELEASE}" +echo "working on release branch: ${RC_TAG}" +echo "will create release candidate: RC${RC}" Review comment: At this point RC is already created right? How about we say: `will download artifacts for RC built by github actions` instead ########## File path: .github/workflows/build_wheels.yml ########## @@ -95,6 +115,50 @@ jobs: with: name: source_zip path: sdks/python/dist + - name: Clear dist + if: steps.is_rc.outputs.is_rc == 1 + working-directory: ./sdks/python + run: | + rm -r ./dist + rm -rd apache-beam-source + - name: Add rc tag Review comment: Let's call it RC Num to avoid conflation with RC tags. How about `Rewrite SDK version to include RC number.` ########## File path: .github/workflows/build_wheels.yml ########## @@ -150,6 +214,12 @@ jobs: with: name: source path: apache-beam-source + - name: Download python source distribution rc tagged from artifacts Review comment: how about: `Download Python SDK RC source distribution from artifacts` ########## File path: release/src/main/scripts/download_github_actions_artifacts.py ########## @@ -49,6 +49,7 @@ def parse_arguments(): parser.add_argument("--release-branch", required=True) parser.add_argument("--release-commit", required=True) parser.add_argument("--artifacts_dir", required=True) + parser.add_argument("--is_rc_version", required=False, action='store_true') Review comment: How about we pass rc_number, make it optional and if not specified pull the artifacts for final release? The filtering can be improved to filter by rc-### substring which I think would additionally make sure we set correct RC version in the tags. ########## File path: .github/workflows/build_wheels.yml ########## @@ -178,6 +248,28 @@ jobs: with: name: wheelhouse-${{ matrix.os_python.os }} path: apache-beam-source/wheelhouse/ + - name: Build wheel rc + if: ${{ needs.build_source.outputs.is_rc == 1 }} + working-directory: apache-beam-source-rc + env: + CIBW_BUILD: ${{ matrix.os_python.python }} + CIBW_BEFORE_BUILD: pip install cython + run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse + shell: bash + - name: Add checksums rc + if: ${{ needs.build_source.outputs.is_rc == 1 }} + working-directory: apache-beam-source-rc/wheelhouse/ + run: | + for file in *.whl; do + sha512sum $file > ${file}.sha512 + done + shell: bash + - name: Upload wheels as artifacts tagged as RC Review comment: `Upload RC wheels as artifacts` ########## File path: release/src/main/scripts/deploy_release_candidate_pypi.sh ########## @@ -0,0 +1,181 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script will deploy a Release Candidate to pypi, includes: +# 1. Download python binary artifacts +# 2. Deploy Release Candidate to pypi + +set -e + +function usage() { + echo 'Usage: deploy_release_candidate.sh --release <version> --rc <rc> --commit <commit> --user <user> [--deploy]' +} + +RELEASE= +RC= +COMMIT= +USER_GITHUB_ID= +DEPLOY=no +PYTHON_ARTIFACTS_DIR=python +BEAM_ROOT_DIR=beam +GIT_REPO_BASE_URL=apache/beam + +while [[ $# -gt 0 ]] ; do + arg="$1" + + case $arg in + --release) + shift + RELEASE=$1 + shift + ;; + + --rc) + shift + RC=$1 + shift + ;; + + --commit) + shift + COMMIT=$1 + shift + ;; + + --user) + shift + USER_GITHUB_ID=$1 + shift + ;; + + --deploy) + DEPLOY=yes + shift + ;; + + *) + usage + exit 1 + ;; + esac +done + +if [[ -z "$RELEASE" ]] ; then + echo 'No release version supplied.' + usage + exit 1 +fi + +if [[ -z "$RC" ]] ; then + echo 'No RC number supplied' + usage + exit 1 +fi + +if [[ -z "$COMMIT" ]] ; then + echo 'No commit hash supplied.' + usage + exit 1 +fi + +if [[ -z "$USER_GITHUB_ID" ]] ; then + echo 'No github user supplied.' + usage + exit 1 +fi + +function clean_up(){ + echo "Do you want to clean local clone repo ${LOCAL_CLONE_DIR}? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + cd ~ + rm -rf ${LOCAL_CLONE_DIR} + echo "Cleaned up local repo." + fi +} + +RC_TAG="v${RELEASE}-RC${RC}" +LOCAL_CLONE_DIR="beam_release_${RC_TAG}" +SCRIPT_DIR=$(dirname $0) + +echo "================Checking Environment Variables==============" +echo "working on release version: ${RELEASE}" +echo "working on release branch: ${RC_TAG}" Review comment: I think you need to pass release branch, e.g. release-2.17.0, and specify a commit on that branch that is marked with RC tag. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
