norm commented on code in PR #27829: URL: https://github.com/apache/airflow/pull/27829#discussion_r1032252036
########## dev/breeze/src/airflow_breeze/commands/minor_release_command.py: ########## @@ -0,0 +1,174 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os + +import click + +from airflow_breeze.utils.common_options import option_answer +from airflow_breeze.utils.confirm import user_confirm_bool +from airflow_breeze.utils.console import console_print +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.run_utils import run_command + +CI = os.environ.get("CI") + + +def create_branch(version_branch): + if CI: + console_print("Skipping creating branch in CI") + return + if user_confirm_bool(f"Create version branch: {version_branch}?"): + run_command(["git", "checkout", "main"]) + run_command(["git", "checkout", "-b", f"v{version_branch}-test"]) + console_print(f"Created branch: v{version_branch}-test") + + +def update_default_branch(version_branch): + if user_confirm_bool("Update default branches?"): + console_print() + console_print("You need to update the default branch at:") + console_print("./dev/breeze/src/airflow_breeze/branch_defaults.py") + console_print("Change the following:") + console_print("AIRFLOW_BRANCH = 'main'") + console_print("DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = 'constraints-main'") + console_print() + console_print("To:") + console_print() + console_print(f"AIRFLOW_BRANCH = 'v{version_branch}-test'") + console_print(f"DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = 'constraints-{version_branch}'") + + +def commit_changes(version_branch): + if CI: + console_print("Skipping committing changes in CI") + return + if user_confirm_bool("Commit the above changes?"): + run_command(["git", "add", "-p", "."]) + run_command(["git", "commit", "-m", f"Update default branches for {version_branch}"]) + + +def create_stable_branch(version_branch): + if CI: + console_print("Skipping creating stable branch in CI") + return + if user_confirm_bool(f"Create stable branch: v{version_branch}-stable?"): + run_command(["git", "branch", f"v{version_branch}-stable"]) + console_print(f"Created branch: v{version_branch}-stable") + + +def push_test_and_stable_branch(version_branch): + if CI: + console_print("Skipping pushing test and stable branches in CI") + return + if user_confirm_bool("Push test and stable branches?"): + run_command(["git", "checkout", f"v{version_branch}-test"]) + run_command(["git", "push", "--set-upstream", "origin", f"v{version_branch}-test"]) + run_command(["git", "checkout", f"v{version_branch}-stable"]) + run_command(["git", "push", "--set-upstream", "origin", f"v{version_branch}-stable"]) + + +def checkout_main(): + if user_confirm_bool("We now need to checkout main. Continue?"): + run_command(["git", "checkout", "main"]) + run_command(["git", "pull"]) + + +def instruction_update_version_branch(version_branch): + if user_confirm_bool("Now, we need to manually update the version branches in main. Continue?"): + console_print() + console_print( + f"Add v{version_branch}-stable and v{version_branch}-test branches " + "in codecov.yml (there are 2 places in the file!)" + ) + console_print( + """ + branches: + - main + - v2-0-stable + - v2-0-test + - v2-1-stable + - v2-1-test + - v2-2-stable + - v2-2-test + """ + ) + console_print(f"Add v{version_branch}-stable to .asf.yaml ({version_branch} is your new branch)") + console_print( + f""" + protected_branches: + main: + required_pull_request_reviews: + required_approving_review_count: 1 + ... + v{version_branch}-stable: + required_pull_request_reviews: + required_approving_review_count: 1 + """ + ) + console_print("Once you finish with the above. Commit the changes and make a PR against main") + user_confirm_bool("I'm done with the changes. Continue?", abort=True) + + +def create_constraints(version_branch): + if CI: + console_print("Skipping creating constraints in CI") + return + if user_confirm_bool("Do you want to create branches from the constraints main?"): + run_command(["git", "checkout", "constraints-main"]) + run_command(["git", "checkout", "-b", f"constraints-{version_branch}"]) + if user_confirm_bool("Push the new branch?"): + run_command(["git", "push", "--set-upstream", "origin", f"constraints-{version_branch}"]) + + [email protected]( + name="create-minor-branch", + help="Create a new version branch and update the default branches in main", + hidden=True, +) [email protected]("--version-branch", help="The version branch you want to create e.g 2-4", required=True) +@option_answer +def create_minor_version_branch(version_branch): + for obj in version_branch.split("-"): + assert isinstance(int(obj), int) + os.chdir(AIRFLOW_SOURCES_ROOT) + repo_root = os.getcwd() + console_print() + console_print(f"Repo root: {repo_root}") + console_print(f"Version branch: {version_branch}") + user_confirm_bool("Verify that the above information are correct. Do you want to continue?", abort=True) Review Comment: s/are/is/ — information is singular in English :) ########## dev/breeze/src/airflow_breeze/commands/minor_release_command.py: ########## @@ -0,0 +1,174 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os + +import click + +from airflow_breeze.utils.common_options import option_answer +from airflow_breeze.utils.confirm import user_confirm_bool +from airflow_breeze.utils.console import console_print +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.run_utils import run_command + +CI = os.environ.get("CI") + + +def create_branch(version_branch): + if CI: + console_print("Skipping creating branch in CI") + return + if user_confirm_bool(f"Create version branch: {version_branch}?"): + run_command(["git", "checkout", "main"]) + run_command(["git", "checkout", "-b", f"v{version_branch}-test"]) + console_print(f"Created branch: v{version_branch}-test") + + +def update_default_branch(version_branch): + if user_confirm_bool("Update default branches?"): + console_print() + console_print("You need to update the default branch at:") + console_print("./dev/breeze/src/airflow_breeze/branch_defaults.py") + console_print("Change the following:") + console_print("AIRFLOW_BRANCH = 'main'") + console_print("DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = 'constraints-main'") + console_print() + console_print("To:") + console_print() + console_print(f"AIRFLOW_BRANCH = 'v{version_branch}-test'") + console_print(f"DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = 'constraints-{version_branch}'") + + +def commit_changes(version_branch): + if CI: + console_print("Skipping committing changes in CI") + return + if user_confirm_bool("Commit the above changes?"): + run_command(["git", "add", "-p", "."]) + run_command(["git", "commit", "-m", f"Update default branches for {version_branch}"]) + + +def create_stable_branch(version_branch): + if CI: + console_print("Skipping creating stable branch in CI") + return + if user_confirm_bool(f"Create stable branch: v{version_branch}-stable?"): + run_command(["git", "branch", f"v{version_branch}-stable"]) + console_print(f"Created branch: v{version_branch}-stable") + + +def push_test_and_stable_branch(version_branch): + if CI: + console_print("Skipping pushing test and stable branches in CI") + return + if user_confirm_bool("Push test and stable branches?"): + run_command(["git", "checkout", f"v{version_branch}-test"]) + run_command(["git", "push", "--set-upstream", "origin", f"v{version_branch}-test"]) + run_command(["git", "checkout", f"v{version_branch}-stable"]) + run_command(["git", "push", "--set-upstream", "origin", f"v{version_branch}-stable"]) + + +def checkout_main(): + if user_confirm_bool("We now need to checkout main. Continue?"): + run_command(["git", "checkout", "main"]) + run_command(["git", "pull"]) + + +def instruction_update_version_branch(version_branch): + if user_confirm_bool("Now, we need to manually update the version branches in main. Continue?"): + console_print() + console_print( + f"Add v{version_branch}-stable and v{version_branch}-test branches " + "in codecov.yml (there are 2 places in the file!)" + ) + console_print( + """ + branches: Review Comment: I assume this is illustrating what to look for, so it could be a little clearer that this is not the code to *add* but the areas to *add to*. ########## dev/breeze/src/airflow_breeze/commands/release_command.py: ########## @@ -0,0 +1,284 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os + +import click + +from airflow_breeze.utils.common_options import option_answer +from airflow_breeze.utils.confirm import user_confirm_bool +from airflow_breeze.utils.console import console_print +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.run_utils import run_command + +CI = os.environ.get("CI") + + +def clone_asf_repo(working_dir): + if user_confirm_bool("Clone ASF repo?"): + run_command(["rm", "-rf", f"{working_dir}/asf-dist"]) + run_command( + ["svn", "checkout", "--depth=immediates", "https://dist.apache.org/repos/dist";, "asf-dist"] + ) + dev_dir = f"{working_dir}/asf-dist/dev/airflow" + release_dir = f"{working_dir}/asf-dist/release/airflow" + run_command(["svn", "update", "--set-depth", "infinity", dev_dir]) + run_command(["svn", "update", "--set-depth", "infinity", release_dir]) + + +def create_version_dir(version): + if CI: + console_print("Skipping creation of version dir in CI") + return + if user_confirm_bool(f"Create SVN version directory for {version}?"): + run_command(["svn", "mkdir", f"{version}"]) + console_print(f"{version} directory created") + + +def copy_artifacts_to_svn(rc, svn_dev_repo): + if CI: + console_print("Skipping copying artifacts to SVN in CI") + return + if user_confirm_bool(f"Copy artifacts to SVN for {rc}?"): + run_command( + [ + "for", + "f", + "in", + f"{svn_dev_repo}/{rc}/*", + ";", + "do", + "svn", + "cp", + "$f", + "${$(basename $f)/}", + "done", + ] + ) + console_print("Artifacts copied to SVN:") + run_command(["ls"]) + + +def commit_release(version, rc, svn_release_version_dir): + if CI: + console_print("Skipping commit in CI") + return + if user_confirm_bool(f"Commit release {version} to SVN?"): + run_command(["svn", "commit", "-m", f"'Release Airflow {version} from {rc}'"]) + + +def remove_old_release(previous_release): + if CI: + console_print("Skipping removing old release in CI") + return + if user_confirm_bool(f"Remove old release {previous_release}?"): + run_command(["svn", "rm", f"'{previous_release}'"]) + run_command(["svn", "commit", "-m", f"'Remove old release: {previous_release}'"]) + user_confirm_bool( + "Verify that the packages appear in " + "[airflow](https://dist.apache.org/repos/dist/release/airflow/). Continue?", + abort=True, + ) + + +def verify_pypi_package(version): + if user_confirm_bool("Verify PyPI package?"): + run_command(["twine", "check", "*.whl", f"*{version}.tar.gz"]) + + +def upload_to_pypi_test(version): + if CI: + console_print("Skipping upload to PyPI test in CI") + return + if user_confirm_bool("Upload to PyPI test?"): + run_command(["twine", "upload", "-r", "pypitest", "*.whl", f"*{version}.tar.gz"]) + console_print("Packages pushed to test PyPI") + console_print( + "Verify that the test package looks good by downloading it and installing it into a virtual " + "environment. The package download link is available at: " + "https://test.pypi.org/project/apache-airflow/#files"; + ) + + +def upload_to_pypi(version): + if CI: + console_print("Skipping upload to PyPI in CI") + return + if user_confirm_bool("Upload to PyPI?"): + + user_confirm_bool( + "I have tested the package I uploaded to test PyPI. " + "I installed and ran a dag with it and there's no issue. Do you agree to the above?", Review Comment: I think dag is written DAG in (most of) the documentation, do the same here? ########## dev/breeze/src/airflow_breeze/commands/minor_release_command.py: ########## @@ -0,0 +1,174 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os + +import click + +from airflow_breeze.utils.common_options import option_answer +from airflow_breeze.utils.confirm import user_confirm_bool +from airflow_breeze.utils.console import console_print +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.run_utils import run_command + +CI = os.environ.get("CI") + + +def create_branch(version_branch): + if CI: + console_print("Skipping creating branch in CI") + return + if user_confirm_bool(f"Create version branch: {version_branch}?"): + run_command(["git", "checkout", "main"]) + run_command(["git", "checkout", "-b", f"v{version_branch}-test"]) + console_print(f"Created branch: v{version_branch}-test") + + +def update_default_branch(version_branch): + if user_confirm_bool("Update default branches?"): + console_print() + console_print("You need to update the default branch at:") + console_print("./dev/breeze/src/airflow_breeze/branch_defaults.py") + console_print("Change the following:") + console_print("AIRFLOW_BRANCH = 'main'") + console_print("DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = 'constraints-main'") + console_print() + console_print("To:") + console_print() + console_print(f"AIRFLOW_BRANCH = 'v{version_branch}-test'") + console_print(f"DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = 'constraints-{version_branch}'") + + +def commit_changes(version_branch): + if CI: + console_print("Skipping committing changes in CI") + return + if user_confirm_bool("Commit the above changes?"): + run_command(["git", "add", "-p", "."]) + run_command(["git", "commit", "-m", f"Update default branches for {version_branch}"]) + + +def create_stable_branch(version_branch): + if CI: + console_print("Skipping creating stable branch in CI") + return + if user_confirm_bool(f"Create stable branch: v{version_branch}-stable?"): + run_command(["git", "branch", f"v{version_branch}-stable"]) + console_print(f"Created branch: v{version_branch}-stable") + + +def push_test_and_stable_branch(version_branch): + if CI: + console_print("Skipping pushing test and stable branches in CI") + return + if user_confirm_bool("Push test and stable branches?"): + run_command(["git", "checkout", f"v{version_branch}-test"]) + run_command(["git", "push", "--set-upstream", "origin", f"v{version_branch}-test"]) Review Comment: Oh, I see later that that is already called out in the script. Never mind :) ########## dev/breeze/src/airflow_breeze/commands/release_candidate_command.py: ########## @@ -0,0 +1,343 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os + +import click + +from airflow_breeze.utils.common_options import option_answer +from airflow_breeze.utils.confirm import user_confirm_bool +from airflow_breeze.utils.console import console_print +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.run_utils import run_command + +CI = os.environ.get("CI") + + +def merge_pr(version_branch): + if CI: + console_print("Skipping merge PR in CI") + return + if user_confirm_bool("Do you want to merge the Sync PR?"): + run_command( + [ + "git", + "checkout", + f"v{version_branch}-stable", + ] + ) + run_command(["git", "reset", "--hard", f"origin/v{version_branch}-stable"]) + run_command(["git", "merge", "--ff-only", f"v{version_branch}-test"]) + if user_confirm_bool("Do you want to push the changes? Pushing the changes closes the PR"): + run_command(["git", "push", "origin", f"v{version_branch}-stable"]) + + +def git_tag(version): + if CI: + console_print("Skipping git tag in CI") + return + if user_confirm_bool(f"Tag {version}?"): + run_command(["git", "tag", "-s", f"{version}", "-m", f"'Apache Airflow {version}'"]) + console_print("Tagged") + + +def git_clean(): + if user_confirm_bool("Clean git repo?"): + run_command(["git", "clean", "-fxd"]) + console_print("Git repo cleaned") + + +def tarball_release(version, version_without_rc): + if user_confirm_bool("Create tarball?"): + run_command(["rm", "-rf", "dist"]) + + run_command(["mkdir", "dist"]) + run_command( + [ + "git", + "archive", + "--format=tar.gz", + f"{version}", + f"--prefix=apache-airflow-{version_without_rc}/", + "-o", + f"dist/apache-airflow-{version_without_rc}-source.tar.gz", + ] + ) + console_print("Tarball created") + + +def create_artifacts_with_sdist(): + run_command(["python3", "setup.py", "compile_assets", "sdist", "bdist_wheel"]) + console_print("Artifacts created") + + +def create_artifacts_with_breeze(): + run_command(["breeze", "release-management", "prepare-airflow-package", "--package-format", "both"]) + console_print("Artifacts created") + + +def sign_the_release(repo_root): + if CI: + console_print("Skipping sign the release in CI") + return + if user_confirm_bool("Do you want to sign the release?"): + os.chdir(repo_root) + run_command(["pushd", "dist"]) + run_command(["./dev/sign.sh", "*"]) + run_command("popd") + console_print("Release signed") + + +def tag_and_push_constraints(version, version_branch): + if CI: + console_print("Skipping tag and push constraints in CI") + return + if user_confirm_bool("Do you want to tag and push constraints?"): + run_command(["git", "checkout", f"origin/constraints-{version_branch}"]) + run_command( + [ + "git", + "tag", + "-s", + f"'constraints-{version}'", + "-m", + f"'Constraints for Apache Airflow {version}'", + ] + ) + run_command(["git", "push", "origin", "tag", f"'constraints-{version}'"]) + console_print("Constraints tagged and pushed") + + +def clone_asf_repo(version, repo_root): + if user_confirm_bool("Do you want to clone asf repo?"): + os.chdir(repo_root) + run_command( + ["svn", "checkout", "--depth=immediates", "https://dist.apache.org/repos/dist";, "asf-dist"] + ) + run_command(["svn", "update", "--set-depth=infinity", "asf-dist/dev/airflow"]) + console_print("Cloned ASF repo successfully") + + +def move_artifacts_to_svn(version, repo_root): + if CI: + console_print("Skipping move artifacts to svn in CI") + return + if user_confirm_bool("Do you want to move artifacts to SVN?"): + os.chdir(f"{repo_root}/asf-dist/dev/airflow") + run_command(["svn", "mkdir", f"{version}"]) + run_command(["mv", f"{repo_root}/dist/*", f"{version}/"]) + console_print("Moved artifacts to SVN:") + run_command(["ls"]) + + +def push_artifacts_to_asf_repo(version, repo_root): + if CI: + console_print("Skipping push artifacts to ASF repo in CI") + return + if user_confirm_bool("Do you want to push artifacts to ASF repo?"): + console_print("Files to push to svn:") + os.chdir(f"{repo_root}/asf-dist/dev/airflow/{version}") + run_command(["ls"]) + user_confirm_bool("Do you want to continue?", abort=True) + run_command(["svn", "add", "*"]) + run_command(["svn", "commit", "-m", f"'Add artifacts for Airflow {version}'"]) + console_print("Files pushed to svn") + os.chdir(repo_root) + run_command(["rm", "-rf", "asf-dist"]) + + +def prepare_pypi_packages(version, version_suffix, repo_root): + if user_confirm_bool("Prepare pypi packages?"): + console_print("Preparing PyPI packages") + os.chdir(repo_root) + run_command(["git", "checkout", f"{version}"]) + run_command( + [ + "breeze", + "release-management", + "prepare-airflow-package", + "--version-suffix-for-pypi", + f"{version_suffix}", + "--package-format", + "both", + ] + ) + run_command(["twine", "check", "dist/*"]) + console_print("PyPI packages prepared") + + +def push_packages_to_test_pypi(): + if CI: + console_print("Skipping push packages to test PyPI in CI") + return + if user_confirm_bool("Do you want to push packages to test PyPI?"): + run_command(["twine", "upload", "-r", "pypitest", "dist/*"]) + console_print("Packages pushed to test PyPI") + console_print( + "Verify that the test package looks good by downloading it and installing it into a virtual " + "environment. The package download link is available at: " + "https://test.pypi.org/project/apache-airflow/#files"; + ) + + +def push_packages_to_pypi(): + if CI: + console_print("Skipping push packages to PyPI in CI") + return + if user_confirm_bool("Do you want to push packages to production PyPI?"): + user_confirm_bool( + "I have tested the package I uploaded to test PyPI. " + "I installed and ran a dag with it and there's no issue. Do you agree to the above?", Review Comment: Another lower-case dag. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
