This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to tag apache in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7ed31bf57669e3a146c7918f123ef743d9acb40f Author: Jarek Potiuk <[email protected]> AuthorDate: Mon May 18 00:24:06 2020 +0200 Correct generation and package generation scripts --- BREEZE.rst | 25 ++++++--- backport_packages/setup_backport_packages.py | 12 ++++- breeze | 33 ++++++++---- breeze-complete | 4 +- dev/BACKPORT_PACKAGES.md | 63 ++++++++++++++-------- scripts/ci/_utils.sh | 6 ++- scripts/ci/docker-compose/local.yml | 2 +- .../in_container/run_prepare_backport_packages.sh | 28 +++++----- 8 files changed, 117 insertions(+), 56 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index 36dd3ff..dc16336 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -917,19 +917,20 @@ This is the current syntax for `./breeze <./breeze>`_: Builds backport packages. You can provide (after --) optional list of packages to prepare. If no packages are specified, readme for all packages are generated. You can specify optional - --version-suffix flag to generate rc candidates for the packages. - - Make sure to set the right version in './backport_packages/setup_backport_packages.py' + --artifact-name-suffix to generate rc candidate packages to upload to SVN or + --version-suffix flag to generate rc candidates for PyPI packages. Examples: 'breeze prepare-backport-packages' or 'breeze prepare-backport-packages -- google' or + 'breeze prepare-backport-packages --artifact-name-suffix rc1 -- http google amazon' or 'breeze prepare-backport-packages --version-suffix rc1 -- http google amazon' General form: - 'breeze prepare-backport-packages -- <PACKAGE_ID> ...' + 'breeze prepare-backport-packages [--artifact-name-suffix|--version-suffix] \ + -- <PACKAGE_ID> ...' * <PACKAGE_ID> is usually directory in the airflow/providers folder (for example 'google'), but in several cases, it might be one level deeper separated with '.' @@ -938,8 +939,12 @@ This is the current syntax for `./breeze <./breeze>`_: Flags: -S, --version-suffix - Adds optional suffix to the generated backport package version. It can be used to generate - rc1/rc2 ... versions of the packages. + Adds optional suffix to the version in the generated backport package. It can be used + to generate rc1/rc2 ... versions of the packages to be uploaded to PyPI. + + -N, --artifact-name-suffix + Adds optional suffix to the generated names of package. It can be used to generate + rc1/rc2 ... versions of the packages to be uploaded to SVN. -v, --verbose Show verbose information about executed commands (enabled by default for running test). @@ -1424,8 +1429,12 @@ This is the current syntax for `./breeze <./breeze>`_: Flags for generation of the backport packages -S, --version-suffix - Adds optional suffix to the generated backport package version. It can be used to generate - rc1/rc2 ... versions of the packages. + Adds optional suffix to the version in the generated backport package. It can be used + to generate rc1/rc2 ... versions of the packages to be uploaded to PyPI. + + -N, --artifact-name-suffix + Adds optional suffix to the generated names of package. It can be used to generate + rc1/rc2 ... versions of the packages to be uploaded to SVN. **************************************************************************************************** Increase verbosity of the scripts diff --git a/backport_packages/setup_backport_packages.py b/backport_packages/setup_backport_packages.py index 6746e80..919bd0b 100644 --- a/backport_packages/setup_backport_packages.py +++ b/backport_packages/setup_backport_packages.py @@ -379,7 +379,8 @@ def do_setup_package_providers(provider_package_id: str, long_description_content_type='text/markdown', license='Apache License 2.0', version=get_package_release_version( - provider_package_id=provider_package_id), + provider_package_id=provider_package_id, + version_suffix=version_suffix), packages=found_packages, zip_safe=False, install_requires=install_requires, @@ -1095,6 +1096,15 @@ if __name__ == "__main__": print() usage() exit(1) + if sys.argv[1] == "--version-suffix": + if len(sys.argv) < 3: + print() + print("ERROR! --version-suffix needs parameter!") + print() + usage() + exit(1) + suffix = sys.argv[2] + sys.argv = [sys.argv[0]] + sys.argv[3:] elif "--help" in sys.argv or "-h" in sys.argv or len(sys.argv) < 2: usage() exit(0) diff --git a/breeze b/breeze index 093f81f..02bb25d 100755 --- a/breeze +++ b/breeze @@ -70,7 +70,7 @@ function setup_default_breeze_variables() { # We have different versions of images depending on the python version used. We keep up with the # Latest patch-level changes in Python (this is done automatically during CI builds) so we have - # To only take into account MAJOR and MINOR version of python. This variable keeps the major/mninor + # To only take into account MAJOR and MINOR version of python. This variable keeps the major/minor # version of python in X.Y format (3.6, 3.7 etc). export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(read_from_file PYTHON_MAJOR_MINOR_VERSION)}" @@ -161,7 +161,7 @@ function setup_default_breeze_variables() { # Initializes development-friendly virtualenv if you are already in such env. It installs all the necessary # packages from PyPI and it case of problems it provides useful hints on what prerequisites should be -# installed. It also removes and resets the existing AIRFOW_HOME installation to make sure that you +# installed. It also removes and resets the existing AIRFLOW_HOME installation to make sure that you # have it synchronized with the version of airflow installed. It resets the airflow's sqlite database to # a clean state. You can use this function if your virtualenv is broken, to clean it up function initialize_virtualenv() { @@ -795,7 +795,15 @@ function parse_arguments() { shift ;; -S|--version-suffix) export VERSION_SUFFIX="${2}" - echo "Dockerhub repo ${VERSION_SUFFIX}" + export ARTIFACT_NAME_SUFFIX="${2}" + echo "Version suffix ${VERSION_SUFFIX}" + echo "Artifact name suffix ${ARTIFACT_NAME_SUFFIX}" + echo + shift 2 ;; + -N|--artifact-name-suffix) + export VERSION_SUFFIX="" + export ARTIFACT_NAME_SUFFIX="${2}" + echo "Artifact name suffix ${ARTIFACT_NAME_SUFFIX}" echo shift 2 ;; --) @@ -1231,19 +1239,20 @@ $(flag_airflow_variants) DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES=" Builds backport packages. You can provide (after --) optional list of packages to prepare. If no packages are specified, readme for all packages are generated. You can specify optional - --version-suffix flag to generate rc candidates for the packages. - - Make sure to set the right version in './backport_packages/setup_backport_packages.py' + --artifact-name-suffix to generate rc candidate packages to upload to SVN or + --version-suffix flag to generate rc candidates for PyPI packages. Examples: '${CMDNAME} prepare-backport-packages' or '${CMDNAME} prepare-backport-packages -- google' or + '${CMDNAME} prepare-backport-packages --artifact-name-suffix rc1 -- http google amazon' or '${CMDNAME} prepare-backport-packages --version-suffix rc1 -- http google amazon' General form: - '${CMDNAME} prepare-backport-packages -- <PACKAGE_ID> ...' + '${CMDNAME} prepare-backport-packages [--artifact-name-suffix|--version-suffix] \\ + -- <PACKAGE_ID> ...' * <PACKAGE_ID> is usually directory in the airflow/providers folder (for example 'google'), but in several cases, it might be one level deeper separated with '.' @@ -1630,8 +1639,12 @@ function flag_push_docker_images() { function flag_version_suffix() { echo " -S, --version-suffix - Adds optional suffix to the generated backport package version. It can be used to generate - rc1/rc2 ... versions of the packages. + Adds optional suffix to the version in the generated backport package. It can be used + to generate rc1/rc2 ... versions of the packages to be uploaded to PyPI. + +-N, --artifact-name-suffix + Adds optional suffix to the generated names of package. It can be used to generate + rc1/rc2 ... versions of the packages to be uploaded to SVN. " } @@ -1904,7 +1917,7 @@ function run_static_checks { fi } -# Runs Build before a comman - it will check and if needed rebuild necessary image, depending on the +# Runs Build before a command - it will check and if needed rebuild necessary image, depending on the # command chosen function run_build_command { case "${COMMAND_TO_RUN}" in diff --git a/breeze-complete b/breeze-complete index 59d173f..2fac4be 100644 --- a/breeze-complete +++ b/breeze-complete @@ -98,7 +98,7 @@ l a: t: d: v y n q f F P I E: C L D: R: u -S: +S: N: " _BREEZE_LONG_OPTIONS=" @@ -110,7 +110,7 @@ verbose assume-yes assume-no assume-quit forward-credentials force-build-images force-pull-images production-image extras: force-clean-images use-local-cache dockerhub-user: dockerhub-repo: push-images postgres-version: mysql-version: -version-suffix: +version-suffix: artifact-name-suffix: " export BREEZE_COMMANDS=" diff --git a/dev/BACKPORT_PACKAGES.md b/dev/BACKPORT_PACKAGES.md index d08c7af..4051a6a 100644 --- a/dev/BACKPORT_PACKAGES.md +++ b/dev/BACKPORT_PACKAGES.md @@ -184,7 +184,7 @@ You can also build all packages by omitting the package id altogether. ## Building an RC The Release Candidate artifacts we vote upon should be the exact ones we vote against, without any -modification than renaming – i.e. the contents of the files must be the same between voted +modification than renaming i.e. the contents of the files must be the same between voted release candidate and final release. Because of this the version in the built artifacts that will become the official Apache releases must not include the rcN suffix. @@ -193,7 +193,7 @@ Note! that the version in PyPI does not contain the leading 0s in version name. * Set environment variables (version and root of airflow repo) ```bash -export VERSION=2020.5.19rc2 +export VERSION=2020.5.19rc1 export AIRFLOW_REPO_ROOT=$(pwd) ``` @@ -201,7 +201,7 @@ export AIRFLOW_REPO_ROOT=$(pwd) * Tag the release ```bash -git tag ${VERSION} +git tag backport-providers-${VERSION} ``` * Clean the checkout @@ -213,17 +213,25 @@ git clean -fxd * Tarball the sources ```bash -git archive --format=tar.gz ${VERSION} --prefix=apache-airflow-${VERSION}/ -o apache-airflow-backport-providers-${VERSION}-source.tar.gz +git archive --format=tar.gz backport-providers-${VERSION} --prefix=apache-airflow-backport-providers-${VERSION}/ \ + -o apache-airflow-backport-providers-${VERSION}-source.tar.gz ``` -* Generate the packages (specify the version suffix, optionally list of packages). +* Generate the packages - since we are preparing packages for SVN repo just change artifact name suffix. This will clean up dist folder before generating the packages ```bash -./breeze prepare-backport-packages --version-suffix rc2 -- [PACKAGE ...] +./breeze prepare-backport-packages --artifact-name-suffix rc1 ``` +if you ony build few packages, run: + +```bash +./breeze prepare-backport-packages --artifact-name-suffix rc1 -- PACKAGE PACKAGE .... +``` + + * Move the source tarball to dist folder ```bash @@ -234,7 +242,7 @@ mv apache-airflow-backport-providers-${VERSION}-source.tar.gz dist ```bash cd dist -../dev/sign.sh *.tar.gz *.whl +../dev/sign.sh * ``` * Push tags @@ -254,15 +262,16 @@ cd airflow-dev/backport-providers svn mkdir ${VERSION} # Move the artifacts to svn folder & commit -mv ${AIRFLOW_REPO_ROOT}/dist/apache{-,_}airflow-${VERSION}* ${VERSION}/ -mv ${AIRFLOW_REPO_ROOT}/apache-airflow-backport-providers-${VERSION}-source.tar.gz ${VERSION}/ - +mv ${AIRFLOW_REPO_ROOT}/dist/*${VERSION}* ${VERSION}/ cd ${VERSION} svn add * svn commit -m "Add artifacts for Airflow ${VERSION}" cd ${AIRFLOW_REPO_ROOT} ``` +Verify that the files are available at +[backport-providers](https://dist.apache.org/repos/dist/dev/airflow/backport-providers/) + ## Make sure your public key is on id.apache.org and in KEYS You will need to sign the release artifacts with your pgp key. After you have created a key, make sure you: @@ -289,8 +298,8 @@ cd airflow svn commit -m "Add PGP keys of Airflow developers" ``` - ## Publishing to PyPi + Create a ~/.pypirc file: ```bash @@ -319,6 +328,20 @@ $ chmod 600 ~/.pypirc pip install twine ``` +* Generate the packages with the right version (specify the version suffix) + +This will clean up dist folder before generating the packages. + +```bash +./breeze prepare-backport-packages --version-suffix rc1 +``` + +if you ony build few packages, run: + +```bash +./breeze prepare-backport-packages --version-suffix rc1 -- PACKAGE PACKAGE .... +``` + * Verify the artifacts that would be uploaded: ```bash @@ -332,7 +355,7 @@ twine upload -r pypitest dist/* ``` * Verify that the test packages looks good by downloading it and installing them into a virtual environment. -Twine prints the package linksas output - separately for each package. +Twine prints the package links as output - separately for each package. * Upload the package to PyPi's production environment: @@ -348,27 +371,25 @@ Copy the list of links to the uploaded packages - they will be useful in prepari Make sure the packages are in https://dist.apache.org/repos/dist/dev/airflow/backport-providers/ -Use the dev/airflow-jira script to generate a list of Airflow JIRAs that were closed in the release. - Send out a vote to the [email protected] mailing list: ```text -[VOTE] Airflow Backport Providers 2020.5.19rc2 +[VOTE] Airflow Backport Providers 2020.5.19rc1 Hey all, -I have cut Airflow Backport Providers 2020.5.19rc2. This email is calling a vote on the release, +I have cut Airflow Backport Providers 2020.5.19rc1. This email is calling a vote on the release, which will last for 72 hours - which means that it will end on YYYY,MM.DD TZN. Consider this my (binding) +1. -Airflow Backport Providers 2020.5.19rc2 are available at: +Airflow Backport Providers 2020.5.19rc1 are available at: https://dist.apache.org/repos/dist/dev/airflow/backport-providers/2020.5.19/ -*apache-airflow-backport-providers-2020.5.19rc2-source.tar.gz* is a source release that comes +*apache-airflow-backport-providers-2020.5.19rc1-source.tar.gz* is a source release that comes with INSTALL instructions. -*apache-airflow-backport-providers-PROVIDER-2020.5.19rc2-bin.tar.gz* is the binary Python "sdist" release. +*apache-airflow-backport-providers-<PROVIDER>-2020.5.19rc1-bin.tar.gz* is the binary Python "sdist" release. Public keys are available at: https://dist.apache.org/repos/dist/release/airflow/KEYS @@ -392,12 +413,12 @@ Cheers, * Once the vote has been passed, you will need to send a result vote to [email protected]: ```text -[RESULT][VOTE] Airflow Backport Providers 2020.5.19rc2 +[RESULT][VOTE] Airflow Backport Providers 2020.5.19rc1 Hey all, -Airflow Backport Providers 2020.5.19 (based on RC2) has been accepted. +Airflow Backport Providers 2020.5.19 (based on rc1) has been accepted. N “+1” binding votes received: - PMC Member (binding) diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index b0bc276..473201b 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -224,6 +224,9 @@ function initialize_common_environment { # Version suffix for the generated backport packages export VERSION_SUFFIX="" + + # Artifact name suffix for the generated backport packages + export ARTIFACT_NAME_SUFFIX="" } # Prints verbose information in case VERBOSE variable is set @@ -253,7 +256,7 @@ LICENSE /opt/airflow/ MANIFEST.in /opt/airflow/ NOTICE /opt/airflow/ airflow /opt/airflow/ -backport_packages/setup_backport_packages.py /opt/airflow/backport_packages/ +backport_packages /opt/airflow/ common /opt/airflow/ dags /opt/airflow/ dev /opt/airflow/ @@ -1650,6 +1653,7 @@ function run_prepare_backport_packages() { --env PYTHON_MAJOR_MINOR_VERSION \ --env CHECK_REQUIREMENTS_ONLY \ --env VERSION_SUFFIX \ + --env ARTIFACT_NAME_SUFFIX \ -t \ -v "${AIRFLOW_SOURCES}:/opt/airflow" \ --rm \ diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 0373d31..8062147 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -40,7 +40,7 @@ services: - ../../../MANIFEST.in:/opt/airflow/MANIFEST.in:cached - ../../../NOTICE:/opt/airflow/NOTICE:cached - ../../../airflow:/opt/airflow/airflow:cached - - ../../../backport_packages/setup_backport_packages.py:/opt/airflow/backport_packages/setup_backport_packages.py:cached + - ../../../backport_packages:/opt/airflow/backport_packages:cached - ../../../common:/opt/airflow/common:cached - ../../../dags:/opt/airflow/dags:cached - ../../../dev:/opt/airflow/dev:cached diff --git a/scripts/ci/in_container/run_prepare_backport_packages.sh b/scripts/ci/in_container/run_prepare_backport_packages.sh index 3bef306..58c06b7 100755 --- a/scripts/ci/in_container/run_prepare_backport_packages.sh +++ b/scripts/ci/in_container/run_prepare_backport_packages.sh @@ -85,7 +85,10 @@ echo "========================================================================== python3 setup_backport_packages.py prepare VERSION_SUFFIX=${VERSION_SUFFIX:=""} +ARTIFACT_NAME_SUFFIX=${ARTIFACT_NAME_SUFFIX:=""} +echo "Version suffix = ${VERSION_SUFFIX}" +echo "Artifact name suffix = ${ARTIFACT_NAME_SUFFIX}" for BACKPORT_PACKAGE in ${BACKPORT_PACKAGES} do @@ -95,7 +98,7 @@ do echo "-----------------------------------------------------------------------------------" python3 setup_backport_packages.py "${BACKPORT_PACKAGE}" clean --all >/dev/null 2>&1 set +e - python3 setup_backport_packages.py \ + python3 setup_backport_packages.py --version-suffix "${VERSION_SUFFIX}" \ "${BACKPORT_PACKAGE}" sdist bdist_wheel >"${LOG_FILE}" 2>&1 RES="${?}" if [[ ${RES} != "0" ]]; then @@ -109,20 +112,21 @@ done cd "${AIRFLOW_SOURCES}" || exit 1 pushd dist -for FILE in *.tar.gz -do - mv "${FILE}" "${FILE//\.tar\.gz/${VERSION_SUFFIX}-bin.tar.gz}" -done -for FILE in *.whl -do - if [[ ${VERSION_SUFFIX} != "" ]]; then - mv "${FILE}" "${FILE//\-py2\.py3/${VERSION_SUFFIX}-py2.py3}" - fi -done +if [[ ${ARTIFACT_NAME_SUFFIX} != "${VERSION_SUFFIX}" ]]; then + for FILE in *.tar.gz + do + mv "${FILE}" "${FILE//\.tar\.gz/${ARTIFACT_NAME_SUFFIX}-bin.tar.gz}" + done + for FILE in *.whl + do + mv "${FILE}" "${FILE//\-py2\.py3/${ARTIFACT_NAME_SUFFIX}-py2.py3}" + done +fi + popd -AIRFLOW_PACKAGES_TGZ_FILE="/tmp/airflow-packages-$(date +"%Y%m%d-%H%M%S")-${VERSION_SUFFIX}.tar.gz" +AIRFLOW_PACKAGES_TGZ_FILE="/tmp/airflow-packages-$(date +"%Y%m%d-%H%M%S")-${ARTIFACT_NAME_SUFFIX}.tar.gz" tar -cvzf "${AIRFLOW_PACKAGES_TGZ_FILE}" dist/*.whl dist/*.tar.gz echo
