This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-2-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 1cf852816cc236208253ee6039ece55c0e6bcd90 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Jan 2 11:37:40 2022 +0100 Add twine check for provider packages (#20619) Twine (which we use to upload packages to PyPI) has the ability to run checks of packages before uploading them. This allows to detect cases like when we are using forbidden directives in README.rst (which delayed slightly preparing the December 2021 provider packages and resulted in #20614 With this PR Twine check will be run for all packages in CI before we even attempt to merge such change that could break them. (cherry picked from commit f011f66f763cb9bfcccea085dbd2cb2b44614d20) --- .github/workflows/ci.yml | 1 + breeze | 1 + scripts/ci/docker-compose/_docker.env | 1 + scripts/ci/docker-compose/base.yml | 1 + scripts/ci/libraries/_initialization.sh | 3 +++ scripts/in_container/_in_container_utils.sh | 14 ++++++++++++++ .../run_install_and_test_provider_packages.sh | 22 ++++++++++++++++++++++ setup.py | 1 + 8 files changed, 44 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9002b28..d5d75ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -578,6 +578,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh env: USE_AIRFLOW_VERSION: "2.1.0" + SKIP_TWINE_CHECK: "true" PACKAGE_FORMAT: "wheel" prepare-test-provider-packages-sdist: diff --git a/breeze b/breeze index dd72e6e..c646e08 100755 --- a/breeze +++ b/breeze @@ -647,6 +647,7 @@ export AIRFLOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}" export AIRFLOW_IMAGE_KUBERNETES="${AIRFLOW_IMAGE_KUBERNETES}" export SQLITE_URL="${SQLITE_URL}" export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}" +export SKIP_TWINE_CHECK="${SKIP_TWINE_CHECK}" export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}" export EXECUTOR="${EXECUTOR}" export START_AIRFLOW="${START_AIRFLOW}" diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index 72a1afc..0f3a940 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -57,6 +57,7 @@ RUN_TESTS LIST_OF_INTEGRATION_TESTS_TO_RUN RUN_SYSTEM_TESTS START_AIRFLOW +SKIP_TWINE_CHECK TEST_TYPE UPGRADE_TO_NEWER_DEPENDENCIES VERBOSE diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 8413179..5125dd6 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -70,6 +70,7 @@ services: - LIST_OF_INTEGRATION_TESTS_TO_RUN=${LIST_OF_INTEGRATION_TESTS_TO_RUN} - RUN_SYSTEM_TESTS=${RUN_SYSTEM_TESTS} - START_AIRFLOW=${START_AIRFLOW} + - SKIP_TWINE_CHECK=${SKIP_TWINE_CHECK} - TEST_TYPE=${TEST_TYPE} - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} - VERBOSE=${VERBOSE} diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 2011f6e..a95ce19 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -416,6 +416,9 @@ function initialization::initialize_image_build_variables() { INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES:="true"} export INSTALL_PROVIDERS_FROM_SOURCES + SKIP_TWINE_CHECK=${SKIP_TWINE_CHECK:=""} + export SKIP_TWINE_CHECK + export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.2.4"} diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index ca0a84f..1cf30dd 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -301,6 +301,20 @@ function install_all_provider_packages_from_sdist() { pip install /dist/apache-airflow-*providers-*.tar.gz } +function twine_check_provider_packages_from_wheels() { + echo + echo "Twine check of all provider packages from wheels" + echo + twine check /dist/apache_airflow*providers_*.whl +} + +function twine_check_provider_packages_from_sdist() { + echo + echo "Twine check all provider packages from sdist" + echo + twine check /dist/apache-airflow-*providers-*.tar.gz +} + function setup_provider_packages() { export PACKAGE_TYPE="regular" export PACKAGE_PREFIX_UPPERCASE="" diff --git a/scripts/in_container/run_install_and_test_provider_packages.sh b/scripts/in_container/run_install_and_test_provider_packages.sh index 1e80dd3..d556581 100755 --- a/scripts/in_container/run_install_and_test_provider_packages.sh +++ b/scripts/in_container/run_install_and_test_provider_packages.sh @@ -90,6 +90,22 @@ function install_provider_packages() { group_end } +function twine_check_provider_packages() { + group_start "Twine check provider packages" + if [[ ${PACKAGE_FORMAT} == "wheel" ]]; then + twine_check_provider_packages_from_wheels + elif [[ ${PACKAGE_FORMAT} == "sdist" ]]; then + twine_check_provider_packages_from_sdist + else + echo + echo "${COLOR_RED}ERROR: Wrong package format ${PACKAGE_FORMAT}. Should be wheel or sdist${COLOR_RESET}" + echo + exit 1 + fi + group_end +} + + function discover_all_provider_packages() { group_start "Listing available providers via 'airflow providers list'" # Columns is to force it wider, so it doesn't wrap at 80 characters @@ -225,6 +241,12 @@ function ver() { setup_provider_packages verify_parameters install_airflow_as_specified + +if [[ ${SKIP_TWINE_CHECK=""} != "true" ]]; then + # Airflow 2.1.0 installs importlib_metadata version that does not work well with twine + # So we should skip twine check in this case + twine_check_provider_packages +fi install_provider_packages import_all_provider_classes diff --git a/setup.py b/setup.py index 9095b33..59ddfc3 100644 --- a/setup.py +++ b/setup.py @@ -559,6 +559,7 @@ devel_only = [ 'pytest-httpx', 'requests_mock', 'semver', + 'twine', 'wheel', 'yamllint', ]
