This is an automated email from the ASF dual-hosted git repository. kaxilnaik pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5fd52b8e4d05456f392758171eb58ac3a19c5817 Author: James Timmins <[email protected]> AuthorDate: Fri Jun 5 10:39:28 2020 -0700 Add 3.8 to the test matrices (#8836) (cherry picked from commit 5918efc86a2217caa641a6ada289eee1c21407f8) --- .github/workflows/ci.yml | 14 +++--- BREEZE.rst | 14 +++--- Dockerfile.ci | 1 + README.md | 6 +-- breeze | 4 +- breeze-complete | 2 +- requirements/requirements-python2.7.txt | 4 +- requirements/requirements-python3.5.txt | 4 +- ...ts-python3.5.txt => requirements-python3.8.txt} | 50 +++++++++++----------- requirements/setup-2.7.md5 | 2 +- requirements/setup-3.5.md5 | 2 +- requirements/setup-3.6.md5 | 2 +- requirements/setup-3.7.md5 | 2 +- requirements/setup-3.8.md5 | 1 + setup.py | 10 +++-- .../operators/test_mssql_to_gcs_operator.py | 7 ++- 16 files changed, 66 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc86961..d091d2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} CI_JOB_TYPE: "Prod image" @@ -179,7 +179,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') needs: [static-checks, pyfiles] strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] postgres-version: [9.6, 10] test-type: [Core, Integration] fail-fast: false @@ -211,7 +211,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') needs: [static-checks, pyfiles] strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] mysql-version: [5.6, 5.7] test-type: [Core, Integration] fail-fast: false @@ -243,7 +243,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') needs: [static-checks, pyfiles] strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] test-type: [Core, Integration] fail-fast: false env: @@ -303,7 +303,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] fail-fast: false env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} @@ -334,7 +334,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') if: github.ref == 'refs/heads/master' && github.event_name != 'schedule' strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} CI_JOB_TYPE: "Prod image" @@ -364,7 +364,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') github.event_name != 'schedule' strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] env: PULL_PYTHON_BASE_IMAGES_FROM_CACHE: "false" PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} diff --git a/BREEZE.rst b/BREEZE.rst index ee0ff01..9d2d719 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -802,7 +802,7 @@ This is the current syntax for `./breeze <./breeze>`_: One of: - 2.7 3.5 3.6 3.7 + 2.7 3.5 3.6 3.7 3.8 -a, --install-airflow-version <INSTALL_AIRFLOW_VERSION> If specified, installs Airflow directly from PIP released version. This happens at @@ -922,7 +922,7 @@ This is the current syntax for `./breeze <./breeze>`_: One of: - 2.7 3.5 3.6 3.7 + 2.7 3.5 3.6 3.7 3.8 -I, --production-image Use production image for entering the environment and builds (not for tests). @@ -968,7 +968,7 @@ This is the current syntax for `./breeze <./breeze>`_: One of: - 2.7 3.5 3.6 3.7 + 2.7 3.5 3.6 3.7 3.8 -v, --verbose Show verbose information about executed commands (enabled by default for running test). @@ -1047,7 +1047,7 @@ This is the current syntax for `./breeze <./breeze>`_: One of: - 2.7 3.5 3.6 3.7 + 2.7 3.5 3.6 3.7 3.8 #################################################################################################### @@ -1136,7 +1136,7 @@ This is the current syntax for `./breeze <./breeze>`_: One of: - 2.7 3.5 3.6 3.7 + 2.7 3.5 3.6 3.7 3.8 -b, --backend <BACKEND> Backend to use for tests - it determines which database is used. @@ -1186,7 +1186,7 @@ This is the current syntax for `./breeze <./breeze>`_: One of: - 2.7 3.5 3.6 3.7 + 2.7 3.5 3.6 3.7 3.8 -F, --force-build-images Forces building of the local docker images. The images are rebuilt @@ -1351,7 +1351,7 @@ This is the current syntax for `./breeze <./breeze>`_: One of: - 2.7 3.5 3.6 3.7 + 2.7 3.5 3.6 3.7 3.8 **************************************************************************************************** Choose backend to run for Airflow diff --git a/Dockerfile.ci b/Dockerfile.ci index f4ba142..2b2157a 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -228,6 +228,7 @@ RUN pip install \ --constraint "https://raw.githubusercontent.com/${AIRFLOW_REPO}/${AIRFLOW_BRANCH}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt"; \ && pip uninstall --yes apache-airflow; + # Link dumb-init for backwards compatibility (so that older images also work) RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init diff --git a/README.md b/README.md index 0c0aa7c..d5883bf 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Apache Airflow is tested with: ### Master version (2.0.0dev) -* Python versions: 3.6, 3.7 +* Python versions: 3.6, 3.7, 3.8 * Postgres DB: 9.6, 10 * MySQL DB: 5.7 * Sqlite - latest stable (it is used mainly for development purpose) @@ -69,7 +69,7 @@ Apache Airflow is tested with: ### Stable version (1.10.10) -* Python versions: 2.7, 3.5, 3.6, 3.7 +* Python versions: 2.7, 3.5, 3.6, 3.7, 3.8 * Postgres DB: 9.6, 10 * MySQL DB: 5.6, 5.7 * Sqlite - latest stable (it is used mainly for development purpose) @@ -100,7 +100,7 @@ produce unusable Airflow installation. In order to have repeatable installation, however, starting from **Airflow 1.10.10** we also keep a set of "known-to-be-working" requirement files in the `requirements` folder. Those "known-to-be-working" -requirements are per major/minor python version (3.6/3.7). You can use them as constraint files +requirements are per major/minor python version (3.6/3.7/3.8). You can use them as constraint files when installing Airflow from PyPI. Note that you have to specify correct Airflow version and python versions in the URL. diff --git a/breeze b/breeze index 16826bc..4321c49 100755 --- a/breeze +++ b/breeze @@ -68,7 +68,7 @@ function setup_default_breeze_variables() { # We have different versions of images depending on the python version used. We keep up with the # Latest patch-level changes in Python (this is done automatically during CI builds) so we have # To only take into account MAJOR and MINOR version of python. This variable keeps the major/minor - # version of python in X.Y format (2.7, 3.5, 3.6, 3.7 etc). + # version of python in X.Y format (2.7, 3.5, 3.6, 3.7, 3.8 etc). export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(read_from_file PYTHON_MAJOR_MINOR_VERSION)}" # When we generate documentation for README files, we want to force the width of terminal so that @@ -197,7 +197,7 @@ function initialize_virtualenv() { if [[ ${SYSTEM} == "Darwin" ]]; then echo " brew install sqlite mysql postgresql openssl" else - echo " sudo apt install build-essentials python3.6-dev python3.7-dev python-dev openssl \\" + echo " sudo apt install build-essentials python3.6-dev python3.7-dev python3.8-dev python-dev openssl \\" echo " sqlite sqlite-dev default-libmysqlclient-dev libmysqld-dev postgresql" fi echo diff --git a/breeze-complete b/breeze-complete index a3cc1eb..d9227a5 100644 --- a/breeze-complete +++ b/breeze-complete @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS="2.7 3.5 3.6 3.7" +_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS="2.7 3.5 3.6 3.7 3.8" _BREEZE_ALLOWED_BACKENDS="sqlite mysql postgres" _BREEZE_ALLOWED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis" _BREEZE_ALLOWED_KUBERNETES_MODES="image git" diff --git a/requirements/requirements-python2.7.txt b/requirements/requirements-python2.7.txt index 314c660..e1d8154 100644 --- a/requirements/requirements-python2.7.txt +++ b/requirements/requirements-python2.7.txt @@ -69,9 +69,9 @@ beautifulsoup4==4.7.1 billiard==3.6.3.0 bleach==3.1.5 blinker==1.4 -boto3==1.14.13 +boto3==1.14.14 boto==2.49.0 -botocore==1.17.13 +botocore==1.17.14 cached-property==1.5.1 cachetools==3.1.1 cassandra-driver==3.20.2 diff --git a/requirements/requirements-python3.5.txt b/requirements/requirements-python3.5.txt index 9cc708b..20c9b75 100644 --- a/requirements/requirements-python3.5.txt +++ b/requirements/requirements-python3.5.txt @@ -60,9 +60,9 @@ bcrypt==3.1.7 beautifulsoup4==4.7.1 billiard==3.6.3.0 blinker==1.4 -boto3==1.14.13 +boto3==1.14.14 boto==2.49.0 -botocore==1.17.13 +botocore==1.17.14 cached-property==1.5.1 cachetools==4.1.1 cassandra-driver==3.20.2 diff --git a/requirements/requirements-python3.5.txt b/requirements/requirements-python3.8.txt similarity index 92% copy from requirements/requirements-python3.5.txt copy to requirements/requirements-python3.8.txt index 9cc708b..56f161d 100644 --- a/requirements/requirements-python3.5.txt +++ b/requirements/requirements-python3.8.txt @@ -1,8 +1,8 @@ # Editable install with no version control (apache-airflow==1.10.10) Babel==2.8.0 Flask-Admin==1.5.4 -Flask-AppBuilder==1.13.1 -Flask-Babel==0.12.2 +Flask-AppBuilder==2.3.4 +Flask-Babel==1.0.0 Flask-Bcrypt==0.7.1 Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 @@ -24,6 +24,7 @@ PySmbClient==0.1.5 PyYAML==5.3.1 Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 +SQLAlchemy-Utils==0.36.6 SQLAlchemy==1.3.18 Sphinx==3.1.1 Unidecode==1.1.1 @@ -36,12 +37,12 @@ amqp==2.6.0 analytics-python==1.2.9 ansiwrap==0.8.4 apipkg==1.5 -apispec==3.3.1 +apispec==1.3.3 appdirs==1.4.4 argcomplete==1.11.1 asn1crypto==1.3.0 -aspy.yaml==1.3.0 astroid==2.4.2 +async-generator==1.10 atlasclient==1.0.0 attrs==19.3.0 aws-sam-translator==1.25.0 @@ -59,10 +60,11 @@ backcall==0.2.0 bcrypt==3.1.7 beautifulsoup4==4.7.1 billiard==3.6.3.0 +black==19.10b0 blinker==1.4 -boto3==1.14.13 +boto3==1.14.14 boto==2.49.0 -botocore==1.17.13 +botocore==1.17.14 cached-property==1.5.1 cachetools==4.1.1 cassandra-driver==3.20.2 @@ -70,7 +72,7 @@ cattrs==1.0.0 celery==4.4.6 certifi==2020.6.20 cffi==1.14.0 -cfgv==2.0.1 +cfgv==3.1.0 cfn-lint==0.33.2 cgroupspy==0.1.6 chardet==3.0.4 @@ -145,14 +147,12 @@ humanize==0.5.1 hvac==0.10.4 identify==1.4.20 idna==2.10 -ijson==2.6.1 imagesize==1.2.0 importlib-metadata==1.7.0 -importlib-resources==2.0.1 inflection==0.5.0 ipdb==0.13.3 ipython-genutils==0.2.0 -ipython==7.9.0 +ipython==7.16.1 iso8601==0.1.12 isodate==0.6.0 itsdangerous==1.1.0 @@ -174,10 +174,10 @@ lazy-object-proxy==1.5.0 ldap3==2.7 lockfile==0.12.2 marshmallow-enum==1.5.1 -marshmallow-sqlalchemy==0.18.0 -marshmallow==2.19.5 +marshmallow-sqlalchemy==0.23.1 +marshmallow==2.21.0 mccabe==0.6.1 -mock==3.0.5 +mock==4.0.2 mongomock==3.19.0 more-itertools==8.4.0 moto==1.3.14 @@ -188,23 +188,23 @@ mypy-extensions==0.4.3 mypy==0.720 mysqlclient==1.3.14 natsort==7.0.1 -nbclient==0.1.0 +nbclient==0.4.0 nbformat==5.0.7 +nest-asyncio==1.3.3 networkx==2.4 nodeenv==1.4.0 nteract-scrapbook==0.4.1 ntlm-auth==1.5.0 -numpy==1.18.5 +numpy==1.19.0 oauthlib==3.1.0 oscrypto==1.2.0 packaging==20.4 pandas-gbq==0.13.2 -pandas==0.25.3 -papermill==2.0.0 +pandas==1.0.5 +papermill==2.1.2 parameterized==0.7.4 paramiko==2.7.1 parso==0.7.0 -pathlib2==2.3.5 pathspec==0.8.0 pbr==5.4.5 pendulum==1.4.4 @@ -212,10 +212,10 @@ pexpect==4.8.0 pickleshare==0.7.5 pinotdb==0.1.1 pluggy==0.13.1 -pre-commit==1.21.0 +pre-commit==2.5.1 presto-python-client==0.7.0 -prison==0.1.0 -prompt-toolkit==2.0.10 +prison==0.1.3 +prompt-toolkit==3.0.5 protobuf==3.12.2 psutil==5.7.0 psycopg2-binary==2.8.5 @@ -233,7 +233,6 @@ pydruid==0.5.8 pyflakes==2.2.0 pykerberos==1.2.1 pymongo==3.10.1 -pymssql==2.1.4 pyparsing==2.4.7 pyrsistent==0.16.0 pysftp==0.2.9 @@ -259,6 +258,7 @@ pywinrm==0.4.1 pyzmq==19.0.1 qds-sdk==1.16.0 redis==3.5.3 +regex==2020.6.8 requests-futures==0.9.4 requests-kerberos==0.12.0 requests-mock==1.8.0 @@ -282,6 +282,7 @@ snowflake-sqlalchemy==1.2.3 soupsieve==2.0.1 sphinx-argparse==0.2.5 sphinx-autoapi==1.0.0 +sphinx-copybutton==0.2.12 sphinx-jinja==1.1.1 sphinx-rtd-theme==0.5.0 sphinxcontrib-applehelp==1.0.2 @@ -297,19 +298,16 @@ sshpubkeys==3.1.0 sshtunnel==0.1.5 tabulate==0.8.7 tenacity==4.12.0 -termcolor==1.1.0 text-unidecode==1.3 textwrap3==0.9.2 thrift-sasl==0.4.2 thrift==0.13.0 -tokenize-rt==3.2.0 toml==0.10.1 tornado==5.1.1 tqdm==4.47.0 traitlets==4.3.3 typed-ast==1.4.1 typing-extensions==3.7.4.2 -typing==3.7.4.1 tzlocal==1.5.1 unicodecsv==0.14.1 uritemplate==3.0.1 @@ -323,5 +321,5 @@ wrapt==1.12.1 xmltodict==0.12.0 yamllint==1.23.0 zdesk==2.7.1 -zipp==1.2.0 +zipp==3.1.0 zope.deprecation==4.4.0 diff --git a/requirements/setup-2.7.md5 b/requirements/setup-2.7.md5 index 02fe09a..7cb2b1c 100644 --- a/requirements/setup-2.7.md5 +++ b/requirements/setup-2.7.md5 @@ -1 +1 @@ -f86f24b093d55ecc490bb77e74b332eb /opt/airflow/setup.py +9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py diff --git a/requirements/setup-3.5.md5 b/requirements/setup-3.5.md5 index 02fe09a..7cb2b1c 100644 --- a/requirements/setup-3.5.md5 +++ b/requirements/setup-3.5.md5 @@ -1 +1 @@ -f86f24b093d55ecc490bb77e74b332eb /opt/airflow/setup.py +9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py diff --git a/requirements/setup-3.6.md5 b/requirements/setup-3.6.md5 index 02fe09a..7cb2b1c 100644 --- a/requirements/setup-3.6.md5 +++ b/requirements/setup-3.6.md5 @@ -1 +1 @@ -f86f24b093d55ecc490bb77e74b332eb /opt/airflow/setup.py +9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py diff --git a/requirements/setup-3.7.md5 b/requirements/setup-3.7.md5 index 02fe09a..7cb2b1c 100644 --- a/requirements/setup-3.7.md5 +++ b/requirements/setup-3.7.md5 @@ -1 +1 @@ -f86f24b093d55ecc490bb77e74b332eb /opt/airflow/setup.py +9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py diff --git a/requirements/setup-3.8.md5 b/requirements/setup-3.8.md5 new file mode 100644 index 0000000..7cb2b1c --- /dev/null +++ b/requirements/setup-3.8.md5 @@ -0,0 +1 @@ +9688bea64971365f2ab7771658b1fc80 /opt/airflow/setup.py diff --git a/setup.py b/setup.py index 3c6defc..fc14c3c 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ logger = logging.getLogger(__name__) version = imp.load_source('airflow.version', os.path.join('airflow', 'version.py')).version # type: ignore PY3 = sys.version_info[0] == 3 +PY38 = PY3 and sys.version_info[1] >= 8 my_dir = dirname(__file__) @@ -184,7 +185,7 @@ aws = [ 'boto3~=1.10', ] azure_blob_storage = [ - 'azure-storage>=0.34.0,<0.37.0', + 'azure-storage>=0.34.0, <0.37.0', 'azure-storage-blob<12.0', ] azure_container_instances = [ @@ -456,8 +457,10 @@ devel_all = (all_dbs + atlas + aws + # Snakebite is not Python 3 compatible :'( if PY3: - devel_all = [package for package in devel_all if package not in - ['snakebite>=2.7.8', 'snakebite[kerberos]>=2.7.8']] + package_to_excludes = ['snakebite>=2.7.8', 'snakebite[kerberos]>=2.7.8'] + if PY38: + package_to_excludes.extend(['pymssql~=2.1.1']) + devel_all = [package for package in devel_all if package not in package_to_excludes] devel_ci = devel_all else: devel_ci = devel_all + ['unittest2'] @@ -642,6 +645,7 @@ def do_setup(): 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: System :: Monitoring', ], author='Apache Software Foundation', diff --git a/tests/contrib/operators/test_mssql_to_gcs_operator.py b/tests/contrib/operators/test_mssql_to_gcs_operator.py index 19ef774..27d7bad 100644 --- a/tests/contrib/operators/test_mssql_to_gcs_operator.py +++ b/tests/contrib/operators/test_mssql_to_gcs_operator.py @@ -20,11 +20,13 @@ import sys import unittest -from airflow.contrib.operators.mssql_to_gcs import \ - MsSqlToGoogleCloudStorageOperator from tests.compat import mock PY3 = sys.version_info[0] == 3 +PY38 = PY3 and sys.version_info[1] >= 8 + +if not PY38: + from airflow.contrib.operators.mssql_to_gcs import MsSqlToGoogleCloudStorageOperator TASK_ID = 'test-mssql-to-gcs' MSSQL_CONN_ID = 'mssql_conn_test' @@ -54,6 +56,7 @@ SCHEMA_JSON = [ ] [email protected](PY38, "Mssql package not available when Python >= 3.8.") class MsSqlToGoogleCloudStorageOperatorTest(unittest.TestCase): def test_init(self):
