This is an automated email from the ASF dual-hosted git repository. ash pushed a commit to branch v2-0-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7ff9b8c4bbfe0376fcb9a8ecd3c70bbedb511b4e Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Apr 5 20:25:11 2021 +0200 Updates 3.6 limits for latest versions of a few libraries (#15209) This PR sets Pythong 3.6 specific limits for some of the packages that recently dropped support for Python 3.6 binary packages released via PyPI. Even if those packages did not drop the Python 3.6 support entirely, it gets more and more difficult to get those packages installed (both locally and in the Docker image) because the require the packages to be compiled and they often require a number of external dependencies to do so. This makes it difficult to automatically upgrade dependencies, because such upgrade fails for Python 3.6 images if we attempt to do so. This PR limits several of those dependencies (dask/pandas/numpy) to not use the lates major releases for those packages but limits them to the latest released versions. Also comment/clarification was added to recently (#15114) added limit for `pandas-gbq`. This limit has been added because of broken import for bigquery provider, but the comment about it was missing so the comment is added now. (cherry picked from commit e49722859b81cfcdd7e4bb8e8aba4efb049a8590) --- airflow/models/baseoperator.py | 2 +- airflow/providers/google/cloud/operators/dataflow.py | 6 +++--- setup.cfg | 7 ++++++- setup.py | 11 +++++++++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 06094a1..8bda785 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -1493,7 +1493,7 @@ def cross_downstream( class BaseOperatorLink(metaclass=ABCMeta): """Abstract base class that defines how we get an operator link.""" - operators: ClassVar[List[Type[BaseOperator]]] = [] + operators: ClassVar[List[Type[BaseOperator]]] = [] # pylint: disable=invalid-name """ This property will be used by Airflow Plugins to find the Operators to which you want to assign this Operator Link diff --git a/airflow/providers/google/cloud/operators/dataflow.py b/airflow/providers/google/cloud/operators/dataflow.py index 92ae77e..513fea3 100644 --- a/airflow/providers/google/cloud/operators/dataflow.py +++ b/airflow/providers/google/cloud/operators/dataflow.py @@ -43,9 +43,9 @@ class CheckJobRunning(Enum): WaitForRun - wait for job to finish and then continue with new job """ - IgnoreJob = 1 - FinishIfRunning = 2 - WaitForRun = 3 + IgnoreJob = 1 # pylint: disable=invalid-name + FinishIfRunning = 2 # pylint: disable=invalid-name + WaitForRun = 3 # pylint: disable=invalid-name class DataflowConfiguration: diff --git a/setup.cfg b/setup.cfg index fbb2276..ac103ba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -110,7 +110,12 @@ install_requires = markdown>=2.5.2, <4.0 markupsafe>=1.1.1, <2.0 marshmallow-oneofschema>=2.0.1 - pandas>=0.17.1, <2.0 + # Numpy stopped releasing 3.6 binaries for 1.20.* series. + numpy<1.20;python_version<"3.7" + numpy;python_version>="3.7" + # Pandas stopped releasing 3.6 binaries for 1.2.* series. + pandas>=0.17.1, <1.2;python_version<"3.7" + pandas>=0.17.1, <2.0;python_version>="3.7" pendulum~=2.0 pep562~=1.0;python_version<"3.7" psutil>=4.2.0, <6.0.0 diff --git a/setup.py b/setup.py index 0f421bc..51bd9be 100644 --- a/setup.py +++ b/setup.py @@ -237,7 +237,12 @@ cgroups = [ cloudant = [ 'cloudant>=2.0', ] -dask = ['cloudpickle>=1.4.1, <1.5.0', 'distributed>=2.11.1, <2.20'] +dask = [ + 'cloudpickle>=1.4.1, <1.5.0', + 'dask<2021.3.1;python_version>"3.7"', # dask stopped supporting python 3.6 in 2021.3.1 version + 'dask>=2.9.0;python_version>="3.7"', + 'distributed>=2.11.1, <2.20', +] databricks = [ 'requests>=2.20.0, <3', ] @@ -313,7 +318,9 @@ google = [ 'google-cloud-workflows>=0.1.0,<2.0.0', 'grpcio-gcp>=0.2.2', 'json-merge-patch~=0.2', - 'pandas-gbq', + # pandas-gbq 0.15.0 release broke google provider's bigquery import + # _check_google_client_version (airflow/providers/google/cloud/hooks/bigquery.py:49) + 'pandas-gbq<0.15.0', 'plyvel', ] grpc = [
