This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new e497228 Updates 3.6 limits for latest versions of a few libraries
(#15209)
e497228 is described below
commit e49722859b81cfcdd7e4bb8e8aba4efb049a8590
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Apr 5 20:25:11 2021 +0200
Updates 3.6 limits for latest versions of a few libraries (#15209)
This PR sets Pythong 3.6 specific limits for some of the packages
that recently dropped support for Python 3.6 binary packages
released via PyPI. Even if those packages did not drop the
Python 3.6 support entirely, it gets more and more difficult to
get those packages installed (both locally and in the Docker image)
because the require the packages to be compiled and they often
require a number of external dependencies to do so.
This makes it difficult to automatically upgrade dependencies,
because such upgrade fails for Python 3.6 images if we attempt
to do so.
This PR limits several of those dependencies (dask/pandas/numpy)
to not use the lates major releases for those packages but limits
them to the latest released versions.
Also comment/clarification was added to recently (#15114) added limit
for `pandas-gbq`. This limit has been added because of broken
import for bigquery provider, but the comment about it was missing
so the comment is added now.
---
airflow/models/baseoperator.py | 2 +-
airflow/providers/google/cloud/operators/dataflow.py | 6 +++---
setup.cfg | 7 ++++++-
setup.py | 9 ++++++++-
4 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index 1d595c7..faf7a13 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -1523,7 +1523,7 @@ def cross_downstream(
class BaseOperatorLink(metaclass=ABCMeta):
"""Abstract base class that defines how we get an operator link."""
- operators: ClassVar[List[Type[BaseOperator]]] = []
+ operators: ClassVar[List[Type[BaseOperator]]] = [] # pylint:
disable=invalid-name
"""
This property will be used by Airflow Plugins to find the Operators to
which you want
to assign this Operator Link
diff --git a/airflow/providers/google/cloud/operators/dataflow.py
b/airflow/providers/google/cloud/operators/dataflow.py
index 4b4a898..81ee281 100644
--- a/airflow/providers/google/cloud/operators/dataflow.py
+++ b/airflow/providers/google/cloud/operators/dataflow.py
@@ -43,9 +43,9 @@ class CheckJobRunning(Enum):
WaitForRun - wait for job to finish and then continue with new job
"""
- IgnoreJob = 1
- FinishIfRunning = 2
- WaitForRun = 3
+ IgnoreJob = 1 # pylint: disable=invalid-name
+ FinishIfRunning = 2 # pylint: disable=invalid-name
+ WaitForRun = 3 # pylint: disable=invalid-name
class DataflowConfiguration:
diff --git a/setup.cfg b/setup.cfg
index 9c051af..1dcdef9 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -110,7 +110,12 @@ install_requires =
markdown>=2.5.2, <4.0
markupsafe>=1.1.1, <2.0
marshmallow-oneofschema>=2.0.1
- pandas>=0.17.1, <2.0
+ # Numpy stopped releasing 3.6 binaries for 1.20.* series.
+ numpy<1.20;python_version<"3.7"
+ numpy;python_version>="3.7"
+ # Pandas stopped releasing 3.6 binaries for 1.2.* series.
+ pandas>=0.17.1, <1.2;python_version<"3.7"
+ pandas>=0.17.1, <2.0;python_version>="3.7"
pendulum~=2.0
pep562~=1.0;python_version<"3.7"
psutil>=4.2.0, <6.0.0
diff --git a/setup.py b/setup.py
index 84c394b..e1f1ebb 100644
--- a/setup.py
+++ b/setup.py
@@ -237,7 +237,12 @@ cgroups = [
cloudant = [
'cloudant>=2.0',
]
-dask = ['cloudpickle>=1.4.1, <1.5.0', 'distributed>=2.11.1, <2.20']
+dask = [
+ 'cloudpickle>=1.4.1, <1.5.0',
+ 'dask<2021.3.1;python_version>"3.7"', # dask stopped supporting python
3.6 in 2021.3.1 version
+ 'dask>=2.9.0;python_version>="3.7"',
+ 'distributed>=2.11.1, <2.20',
+]
databricks = [
'requests>=2.20.0, <3',
]
@@ -313,6 +318,8 @@ google = [
'google-cloud-workflows>=0.1.0,<2.0.0',
'grpcio-gcp>=0.2.2',
'json-merge-patch~=0.2',
+ # pandas-gbq 0.15.0 release broke google provider's bigquery import
+ # _check_google_client_version
(airflow/providers/google/cloud/hooks/bigquery.py:49)
'pandas-gbq<0.15.0',
'plyvel',
]
