This is an automated email from the ASF dual-hosted git repository.
uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d9ae90fc64 Make pandas dependency optional for Amazon Provider (#28505)
d9ae90fc64 is described below
commit d9ae90fc6478133767e29774920ed797175146bc
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Dec 21 09:59:52 2022 +0100
Make pandas dependency optional for Amazon Provider (#28505)
---
airflow/providers/amazon/CHANGELOG.rst | 13 +++++++++++++
airflow/providers/amazon/aws/transfers/sql_to_s3.py | 8 +++++++-
airflow/providers/amazon/provider.yaml | 7 ++++++-
airflow/providers/apache/hive/hooks/hive.py | 8 +++++++-
generated/provider_dependencies.json | 1 -
5 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/airflow/providers/amazon/CHANGELOG.rst
b/airflow/providers/amazon/CHANGELOG.rst
index e8f7d9834a..c3b96a7c79 100644
--- a/airflow/providers/amazon/CHANGELOG.rst
+++ b/airflow/providers/amazon/CHANGELOG.rst
@@ -24,6 +24,19 @@
Changelog
---------
+7.0.0
+-----
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+TODO: add good description of Secrets Backend breaking changes as implemented
in
+https://github.com/apache/airflow/pull/27920
+
+Pandas is now an optional dependency of the provider. The ``SqlToS3Operator``
and ``HiveToDynamoDBOperator``
+require Pandas to be installed (you can install it automatically by adding
``[pandas]`` extra when installing
+the provider.
+
6.2.0
.....
diff --git a/airflow/providers/amazon/aws/transfers/sql_to_s3.py
b/airflow/providers/amazon/aws/transfers/sql_to_s3.py
index 713fbc0059..1ffc736ba3 100644
--- a/airflow/providers/amazon/aws/transfers/sql_to_s3.py
+++ b/airflow/providers/amazon/aws/transfers/sql_to_s3.py
@@ -23,7 +23,13 @@ from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Iterable, Mapping, Sequence
import numpy as np
-import pandas as pd
+
+try:
+ import pandas as pd
+except ImportError as e:
+ from airflow.exceptions import AirflowOptionalProviderFeatureException
+
+ raise AirflowOptionalProviderFeatureException(e)
from typing_extensions import Literal
from airflow.exceptions import AirflowException
diff --git a/airflow/providers/amazon/provider.yaml
b/airflow/providers/amazon/provider.yaml
index 230bc135ae..dc401b8884 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -22,6 +22,7 @@ description: |
Amazon integration (including `Amazon Web Services (AWS)
<https://aws.amazon.com/>`__).
versions:
+ - 7.0.0
- 6.2.0
- 6.1.0
- 6.0.0
@@ -59,7 +60,6 @@ dependencies:
- jsonpath_ng>=1.5.3
- redshift_connector>=2.0.888
- sqlalchemy_redshift>=0.8.6
- - pandas>=0.17.1
- mypy-boto3-rds>=1.24.0
- mypy-boto3-redshift-data>=1.24.0
- mypy-boto3-appflow>=1.24.0
@@ -576,3 +576,8 @@ secrets-backends:
logging:
- airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler
-
airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler
+
+additional-extras:
+ - name: pandas
+ dependencies:
+ - pandas>=0.17.1
diff --git a/airflow/providers/apache/hive/hooks/hive.py
b/airflow/providers/apache/hive/hooks/hive.py
index 8621098de4..d083dba952 100644
--- a/airflow/providers/apache/hive/hooks/hive.py
+++ b/airflow/providers/apache/hive/hooks/hive.py
@@ -28,7 +28,13 @@ from collections import OrderedDict
from tempfile import NamedTemporaryFile, TemporaryDirectory
from typing import Any, Iterable, Mapping
-import pandas
+try:
+ import pandas
+except ImportError as e:
+ from airflow.exceptions import AirflowOptionalProviderFeatureException
+
+ raise AirflowOptionalProviderFeatureException(e)
+
import unicodecsv as csv
from airflow.configuration import conf
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index ce549a6d07..5d538cfaf7 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -24,7 +24,6 @@
"mypy-boto3-appflow>=1.24.0",
"mypy-boto3-rds>=1.24.0",
"mypy-boto3-redshift-data>=1.24.0",
- "pandas>=0.17.1",
"redshift_connector>=2.0.888",
"sqlalchemy_redshift>=0.8.6",
"watchtower~=2.0.1"
