This is an automated email from the ASF dual-hosted git repository.
dabla pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d6d425da028 Declare pyarrow for Parquet output of SqlToS3Operator
(#74691)
d6d425da028 is described below
commit d6d425da028455548c12a6848f2821f7bcc52012
Author: Dev-iL <[email protected]>
AuthorDate: Fri Oct 16 19:48:22 2026 +0300
Declare pyarrow for Parquet output of SqlToS3Operator (#74691)
Writing Parquet files and reading with the pyarrow dtype backend both need
pyarrow, but the provider never declared it. It only worked when another
installed package happened to bring pyarrow in, and otherwise failed at runtime
with pandas' "Unable to find a usable engine" error. A separate extra keeps the
roughly 40 MB dependency out of installs that only write CSV or JSON.
---
providers/amazon/docs/index.rst | 7 ++++---
providers/amazon/docs/transfer/sql_to_s3.rst | 4 ++++
providers/amazon/pyproject.toml | 6 ++++++
uv.lock | 11 +++++++++--
4 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/providers/amazon/docs/index.rst b/providers/amazon/docs/index.rst
index 2182278e29d..38a1fcb59c2 100644
--- a/providers/amazon/docs/index.rst
+++ b/providers/amazon/docs/index.rst
@@ -174,9 +174,9 @@ Install them when installing from PyPI. For example:
pip install apache-airflow-providers-amazon[aiobotocore]
-====================
============================================================================================================================================================
+====================
======================================================================================================================================================================
Extra Dependencies
-====================
============================================================================================================================================================
+====================
======================================================================================================================================================================
``aiobotocore`` ``aiobotocore>=3.0.0``
``cncf.kubernetes`` ``apache-airflow-providers-cncf-kubernetes>=7.2.0``
``s3fs`` ``s3fs>=2023.10.0``
@@ -191,6 +191,7 @@ Extra Dependencies
``microsoft.azure`` ``apache-airflow-providers-microsoft-azure``
``mongo`` ``apache-airflow-providers-mongo``
``pandas`` ``pandas>=2.1.2; python_version <"3.13"``,
``pandas>=2.2.3; python_version >="3.13" and python_version <"3.14"``,
``pandas>=2.3.3; python_version >="3.14"``
+``parquet`` ``pyarrow>=16.1.0; python_version < '3.13'``,
``pyarrow>=18.0.0; python_version >= '3.13' and python_version < '3.14'``,
``pyarrow>=22.0.0; python_version >= '3.14'``
``openlineage`` ``apache-airflow-providers-openlineage>=2.3.0``
``salesforce`` ``apache-airflow-providers-salesforce``
``ssh`` ``apache-airflow-providers-ssh``
@@ -199,7 +200,7 @@ Extra Dependencies
``common.messaging`` ``apache-airflow-providers-common-messaging>=2.0.0``
``duckdb`` ``apache-airflow-providers-duckdb>=0.1.0``
``sqlalchemy`` ``sqlalchemy>=1.4.54,!=2.1.0``
-====================
============================================================================================================================================================
+====================
======================================================================================================================================================================
Downloading official packages
-----------------------------
diff --git a/providers/amazon/docs/transfer/sql_to_s3.rst
b/providers/amazon/docs/transfer/sql_to_s3.rst
index 8abb98b44ac..9aa9db59384 100644
--- a/providers/amazon/docs/transfer/sql_to_s3.rst
+++ b/providers/amazon/docs/transfer/sql_to_s3.rst
@@ -24,6 +24,10 @@ Use ``SqlToS3Operator`` to copy data from a SQL server to an
Amazon Simple Stora
converts the SQL result to `pandas dataframe
<https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html>`__
(e.g. MySQL, Hive, ...).
+With the default ``df_type="pandas"`` the operator needs the ``pandas`` extra.
Writing Parquet files
+(``file_format="parquet"``) or reading with ``read_kwargs={"dtype_backend":
"pyarrow"}`` also needs
+pyarrow, which the ``parquet`` extra installs:
``apache-airflow-providers-amazon[pandas,parquet]``.
+
Prerequisite Tasks
------------------
diff --git a/providers/amazon/pyproject.toml b/providers/amazon/pyproject.toml
index f0317b9389b..cc144b0bb40 100644
--- a/providers/amazon/pyproject.toml
+++ b/providers/amazon/pyproject.toml
@@ -136,6 +136,11 @@ dependencies = [
'pandas>=2.2.3; python_version >="3.13" and python_version <"3.14"',
'pandas>=2.3.3; python_version >="3.14"',
]
+"parquet" = [
+ "pyarrow>=16.1.0; python_version < '3.13'",
+ "pyarrow>=18.0.0; python_version >= '3.13' and python_version < '3.14'",
+ "pyarrow>=22.0.0; python_version >= '3.14'",
+]
"openlineage" = [
"apache-airflow-providers-openlineage>=2.3.0"
]
@@ -201,6 +206,7 @@ dev = [
"responses>=0.25.0",
"apache-airflow-providers-common-sql[pandas,polars]",
"apache-airflow-providers-amazon[sqlalchemy]",
+ "apache-airflow-providers-amazon[parquet]",
"apache-airflow-providers-duckdb"
]
diff --git a/uv.lock b/uv.lock
index 545f7077260..068d8c7bb20 100644
--- a/uv.lock
+++ b/uv.lock
@@ -3126,6 +3126,9 @@ openlineage = [
pandas = [
{ name = "pandas" },
]
+parquet = [
+ { name = "pyarrow" },
+]
python3-saml = [
{ name = "lxml", marker = "python_full_version < '3.13'" },
{ name = "python3-saml", marker = "python_full_version < '3.13'" },
@@ -3151,7 +3154,7 @@ standard = [
dev = [
{ name = "apache-airflow" },
{ name = "apache-airflow-devel-common" },
- { name = "apache-airflow-providers-amazon", extra = ["aiobotocore",
"sqlalchemy"] },
+ { name = "apache-airflow-providers-amazon", extra = ["aiobotocore",
"parquet", "sqlalchemy"] },
{ name = "apache-airflow-providers-apache-hive" },
{ name = "apache-airflow-providers-cncf-kubernetes" },
{ name = "apache-airflow-providers-common-ai" },
@@ -3223,6 +3226,9 @@ requires-dist = [
{ name = "pandas", marker = "python_full_version == '3.13.*' and extra ==
'pandas'", specifier = ">=2.2.3" },
{ name = "pandas", marker = "python_full_version < '3.13' and extra ==
'pandas'", specifier = ">=2.1.2" },
{ name = "pandas", marker = "python_full_version >= '3.14' and extra ==
'pandas'", specifier = ">=2.3.3" },
+ { name = "pyarrow", marker = "python_full_version == '3.13.*' and extra ==
'parquet'", specifier = ">=18.0.0" },
+ { name = "pyarrow", marker = "python_full_version < '3.13' and extra ==
'parquet'", specifier = ">=16.1.0" },
+ { name = "pyarrow", marker = "python_full_version >= '3.14' and extra ==
'parquet'", specifier = ">=22.0.0" },
{ name = "pyathena", specifier = ">=3.10.0" },
{ name = "python3-saml", marker = "python_full_version < '3.13' and extra
== 'python3-saml'", specifier = ">=1.16.0" },
{ name = "redshift-connector", specifier = ">=2.1.3" },
@@ -3233,13 +3239,14 @@ requires-dist = [
{ name = "watchtower", specifier = ">=3.3.1,<4" },
{ name = "xmlsec", marker = "python_full_version < '3.13' and extra ==
'python3-saml'", specifier = ">=1.3.14" },
]
-provides-extras = ["aiobotocore", "cncf-kubernetes", "s3fs", "msk",
"python3-saml", "apache-hive", "exasol", "fab", "ftp", "google", "imap",
"microsoft-azure", "mongo", "pandas", "openlineage", "salesforce", "ssh",
"standard", "common-ai", "common-messaging", "duckdb", "sqlalchemy"]
+provides-extras = ["aiobotocore", "cncf-kubernetes", "s3fs", "msk",
"python3-saml", "apache-hive", "exasol", "fab", "ftp", "google", "imap",
"microsoft-azure", "mongo", "pandas", "parquet", "openlineage", "salesforce",
"ssh", "standard", "common-ai", "common-messaging", "duckdb", "sqlalchemy"]
[package.metadata.requires-dev]
dev = [
{ name = "apache-airflow", editable = "." },
{ name = "apache-airflow-devel-common", editable = "devel-common" },
{ name = "apache-airflow-providers-amazon", extras = ["aiobotocore"],
editable = "providers/amazon" },
+ { name = "apache-airflow-providers-amazon", extras = ["parquet"], editable
= "providers/amazon" },
{ name = "apache-airflow-providers-amazon", extras = ["sqlalchemy"],
editable = "providers/amazon" },
{ name = "apache-airflow-providers-apache-hive", editable =
"providers/apache/hive" },
{ name = "apache-airflow-providers-cncf-kubernetes", editable =
"providers/cncf/kubernetes" },
