This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 4071d39f Build: Bump datafusion to 52.3.0 and pyiceberg-core to 0.9.0
(#3171)
4071d39f is described below
commit 4071d39f13fb8004d8d84827fca0d51d766ff114
Author: Kevin Liu <[email protected]>
AuthorDate: Fri Mar 20 18:10:08 2026 -0700
Build: Bump datafusion to 52.3.0 and pyiceberg-core to 0.9.0 (#3171)
<!--
Thanks for opening a pull request!
-->
<!-- In the case this PR will resolve an issue, please replace
${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
<!-- Closes #${GITHUB_ISSUE_ID} -->
# Rationale for this change
Upgrade deps
* datafusion 51 -> 52.3
* pyiceberg-core 0.8.0 -> 0.9.0
Update `__datafusion_table_provider__` to take in `session` param as per
[df 52 migration
guide](https://github.com/apache/datafusion-python/blob/85a3595444e7946dc4eaa166cb4843bee2bf2f07/docs/source/user-guide/upgrade-guides.rst#datafusion-5200)
Prevent `dependabot` from auto update `datafusion`, we do it manually
## Are these changes tested?
## Are there any user-facing changes?
<!-- In the case of user-facing changes, please add the changelog label.
-->
---------
Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot]
<49699333+dependabot[bot]@users.noreply.github.com>
---
.github/dependabot.yml | 2 ++
pyiceberg/table/__init__.py | 7 ++++---
pyproject.toml | 6 +++---
uv.lock | 37 ++++++++++++++++++-------------------
4 files changed, 27 insertions(+), 25 deletions(-)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 7f5b7403..4416fc89 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -24,6 +24,8 @@ updates:
schedule:
interval: "weekly"
open-pull-requests-limit: 50
+ ignore:
+ - dependency-name: "datafusion"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py
index 68089beb..bb8765b6 100644
--- a/pyiceberg/table/__init__.py
+++ b/pyiceberg/table/__init__.py
@@ -1559,7 +1559,7 @@ class Table:
return pl.scan_iceberg(self)
- def __datafusion_table_provider__(self) -> IcebergDataFusionTable:
+ def __datafusion_table_provider__(self, session: Any | None = None) ->
IcebergDataFusionTable:
"""Return the DataFusion table provider PyCapsule interface.
To support DataFusion features such as push down filtering, this
function will return a PyCapsule
@@ -1598,11 +1598,12 @@ class Table:
"""
from pyiceberg_core.datafusion import IcebergDataFusionTable
- return IcebergDataFusionTable(
+ provider = IcebergDataFusionTable(
identifier=self.name(),
metadata_location=self.metadata_location,
file_io_properties=self.io.properties,
- ).__datafusion_table_provider__()
+ ).__datafusion_table_provider__
+ return provider(session)
class StaticTable(Table):
diff --git a/pyproject.toml b/pyproject.toml
index 6eafa379..ad9bb7cb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -55,7 +55,7 @@ pyiceberg = "pyiceberg.cli.console:run"
[project.optional-dependencies]
pyarrow = [
"pyarrow>=17.0.0",
- "pyiceberg-core>=0.5.1,<0.9.0",
+ "pyiceberg-core>=0.5.1,<0.10.0",
]
pandas = [
"pandas>=1.0.0",
@@ -93,8 +93,8 @@ sql-sqlite = ["sqlalchemy>=2.0.18,<3"]
gcsfs = ["gcsfs>=2023.1.0"]
rest-sigv4 = ["boto3>=1.24.59"]
hf = ["huggingface-hub>=0.24.0"]
-pyiceberg-core = ["pyiceberg-core>=0.5.1,<0.9.0"]
-datafusion = ["datafusion>=51,<52"]
+pyiceberg-core = ["pyiceberg-core>=0.5.1,<0.10.0"]
+datafusion = ["datafusion>=52,<53"]
gcp-auth = ["google-auth>=2.4.0"]
entra-auth = ["azure-identity>=1.25.1"]
geoarrow = ["geoarrow-pyarrow>=0.2.0"]
diff --git a/uv.lock b/uv.lock
index f0b1c890..f01fc1c6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1192,19 +1192,19 @@ wheels = [
[[package]]
name = "datafusion"
-version = "51.0.0"
+version = "52.3.0"
source = { registry = "https://pypi.org/simple"; }
dependencies = [
{ name = "pyarrow" },
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
-sdist = { url =
"https://files.pythonhosted.org/packages/2c/6d/d0e2632c93bbcca0687eeda672af3f92042ecd349df7be55da86253594a9/datafusion-51.0.0.tar.gz";,
hash =
"sha256:1887c7d5ed3ae5d9f389e62ba869864afad4006a3f7c99ef0ca4707782a7838f", size
= 193751, upload-time = "2026-01-09T13:23:41.562Z" }
+sdist = { url =
"https://files.pythonhosted.org/packages/db/d4/a5ad7b665a80008901892fde61dc667318db0652a955d706ddca3a224b5a/datafusion-52.3.0.tar.gz";,
hash =
"sha256:2e8b02ad142b1a0d673f035d96a0944a640ac78275003d7e453cee4afe4a20a4", size
= 205026, upload-time = "2026-03-16T10:54:07.739Z" }
wheels = [
- { url =
"https://files.pythonhosted.org/packages/cf/a9/7717cec053a3309be3020fe3147e3f76e5bf21295fa8adf9b52dd44ea3ff/datafusion-51.0.0-cp310-abi3-macosx_10_12_x86_64.whl";,
hash =
"sha256:0c0d265fe3ee0dcbfa7cc3c64c7cd94fc493f38418bd79debb7ec29f29b7176e", size
= 30389413, upload-time = "2026-01-09T13:23:23.266Z" },
- { url =
"https://files.pythonhosted.org/packages/55/45/72c9874fd3740a4cb9d55049fdbae0df512dc5433e9f1176f3cfd970f1a1/datafusion-51.0.0-cp310-abi3-macosx_11_0_arm64.whl";,
hash =
"sha256:43e6011db86e950bf9a21ed73cc089c2346b340a41a4f1044268af6c3a357acc", size
= 26982206, upload-time = "2026-01-09T13:23:27.437Z" },
- { url =
"https://files.pythonhosted.org/packages/21/ac/b32ba1f25d38fc16e7623cc4bfb7bd68db61be2ef27b2d9969ea5c865765/datafusion-51.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl";,
hash =
"sha256:e76803907150159aa059d5cc9291645bbaac1b6a46d07e56035118d327b741ae", size
= 33246117, upload-time = "2026-01-09T13:23:30.981Z" },
- { url =
"https://files.pythonhosted.org/packages/0b/4e/437121422ef010690fc3cdd7f080203e986ba00e0e3c3b577e03f5b54ca2/datafusion-51.0.0-cp310-abi3-manylinux_2_28_aarch64.whl";,
hash =
"sha256:9d0cfabfe1853994adc2e6e9da5f36c1eb061102e34a2f1101fa935c6991c9e1", size
= 31421867, upload-time = "2026-01-09T13:23:34.436Z" },
- { url =
"https://files.pythonhosted.org/packages/db/fc/58cf27fcb85b2fd2a698253ae46213b1cbda784407e205c148f4006c1429/datafusion-51.0.0-cp310-abi3-win_amd64.whl";,
hash =
"sha256:fd5f9abfd6669062debf0658d13e4583234c89d4df95faf381927b11cea411f5", size
= 32517679, upload-time = "2026-01-09T13:23:39.615Z" },
+ { url =
"https://files.pythonhosted.org/packages/55/63/1bb0737988cefa77274b459d64fa4b57ba4cf755639a39733e9581b5d599/datafusion-52.3.0-cp310-abi3-macosx_10_12_x86_64.whl";,
hash =
"sha256:a73f02406b2985b9145dd97f8221a929c9ef3289a8ba64c6b52043e240938528", size
= 31503230, upload-time = "2026-03-16T10:53:50.312Z" },
+ { url =
"https://files.pythonhosted.org/packages/d6/e3/ea3b79239953c3044d19d8e9581015da025b6640796db03799e435b17910/datafusion-52.3.0-cp310-abi3-macosx_11_0_arm64.whl";,
hash =
"sha256:118a1f0add6a3f91fcbc90c71819fe08750e2981637d5e7b346e099e94a20b8b", size
= 28159497, upload-time = "2026-03-16T10:53:54.032Z" },
+ { url =
"https://files.pythonhosted.org/packages/24/c8/7d325feb4b7509ae03857fd7e164e95ec72e8c9f3dfd3178ec7f80d53977/datafusion-52.3.0-cp310-abi3-manylinux_2_28_aarch64.whl";,
hash =
"sha256:253ce7aee5fe84bd6ee290c20608114114bdb5115852617f97d3855d36ad9341", size
= 30769154, upload-time = "2026-03-16T10:53:57.835Z" },
+ { url =
"https://files.pythonhosted.org/packages/37/ee/478689c69b3cb1ccabb2d52feac0c181f6cdf20b51a81df35344b1dab9a6/datafusion-52.3.0-cp310-abi3-manylinux_2_28_x86_64.whl";,
hash =
"sha256:2af3469d2f06959bec88579ab107a72f965de18b32e607069bbdd0b859ed8dbb", size
= 33060335, upload-time = "2026-03-16T10:54:01.715Z" },
+ { url =
"https://files.pythonhosted.org/packages/1c/48/01906ab5c1a70373c6874ac5192d03646fa7b94d9ff06e3f676cb6b0f43f/datafusion-52.3.0-cp310-abi3-win_amd64.whl";,
hash =
"sha256:9fb35738cf4dbff672dbcfffc7332813024cb0ad2ab8cda1fb90b9054277ab0c", size
= 33765807, upload-time = "2026-03-16T10:54:05.728Z" },
]
[[package]]
@@ -4692,7 +4692,7 @@ requires-dist = [
{ name = "cachetools", specifier = ">=5.5,<8.0" },
{ name = "click", specifier = ">=7.1.1,<9.0.0" },
{ name = "daft", marker = "extra == 'daft'", specifier = ">=0.5.0" },
- { name = "datafusion", marker = "extra == 'datafusion'", specifier =
">=51,<52" },
+ { name = "datafusion", marker = "extra == 'datafusion'", specifier =
">=52,<53" },
{ name = "duckdb", marker = "extra == 'duckdb'", specifier =
">=0.5.0,<2.0.0" },
{ name = "fsspec", specifier = ">=2023.1.0" },
{ name = "gcsfs", marker = "extra == 'gcsfs'", specifier = ">=2023.1.0" },
@@ -4711,8 +4711,8 @@ requires-dist = [
{ name = "pyarrow", marker = "extra == 'pyarrow'", specifier = ">=17.0.0"
},
{ name = "pyarrow", marker = "extra == 'ray'", specifier = ">=17.0.0" },
{ name = "pydantic", specifier =
">=2.0,!=2.4.0,!=2.4.1,!=2.12.0,!=2.12.1,<3.0" },
- { name = "pyiceberg-core", marker = "extra == 'pyarrow'", specifier =
">=0.5.1,<0.9.0" },
- { name = "pyiceberg-core", marker = "extra == 'pyiceberg-core'", specifier
= ">=0.5.1,<0.9.0" },
+ { name = "pyiceberg-core", marker = "extra == 'pyarrow'", specifier =
">=0.5.1,<0.10.0" },
+ { name = "pyiceberg-core", marker = "extra == 'pyiceberg-core'", specifier
= ">=0.5.1,<0.10.0" },
{ name = "pyparsing", specifier = ">=3.1.0,<4.0.0" },
{ name = "pyroaring", specifier = ">=1.0.0,<2.0.0" },
{ name = "python-snappy", marker = "extra == 'snappy'", specifier =
">=0.6.0,<1.0.0" },
@@ -4771,17 +4771,16 @@ notebook = [{ name = "jupyterlab", specifier =
">=4.0.0" }]
[[package]]
name = "pyiceberg-core"
-version = "0.8.0"
+version = "0.9.0"
source = { registry = "https://pypi.org/simple"; }
-sdist = { url =
"https://files.pythonhosted.org/packages/9c/a0/0bcedbbe901484aacb6c605505f8574fd65954826e592fdb163e1cfb09f2/pyiceberg_core-0.8.0.tar.gz";,
hash =
"sha256:59021ca5bc7ca95f2b06fb0730280fb3f60ed898060bcd874c156d093853b5f3", size
= 618882, upload-time = "2026-01-20T00:50:40.076Z" }
+sdist = { url =
"https://files.pythonhosted.org/packages/ab/a3/2fd5953aa1cdc8dca3b498b4d7691db8431a1199df95a4e826d120860880/pyiceberg_core-0.9.0.tar.gz";,
hash =
"sha256:c8493b99160c269e175757f46fc0ff06bc79314106865ec3cc49da07099194f4", size
= 687481, upload-time = "2026-03-20T18:37:08.758Z" }
wheels = [
- { url =
"https://files.pythonhosted.org/packages/77/e0/9a8fa537d29d34e3265682056d6517b926975107b5b1af6057d1713557d6/pyiceberg_core-0.8.0-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl";,
hash =
"sha256:d60c75a741a1d9199277a9e50fc3adbc84ab286a881f9b1f721fa120e7197912", size
= 24733948, upload-time = "2026-01-20T00:50:20.566Z" },
- { url =
"https://files.pythonhosted.org/packages/6d/3e/f5522c1e9c20c3e89bfd76b2f54ba38e57389e5a2872233e49e60a131e04/pyiceberg_core-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl";,
hash =
"sha256:1d71e566b2d56141760ff8734667eede5a5d60963dfbcdce80c2dd3cf2edb39d", size
= 11682041, upload-time = "2026-01-20T00:50:22.843Z" },
- { url =
"https://files.pythonhosted.org/packages/95/4b/f799e5c7a2b2ede75514e64901503358a7a134ca1ea217fd86535af533b6/pyiceberg_core-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl";,
hash =
"sha256:82782d1b974200c5526d069391ba2bc235a868b5d0d6ac17ca406df735ab89a3", size
= 13835428, upload-time = "2026-01-20T00:50:25.021Z" },
- { url =
"https://files.pythonhosted.org/packages/0b/ff/2dbd6f7c99a2f782f908be2cc997371de45cc1df61abeeff1fc0165c05b6/pyiceberg_core-0.8.0-cp310-abi3-manylinux_2_28_aarch64.whl";,
hash =
"sha256:0e14b2aea26293ba5878c398adc880fff0f1ce5d989e00d4b1a930c143541114", size
= 14580807, upload-time = "2026-01-20T00:50:27.158Z" },
- { url =
"https://files.pythonhosted.org/packages/bc/13/176c2b00a9b804af79d8b697ba1a2525f4390e959be777076972071ca069/pyiceberg_core-0.8.0-cp310-abi3-win_amd64.whl";,
hash =
"sha256:a5726cc62f9ac2582a0d5dde92e4140b711b5e29ec0c6c636d6d2782d984031b", size
= 13354110, upload-time = "2026-01-20T00:50:29.785Z" },
- { url =
"https://files.pythonhosted.org/packages/43/33/2c93b4d40f38e173dcfb2f555a2d992e3345d38eba8f8fbb2731c2e18462/pyiceberg_core-0.8.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl";,
hash =
"sha256:0c51f1e6a4112864c8fface904429814b26baa45efab7efbae8013796e815915", size
= 11682427, upload-time = "2026-01-20T00:50:31.714Z" },
- { url =
"https://files.pythonhosted.org/packages/7c/97/25e835c2bf9b090bf97174638bc26a034952208e2767730035e285766bb6/pyiceberg_core-0.8.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl";,
hash =
"sha256:6d2e71fb83ae578d93f24699ee131b77e5ee630b8ec6310acf570d04062ef2fb", size
= 14581679, upload-time = "2026-01-20T00:50:33.869Z" },
+ { url =
"https://files.pythonhosted.org/packages/fa/2f/ccb605b8f09dc417dc01345f13273688efcd08e9a4c8297749fab20a1441/pyiceberg_core-0.9.0-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl";,
hash =
"sha256:c359af999247087119f5986c11420eaa41cc31d30de89cf59ddc3077c4498523", size
= 18447345, upload-time = "2026-03-20T18:36:46.952Z" },
+ { url =
"https://files.pythonhosted.org/packages/c7/c0/f1b0d969b8c29d76c2a5bce8bdc5cc7ad50d91f221f287b5a00f144475b0/pyiceberg_core-0.9.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl";,
hash =
"sha256:117e38359219d2834b949124027d0c9d82d42bb5b41e7475bd484bb1769703ea", size
= 8796400, upload-time = "2026-03-20T18:36:49.444Z" },
+ { url =
"https://files.pythonhosted.org/packages/c2/70/1d80a9f612748f8fdd62c5acddddac89b8c5834f44d10a522657e3b7b041/pyiceberg_core-0.9.0-cp310-abi3-manylinux_2_28_aarch64.whl";,
hash =
"sha256:125272db73593b4153d7a95f559ab1e5623b60564c5e45fb89cf638f4b57870a", size
= 10869079, upload-time = "2026-03-20T18:36:52.182Z" },
+ { url =
"https://files.pythonhosted.org/packages/46/30/5b65a2748ed6d96ca9fbc68a99eb9430127a0387393c92efb363fc709c98/pyiceberg_core-0.9.0-cp310-abi3-win_amd64.whl";,
hash =
"sha256:b0d126bdd901a7a3e9b42901a22be9c221b46a8e9931ee2da404508f891e03a2", size
= 9924305, upload-time = "2026-03-20T18:36:54.28Z" },
+ { url =
"https://files.pythonhosted.org/packages/c1/6e/9a2bdad1efc1aefe6bf9651eb2251ca28a09ed4309ce50d365abb3640e5a/pyiceberg_core-0.9.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl";,
hash =
"sha256:c67d9e4d2c5d831796fe51d63d7edb95fb4db4d6c05ff9c653873d52be8816cf", size
= 8796030, upload-time = "2026-03-20T18:36:58.98Z" },
+ { url =
"https://files.pythonhosted.org/packages/d3/12/59167a1ec7868eb06178bf91c16ffbc0f646ea8a09bef04b3b2433c5e290/pyiceberg_core-0.9.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl";,
hash =
"sha256:cad7fc1faac17455122d4abf6f7f83e818cbc66d20720a6c53395cd07e154370", size
= 10869219, upload-time = "2026-03-20T18:37:00.98Z" },
]
[[package]]
