This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 12d4a9a3c FsspecFileIO: Resolve the ADLS account per location instead
of caching it in properties (#3886)
12d4a9a3c is described below
commit 12d4a9a3ced73da290626bde6019ac2835d5c636
Author: Kaanchan Krishna <[email protected]>
AuthorDate: Sat Oct 24 23:00:19 2026 +0000
FsspecFileIO: Resolve the ADLS account per location instead of caching it
in properties (#3886)
* FsspecFileIO: Resolve the ADLS account per location instead of caching it
in properties
_adls wrote the account name it inferred back into the properties dict it
was
given, and the caller passes the FileIO's own properties. The first location
therefore pinned the account for every later one, so a second storage
account
was served a filesystem built for the first. Resolve the account and the SAS
token into locals and leave the caller's dict alone. Precedence is
unchanged:
an explicit adls.account-name still wins, then a per account SAS token key,
then the hostname.
* Remove unnecessary comment
* Cover SAS token write-back in ADLS no-mutation test
Add a per-account SAS key so the test also guards against adls.sas-token
being written back into the FileIO properties.
Co-authored-by: Copilot App <[email protected]>
---------
Co-authored-by: Kevin Liu <[email protected]>
Co-authored-by: Copilot App <[email protected]>
---
pyiceberg/io/fsspec.py | 21 ++++++++++++---------
tests/io/test_fsspec.py | 31 +++++++++++++++++++++++++++++++
2 files changed, 43 insertions(+), 9 deletions(-)
diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py
index 09bbe6f1d..98275f25a 100644
--- a/pyiceberg/io/fsspec.py
+++ b/pyiceberg/io/fsspec.py
@@ -265,17 +265,20 @@ def _adls(properties: Properties, hostname: str | None =
None) -> AbstractFileSy
from azure.core.credentials import AccessToken
from azure.core.credentials_async import AsyncTokenCredential
- for key, sas_token in {
+ account_name = properties.get(ADLS_ACCOUNT_NAME)
+ sas_token = properties.get(ADLS_SAS_TOKEN)
+
+ for key, per_account_sas_token in {
key.replace(f"{ADLS_SAS_TOKEN}.", ""): value for key, value in
properties.items() if key.startswith(f"{ADLS_SAS_TOKEN}.")
}.items():
- if ADLS_ACCOUNT_NAME not in properties:
- properties[ADLS_ACCOUNT_NAME] = key.split(".")[0]
- if ADLS_SAS_TOKEN not in properties:
- properties[ADLS_SAS_TOKEN] = sas_token
+ if account_name is None:
+ account_name = key.split(".")[0]
+ if sas_token is None:
+ sas_token = per_account_sas_token
# Fallback: extract account_name from URI hostname (e.g.
"account.dfs.core.windows.net" -> "account")
- if hostname and ADLS_ACCOUNT_NAME not in properties:
- properties[ADLS_ACCOUNT_NAME] = hostname.split(".")[0]
+ if hostname and account_name is None:
+ account_name = hostname.split(".")[0]
class StaticTokenCredential(AsyncTokenCredential):
_DEFAULT_EXPIRY_SECONDS = 3600
@@ -298,9 +301,9 @@ def _adls(properties: Properties, hostname: str | None =
None) -> AbstractFileSy
return AzureBlobFileSystem(
connection_string=properties.get(ADLS_CONNECTION_STRING),
credential=credential,
- account_name=properties.get(ADLS_ACCOUNT_NAME),
+ account_name=account_name,
account_key=properties.get(ADLS_ACCOUNT_KEY),
- sas_token=properties.get(ADLS_SAS_TOKEN),
+ sas_token=sas_token,
tenant_id=properties.get(ADLS_TENANT_ID),
client_id=properties.get(ADLS_CLIENT_ID),
client_secret=properties.get(ADLS_CLIENT_SECRET),
diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py
index 45835a08e..7b9d87155 100644
--- a/tests/io/test_fsspec.py
+++ b/tests/io/test_fsspec.py
@@ -697,6 +697,37 @@ def test_adls_account_name_extracted_from_uri_hostname()
-> None:
)
+def test_adls_account_name_resolved_per_location() -> None:
+ """Two locations in different accounts must each get a filesystem for
their own account."""
+ session_properties: Properties = {"adls.tenant-id": "test-tenant-id"}
+
+ with mock.patch("adlfs.AzureBlobFileSystem") as mock_adlfs:
+ adls_fileio = FsspecFileIO(properties=session_properties)
+
+
adls_fileio.new_input(location="abfss://[email protected]/wh/t/a.parquet")
+
adls_fileio.new_input(location="abfss://[email protected]/wh/t/b.parquet")
+
+ account_names = [call.kwargs["account_name"] for call in
mock_adlfs.call_args_list]
+ assert account_names == ["accountone", "accounttwo"]
+
+
+def test_adls_does_not_mutate_properties() -> None:
+ """Resolving an account or SAS token from a location must not write it
back into the shared properties."""
+ original_properties: Properties = {
+ "adls.tenant-id": "test-tenant-id",
+ "adls.sas-token.accountone.dfs.core.windows.net": "test-sas-token",
+ }
+ session_properties: Properties = dict(original_properties)
+
+ with mock.patch("adlfs.AzureBlobFileSystem"):
+ adls_fileio = FsspecFileIO(properties=session_properties)
+
adls_fileio.new_input(location="abfss://[email protected]/wh/t/a.parquet")
+
+ assert "adls.account-name" not in adls_fileio.properties
+ assert "adls.sas-token" not in adls_fileio.properties
+ assert session_properties == original_properties
+
+
def test_adls_account_name_not_overridden_when_in_properties() -> None:
"""Test that explicit adls.account-name in properties is not overridden by
URI hostname."""
session_properties: Properties = {
