This is an automated email from the ASF dual-hosted git repository.
honahx pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 8968996f Add azure configuration variables (#745)
8968996f is described below
commit 8968996fbde6e7892e0b576837acb335a52d9caf
Author: Kev Wang <[email protected]>
AuthorDate: Mon May 27 22:47:46 2024 -0700
Add azure configuration variables (#745)
---
pyiceberg/io/__init__.py | 7 +++++++
pyiceberg/io/fsspec.py | 21 ++++++++++++++-------
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py
index 4b5e99d3..1a78f306 100644
--- a/pyiceberg/io/__init__.py
+++ b/pyiceberg/io/__init__.py
@@ -57,6 +57,13 @@ HDFS_HOST = "hdfs.host"
HDFS_PORT = "hdfs.port"
HDFS_USER = "hdfs.user"
HDFS_KERB_TICKET = "hdfs.kerberos_ticket"
+ADLFS_CONNECTION_STRING = "adlfs.connection-string"
+ADLFS_ACCOUNT_NAME = "adlfs.account-name"
+ADLFS_ACCOUNT_KEY = "adlfs.account-key"
+ADLFS_SAS_TOKEN = "adlfs.sas-token"
+ADLFS_TENANT_ID = "adlfs.tenant-id"
+ADLFS_CLIENT_ID = "adlfs.client-id"
+ADLFS_ClIENT_SECRET = "adlfs.client-secret"
GCS_TOKEN = "gcs.oauth2.token"
GCS_TOKEN_EXPIRES_AT_MS = "gcs.oauth2.token-expires-at"
GCS_PROJECT_ID = "gcs.project-id"
diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py
index ee97829c..1089c9fe 100644
--- a/pyiceberg/io/fsspec.py
+++ b/pyiceberg/io/fsspec.py
@@ -40,6 +40,12 @@ from requests import HTTPError
from pyiceberg.catalog import TOKEN
from pyiceberg.exceptions import SignError
from pyiceberg.io import (
+ ADLFS_ACCOUNT_KEY,
+ ADLFS_ACCOUNT_NAME,
+ ADLFS_CLIENT_ID,
+ ADLFS_CONNECTION_STRING,
+ ADLFS_SAS_TOKEN,
+ ADLFS_TENANT_ID,
GCS_ACCESS,
GCS_CACHE_TIMEOUT,
GCS_CONSISTENCY,
@@ -57,6 +63,7 @@ from pyiceberg.io import (
S3_REGION,
S3_SECRET_ACCESS_KEY,
S3_SESSION_TOKEN,
+ ADLFS_ClIENT_SECRET,
FileIO,
InputFile,
InputStream,
@@ -163,13 +170,13 @@ def _adlfs(properties: Properties) -> AbstractFileSystem:
from adlfs import AzureBlobFileSystem
return AzureBlobFileSystem(
- connection_string=properties.get("adlfs.connection-string"),
- account_name=properties.get("adlfs.account-name"),
- account_key=properties.get("adlfs.account-key"),
- sas_token=properties.get("adlfs.sas-token"),
- tenant_id=properties.get("adlfs.tenant-id"),
- client_id=properties.get("adlfs.client-id"),
- client_secret=properties.get("adlfs.client-secret"),
+ connection_string=properties.get(ADLFS_CONNECTION_STRING),
+ account_name=properties.get(ADLFS_ACCOUNT_NAME),
+ account_key=properties.get(ADLFS_ACCOUNT_KEY),
+ sas_token=properties.get(ADLFS_SAS_TOKEN),
+ tenant_id=properties.get(ADLFS_TENANT_ID),
+ client_id=properties.get(ADLFS_CLIENT_ID),
+ client_secret=properties.get(ADLFS_ClIENT_SECRET),
)
