This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 20f6afdf Update `fsspec.py`to respect `s3.signer.uri property` (#741)
20f6afdf is described below
commit 20f6afdf5f000ea5b167e804012f2000aa5b8573
Author: Christian <[email protected]>
AuthorDate: Fri May 31 17:19:40 2024 +0200
Update `fsspec.py`to respect `s3.signer.uri property` (#741)
* Update fsspec.py to respect s3.signer.uri property
* Add S3_SIGNER_URI constant, add docs
---------
Co-authored-by: Fokko Driesprong <[email protected]>
---
mkdocs/docs/configuration.md | 1 +
pyiceberg/io/__init__.py | 1 +
pyiceberg/io/fsspec.py | 3 ++-
3 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md
index c0879b1d..f8a69119 100644
--- a/mkdocs/docs/configuration.md
+++ b/mkdocs/docs/configuration.md
@@ -89,6 +89,7 @@ For the FileIO there are several configuration options
available:
| s3.access-key-id | admin | Configure the static
secret access key used to access the FileIO.
|
| s3.secret-access-key | password | Configure the static
session token used to access the FileIO.
|
| s3.signer | bearer | Configure the signature
version of the FileIO.
|
+| s3.signer.uri | http://my.signer:8080/s3 | Configure the remote
signing uri if it differs from the catalog uri. Remote signing is only
implemented for `FsspecFileIO`. The final request is sent to
`<s3.singer.uri>/v1/aws/s3/sign`.
|
| s3.region | us-west-2 | Sets the region of the
bucket
|
| s3.proxy-uri | http://my.proxy.com:8080 | Configure the proxy server
to be used by the FileIO.
|
| s3.connect-timeout | 60.0 | Configure socket
connection timeout, in seconds.
|
diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py
index 9143cf66..36c3e625 100644
--- a/pyiceberg/io/__init__.py
+++ b/pyiceberg/io/__init__.py
@@ -53,6 +53,7 @@ S3_SESSION_TOKEN = "s3.session-token"
S3_REGION = "s3.region"
S3_PROXY_URI = "s3.proxy-uri"
S3_CONNECT_TIMEOUT = "s3.connect-timeout"
+S3_SIGNER_URI = "s3.signer.uri"
HDFS_HOST = "hdfs.host"
HDFS_PORT = "hdfs.port"
HDFS_USER = "hdfs.user"
diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py
index 1089c9fe..bb76f043 100644
--- a/pyiceberg/io/fsspec.py
+++ b/pyiceberg/io/fsspec.py
@@ -63,6 +63,7 @@ from pyiceberg.io import (
S3_REGION,
S3_SECRET_ACCESS_KEY,
S3_SESSION_TOKEN,
+ S3_SIGNER_URI,
ADLFS_ClIENT_SECRET,
FileIO,
InputFile,
@@ -79,7 +80,7 @@ def s3v4_rest_signer(properties: Properties, request:
AWSRequest, **_: Any) -> A
if TOKEN not in properties:
raise SignError("Signer set, but token is not available")
- signer_url = properties["uri"].rstrip("/")
+ signer_url = properties.get(S3_SIGNER_URI, properties["uri"]).rstrip("/")
signer_headers = {"Authorization": f"Bearer {properties[TOKEN]}"}
signer_body = {
"method": request.method,
