This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs-object-store.git
The following commit(s) were added to refs/heads/main by this push:
new 7e854ef fix(aws): recognize S3 dual-stack endpoints in URL parsing
(#780)
7e854ef is described below
commit 7e854ef3c728fd7dca4a705bf26141c9a40e3a55
Author: Andrew Duffy <[email protected]>
AuthorDate: Wed Jul 8 21:55:24 2026 +0100
fix(aws): recognize S3 dual-stack endpoints in URL parsing (#780)
`AmazonS3Builder::parse_url` matched S3 hostnames against a fixed set of
patterns via `splitn(4, '.')`. Dual-stack endpoints insert a `dualstack`
label after `s3` (e.g. `s3.dualstack.<region>.amazonaws.com` for path-style
and `<bucket>.s3.dualstack.<region>.amazonaws.com` for virtual-hosted),
which matched none of the patterns and fell through to `UrlNotRecognised`.
Because `ObjectStoreScheme::parse` already classifies these URLs as S3,
`DefaultObjectStoreRegistry::resolve` would call into the S3 builder and
then fail with `ObjectStore not found`.
Add handling for both dual-stack forms, setting the endpoint explicitly so
the `dualstack` label is preserved (IPv6 addressing) rather than being lost
when the endpoint is reconstructed from the region. Includes a registry
regression test plus builder and scheme-parse coverage.
Claude-Session: https://claude.ai/code/session_014n2xnhDiTk4T8RESdRDgDS
Signed-off-by: Andrew Duffy <[email protected]>
Co-authored-by: Claude <[email protected]>
---
src/aws/builder.rs | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
src/parse.rs | 8 ++++++++
2 files changed, 60 insertions(+)
diff --git a/src/aws/builder.rs b/src/aws/builder.rs
index b660a6c..b09c665 100644
--- a/src/aws/builder.rs
+++ b/src/aws/builder.rs
@@ -625,6 +625,8 @@ impl AmazonS3Builder {
/// - `s3a://<bucket>/<path>`
/// - `https://s3.<region>.amazonaws.com/<bucket>`
/// - `https://<bucket>.s3.<region>.amazonaws.com`
+ /// - `https://s3.dualstack.<region>.amazonaws.com/<bucket>`
+ /// - `https://<bucket>.s3.dualstack.<region>.amazonaws.com`
/// - `https://ACCOUNT_ID.r2.cloudflarestorage.com/bucket`
///
/// Note: Settings derived from the URL will override any others set on
this builder
@@ -807,10 +809,34 @@ impl AmazonS3Builder {
self.bucket_name = Some(bucket.into());
}
}
+ // Path-style dual-stack endpoint, e.g.
`s3.dualstack.<region>.amazonaws.com`.
+ Some(("s3", "dualstack", region, "amazonaws.com")) => {
+ self.region = Some(region.to_string());
+ // Override endpoint to explicitly preserve dualstack
address
+ self.endpoint =
Some(format!("https://s3.dualstack.{region}.amazonaws.com";));
+ let bucket =
parsed.path_segments().into_iter().flatten().next();
+ if let Some(bucket) = bucket {
+ self.bucket_name = Some(bucket.into());
+ }
+ }
Some((bucket, "s3", "amazonaws", "com")) => {
self.bucket_name = Some(bucket.to_string());
self.virtual_hosted_style_request = true.into();
}
+ // Virtual-hosted dual-stack endpoint, e.g.
+ // `<bucket>.s3.dualstack.<region>.amazonaws.com`
+ Some((bucket, "s3", "dualstack", rest)) => {
+ let region = rest
+ .strip_suffix(".amazonaws.com")
+ .ok_or_else(|| Error::UrlNotRecognised { url:
url.into() })?;
+ self.bucket_name = Some(bucket.to_string());
+ self.region = Some(region.to_string());
+ // Override endpoint to explicitly preserve dualstack
address
+ self.endpoint = Some(format!(
+ "https://{bucket}.s3.dualstack.{region}.amazonaws.com";
+ ));
+ self.virtual_hosted_style_request = true.into();
+ }
Some((bucket, "s3", region, "amazonaws.com")) => {
self.bucket_name = Some(bucket.to_string());
self.region = Some(region.to_string());
@@ -1782,6 +1808,32 @@ mod tests {
assert_eq!(builder.region, Some("region".to_string()));
assert!(builder.virtual_hosted_style_request.get().unwrap());
+ // Path-style dual-stack endpoint
+ let mut builder = AmazonS3Builder::new();
+ builder
+
.parse_url("https://s3.dualstack.us-east-1.amazonaws.com/bucket/path";)
+ .unwrap();
+ assert_eq!(builder.region, Some("us-east-1".to_string()));
+ assert_eq!(builder.bucket_name, Some("bucket".to_string()));
+ assert_eq!(
+ builder.endpoint,
+ Some("https://s3.dualstack.us-east-1.amazonaws.com".to_string())
+ );
+ assert!(!builder.virtual_hosted_style_request.get().unwrap());
+
+ // Virtual-hosted dual-stack endpoint
+ let mut builder = AmazonS3Builder::new();
+ builder
+ .parse_url("https://bucket.s3.dualstack.us-east-1.amazonaws.com";)
+ .unwrap();
+ assert_eq!(builder.bucket_name, Some("bucket".to_string()));
+ assert_eq!(builder.region, Some("us-east-1".to_string()));
+ assert_eq!(
+ builder.endpoint,
+
Some("https://bucket.s3.dualstack.us-east-1.amazonaws.com".to_string())
+ );
+ assert!(builder.virtual_hosted_style_request.get().unwrap());
+
let mut builder = AmazonS3Builder::new();
builder
.parse_url("https://account123.r2.cloudflarestorage.com/bucket-123";)
diff --git a/src/parse.rs b/src/parse.rs
index 2265547..218fe69 100644
--- a/src/parse.rs
+++ b/src/parse.rs
@@ -263,6 +263,14 @@ mod tests {
"https://bucket.s3.region.amazonaws.com";,
(ObjectStoreScheme::AmazonS3, ""),
),
+ (
+ "https://s3.dualstack.us-east-1.amazonaws.com/bucket/path";,
+ (ObjectStoreScheme::AmazonS3, "path"),
+ ),
+ (
+ "https://bucket.s3.dualstack.us-east-1.amazonaws.com/path";,
+ (ObjectStoreScheme::AmazonS3, "path"),
+ ),
(
"https://ACCOUNT_ID.r2.cloudflarestorage.com/bucket";,
(ObjectStoreScheme::AmazonS3, ""),
