This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs-object-store.git
The following commit(s) were added to refs/heads/main by this push:
new 7a7deea Do not explicitly add `Host` header when making requests with
S3Builder (#790)
7a7deea is described below
commit 7a7deeaee23ea35d6c3bcf3138c001836a6f44de
Author: Nick <[email protected]>
AuthorDate: Fri Jul 10 09:03:58 2026 -0400
Do not explicitly add `Host` header when making requests with S3Builder
(#790)
* fix: don't pin Host header when signing S3 requests
The SigV4 signer inserted an explicit `Host` header on the outgoing
request. reqwest carries that header across redirects, so when an
S3-compatible gateway answers GET with a cross-host 302 — e.g. the
Hugging Face gateway at `s3.hf.co` redirecting to a Xet CDN on
`cas-bridge.xethub.hf.co` — the stale `Host: s3.hf.co` header poisoned
the redirected request and the download failed with a connection error.
`list`/`head` (no redirect) worked, but every `get` failed.
Host must still be part of the SigV4 signature, so it is now added to a
throwaway copy of the headers used only to compute the signature; the
actual Host header is left to the transport (hyper), which derives it
from the request URL and regenerates it on redirect. The produced
signature is unchanged (existing signing tests still pass), so AWS S3 and
other providers are unaffected.
Addresses #706 (Hugging Face Storage Buckets).
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
* test: add throwaway HF S3 gateway verification harness
Adds `verification/hf-smoke`, the standalone harness used to verify the
Host-header
fix end-to-end against a real Hugging Face Storage Bucket (issue #706):
- exercises get / ranged get / head / list / list_with_delimiter / copy /
multipart /
conditional create / bulk delete, plus presigned-URL download (Signer
trait);
- has a read-only mode (get/head/list only) for Read-scoped tokens;
- `gather.py` (a uv PEP-723 script) collects + boto3-validates credentials;
- `repro.rs` isolates the reqwest cross-host-redirect behaviour;
- `live-results.sample.md` is an actual passing run against `s3.hf.co`.
This is committed only for reviewer inspection and is reverted by the next
commit;
it is not part of the object_store crate.
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
* Revert "test: add throwaway HF S3 gateway verification harness"
This reverts commit d5fa4550a4877c329b3ef94bab791f59a6c69e81.
* Update comment on why headers are handled in this specific way
Co-authored-by: Andrew Lamb <[email protected]>
* Clarify HOST mutation
* Cargo fmt
---------
Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]>
Co-authored-by: Andrew Lamb <[email protected]>
---
src/aws/credential.rs | 60 ++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 55 insertions(+), 5 deletions(-)
diff --git a/src/aws/credential.rs b/src/aws/credential.rs
index c16330a..7e2245e 100644
--- a/src/aws/credential.rs
+++ b/src/aws/credential.rs
@@ -28,7 +28,7 @@ use crate::{CredentialProvider, Result, RetryConfig};
use async_trait::async_trait;
use bytes::Buf;
use chrono::{DateTime, Utc};
-use http::header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue};
+use http::header::{AUTHORIZATION, HOST, HeaderMap, HeaderName, HeaderValue};
use http::{Method, StatusCode};
use percent_encoding::utf8_percent_encode;
use serde::Deserialize;
@@ -227,10 +227,6 @@ impl<'a> AwsAuthorizer<'a> {
request.headers_mut().insert(header, token_val);
}
- let host = &url[url::Position::BeforeHost..url::Position::AfterPort];
- let host_val = HeaderValue::from_str(host).unwrap();
- request.headers_mut().insert("host", host_val);
-
let date = self.date.unwrap_or_else(Utc::now);
let date_str = date.format("%Y%m%dT%H%M%SZ").to_string();
let date_val = HeaderValue::from_str(&date_str).unwrap();
@@ -262,7 +258,15 @@ impl<'a> AwsAuthorizer<'a> {
.insert(&REQUEST_PAYER_HEADER,
REQUEST_PAYER_HEADER_VALUE.clone());
}
+ // the SigV4 must include a value for `host`, but the actual Host
header may need to change
+ // due to a redirect. Therefore do not override the Host header for
the http
+ // request, let the client set it.
+ let host = &url[url::Position::BeforeHost..url::Position::AfterPort];
+ request
+ .headers_mut()
+ .insert(&HOST, HeaderValue::from_str(host).unwrap());
let (signed_headers, canonical_headers) =
canonicalize_headers(request.headers());
+ request.headers_mut().remove(&HOST);
let scope = self.scope(date);
@@ -974,6 +978,52 @@ mod tests {
)
}
+ /// `host` must be part of the SigV4 signature, but must NOT be pinned as
a header on the
+ /// request itself — the transport (hyper) sets Host from the URL, so a
pinned Host header
+ /// would be carried onto a cross-host redirect (e.g. an S3-compatible
gateway 302 to a CDN)
+ /// and break the redirected request.
+ #[cfg(feature = "reqwest")]
+ #[test]
+ fn test_host_is_signed_but_not_pinned_on_request() {
+ let client = HttpClient::new(Client::new());
+ let credential = AwsCredential {
+ key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
+ secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
+ token: None,
+ };
+ let date = DateTime::parse_from_rfc3339("2022-08-06T18:01:34Z")
+ .unwrap()
+ .with_timezone(&Utc);
+ let mut request = client
+ .request(Method::GET, "https://ec2.amazon.com/";)
+ .into_parts()
+ .1
+ .unwrap();
+ let signer = AwsAuthorizer {
+ date: Some(date),
+ crypto: None,
+ credential: &credential,
+ service: "ec2",
+ region: "us-east-1",
+ sign_payload: true,
+ token_header: None,
+ request_payer: false,
+ };
+
+ signer.try_authorize(&mut request, None).unwrap();
+
+ // host is signed ...
+ let auth = request
+ .headers()
+ .get(&AUTHORIZATION)
+ .unwrap()
+ .to_str()
+ .unwrap();
+ assert!(auth.contains("SignedHeaders=host;"), "auth was: {auth}");
+ // ... but not pinned as a request header (so it is regenerated across
redirects).
+ assert!(request.headers().get(&HOST).is_none());
+ }
+
#[cfg(feature = "reqwest")]
#[test]
fn test_sign_with_signed_payload_request_payer() {
