This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 57cd0945db Allow opting out of request signing (#4927) (#4929)
57cd0945db is described below
commit 57cd0945db863059d30d31a890b692a6844038fd
Author: Raphael Taylor-Davies <[email protected]>
AuthorDate: Mon Oct 16 10:56:10 2023 +0100
Allow opting out of request signing (#4927) (#4929)
---
object_store/src/aws/client.rs | 24 ++++++++++++---------
object_store/src/aws/credential.rs | 21 +++++++++++-------
object_store/src/aws/mod.rs | 44 ++++++++++++++++++++++++++++++++++++++
3 files changed, 71 insertions(+), 18 deletions(-)
diff --git a/object_store/src/aws/client.rs b/object_store/src/aws/client.rs
index ac07f9ab9a..8a45a9f3ac 100644
--- a/object_store/src/aws/client.rs
+++ b/object_store/src/aws/client.rs
@@ -207,6 +207,7 @@ pub struct S3Config {
pub retry_config: RetryConfig,
pub client_options: ClientOptions,
pub sign_payload: bool,
+ pub skip_signature: bool,
pub checksum: Option<Checksum>,
pub copy_if_not_exists: Option<S3CopyIfNotExists>,
}
@@ -234,8 +235,11 @@ impl S3Client {
&self.config
}
- async fn get_credential(&self) -> Result<Arc<AwsCredential>> {
- self.config.credentials.get_credential().await
+ async fn get_credential(&self) -> Result<Option<Arc<AwsCredential>>> {
+ Ok(match self.config.skip_signature {
+ false => Some(self.config.credentials.get_credential().await?),
+ true => None,
+ })
}
/// Make an S3 PUT request
<https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html>
@@ -271,7 +275,7 @@ impl S3Client {
let response = builder
.query(query)
.with_aws_sigv4(
- credential.as_ref(),
+ credential.as_deref(),
&self.config.region,
"s3",
self.config.sign_payload,
@@ -299,7 +303,7 @@ impl S3Client {
.request(Method::DELETE, url)
.query(query)
.with_aws_sigv4(
- credential.as_ref(),
+ credential.as_deref(),
&self.config.region,
"s3",
self.config.sign_payload,
@@ -390,7 +394,7 @@ impl S3Client {
.header(CONTENT_TYPE, "application/xml")
.body(body)
.with_aws_sigv4(
- credential.as_ref(),
+ credential.as_deref(),
&self.config.region,
"s3",
self.config.sign_payload,
@@ -459,7 +463,7 @@ impl S3Client {
builder
.with_aws_sigv4(
- credential.as_ref(),
+ credential.as_deref(),
&self.config.region,
"s3",
self.config.sign_payload,
@@ -490,7 +494,7 @@ impl S3Client {
.client
.request(Method::POST, url)
.with_aws_sigv4(
- credential.as_ref(),
+ credential.as_deref(),
&self.config.region,
"s3",
self.config.sign_payload,
@@ -535,7 +539,7 @@ impl S3Client {
.query(&[("uploadId", upload_id)])
.body(body)
.with_aws_sigv4(
- credential.as_ref(),
+ credential.as_deref(),
&self.config.region,
"s3",
self.config.sign_payload,
@@ -567,7 +571,7 @@ impl GetClient for S3Client {
let response = builder
.with_get_options(options)
.with_aws_sigv4(
- credential.as_ref(),
+ credential.as_deref(),
&self.config.region,
"s3",
self.config.sign_payload,
@@ -621,7 +625,7 @@ impl ListClient for S3Client {
.request(Method::GET, &url)
.query(&query)
.with_aws_sigv4(
- credential.as_ref(),
+ credential.as_deref(),
&self.config.region,
"s3",
self.config.sign_payload,
diff --git a/object_store/src/aws/credential.rs
b/object_store/src/aws/credential.rs
index e27b71f7c4..e0c5de5fe7 100644
--- a/object_store/src/aws/credential.rs
+++ b/object_store/src/aws/credential.rs
@@ -291,7 +291,7 @@ pub trait CredentialExt {
/// Sign a request
<https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html>
fn with_aws_sigv4(
self,
- credential: &AwsCredential,
+ credential: Option<&AwsCredential>,
region: &str,
service: &str,
sign_payload: bool,
@@ -302,20 +302,25 @@ pub trait CredentialExt {
impl CredentialExt for RequestBuilder {
fn with_aws_sigv4(
self,
- credential: &AwsCredential,
+ credential: Option<&AwsCredential>,
region: &str,
service: &str,
sign_payload: bool,
payload_sha256: Option<&[u8]>,
) -> Self {
- let (client, request) = self.build_split();
- let mut request = request.expect("request valid");
+ match credential {
+ Some(credential) => {
+ let (client, request) = self.build_split();
+ let mut request = request.expect("request valid");
- AwsAuthorizer::new(credential, service, region)
- .with_sign_payload(sign_payload)
- .authorize(&mut request, payload_sha256);
+ AwsAuthorizer::new(credential, service, region)
+ .with_sign_payload(sign_payload)
+ .authorize(&mut request, payload_sha256);
- Self::from_parts(client, request)
+ Self::from_parts(client, request)
+ }
+ None => self,
+ }
}
}
diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs
index 285ee2f59d..70170a3cf4 100644
--- a/object_store/src/aws/mod.rs
+++ b/object_store/src/aws/mod.rs
@@ -448,6 +448,8 @@ pub struct AmazonS3Builder {
client_options: ClientOptions,
/// Credentials
credentials: Option<AwsCredentialProvider>,
+ /// Skip signing requests
+ skip_signature: ConfigValue<bool>,
/// Copy if not exists
copy_if_not_exists: Option<ConfigValue<S3CopyIfNotExists>>,
}
@@ -586,6 +588,9 @@ pub enum AmazonS3ConfigKey {
/// See [`S3CopyIfNotExists`]
CopyIfNotExists,
+ /// Skip signing request
+ SkipSignature,
+
/// Client options
Client(ClientConfigKey),
}
@@ -608,6 +613,7 @@ impl AsRef<str> for AmazonS3ConfigKey {
Self::ContainerCredentialsRelativeUri => {
"aws_container_credentials_relative_uri"
}
+ Self::SkipSignature => "aws_skip_signature",
Self::CopyIfNotExists => "copy_if_not_exists",
Self::Client(opt) => opt.as_ref(),
}
@@ -642,6 +648,7 @@ impl FromStr for AmazonS3ConfigKey {
"aws_container_credentials_relative_uri" => {
Ok(Self::ContainerCredentialsRelativeUri)
}
+ "aws_skip_signature" | "skip_signature" => Ok(Self::SkipSignature),
"copy_if_not_exists" => Ok(Self::CopyIfNotExists),
// Backwards compatibility
"aws_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)),
@@ -753,6 +760,7 @@ impl AmazonS3Builder {
AmazonS3ConfigKey::Client(key) => {
self.client_options = self.client_options.with_config(key,
value)
}
+ AmazonS3ConfigKey::SkipSignature =>
self.skip_signature.parse(value),
AmazonS3ConfigKey::CopyIfNotExists => {
self.copy_if_not_exists =
Some(ConfigValue::Deferred(value.into()))
}
@@ -823,6 +831,7 @@ impl AmazonS3Builder {
AmazonS3ConfigKey::ContainerCredentialsRelativeUri => {
self.container_credentials_relative_uri.clone()
}
+ AmazonS3ConfigKey::SkipSignature =>
Some(self.skip_signature.to_string()),
AmazonS3ConfigKey::CopyIfNotExists => {
self.copy_if_not_exists.as_ref().map(ToString::to_string)
}
@@ -977,6 +986,14 @@ impl AmazonS3Builder {
self
}
+ /// If enabled, [`AmazonS3`] will not fetch credentials and will not sign
requests
+ ///
+ /// This can be useful when interacting with public S3 buckets that deny
authorized requests
+ pub fn with_skip_signature(mut self, skip_signature: bool) -> Self {
+ self.skip_signature = skip_signature.into();
+ self
+ }
+
/// Sets the [checksum algorithm] which has to be used for object
integrity check during upload.
///
/// [checksum algorithm]:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
@@ -1146,6 +1163,7 @@ impl AmazonS3Builder {
retry_config: self.retry_config,
client_options: self.client_options,
sign_payload: !self.unsigned_payload.get()?,
+ skip_signature: self.skip_signature.get()?,
checksum,
copy_if_not_exists,
};
@@ -1505,4 +1523,30 @@ mod s3_resolve_bucket_region_tests {
assert!(result.is_err());
}
+
+ #[tokio::test]
+ #[ignore = "Tests shouldn't call use remote services by default"]
+ async fn test_disable_creds() {
+ // https://registry.opendata.aws/daylight-osm/
+ let v1 = AmazonS3Builder::new()
+ .with_bucket_name("daylight-map-distribution")
+ .with_region("us-west-1")
+ .with_access_key_id("local")
+ .with_secret_access_key("development")
+ .build()
+ .unwrap();
+
+ let prefix = Path::from("release");
+
+ v1.list_with_delimiter(Some(&prefix)).await.unwrap_err();
+
+ let v2 = AmazonS3Builder::new()
+ .with_bucket_name("daylight-map-distribution")
+ .with_region("us-west-1")
+ .with_skip_signature(true)
+ .build()
+ .unwrap();
+
+ v2.list_with_delimiter(Some(&prefix)).await.unwrap();
+ }
}
