This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new e7eb304da Add support for unsigned payloads in aws (#3741)
e7eb304da is described below
commit e7eb304dac442a943c434f8ea248de909f82aa88
Author: Satyam Singh <[email protected]>
AuthorDate: Tue Feb 28 04:30:55 2023 +0530
Add support for unsigned payloads in aws (#3741)
* Add support for unsigned payloads in aws
* Add unsigned payload to AmazonS3ConfigKey
* Link to aws doc
* Add env test
* Add test
* Add integration test
* Take boolean argument
* Fix doc
* Clippy fixes
* Merge into s3 test
---
object_store/src/aws/client.rs | 50 ++++++++++++++++++++++++++++-----
object_store/src/aws/credential.rs | 57 +++++++++++++++++++++++++++++++++++---
object_store/src/aws/mod.rs | 38 ++++++++++++++++++++++++-
3 files changed, 133 insertions(+), 12 deletions(-)
diff --git a/object_store/src/aws/client.rs b/object_store/src/aws/client.rs
index b40bcbacf..0b0f883b7 100644
--- a/object_store/src/aws/client.rs
+++ b/object_store/src/aws/client.rs
@@ -204,6 +204,7 @@ pub struct S3Config {
pub credentials: Box<dyn CredentialProvider>,
pub retry_config: RetryConfig,
pub client_options: ClientOptions,
+ pub sign_payload: bool,
}
impl S3Config {
@@ -256,7 +257,12 @@ impl S3Client {
}
let response = builder
- .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+ .with_aws_sigv4(
+ credential.as_ref(),
+ &self.config.region,
+ "s3",
+ self.config.sign_payload,
+ )
.send_retry(&self.config.retry_config)
.await
.context(GetRequestSnafu {
@@ -287,7 +293,12 @@ impl S3Client {
let response = builder
.query(query)
- .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+ .with_aws_sigv4(
+ credential.as_ref(),
+ &self.config.region,
+ "s3",
+ self.config.sign_payload,
+ )
.send_retry(&self.config.retry_config)
.await
.context(PutRequestSnafu {
@@ -309,7 +320,12 @@ impl S3Client {
self.client
.request(Method::DELETE, url)
.query(query)
- .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+ .with_aws_sigv4(
+ credential.as_ref(),
+ &self.config.region,
+ "s3",
+ self.config.sign_payload,
+ )
.send_retry(&self.config.retry_config)
.await
.context(DeleteRequestSnafu {
@@ -328,7 +344,12 @@ impl S3Client {
self.client
.request(Method::PUT, url)
.header("x-amz-copy-source", source)
- .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+ .with_aws_sigv4(
+ credential.as_ref(),
+ &self.config.region,
+ "s3",
+ self.config.sign_payload,
+ )
.send_retry(&self.config.retry_config)
.await
.context(CopyRequestSnafu {
@@ -369,7 +390,12 @@ impl S3Client {
.client
.request(Method::GET, &url)
.query(&query)
- .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+ .with_aws_sigv4(
+ credential.as_ref(),
+ &self.config.region,
+ "s3",
+ self.config.sign_payload,
+ )
.send_retry(&self.config.retry_config)
.await
.context(ListRequestSnafu)?
@@ -407,7 +433,12 @@ impl S3Client {
let response = self
.client
.request(Method::POST, url)
- .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+ .with_aws_sigv4(
+ credential.as_ref(),
+ &self.config.region,
+ "s3",
+ self.config.sign_payload,
+ )
.send_retry(&self.config.retry_config)
.await
.context(CreateMultipartRequestSnafu)?
@@ -446,7 +477,12 @@ impl S3Client {
.request(Method::POST, url)
.query(&[("uploadId", upload_id)])
.body(body)
- .with_aws_sigv4(credential.as_ref(), &self.config.region, "s3")
+ .with_aws_sigv4(
+ credential.as_ref(),
+ &self.config.region,
+ "s3",
+ self.config.sign_payload,
+ )
.send_retry(&self.config.retry_config)
.await
.context(CompleteMultipartRequestSnafu)?;
diff --git a/object_store/src/aws/credential.rs
b/object_store/src/aws/credential.rs
index e2332d0fa..05f2c535b 100644
--- a/object_store/src/aws/credential.rs
+++ b/object_store/src/aws/credential.rs
@@ -39,6 +39,7 @@ type StdError = Box<dyn std::error::Error + Send + Sync>;
/// SHA256 hash of empty string
static EMPTY_SHA256_HASH: &str =
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+static UNSIGNED_PAYLOAD_LITERAL: &str = "UNSIGNED-PAYLOAD";
#[derive(Debug)]
pub struct AwsCredential {
@@ -72,6 +73,7 @@ struct RequestSigner<'a> {
credential: &'a AwsCredential,
service: &'a str,
region: &'a str,
+ sign_payload: bool,
}
const DATE_HEADER: &str = "x-amz-date";
@@ -98,9 +100,13 @@ impl<'a> RequestSigner<'a> {
let date_val = HeaderValue::from_str(&date_str).unwrap();
request.headers_mut().insert(DATE_HEADER, date_val);
- let digest = match request.body() {
- None => EMPTY_SHA256_HASH.to_string(),
- Some(body) => hex_digest(body.as_bytes().unwrap()),
+ let digest = if self.sign_payload {
+ match request.body() {
+ None => EMPTY_SHA256_HASH.to_string(),
+ Some(body) => hex_digest(body.as_bytes().unwrap()),
+ }
+ } else {
+ UNSIGNED_PAYLOAD_LITERAL.to_string()
};
let header_digest = HeaderValue::from_str(&digest).unwrap();
@@ -158,6 +164,7 @@ pub trait CredentialExt {
credential: &AwsCredential,
region: &str,
service: &str,
+ sign_payload: bool,
) -> Self;
}
@@ -167,6 +174,7 @@ impl CredentialExt for RequestBuilder {
credential: &AwsCredential,
region: &str,
service: &str,
+ sign_payload: bool,
) -> Self {
// Hack around lack of access to underlying request
// https://github.com/seanmonstar/reqwest/issues/1212
@@ -182,6 +190,7 @@ impl CredentialExt for RequestBuilder {
credential,
service,
region,
+ sign_payload,
};
signer.sign(&mut request);
@@ -585,7 +594,7 @@ mod tests {
// Test generated using
https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
#[test]
- fn test_sign() {
+ fn test_sign_with_signed_payload() {
let client = Client::new();
// Test credentials from
https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html
@@ -615,12 +624,51 @@ mod tests {
credential: &credential,
service: "ec2",
region: "us-east-1",
+ sign_payload: true,
};
signer.sign(&mut request);
assert_eq!(request.headers().get(AUTH_HEADER).unwrap(),
"AWS4-HMAC-SHA256
Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request,
SignedHeaders=host;x-amz-content-sha256;x-amz-date,
Signature=a3c787a7ed37f7fdfbfd2d7056a3d7c9d85e6d52a2bfbec73793c0be6e7862d4")
}
+ #[test]
+ fn test_sign_with_unsigned_payload() {
+ let client = Client::new();
+
+ // Test credentials from
https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html
+ let credential = AwsCredential {
+ key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
+ secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
+ token: None,
+ };
+
+ // method = 'GET'
+ // service = 'ec2'
+ // host = 'ec2.amazonaws.com'
+ // region = 'us-east-1'
+ // endpoint = 'https://ec2.amazonaws.com'
+ // request_parameters = ''
+ let date = DateTime::parse_from_rfc3339("2022-08-06T18:01:34Z")
+ .unwrap()
+ .with_timezone(&Utc);
+
+ let mut request = client
+ .request(Method::GET, "https://ec2.amazon.com/";)
+ .build()
+ .unwrap();
+
+ let signer = RequestSigner {
+ date,
+ credential: &credential,
+ service: "ec2",
+ region: "us-east-1",
+ sign_payload: false,
+ };
+
+ signer.sign(&mut request);
+ assert_eq!(request.headers().get(AUTH_HEADER).unwrap(),
"AWS4-HMAC-SHA256
Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request,
SignedHeaders=host;x-amz-content-sha256;x-amz-date,
Signature=653c3d8ea261fd826207df58bc2bb69fbb5003e9eb3c0ef06e4a51f2a81d8699")
+ }
+
#[test]
fn test_sign_port() {
let client = Client::new();
@@ -651,6 +699,7 @@ mod tests {
credential: &credential,
service: "s3",
region: "us-east-1",
+ sign_payload: true,
};
signer.sign(&mut request);
diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs
index a1c9eae84..c724886cf 100644
--- a/object_store/src/aws/mod.rs
+++ b/object_store/src/aws/mod.rs
@@ -385,6 +385,7 @@ pub struct AmazonS3Builder {
retry_config: RetryConfig,
imdsv1_fallback: bool,
virtual_hosted_style_request: bool,
+ unsigned_payload: bool,
metadata_endpoint: Option<String>,
profile: Option<String>,
client_options: ClientOptions,
@@ -504,6 +505,15 @@ pub enum AmazonS3ConfigKey {
/// - `virtual_hosted_style_request`
VirtualHostedStyleRequest,
+ /// Avoid computing payload checksum when calculating signature.
+ ///
+ /// See [`AmazonS3Builder::with_unsigned_payload`] for details.
+ ///
+ /// Supported keys:
+ /// - `aws_unsigned_payload`
+ /// - `unsigned_payload`
+ UnsignedPayload,
+
/// Set the instance metadata endpoint
///
/// See [`AmazonS3Builder::with_metadata_endpoint`] for details.
@@ -535,6 +545,7 @@ impl AsRef<str> for AmazonS3ConfigKey {
Self::DefaultRegion => "aws_default_region",
Self::MetadataEndpoint => "aws_metadata_endpoint",
Self::Profile => "aws_profile",
+ Self::UnsignedPayload => "aws_unsigned_payload",
}
}
}
@@ -563,6 +574,7 @@ impl FromStr for AmazonS3ConfigKey {
"aws_profile" | "profile" => Ok(Self::Profile),
"aws_imdsv1_fallback" | "imdsv1_fallback" =>
Ok(Self::ImdsV1Fallback),
"aws_metadata_endpoint" | "metadata_endpoint" =>
Ok(Self::MetadataEndpoint),
+ "aws_unsigned_payload" | "unsigned_payload" =>
Ok(Self::UnsignedPayload),
_ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
}
}
@@ -679,6 +691,9 @@ impl AmazonS3Builder {
self.metadata_endpoint = Some(value.into())
}
AmazonS3ConfigKey::Profile => self.profile = Some(value.into()),
+ AmazonS3ConfigKey::UnsignedPayload => {
+ self.unsigned_payload = str_is_truthy(&value.into())
+ }
};
Ok(self)
}
@@ -822,6 +837,15 @@ impl AmazonS3Builder {
self
}
+ /// Sets if unsigned payload option has to be used.
+ /// See [unsigned payload
option](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html)
+ /// * false (default): Signed payload option is used, where the checksum
for the request body is computed and included when constructing a canonical
request.
+ /// * true: Unsigned payload option is used. `UNSIGNED-PAYLOAD` literal is
included when constructing a canonical request,
+ pub fn with_unsigned_payload(mut self, unsigned_payload: bool) -> Self {
+ self.unsigned_payload = unsigned_payload;
+ self
+ }
+
/// Set the [instance metadata
endpoint](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html),
/// used primarily within AWS EC2.
///
@@ -967,6 +991,7 @@ impl AmazonS3Builder {
credentials,
retry_config: self.retry_config,
client_options: self.client_options,
+ sign_payload: !self.unsigned_payload,
};
let client = Arc::new(S3Client::new(config)?);
@@ -1125,6 +1150,7 @@ mod tests {
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
&container_creds_relative_uri,
);
+ env::set_var("AWS_UNSIGNED_PAYLOAD", "true");
let builder = AmazonS3Builder::from_env();
assert_eq!(builder.access_key_id.unwrap(), aws_access_key_id.as_str());
@@ -1136,9 +1162,9 @@ mod tests {
assert_eq!(builder.endpoint.unwrap(), aws_endpoint);
assert_eq!(builder.token.unwrap(), aws_session_token);
-
let metadata_uri =
format!("{METADATA_ENDPOINT}{container_creds_relative_uri}");
assert_eq!(builder.metadata_endpoint.unwrap(), metadata_uri);
+ assert!(builder.unsigned_payload);
}
#[test]
@@ -1154,6 +1180,7 @@ mod tests {
("aws_default_region", aws_default_region.clone()),
("aws_endpoint", aws_endpoint.clone()),
("aws_session_token", aws_session_token.clone()),
+ ("aws_unsigned_payload", "true".to_string()),
]);
let builder = AmazonS3Builder::new()
@@ -1166,6 +1193,7 @@ mod tests {
assert_eq!(builder.region.unwrap(), aws_default_region);
assert_eq!(builder.endpoint.unwrap(), aws_endpoint);
assert_eq!(builder.token.unwrap(), aws_session_token);
+ assert!(builder.unsigned_payload);
}
#[test]
@@ -1181,6 +1209,7 @@ mod tests {
(AmazonS3ConfigKey::DefaultRegion, aws_default_region.clone()),
(AmazonS3ConfigKey::Endpoint, aws_endpoint.clone()),
(AmazonS3ConfigKey::Token, aws_session_token.clone()),
+ (AmazonS3ConfigKey::UnsignedPayload, "true".to_string()),
]);
let builder = AmazonS3Builder::new()
@@ -1193,6 +1222,7 @@ mod tests {
assert_eq!(builder.region.unwrap(), aws_default_region);
assert_eq!(builder.endpoint.unwrap(), aws_endpoint);
assert_eq!(builder.token.unwrap(), aws_session_token);
+ assert!(builder.unsigned_payload);
}
#[test]
@@ -1220,6 +1250,12 @@ mod tests {
list_with_delimiter(&integration).await;
rename_and_copy(&integration).await;
stream_get(&integration).await;
+
+ // run integration test with unsigned payload enabled
+ let config = maybe_skip_integration!().with_unsigned_payload(true);
+ let is_local = matches!(&config.endpoint, Some(e) if
e.starts_with("http://";));
+ let integration = config.build().unwrap();
+ put_get_delete_list_opts(&integration, is_local).await;
}
#[tokio::test]
