This is an automated email from the ASF dual-hosted git repository.
wjones127 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 8d1f0f5214 feat(object_store): add support for server-side encryption
with customer-provided keys (SSE-C) (#6230)
8d1f0f5214 is described below
commit 8d1f0f52141da8db3a8cd6846d8a7e73fce236cd
Author: Jiacheng Yang <[email protected]>
AuthorDate: Thu Aug 15 13:30:55 2024 -0700
feat(object_store): add support for server-side encryption with
customer-provided keys (SSE-C) (#6230)
* Add support for server-side encryption with customer-provided keys
(SSE-C).
* Add SSE-C test using MinIO.
* Visibility change
* add nocapture to verify the test indeed runs
* cargo fmt
* Update object_store/src/aws/mod.rs
use environment variables
Co-authored-by: Will Jones <[email protected]>
* Update object_store/CONTRIBUTING.md
use environment variables
Co-authored-by: Will Jones <[email protected]>
* Fix api
---------
Co-authored-by: Will Jones <[email protected]>
---
object_store/CONTRIBUTING.md | 48 +++++++++++++
object_store/src/aws/builder.rs | 148 ++++++++++++++++++++++++++++++++--------
object_store/src/aws/client.rs | 56 +++++++++++++--
object_store/src/aws/mod.rs | 68 +++++++++++++++++-
4 files changed, 285 insertions(+), 35 deletions(-)
diff --git a/object_store/CONTRIBUTING.md b/object_store/CONTRIBUTING.md
index 4b0ef1f0e6..5444ec7434 100644
--- a/object_store/CONTRIBUTING.md
+++ b/object_store/CONTRIBUTING.md
@@ -101,6 +101,54 @@ export AWS_SERVER_SIDE_ENCRYPTION=aws:kms:dsse
cargo test --features aws
```
+#### SSE-C Encryption tests
+
+Unfortunately, localstack does not support SSE-C encryption
(https://github.com/localstack/localstack/issues/11356).
+
+We will use
[MinIO](https://min.io/docs/minio/container/operations/server-side-encryption.html)
to test SSE-C encryption.
+
+First, create a self-signed certificate to enable HTTPS for MinIO, as SSE-C
requires HTTPS.
+
+```shell
+mkdir ~/certs
+cd ~/certs
+openssl genpkey -algorithm RSA -out private.key
+openssl req -new -key private.key -out request.csr -subj
"/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=example.com/[email protected]"
+openssl x509 -req -days 365 -in request.csr -signkey private.key -out
public.crt
+rm request.csr
+```
+
+Second, start MinIO with the self-signed certificate.
+
+```shell
+docker run -d \
+ -p 9000:9000 \
+ --name minio \
+ -v ${HOME}/certs:/root/.minio/certs \
+ -e "MINIO_ROOT_USER=minio" \
+ -e "MINIO_ROOT_PASSWORD=minio123" \
+ minio/minio server /data
+```
+
+Create a test bucket.
+
+```shell
+export AWS_BUCKET_NAME=test-bucket
+export AWS_ACCESS_KEY_ID=minio
+export AWS_SECRET_ACCESS_KEY=minio123
+export AWS_ENDPOINT=https://localhost:9000
+aws s3 mb s3://test-bucket --endpoint-url=https://localhost:9000
--no-verify-ssl
+```
+
+Run the tests. The real test is `test_s3_ssec_encryption_with_minio()`
+
+```shell
+export TEST_S3_SSEC_ENCRYPTION=1
+cargo test --features aws --package object_store --lib
aws::tests::test_s3_ssec_encryption_with_minio -- --exact --nocapture
+```
+
+
+
### Azure
To test the Azure integration
diff --git a/object_store/src/aws/builder.rs b/object_store/src/aws/builder.rs
index ffef3fb33d..574345c389 100644
--- a/object_store/src/aws/builder.rs
+++ b/object_store/src/aws/builder.rs
@@ -26,7 +26,10 @@ use crate::aws::{
use crate::client::TokenCredentialProvider;
use crate::config::ConfigValue;
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig,
StaticCredentialProvider};
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
use itertools::Itertools;
+use md5::{Digest, Md5};
use reqwest::header::{HeaderMap, HeaderValue};
use serde::{Deserialize, Serialize};
use snafu::{OptionExt, ResultExt, Snafu};
@@ -73,7 +76,7 @@ enum Error {
#[snafu(display("Invalid Zone suffix for bucket '{bucket}'"))]
ZoneSuffix { bucket: String },
- #[snafu(display("Invalid encryption type: {}. Valid values are \"AES256\",
\"sse:kms\", and \"sse:kms:dsse\".", passed))]
+ #[snafu(display("Invalid encryption type: {}. Valid values are \"AES256\",
\"sse:kms\", \"sse:kms:dsse\" and \"sse-c\".", passed))]
InvalidEncryptionType { passed: String },
#[snafu(display(
@@ -166,6 +169,8 @@ pub struct AmazonS3Builder {
encryption_type: Option<ConfigValue<S3EncryptionType>>,
encryption_kms_key_id: Option<String>,
encryption_bucket_key_enabled: Option<ConfigValue<bool>>,
+ /// base64-encoded 256-bit customer encryption key for SSE-C.
+ encryption_customer_key_base64: Option<String>,
}
/// Configuration keys for [`AmazonS3Builder`]
@@ -394,6 +399,9 @@ impl FromStr for AmazonS3ConfigKey {
"aws_sse_bucket_key_enabled" => {
Ok(Self::Encryption(S3EncryptionConfigKey::BucketKeyEnabled))
}
+ "aws_sse_customer_key_base64" => Ok(Self::Encryption(
+ S3EncryptionConfigKey::CustomerEncryptionKey,
+ )),
_ => match s.parse() {
Ok(key) => Ok(Self::Client(key)),
Err(_) => Err(Error::UnknownConfigurationKey { key: s.into()
}.into()),
@@ -511,6 +519,9 @@ impl AmazonS3Builder {
S3EncryptionConfigKey::BucketKeyEnabled => {
self.encryption_bucket_key_enabled =
Some(ConfigValue::Deferred(value.into()))
}
+ S3EncryptionConfigKey::CustomerEncryptionKey => {
+ self.encryption_customer_key_base64 = Some(value.into())
+ }
},
};
self
@@ -566,6 +577,9 @@ impl AmazonS3Builder {
.encryption_bucket_key_enabled
.as_ref()
.map(ToString::to_string),
+ S3EncryptionConfigKey::CustomerEncryptionKey => {
+ self.encryption_customer_key_base64.clone()
+ }
},
}
}
@@ -813,6 +827,14 @@ impl AmazonS3Builder {
self
}
+ /// Use SSE-C for server side encryption.
+ /// Must pass the *base64-encoded* 256-bit customer encryption key.
+ pub fn with_ssec_encryption(mut self, customer_key_base64: impl
Into<String>) -> Self {
+ self.encryption_type =
Some(ConfigValue::Parsed(S3EncryptionType::SseC));
+ self.encryption_customer_key_base64 =
customer_key_base64.into().into();
+ self
+ }
+
/// Set whether to enable bucket key for server side encryption. This
overrides
/// the bucket default setting for bucket keys.
///
@@ -953,6 +975,7 @@ impl AmazonS3Builder {
self.encryption_bucket_key_enabled
.map(|val| val.get())
.transpose()?,
+ self.encryption_customer_key_base64,
)?
} else {
S3EncryptionHeaders::default()
@@ -994,15 +1017,14 @@ fn parse_bucket_az(bucket: &str) -> Option<&str> {
/// These options are used to configure server-side encryption for S3 objects.
/// To configure them, pass them to [`AmazonS3Builder::with_config`].
///
-/// Both [SSE-KMS] and [DSSE-KMS] are supported. [SSE-C] is not yet supported.
-///
+/// [SSE-S3]:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html
/// [SSE-KMS]:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html
/// [DSSE-KMS]:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingDSSEncryption.html
/// [SSE-C]:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html
#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Serialize, Deserialize)]
#[non_exhaustive]
pub enum S3EncryptionConfigKey {
- /// Type of encryption to use. If set, must be one of "AES256", "aws:kms",
or "aws:kms:dsse".
+ /// Type of encryption to use. If set, must be one of "AES256" (SSE-S3),
"aws:kms" (SSE-KMS), "aws:kms:dsse" (DSSE-KMS) or "sse-c".
ServerSideEncryption,
/// The KMS key ID to use for server-side encryption. If set,
ServerSideEncryption
/// must be "aws:kms" or "aws:kms:dsse".
@@ -1010,6 +1032,10 @@ pub enum S3EncryptionConfigKey {
/// If set to true, will use the bucket's default KMS key for server-side
encryption.
/// If set to false, will disable the use of the bucket's default KMS key
for server-side encryption.
BucketKeyEnabled,
+
+ /// The base64 encoded, 256-bit customer encryption key to use for
server-side encryption.
+ /// If set, ServerSideEncryption must be "sse-c".
+ CustomerEncryptionKey,
}
impl AsRef<str> for S3EncryptionConfigKey {
@@ -1018,6 +1044,7 @@ impl AsRef<str> for S3EncryptionConfigKey {
Self::ServerSideEncryption => "aws_server_side_encryption",
Self::KmsKeyId => "aws_sse_kms_key_id",
Self::BucketKeyEnabled => "aws_sse_bucket_key_enabled",
+ Self::CustomerEncryptionKey => "aws_sse_customer_key_base64",
}
}
}
@@ -1027,6 +1054,7 @@ enum S3EncryptionType {
S3,
SseKms,
DsseKms,
+ SseC,
}
impl crate::config::Parse for S3EncryptionType {
@@ -1035,6 +1063,7 @@ impl crate::config::Parse for S3EncryptionType {
"AES256" => Ok(Self::S3),
"aws:kms" => Ok(Self::SseKms),
"aws:kms:dsse" => Ok(Self::DsseKms),
+ "sse-c" => Ok(Self::SseC),
_ => Err(Error::InvalidEncryptionType { passed: s.into() }.into()),
}
}
@@ -1046,6 +1075,7 @@ impl From<&S3EncryptionType> for &'static str {
S3EncryptionType::S3 => "AES256",
S3EncryptionType::SseKms => "aws:kms",
S3EncryptionType::DsseKms => "aws:kms:dsse",
+ S3EncryptionType::SseC => "sse-c",
}
}
}
@@ -1062,37 +1092,87 @@ impl std::fmt::Display for S3EncryptionType {
/// Whether these headers are sent depends on both the kind of encryption set
/// and the kind of request being made.
#[derive(Default, Clone, Debug)]
-pub struct S3EncryptionHeaders(HeaderMap);
+pub(super) struct S3EncryptionHeaders(pub HeaderMap);
impl S3EncryptionHeaders {
fn try_new(
encryption_type: &S3EncryptionType,
- key_id: Option<String>,
+ encryption_kms_key_id: Option<String>,
bucket_key_enabled: Option<bool>,
+ encryption_customer_key_base64: Option<String>,
) -> Result<Self> {
let mut headers = HeaderMap::new();
- // Note: if we later add support for SSE-C, we should be sure to use
- // HeaderValue::set_sensitive to prevent the key from being logged.
- headers.insert(
- "x-amz-server-side-encryption",
- HeaderValue::from_static(encryption_type.into()),
- );
- if let Some(key_id) = key_id {
- headers.insert(
- "x-amz-server-side-encryption-aws-kms-key-id",
- key_id
- .try_into()
- .map_err(|err| Error::InvalidEncryptionHeader {
- header: "kms-key-id",
- source: Box::new(err),
- })?,
- );
- }
- if let Some(bucket_key_enabled) = bucket_key_enabled {
- headers.insert(
- "x-amz-server-side-encryption-bucket-key-enabled",
- HeaderValue::from_static(if bucket_key_enabled { "true" } else
{ "false" }),
- );
+ match encryption_type {
+ S3EncryptionType::S3 | S3EncryptionType::SseKms |
S3EncryptionType::DsseKms => {
+ headers.insert(
+ "x-amz-server-side-encryption",
+ HeaderValue::from_static(encryption_type.into()),
+ );
+ if let Some(key_id) = encryption_kms_key_id {
+ headers.insert(
+ "x-amz-server-side-encryption-aws-kms-key-id",
+ key_id
+ .try_into()
+ .map_err(|err| Error::InvalidEncryptionHeader {
+ header: "kms-key-id",
+ source: Box::new(err),
+ })?,
+ );
+ }
+ if let Some(bucket_key_enabled) = bucket_key_enabled {
+ headers.insert(
+ "x-amz-server-side-encryption-bucket-key-enabled",
+ HeaderValue::from_static(if bucket_key_enabled {
"true" } else { "false" }),
+ );
+ }
+ }
+ S3EncryptionType::SseC => {
+ headers.insert(
+ "x-amz-server-side-encryption-customer-algorithm",
+ HeaderValue::from_static("AES256"),
+ );
+ if let Some(key) = encryption_customer_key_base64 {
+ let mut header_value: HeaderValue =
+ key.clone()
+ .try_into()
+ .map_err(|err| Error::InvalidEncryptionHeader {
+ header:
"x-amz-server-side-encryption-customer-key",
+ source: Box::new(err),
+ })?;
+ header_value.set_sensitive(true);
+
headers.insert("x-amz-server-side-encryption-customer-key", header_value);
+
+ let decoded_key =
BASE64_STANDARD.decode(key.as_bytes()).map_err(|err| {
+ Error::InvalidEncryptionHeader {
+ header:
"x-amz-server-side-encryption-customer-key",
+ source: Box::new(err),
+ }
+ })?;
+ let mut hasher = Md5::new();
+ hasher.update(decoded_key);
+ let md5 = BASE64_STANDARD.encode(hasher.finalize());
+ let mut md5_header_value: HeaderValue =
+ md5.try_into()
+ .map_err(|err| Error::InvalidEncryptionHeader {
+ header:
"x-amz-server-side-encryption-customer-key-MD5",
+ source: Box::new(err),
+ })?;
+ md5_header_value.set_sensitive(true);
+ headers.insert(
+ "x-amz-server-side-encryption-customer-key-MD5",
+ md5_header_value,
+ );
+ } else {
+ return Err(Error::InvalidEncryptionHeader {
+ header: "x-amz-server-side-encryption-customer-key",
+ source: Box::new(std::io::Error::new(
+ std::io::ErrorKind::InvalidInput,
+ "Missing customer key",
+ )),
+ }
+ .into());
+ }
+ }
}
Ok(Self(headers))
}
@@ -1162,7 +1242,11 @@ mod tests {
.with_config(AmazonS3ConfigKey::UnsignedPayload, "true")
.with_config("aws_server_side_encryption".parse().unwrap(),
"AES256")
.with_config("aws_sse_kms_key_id".parse().unwrap(), "some_key_id")
- .with_config("aws_sse_bucket_key_enabled".parse().unwrap(),
"true");
+ .with_config("aws_sse_bucket_key_enabled".parse().unwrap(), "true")
+ .with_config(
+ "aws_sse_customer_key_base64".parse().unwrap(),
+ "some_customer_key",
+ );
assert_eq!(
builder
@@ -1216,6 +1300,12 @@ mod tests {
.unwrap(),
"true"
);
+ assert_eq!(
+ builder
+
.get_config_value(&"aws_sse_customer_key_base64".parse().unwrap())
+ .unwrap(),
+ "some_customer_key"
+ );
}
#[test]
diff --git a/object_store/src/aws/client.rs b/object_store/src/aws/client.rs
index ab4da86f50..007e271086 100644
--- a/object_store/src/aws/client.rs
+++ b/object_store/src/aws/client.rs
@@ -181,7 +181,7 @@ pub struct S3Config {
pub checksum: Option<Checksum>,
pub copy_if_not_exists: Option<S3CopyIfNotExists>,
pub conditional_put: Option<S3ConditionalPut>,
- pub encryption_headers: S3EncryptionHeaders,
+ pub(super) encryption_headers: S3EncryptionHeaders,
}
impl S3Config {
@@ -522,10 +522,47 @@ impl S3Client {
/// Make an S3 Copy request
<https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html>
pub fn copy_request<'a>(&'a self, from: &Path, to: &'a Path) ->
Request<'a> {
let source = format!("{}/{}", self.config.bucket, encode_path(from));
+
+ let mut copy_source_encryption_headers = HeaderMap::new();
+ if let Some(customer_algorithm) = self
+ .config
+ .encryption_headers
+ .0
+ .get("x-amz-server-side-encryption-customer-algorithm")
+ {
+ copy_source_encryption_headers.insert(
+ "x-amz-copy-source-server-side-encryption-customer-algorithm",
+ customer_algorithm.clone(),
+ );
+ }
+ if let Some(customer_key) = self
+ .config
+ .encryption_headers
+ .0
+ .get("x-amz-server-side-encryption-customer-key")
+ {
+ copy_source_encryption_headers.insert(
+ "x-amz-copy-source-server-side-encryption-customer-key",
+ customer_key.clone(),
+ );
+ }
+ if let Some(customer_key_md5) = self
+ .config
+ .encryption_headers
+ .0
+ .get("x-amz-server-side-encryption-customer-key-MD5")
+ {
+ copy_source_encryption_headers.insert(
+ "x-amz-copy-source-server-side-encryption-customer-key-MD5",
+ customer_key_md5.clone(),
+ );
+ }
+
self.request(Method::PUT, to)
.idempotent(true)
.header(©_SOURCE_HEADER, &source)
.headers(self.config.encryption_headers.clone().into())
+ .headers(copy_source_encryption_headers)
.with_session_creds(false)
}
@@ -562,13 +599,21 @@ impl S3Client {
) -> Result<PartId> {
let part = (part_idx + 1).to_string();
- let response = self
+ let mut request = self
.request(Method::PUT, path)
.with_payload(data)
.query(&[("partNumber", &part), ("uploadId", upload_id)])
- .idempotent(true)
- .send()
- .await?;
+ .idempotent(true);
+ if self
+ .config
+ .encryption_headers
+ .0
+ .contains_key("x-amz-server-side-encryption-customer-algorithm")
+ {
+ // If SSE-C is used, we must include the encryption headers in
every upload request.
+ request = request.with_encryption_headers();
+ }
+ let response = request.send().await?;
let content_id = get_etag(response.headers()).context(MetadataSnafu)?;
Ok(PartId { content_id })
@@ -660,6 +705,7 @@ impl GetClient for S3Client {
};
let mut builder = self.client.request(method, url);
+ builder =
builder.headers(self.config.encryption_headers.clone().into());
if let Some(v) = &options.version {
builder = builder.query(&[("versionId", v)])
diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs
index f5204a5365..4a773e7a18 100644
--- a/object_store/src/aws/mod.rs
+++ b/object_store/src/aws/mod.rs
@@ -60,7 +60,7 @@ mod dynamo;
mod precondition;
mod resolve;
-pub use builder::{AmazonS3Builder, AmazonS3ConfigKey, S3EncryptionHeaders};
+pub use builder::{AmazonS3Builder, AmazonS3ConfigKey};
pub use checksum::Checksum;
pub use dynamo::DynamoCommit;
pub use precondition::{S3ConditionalPut, S3CopyIfNotExists};
@@ -412,6 +412,9 @@ mod tests {
use crate::client::get::GetClient;
use crate::integration::*;
use crate::tests::*;
+ use crate::ClientOptions;
+ use base64::prelude::BASE64_STANDARD;
+ use base64::Engine;
use hyper::HeaderMap;
const NON_EXISTENT_NAME: &str = "nonexistentname";
@@ -605,4 +608,67 @@ mod tests {
store.delete(location).await.unwrap();
}
}
+
+ /// See CONTRIBUTING.md for the MinIO setup for this test.
+ #[tokio::test]
+ async fn test_s3_ssec_encryption_with_minio() {
+ if std::env::var("TEST_S3_SSEC_ENCRYPTION").is_err() {
+ eprintln!("Skipping S3 SSE-C encryption test");
+ return;
+ }
+ eprintln!("Running S3 SSE-C encryption test");
+
+ let customer_key = "1234567890abcdef1234567890abcdef";
+ let expected_md5 = "JMwgiexXqwuPqIPjYFmIZQ==";
+
+ let store = AmazonS3Builder::from_env()
+ .with_ssec_encryption(BASE64_STANDARD.encode(customer_key))
+
.with_client_options(ClientOptions::default().with_allow_invalid_certificates(true))
+ .build()
+ .unwrap();
+
+ let data = PutPayload::from(vec![3u8; 1024]);
+
+ let locations = [
+ Path::from("test-encryption-1"),
+ Path::from("test-encryption-2"),
+ Path::from("test-encryption-3"),
+ ];
+
+ // Test put with sse-c.
+ store.put(&locations[0], data.clone()).await.unwrap();
+
+ // Test copy with sse-c.
+ store.copy(&locations[0], &locations[1]).await.unwrap();
+
+ // Test multipart upload with sse-c.
+ let mut upload = store.put_multipart(&locations[2]).await.unwrap();
+ upload.put_part(data.clone()).await.unwrap();
+ upload.complete().await.unwrap();
+
+ // Test get with sse-c.
+ for location in &locations {
+ let res = store
+ .client
+ .get_request(location, GetOptions::default())
+ .await
+ .unwrap();
+ let headers = res.headers();
+ assert_eq!(
+ headers
+ .get("x-amz-server-side-encryption-customer-algorithm")
+ .expect("object is not encrypted with SSE-C"),
+ "AES256"
+ );
+
+ assert_eq!(
+ headers
+ .get("x-amz-server-side-encryption-customer-key-MD5")
+ .expect("object is not encrypted with SSE-C"),
+ expected_md5
+ );
+
+ store.delete(location).await.unwrap();
+ }
+ }
}
