This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/main by this push:
new e8f8694 Add additional S3 FileIO Attributes (#505)
e8f8694 is described below
commit e8f8694541e2e4b3f09111f047045f966d400363
Author: Christian <[email protected]>
AuthorDate: Tue Jul 30 11:05:38 2024 +0200
Add additional S3 FileIO Attributes (#505)
* Add additional S3 FileIO Attributes
* Remove custom S3SSEType
---
crates/iceberg/src/io/storage.rs | 2 +-
crates/iceberg/src/io/storage_s3.rs | 51 +++++++++++++++++++++++++++++++++++--
2 files changed, 50 insertions(+), 3 deletions(-)
diff --git a/crates/iceberg/src/io/storage.rs b/crates/iceberg/src/io/storage.rs
index d13ac04..870e61e 100644
--- a/crates/iceberg/src/io/storage.rs
+++ b/crates/iceberg/src/io/storage.rs
@@ -54,7 +54,7 @@ impl Storage {
#[cfg(feature = "storage-s3")]
Scheme::S3 => Ok(Self::S3 {
scheme_str,
- config: super::s3_config_parse(props).into(),
+ config: super::s3_config_parse(props)?.into(),
}),
_ => Err(Error::new(
ErrorKind::FeatureUnsupported,
diff --git a/crates/iceberg/src/io/storage_s3.rs
b/crates/iceberg/src/io/storage_s3.rs
index acce18e..6e92b90 100644
--- a/crates/iceberg/src/io/storage_s3.rs
+++ b/crates/iceberg/src/io/storage_s3.rs
@@ -32,9 +32,20 @@ pub const S3_ACCESS_KEY_ID: &str = "s3.access-key-id";
pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret-access-key";
/// S3 region.
pub const S3_REGION: &str = "s3.region";
+/// S3 Path Style Access.
+pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access";
+/// S3 Server Side Encryption Type.
+pub const S3_SSE_TYPE: &str = "s3.sse.type";
+/// S3 Server Side Encryption Key.
+/// If S3 encryption type is kms, input is a KMS Key ID.
+/// In case this property is not set, default key "aws/s3" is used.
+/// If encryption type is custom, input is a custom base-64 AES256 symmetric
key.
+pub const S3_SSE_KEY: &str = "s3.sse.key";
+/// S3 Server Side Encryption MD5.
+pub const S3_SSE_MD5: &str = "s3.sse.md5";
/// Parse iceberg props to s3 config.
-pub(crate) fn s3_config_parse(mut m: HashMap<String, String>) -> S3Config {
+pub(crate) fn s3_config_parse(mut m: HashMap<String, String>) ->
Result<S3Config> {
let mut cfg = S3Config::default();
if let Some(endpoint) = m.remove(S3_ENDPOINT) {
cfg.endpoint = Some(endpoint);
@@ -48,8 +59,44 @@ pub(crate) fn s3_config_parse(mut m: HashMap<String,
String>) -> S3Config {
if let Some(region) = m.remove(S3_REGION) {
cfg.region = Some(region);
};
+ if let Some(path_style_access) = m.remove(S3_PATH_STYLE_ACCESS) {
+ if ["true", "True", "1"].contains(&path_style_access.as_str()) {
+ cfg.enable_virtual_host_style = true;
+ }
+ };
+ let s3_sse_key = m.remove(S3_SSE_KEY);
+ if let Some(sse_type) = m.remove(S3_SSE_TYPE) {
+ match sse_type.to_lowercase().as_str() {
+ // No Server Side Encryption
+ "none" => {}
+ // S3 SSE-S3 encryption (S3 managed keys).
https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
+ "s3" => {
+ cfg.server_side_encryption = Some("AES256".to_string());
+ }
+ // S3 SSE KMS, either using default or custom KMS key.
https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+ "kms" => {
+ cfg.server_side_encryption = Some("aws:kms".to_string());
+ cfg.server_side_encryption_aws_kms_key_id = s3_sse_key;
+ }
+ // S3 SSE-C, using customer managed keys.
https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+ "custom" => {
+ cfg.server_side_encryption_customer_algorithm =
Some("AES256".to_string());
+ cfg.server_side_encryption_customer_key = s3_sse_key;
+ cfg.server_side_encryption_customer_key_md5 =
m.remove(S3_SSE_MD5);
+ }
+ _ => {
+ return Err(Error::new(
+ ErrorKind::DataInvalid,
+ format!(
+ "Invalid {}: {}. Expected one of (custom, kms, s3,
none)",
+ S3_SSE_TYPE, sse_type
+ ),
+ ));
+ }
+ }
+ };
- cfg
+ Ok(cfg)
}
/// Build new opendal operator from give path.
