This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 6c5c34b6f Add get_config_value to AWS/Azure/GCP Builders (#4035)
6c5c34b6f is described below
commit 6c5c34b6fb3d2fa700856077b6ea2555ff5fb598
Author: r.4ntix <[email protected]>
AuthorDate: Mon Apr 10 23:13:00 2023 +0800
Add get_config_value to AWS/Azure/GCP Builders (#4035)
* minor: make struct fields of Builders(S3/Azure/GCS) to pub
* minor: use `get_config_value` method instead of public fields
* fix clippy error
---
object_store/src/aws/checksum.rs | 10 +++-
object_store/src/aws/mod.rs | 103 +++++++++++++++++++++++++++++++++++++++
object_store/src/azure/mod.rs | 81 ++++++++++++++++++++++++++++++
object_store/src/gcp/mod.rs | 56 +++++++++++++++++++++
4 files changed, 249 insertions(+), 1 deletion(-)
diff --git a/object_store/src/aws/checksum.rs b/object_store/src/aws/checksum.rs
index ae35f0612..c787c28a8 100644
--- a/object_store/src/aws/checksum.rs
+++ b/object_store/src/aws/checksum.rs
@@ -39,11 +39,19 @@ impl Checksum {
}
}
+impl std::fmt::Display for Checksum {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match &self {
+ Self::SHA256 => write!(f, "sha256"),
+ }
+ }
+}
+
impl TryFrom<&String> for Checksum {
type Error = ();
fn try_from(value: &String) -> Result<Self, Self::Error> {
- match value.as_str() {
+ match value.to_lowercase().as_str() {
"sha256" => Ok(Self::SHA256),
_ => Err(()),
}
diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs
index f88960b4b..de62360d0 100644
--- a/object_store/src/aws/mod.rs
+++ b/object_store/src/aws/mod.rs
@@ -400,20 +400,35 @@ impl CloudMultiPartUploadImpl for S3MultiPartUpload {
/// ```
#[derive(Debug, Default, Clone)]
pub struct AmazonS3Builder {
+ /// Access key id
access_key_id: Option<String>,
+ /// Secret access_key
secret_access_key: Option<String>,
+ /// Region
region: Option<String>,
+ /// Bucket name
bucket_name: Option<String>,
+ /// Endpoint for communicating with AWS S3
endpoint: Option<String>,
+ /// Token to use for requests
token: Option<String>,
+ /// Url
url: Option<String>,
+ /// Retry config
retry_config: RetryConfig,
+ /// When set to true, fallback to IMDSv1
imdsv1_fallback: bool,
+ /// When set to true, virtual hosted style request has to be used
virtual_hosted_style_request: bool,
+ /// When set to true, unsigned payload option has to be used
unsigned_payload: bool,
+ /// Checksum algorithm which has to be used for object integrity check
during upload
checksum_algorithm: Option<Checksum>,
+ /// Metadata endpoint, see
<https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html>
metadata_endpoint: Option<String>,
+ /// Profile name, see
<https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html>
profile: Option<String>,
+ /// Client options
client_options: ClientOptions,
}
@@ -751,6 +766,38 @@ impl AmazonS3Builder {
Ok(self)
}
+ /// Get config value via a [`AmazonS3ConfigKey`].
+ ///
+ /// # Example
+ /// ```
+ /// use object_store::aws::{AmazonS3Builder, AmazonS3ConfigKey};
+ ///
+ /// let builder = AmazonS3Builder::from_env()
+ /// .with_bucket_name("foo");
+ /// let bucket_name =
builder.get_config_value(&AmazonS3ConfigKey::Bucket).unwrap_or_default();
+ /// assert_eq!("foo", &bucket_name);
+ /// ```
+ pub fn get_config_value(&self, key: &AmazonS3ConfigKey) -> Option<String> {
+ match key {
+ AmazonS3ConfigKey::AccessKeyId => self.access_key_id.clone(),
+ AmazonS3ConfigKey::SecretAccessKey =>
self.secret_access_key.clone(),
+ AmazonS3ConfigKey::Region | AmazonS3ConfigKey::DefaultRegion => {
+ self.region.clone()
+ }
+ AmazonS3ConfigKey::Bucket => self.bucket_name.clone(),
+ AmazonS3ConfigKey::Endpoint => self.endpoint.clone(),
+ AmazonS3ConfigKey::Token => self.token.clone(),
+ AmazonS3ConfigKey::ImdsV1Fallback =>
Some(self.imdsv1_fallback.to_string()),
+ AmazonS3ConfigKey::VirtualHostedStyleRequest => {
+ Some(self.virtual_hosted_style_request.to_string())
+ }
+ AmazonS3ConfigKey::MetadataEndpoint =>
self.metadata_endpoint.clone(),
+ AmazonS3ConfigKey::Profile => self.profile.clone(),
+ AmazonS3ConfigKey::UnsignedPayload =>
Some(self.unsigned_payload.to_string()),
+ AmazonS3ConfigKey::Checksum => self.checksum_algorithm.map(|v|
v.to_string()),
+ }
+ }
+
/// Sets properties on this builder based on a URL
///
/// This is a separate member function to allow fallible computation to
@@ -1272,6 +1319,62 @@ mod tests {
assert!(builder.unsigned_payload);
}
+ #[test]
+ fn s3_test_config_get_value() {
+ let aws_access_key_id = "object_store:fake_access_key_id".to_string();
+ let aws_secret_access_key = "object_store:fake_secret_key".to_string();
+ let aws_default_region =
"object_store:fake_default_region".to_string();
+ let aws_endpoint = "object_store:fake_endpoint".to_string();
+ let aws_session_token = "object_store:fake_session_token".to_string();
+ let options = HashMap::from([
+ (AmazonS3ConfigKey::AccessKeyId, aws_access_key_id.clone()),
+ (
+ AmazonS3ConfigKey::SecretAccessKey,
+ aws_secret_access_key.clone(),
+ ),
+ (AmazonS3ConfigKey::DefaultRegion, aws_default_region.clone()),
+ (AmazonS3ConfigKey::Endpoint, aws_endpoint.clone()),
+ (AmazonS3ConfigKey::Token, aws_session_token.clone()),
+ (AmazonS3ConfigKey::UnsignedPayload, "true".to_string()),
+ ]);
+
+ let builder =
AmazonS3Builder::new().try_with_options(&options).unwrap();
+ assert_eq!(
+ builder
+ .get_config_value(&AmazonS3ConfigKey::AccessKeyId)
+ .unwrap(),
+ aws_access_key_id
+ );
+ assert_eq!(
+ builder
+ .get_config_value(&AmazonS3ConfigKey::SecretAccessKey)
+ .unwrap(),
+ aws_secret_access_key
+ );
+ assert_eq!(
+ builder
+ .get_config_value(&AmazonS3ConfigKey::DefaultRegion)
+ .unwrap(),
+ aws_default_region
+ );
+ assert_eq!(
+ builder
+ .get_config_value(&AmazonS3ConfigKey::Endpoint)
+ .unwrap(),
+ aws_endpoint
+ );
+ assert_eq!(
+ builder.get_config_value(&AmazonS3ConfigKey::Token).unwrap(),
+ aws_session_token
+ );
+ assert_eq!(
+ builder
+ .get_config_value(&AmazonS3ConfigKey::UnsignedPayload)
+ .unwrap(),
+ "true"
+ );
+ }
+
#[test]
fn s3_test_config_fallible_options() {
let aws_access_key_id = "object_store:fake_access_key_id".to_string();
diff --git a/object_store/src/azure/mod.rs b/object_store/src/azure/mod.rs
index c2e72f214..11350a202 100644
--- a/object_store/src/azure/mod.rs
+++ b/object_store/src/azure/mod.rs
@@ -394,24 +394,43 @@ impl CloudMultiPartUploadImpl for AzureMultiPartUpload {
/// ```
#[derive(Default, Clone)]
pub struct MicrosoftAzureBuilder {
+ /// Account name
account_name: Option<String>,
+ /// Access key
access_key: Option<String>,
+ /// Container name
container_name: Option<String>,
+ /// Bearer token
bearer_token: Option<String>,
+ /// Client id
client_id: Option<String>,
+ /// Client secret
client_secret: Option<String>,
+ /// Tenant id
tenant_id: Option<String>,
+ /// Query pairs for shared access signature authorization
sas_query_pairs: Option<Vec<(String, String)>>,
+ /// Shared access signature
sas_key: Option<String>,
+ /// Authority host
authority_host: Option<String>,
+ /// Url
url: Option<String>,
+ /// When set to true, azurite storage emulator has to be used
use_emulator: bool,
+ /// Msi endpoint for acquiring managed identity token
msi_endpoint: Option<String>,
+ /// Object id for use with managed identity authentication
object_id: Option<String>,
+ /// Msi resource id for use with managed identity authentication
msi_resource_id: Option<String>,
+ /// File containing token for Azure AD workload identity federation
federated_token_file: Option<String>,
+ /// When set to true, azure cli has to be used for acquiring access token
use_azure_cli: bool,
+ /// Retry config
retry_config: RetryConfig,
+ /// Client options
client_options: ClientOptions,
}
@@ -747,6 +766,35 @@ impl MicrosoftAzureBuilder {
Ok(self)
}
+ /// Get config value via a [`AzureConfigKey`].
+ ///
+ /// # Example
+ /// ```
+ /// use object_store::azure::{MicrosoftAzureBuilder, AzureConfigKey};
+ ///
+ /// let builder = MicrosoftAzureBuilder::from_env()
+ /// .with_account("foo");
+ /// let account_name =
builder.get_config_value(&AzureConfigKey::AccountName).unwrap_or_default();
+ /// assert_eq!("foo", &account_name);
+ /// ```
+ pub fn get_config_value(&self, key: &AzureConfigKey) -> Option<String> {
+ match key {
+ AzureConfigKey::AccountName => self.account_name.clone(),
+ AzureConfigKey::AccessKey => self.access_key.clone(),
+ AzureConfigKey::ClientId => self.client_id.clone(),
+ AzureConfigKey::ClientSecret => self.client_secret.clone(),
+ AzureConfigKey::AuthorityId => self.tenant_id.clone(),
+ AzureConfigKey::SasKey => self.sas_key.clone(),
+ AzureConfigKey::Token => self.bearer_token.clone(),
+ AzureConfigKey::UseEmulator => Some(self.use_emulator.to_string()),
+ AzureConfigKey::MsiEndpoint => self.msi_endpoint.clone(),
+ AzureConfigKey::ObjectId => self.object_id.clone(),
+ AzureConfigKey::MsiResourceId => self.msi_resource_id.clone(),
+ AzureConfigKey::FederatedTokenFile =>
self.federated_token_file.clone(),
+ AzureConfigKey::UseAzureCli =>
Some(self.use_azure_cli.to_string()),
+ }
+ }
+
/// Sets properties on this builder based on a URL
///
/// This is a separate member function to allow fallible computation to
@@ -1252,6 +1300,39 @@ mod tests {
assert_eq!(builder.bearer_token.unwrap(), azure_storage_token);
}
+ #[test]
+ fn azure_test_config_get_value() {
+ let azure_client_id = "object_store:fake_access_key_id".to_string();
+ let azure_storage_account_name =
"object_store:fake_secret_key".to_string();
+ let azure_storage_token =
"object_store:fake_default_region".to_string();
+ let options = HashMap::from([
+ (AzureConfigKey::ClientId, azure_client_id.clone()),
+ (
+ AzureConfigKey::AccountName,
+ azure_storage_account_name.clone(),
+ ),
+ (AzureConfigKey::Token, azure_storage_token.clone()),
+ ]);
+
+ let builder = MicrosoftAzureBuilder::new()
+ .try_with_options(&options)
+ .unwrap();
+ assert_eq!(
+ builder.get_config_value(&AzureConfigKey::ClientId).unwrap(),
+ azure_client_id
+ );
+ assert_eq!(
+ builder
+ .get_config_value(&AzureConfigKey::AccountName)
+ .unwrap(),
+ azure_storage_account_name
+ );
+ assert_eq!(
+ builder.get_config_value(&AzureConfigKey::Token).unwrap(),
+ azure_storage_token
+ );
+ }
+
#[test]
fn azure_test_config_fallible_options() {
let azure_client_id = "object_store:fake_access_key_id".to_string();
diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs
index 5247693e6..a6cf66022 100644
--- a/object_store/src/gcp/mod.rs
+++ b/object_store/src/gcp/mod.rs
@@ -768,12 +768,19 @@ impl ObjectStore for GoogleCloudStorage {
/// ```
#[derive(Debug, Clone)]
pub struct GoogleCloudStorageBuilder {
+ /// Bucket name
bucket_name: Option<String>,
+ /// Url
url: Option<String>,
+ /// Path to the service account file
service_account_path: Option<String>,
+ /// The serialized service account key
service_account_key: Option<String>,
+ /// Path to the application credentials file.
application_credentials_path: Option<String>,
+ /// Retry config
retry_config: RetryConfig,
+ /// Client options
client_options: ClientOptions,
}
@@ -983,6 +990,28 @@ impl GoogleCloudStorageBuilder {
Ok(self)
}
+ /// Get config value via a [`GoogleConfigKey`].
+ ///
+ /// # Example
+ /// ```
+ /// use object_store::gcp::{GoogleCloudStorageBuilder, GoogleConfigKey};
+ ///
+ /// let builder = GoogleCloudStorageBuilder::from_env()
+ /// .with_service_account_key("foo");
+ /// let service_account_key =
builder.get_config_value(&GoogleConfigKey::ServiceAccountKey).unwrap_or_default();
+ /// assert_eq!("foo", &service_account_key);
+ /// ```
+ pub fn get_config_value(&self, key: &GoogleConfigKey) -> Option<String> {
+ match key {
+ GoogleConfigKey::ServiceAccount =>
self.service_account_path.clone(),
+ GoogleConfigKey::ServiceAccountKey =>
self.service_account_key.clone(),
+ GoogleConfigKey::Bucket => self.bucket_name.clone(),
+ GoogleConfigKey::ApplicationCredentials => {
+ self.application_credentials_path.clone()
+ }
+ }
+ }
+
/// Sets properties on this builder based on a URL
///
/// This is a separate member function to allow fallible computation to
@@ -1452,6 +1481,33 @@ mod test {
assert_eq!(builder.bucket_name.unwrap(), google_bucket_name.as_str());
}
+ #[test]
+ fn gcs_test_config_get_value() {
+ let google_service_account =
"object_store:fake_service_account".to_string();
+ let google_bucket_name = "object_store:fake_bucket".to_string();
+ let options = HashMap::from([
+ (
+ GoogleConfigKey::ServiceAccount,
+ google_service_account.clone(),
+ ),
+ (GoogleConfigKey::Bucket, google_bucket_name.clone()),
+ ]);
+
+ let builder = GoogleCloudStorageBuilder::new()
+ .try_with_options(&options)
+ .unwrap();
+ assert_eq!(
+ builder
+ .get_config_value(&GoogleConfigKey::ServiceAccount)
+ .unwrap(),
+ google_service_account
+ );
+ assert_eq!(
+ builder.get_config_value(&GoogleConfigKey::Bucket).unwrap(),
+ google_bucket_name
+ );
+ }
+
#[test]
fn gcs_test_config_fallible_options() {
let google_service_account =
"object_store:fake_service_account".to_string();
