This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 137b114 feat: support taking env vars for cloud storages (#55)
137b114 is described below
commit 137b114f7a32f48bdbf8c9d71d3edb9afaa6541d
Author: Shiyan Xu <[email protected]>
AuthorDate: Sun Jul 7 21:46:17 2024 -0500
feat: support taking env vars for cloud storages (#55)
---
crates/core/src/table/mod.rs | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs
index 07126f1..8454a1a 100644
--- a/crates/core/src/table/mod.rs
+++ b/crates/core/src/table/mod.rs
@@ -18,6 +18,7 @@
*/
use std::collections::HashMap;
+use std::env;
use std::io::{BufRead, BufReader};
use std::str::FromStr;
use std::sync::Arc;
@@ -87,8 +88,12 @@ impl Table {
base_url: Arc<Url>,
all_options: &HashMap<String, String>,
) -> Result<(HudiConfigs, HashMap<String, String>)> {
+ // TODO: load hudi global config
let mut hudi_options = HashMap::new();
let mut extra_options = HashMap::new();
+
+ Self::imbue_cloud_env_vars(&mut extra_options);
+
for (k, v) in all_options {
if k.starts_with("hoodie.") {
hudi_options.insert(k.clone(), v.clone());
@@ -117,6 +122,15 @@ impl Table {
Self::validate_configs(&hudi_configs).map(|_| (hudi_configs,
extra_options))
}
+ fn imbue_cloud_env_vars(options: &mut HashMap<String, String>) {
+ let prefixes = ["AWS_", "AZURE_", "GOOGLE_"];
+ options.extend(
+ env::vars()
+ .filter(|(key, _)| prefixes.iter().any(|prefix|
key.starts_with(prefix)))
+ .map(|(k, v)| (k.to_ascii_lowercase(), v)),
+ );
+ }
+
fn validate_configs(hudi_configs: &HudiConfigs) -> Result<()> {
if hudi_configs
.get_or_default(SkipConfigValidation)
@@ -242,6 +256,8 @@ mod tests {
use url::Url;
+ use hudi_tests::{assert_not, TestTable};
+
use crate::config::read::HudiReadConfig::AsOfTimestamp;
use crate::config::table::HudiTableConfig::{
BaseFileFormat, Checksum, DatabaseName, DropsPartitionFields,
IsHiveStylePartitioning,
@@ -251,7 +267,6 @@ mod tests {
};
use crate::storage::utils::join_url_segments;
use crate::table::Table;
- use hudi_tests::{assert_not, TestTable};
#[tokio::test]
async fn hudi_table_get_schema() {
