This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 62da0d2 feat: add internal config to skip validation (#51)
62da0d2 is described below
commit 62da0d28b717a905f59eab059af1c1def83a1878
Author: Shiyan Xu <[email protected]>
AuthorDate: Sat Jul 6 01:57:14 2024 -0500
feat: add internal config to skip validation (#51)
- add internal.rs to host hudi internal configs
- add `hoodie.internal.skip.config.validation` to allow skipping
---
crates/core/src/config/internal.rs | 62 ++++++++++++++++++++++++++++++++++++++
crates/core/src/config/mod.rs | 1 +
crates/core/src/table/mod.rs | 24 +++++++--------
3 files changed, 74 insertions(+), 13 deletions(-)
diff --git a/crates/core/src/config/internal.rs
b/crates/core/src/config/internal.rs
new file mode 100644
index 0000000..d6ad814
--- /dev/null
+++ b/crates/core/src/config/internal.rs
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use std::collections::HashMap;
+use std::str::FromStr;
+
+use anyhow::{anyhow, Result};
+use strum_macros::EnumIter;
+
+use crate::config::{ConfigParser, HudiConfigValue};
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash, EnumIter)]
+pub enum HudiInternalConfig {
+ SkipConfigValidation,
+}
+
+impl AsRef<str> for HudiInternalConfig {
+ fn as_ref(&self) -> &str {
+ match self {
+ Self::SkipConfigValidation =>
"hoodie.internal.skip.config.validation",
+ }
+ }
+}
+
+impl ConfigParser for HudiInternalConfig {
+ type Output = HudiConfigValue;
+
+ fn default_value(&self) -> Option<HudiConfigValue> {
+ match self {
+ Self::SkipConfigValidation =>
Some(HudiConfigValue::Boolean(false)),
+ }
+ }
+
+ fn parse_value(&self, configs: &HashMap<String, String>) ->
Result<Self::Output> {
+ let get_result = configs
+ .get(self.as_ref())
+ .map(|v| v.as_str())
+ .ok_or(anyhow!("Config '{}' not found", self.as_ref()));
+
+ match self {
+ Self::SkipConfigValidation => get_result
+ .and_then(|v| bool::from_str(v).map_err(|e| anyhow!(e)))
+ .map(HudiConfigValue::Boolean),
+ }
+ }
+}
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 2399f93..f8975cf 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -22,6 +22,7 @@ use std::sync::Arc;
use anyhow::Result;
+pub mod internal;
pub mod read;
pub mod table;
diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs
index 1493394..db50525 100644
--- a/crates/core/src/table/mod.rs
+++ b/crates/core/src/table/mod.rs
@@ -28,9 +28,11 @@ use arrow_schema::Schema;
use strum::IntoEnumIterator;
use url::Url;
+use HudiInternalConfig::SkipConfigValidation;
use HudiTableConfig::{DropsPartitionFields, TableType, TableVersion};
use TableTypeValue::CopyOnWrite;
+use crate::config::internal::HudiInternalConfig;
use crate::config::read::HudiReadConfig;
use crate::config::table::{HudiTableConfig, TableTypeValue};
use crate::config::HudiConfigs;
@@ -111,17 +113,13 @@ impl Table {
}
let hudi_configs = HudiConfigs::new(hudi_options);
- Self::validate_configs(&hudi_configs, &extra_options).map(|_|
(hudi_configs, extra_options))
+ Self::validate_configs(&hudi_configs).map(|_| (hudi_configs,
extra_options))
}
- fn validate_configs(
- hudi_configs: &HudiConfigs,
- extra_options: &HashMap<String, String>,
- ) -> Result<()> {
- if extra_options
- .get("hoodie_internal.skip.config.validation")
- .and_then(|v| bool::from_str(v).ok())
- .unwrap_or(false)
+ fn validate_configs(hudi_configs: &HudiConfigs) -> Result<()> {
+ if hudi_configs
+ .get_or_default(SkipConfigValidation)
+ .to::<bool>()
{
return Ok(());
}
@@ -389,7 +387,7 @@ mod tests {
let table = Table::new(
base_url.as_str(),
HashMap::from_iter(vec![(
- "hoodie_internal.skip.config.validation".to_string(),
+ "hoodie.internal.skip.config.validation".to_string(),
"true".to_string(),
)]),
)
@@ -448,7 +446,7 @@ mod tests {
let table = Table::new(
base_url.as_str(),
HashMap::from_iter(vec![(
- "hoodie_internal.skip.config.validation".to_string(),
+ "hoodie.internal.skip.config.validation".to_string(),
"true".to_string(),
)]),
)
@@ -480,7 +478,7 @@ mod tests {
let table = Table::new(
base_url.as_str(),
HashMap::from_iter(vec![(
- "hoodie_internal.skip.config.validation".to_string(),
+ "hoodie.internal.skip.config.validation".to_string(),
"true".to_string(),
)]),
)
@@ -518,7 +516,7 @@ mod tests {
let table = Table::new(
base_url.as_str(),
HashMap::from_iter(vec![(
- "hoodie_internal.skip.config.validation".to_string(),
+ "hoodie.internal.skip.config.validation".to_string(),
"true".to_string(),
)]),
)
