comphead commented on code in PR #3111: URL: https://github.com/apache/iceberg-rust/pull/3111#discussion_r4231835012
########## crates/storage/opendal/src/hdfs_native.rs: ########## @@ -0,0 +1,1347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HDFS storage backend via OpenDAL's `services-hdfs-native` (pure Rust, no JNI). + +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::sync::{Arc, PoisonError, RwLock, Weak}; + +use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; +use iceberg::{Error, ErrorKind, Result}; +use opendal::Operator; +use opendal::services::HdfsNativeConfig; +use serde::{Deserialize, Serialize}; +use tokio::runtime::Handle; +use tokio::task::JoinHandle; +use url::Url; + +use crate::OpenDalClientConfig; +use crate::utils::from_opendal_error; + +/// Hadoop's default filesystem, which serves authority-less paths. +const FS_DEFAULT_FS: &str = "fs.defaultFS"; +const HDFS_DEFAULT_PORT: u16 = 8020; +/// PyIceberg keys with no equivalent in opendal's config. +const HDFS_UNSUPPORTED_KEYS: [&str; 2] = ["hdfs.user", "hdfs.kerberos_ticket"]; +/// Hadoop's keys declaring an HA nameservice: `hdfs.name-node.<nameservice>` +/// expands to them, and they are honored as well when passed through `hadoop.`. +const HA_NAMENODES_PREFIX: &str = "dfs.ha.namenodes"; +const HA_NAMENODE_RPC_ADDRESS_PREFIX: &str = "dfs.namenode.rpc-address"; + +/// Normalizes one NameNode spelling to `hdfs://host:port`, the form path +/// authorities take, so every source shares cache keys. `hdfs-native` dials +/// a socket address and has no default port, so anything else is `None`: +/// another scheme, a logical name, port 0, userinfo, a path. +fn hdfs_native_name_node(entry: &str) -> Option<String> { + let rest = entry.trim().trim_end_matches('/'); + let rest = rest.strip_prefix("hdfs://").unwrap_or(rest); + if rest.is_empty() || rest.contains("://") { + return None; + } + let url = Url::parse(&format!("hdfs://{rest}")).ok()?; + let (host, port) = (url.host_str()?, url.port()?); + let plain = host.is_empty() + || port == 0 + || !url.username().is_empty() + || url.password().is_some() + || !matches!(url.path(), "" | "/") + || url.query().is_some() + || url.fragment().is_some(); + (!plain).then(|| format!("hdfs://{host}:{port}")) +} + +/// Parses a comma-separated NameNode list, each entry normalized; an empty +/// list is `Ok` and empty. +fn hdfs_native_name_node_list(property: &str, value: &str) -> Result<Vec<String>> { + value + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(|entry| { + hdfs_native_name_node(entry).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{property}` entry: {}, expected host:port (hdfs:// optional)", + hdfs_native_redact(entry) + ), + ) + }) + }) + .collect() +} + +/// Parse iceberg properties to [`HdfsNativeConfig`]; dropped properties are +/// logged as warnings. +pub(crate) fn hdfs_native_config_parse(m: HashMap<String, String>) -> Result<HdfsNativeConfig> { + hdfs_native_config_parse_with(m, |warning| tracing::warn!("{warning}")) +} + +/// The parser proper, with the warning sink injected so tests can see what +/// was dropped without a tracing subscriber. +fn hdfs_native_config_parse_with( + mut m: HashMap<String, String>, + mut warn: impl FnMut(String), +) -> Result<HdfsNativeConfig> { + let mut cfg = HdfsNativeConfig::default(); + + // Entries are trimmed one by one: opendal splits the list on `,` as is, + // so a space after a comma would break failover to that NameNode. An + // empty result is dropped because `Operator::from_config` bypasses the + // builder's empty-string guard and `Some("")` would shadow the + // path-authority fallback below. + if let Some(name_node) = m.remove(HDFS_NAME_NODE) { + let entries = hdfs_native_name_node_list(HDFS_NAME_NODE, &name_node)?; + if !entries.is_empty() { + cfg.name_node = Some(entries.join(",")); + } + } + + // `hdfs.name-node.<nameservice>` is sugar for Hadoop's own declaration of + // an HA nameservice, which the resolver reads back from the options. + let nameservice_prefix = format!("{HDFS_NAME_NODE}."); + let declared_keys: Vec<String> = m + .keys() + .filter(|key| key.starts_with(&nameservice_prefix)) + .cloned() + .collect(); + let mut declared = Vec::new(); + for key in declared_keys { + let value = m.remove(&key).unwrap_or_default(); + let nameservice = key[nameservice_prefix.len()..].to_string(); Review Comment: **Nit (manual slicing):** `key[nameservice_prefix.len()..]` relies on a `starts_with` check made in a separate statement above. Using `key.strip_prefix(&nameservice_prefix)` here, or collecting `(nameservice, value)` pairs from `strip_prefix` in the first pass, makes it panic-free by construction and avoids the key clones and the `remove(&key).unwrap_or_default()`. The same applies to `s[..i]`, `s[start..]` and `s[start + at..]` in `hdfs_native_redact` (L252 to 258) and `after_scheme[i..]` in `hdfs_native_parse_path` (L300), which could use `split_once` or `strip_prefix`. The indices come from `find`, so I do not see a panic today. ########## crates/storage/opendal/src/hdfs_native.rs: ########## @@ -0,0 +1,1347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HDFS storage backend via OpenDAL's `services-hdfs-native` (pure Rust, no JNI). + +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::sync::{Arc, PoisonError, RwLock, Weak}; + +use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; +use iceberg::{Error, ErrorKind, Result}; +use opendal::Operator; +use opendal::services::HdfsNativeConfig; +use serde::{Deserialize, Serialize}; +use tokio::runtime::Handle; +use tokio::task::JoinHandle; +use url::Url; + +use crate::OpenDalClientConfig; +use crate::utils::from_opendal_error; + +/// Hadoop's default filesystem, which serves authority-less paths. +const FS_DEFAULT_FS: &str = "fs.defaultFS"; +const HDFS_DEFAULT_PORT: u16 = 8020; +/// PyIceberg keys with no equivalent in opendal's config. +const HDFS_UNSUPPORTED_KEYS: [&str; 2] = ["hdfs.user", "hdfs.kerberos_ticket"]; +/// Hadoop's keys declaring an HA nameservice: `hdfs.name-node.<nameservice>` +/// expands to them, and they are honored as well when passed through `hadoop.`. +const HA_NAMENODES_PREFIX: &str = "dfs.ha.namenodes"; +const HA_NAMENODE_RPC_ADDRESS_PREFIX: &str = "dfs.namenode.rpc-address"; + +/// Normalizes one NameNode spelling to `hdfs://host:port`, the form path +/// authorities take, so every source shares cache keys. `hdfs-native` dials +/// a socket address and has no default port, so anything else is `None`: +/// another scheme, a logical name, port 0, userinfo, a path. +fn hdfs_native_name_node(entry: &str) -> Option<String> { + let rest = entry.trim().trim_end_matches('/'); + let rest = rest.strip_prefix("hdfs://").unwrap_or(rest); + if rest.is_empty() || rest.contains("://") { + return None; + } + let url = Url::parse(&format!("hdfs://{rest}")).ok()?; + let (host, port) = (url.host_str()?, url.port()?); + let plain = host.is_empty() + || port == 0 + || !url.username().is_empty() + || url.password().is_some() + || !matches!(url.path(), "" | "/") + || url.query().is_some() + || url.fragment().is_some(); + (!plain).then(|| format!("hdfs://{host}:{port}")) +} + +/// Parses a comma-separated NameNode list, each entry normalized; an empty +/// list is `Ok` and empty. +fn hdfs_native_name_node_list(property: &str, value: &str) -> Result<Vec<String>> { + value + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(|entry| { + hdfs_native_name_node(entry).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{property}` entry: {}, expected host:port (hdfs:// optional)", + hdfs_native_redact(entry) + ), + ) + }) + }) + .collect() +} + +/// Parse iceberg properties to [`HdfsNativeConfig`]; dropped properties are +/// logged as warnings. +pub(crate) fn hdfs_native_config_parse(m: HashMap<String, String>) -> Result<HdfsNativeConfig> { + hdfs_native_config_parse_with(m, |warning| tracing::warn!("{warning}")) +} + +/// The parser proper, with the warning sink injected so tests can see what +/// was dropped without a tracing subscriber. +fn hdfs_native_config_parse_with( + mut m: HashMap<String, String>, + mut warn: impl FnMut(String), +) -> Result<HdfsNativeConfig> { + let mut cfg = HdfsNativeConfig::default(); + + // Entries are trimmed one by one: opendal splits the list on `,` as is, + // so a space after a comma would break failover to that NameNode. An + // empty result is dropped because `Operator::from_config` bypasses the + // builder's empty-string guard and `Some("")` would shadow the + // path-authority fallback below. + if let Some(name_node) = m.remove(HDFS_NAME_NODE) { + let entries = hdfs_native_name_node_list(HDFS_NAME_NODE, &name_node)?; + if !entries.is_empty() { + cfg.name_node = Some(entries.join(",")); + } + } + + // `hdfs.name-node.<nameservice>` is sugar for Hadoop's own declaration of + // an HA nameservice, which the resolver reads back from the options. + let nameservice_prefix = format!("{HDFS_NAME_NODE}."); + let declared_keys: Vec<String> = m + .keys() + .filter(|key| key.starts_with(&nameservice_prefix)) + .cloned() + .collect(); + let mut declared = Vec::new(); + for key in declared_keys { + let value = m.remove(&key).unwrap_or_default(); + let nameservice = key[nameservice_prefix.len()..].to_string(); + if nameservice.is_empty() || nameservice.chars().any(char::is_whitespace) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid property `{key}`: a nameservice name must follow `{nameservice_prefix}`" + ), + )); + } + let entries = hdfs_native_name_node_list(&key, &value)?; + if entries.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid property `{key}`: no NameNodes"), + )); + } + declared.push((nameservice, entries)); + } + + // A config carried over from PyIceberg would otherwise change identity + // silently; the client reads `HADOOP_USER_NAME` and the Kerberos cache. + for key in HDFS_UNSUPPORTED_KEYS { + if m.remove(key).is_some() { + warn(format!( + "`{key}` is not supported by the hdfs-native backend and is ignored" + )); + } + } + if m.contains_key(HDFS_HADOOP_CONF_PREFIX) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid property `{HDFS_HADOOP_CONF_PREFIX}`: a Hadoop key must follow the prefix" + ), + )); + } + + let host = m + .remove(HDFS_HOST) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + let port = m + .remove(HDFS_PORT) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .map(|port| { + port.parse::<u16>().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid `{HDFS_PORT}`: {port}: {e}"), + ) + }) + }) + .transpose()?; + + let mut options: HashMap<String, String> = m + .into_iter() + .filter_map(|(key, value)| { + key.strip_prefix(HDFS_HADOOP_CONF_PREFIX) + .map(|stripped| (stripped.to_string(), value)) + }) + .collect(); + // Explicit `hadoop.` keys win over the sugar. + for (nameservice, entries) in declared { + let ids: Vec<String> = (0..entries.len()).map(|i| format!("nn{i}")).collect(); + options + .entry(format!("{HA_NAMENODES_PREFIX}.{nameservice}")) + .or_insert_with(|| ids.join(",")); + for (id, entry) in ids.iter().zip(&entries) { + options + .entry(format!( + "{HA_NAMENODE_RPC_ADDRESS_PREFIX}.{nameservice}.{id}" + )) + .or_insert_with(|| entry.trim_start_matches("hdfs://").to_string()); + } + } + // PyIceberg's `hdfs.host`/`hdfs.port` name the filesystem for + // authority-less paths, which is what Hadoop's `fs.defaultFS` means; an + // explicit `hadoop.fs.defaultFS` wins. + match host { + Some(host) => { + // An IPv6 literal needs brackets in a URI authority. + let bracketed = if host.contains(':') && !host.starts_with('[') { + format!("[{host}]") + } else { + host.clone() + }; + let port = port.unwrap_or(HDFS_DEFAULT_PORT); + // Validated like every other NameNode spelling, so a host that + // carries a scheme or port fails here and not at the first I/O. + let default_fs = hdfs_native_name_node(&format!("hdfs://{bracketed}:{port}")) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{HDFS_HOST}`/`{HDFS_PORT}`: {}, expected a host name or IP and a port", + hdfs_native_redact(&format!("{host}:{port}")) + ), + ) + })?; + options + .entry(FS_DEFAULT_FS.to_string()) + .or_insert(default_fs); + } + None if port.is_some() => { + warn(format!( + "`{HDFS_PORT}` has no effect without `{HDFS_HOST}` and is ignored" + )); + } + None => {} + } + if !options.is_empty() { + cfg.options = Some(options); + } + + Ok(cfg) +} + +/// `s` with everything between its scheme and its last `@` masked, so an +/// error message never echoes userinfo, even a password holding a raw `/`, +/// `?`, `#` or `://` that a URL parser would split elsewhere. An `@` further +/// along the path is masked the same way, which only costs context. +fn hdfs_native_redact(s: &str) -> String { + let start = s + .find("://") + .filter(|&i| { + s[..i] + .chars() + .all(|c| c.is_ascii_alphanumeric() || "+-.".contains(c)) + }) + .map_or(0, |i| i + 3); + match s[start..].rfind('@') { + Some(at) => format!("{}***{}", &s[..start], &s[start + at..]), + None => s.to_string(), + } +} + +/// A `DataInvalid` error for an HDFS path, its userinfo masked. +fn hdfs_native_invalid_path(path: &str, reason: impl std::fmt::Display) -> Error { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid hdfs path: {}, {reason}", hdfs_native_redact(path)), + ) +} + +/// Parse an HDFS path into `Some("hdfs://<authority>")` (`None` when +/// authority-less) and the relative path (no leading `/`, opendal style). +pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option<String>, &str)> { + let url = Url::parse(path).map_err(|e| hdfs_native_invalid_path(path, e))?; + // Non-special schemes parse even without `//` (e.g. `hdfs:x` is a valid + // non-hierarchical URL), so require the literal prefix before slicing. + let (Some(after_scheme), "hdfs") = (path.strip_prefix("hdfs://"), url.scheme()) else { + return Err(hdfs_native_invalid_path(path, "expected scheme `hdfs://`")); + }; + // Userinfo has nowhere to go and port 0 cannot be dialed; silently + // dropping the one or reading the other as a logical name would mislead. + if !url.username().is_empty() || url.password().is_some() { + return Err(hdfs_native_invalid_path(path, "userinfo is not supported")); + } + if url.port() == Some(0) { + return Err(hdfs_native_invalid_path(path, "port 0 cannot be dialed")); + } + + let name_node = url.host_str().filter(|h| !h.is_empty()).map(|host| { + url.port() + .map(|port| format!("hdfs://{host}:{port}")) + .unwrap_or_else(|| format!("hdfs://{host}")) + }); + + // `url.path()` borrows from `url` and can't be returned with the input's + // lifetime. Slice the path component out of the original input instead; + // it starts after the first `/` following the `hdfs://` prefix. Opendal + // paths must not start with `/` (`Deleter::delete` rejects them). + let rel = match after_scheme.find('/') { + Some(i) => after_scheme[i..].trim_start_matches('/'), + None => "", + }; + + Ok((name_node, rel)) +} + +/// Resolves the effective NameNode for a path, plus the relative path. An +/// authority with a port is used as is, a portless one must be a declared +/// nameservice, and an authority-less path uses `hdfs.name-node`, else +/// `fs.defaultFS`. The operator cache, `delete_stream` batching and +/// `relativize_path` all go through this, so they cannot drift apart. +pub(crate) fn hdfs_native_effective_name_node<'a>( Review Comment: **Should fix (strong types):** following up on my earlier comment here. cheats.rs recommends `struct Charge(f32)` over a bare `f32` and enums over flags, so that illegal states cannot be built. A NameNode is currently a `String` that can be an endpoint, a logical nameservice or a comma-separated list. `hdfs_native_parse_path` returns `Option<String>`, and this function re-parses that string to learn which kind it is. The results are tuples such as `(String, &'a str)`. A `NameNode` newtype and an `Authority` enum would make the states explicit and remove the re-parse. The `hdfs_native_` prefix on every free function looks like a workaround for the flat `use hdfs_native::*;` in `lib.rs`. With a resolver type these could be methods and the prefix would not be needed. This part is my inference, not a cheats.rs rule. ########## crates/storage/opendal/src/lib.rs: ########## @@ -186,6 +194,9 @@ pub enum OpenDalStorageFactory { /// GCS storage factory. #[cfg(feature = "opendal-gcs")] Gcs, + /// HDFS storage factory. + #[cfg(feature = "opendal-hdfs-native")] + HdfsNative, Review Comment: **Question (API stability):** cheats.rs lists "Adding new enum variants" as breaking and suggests an early `#[non_exhaustive]`. `OpenDalStorageFactory` and `OpenDalStorage` do not have it, so a downstream exhaustive `match` stops compiling when a feature-unified build turns on `opendal-hdfs-native`. Earlier backends did the same and the feature is off by default, so I would not block on it. Would it be worth adding `#[non_exhaustive]` to both enums in a separate PR? ########## crates/storage/opendal/src/hdfs_native.rs: ########## @@ -0,0 +1,1347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HDFS storage backend via OpenDAL's `services-hdfs-native` (pure Rust, no JNI). + +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::sync::{Arc, PoisonError, RwLock, Weak}; + +use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; +use iceberg::{Error, ErrorKind, Result}; +use opendal::Operator; +use opendal::services::HdfsNativeConfig; +use serde::{Deserialize, Serialize}; +use tokio::runtime::Handle; +use tokio::task::JoinHandle; +use url::Url; + +use crate::OpenDalClientConfig; +use crate::utils::from_opendal_error; + +/// Hadoop's default filesystem, which serves authority-less paths. +const FS_DEFAULT_FS: &str = "fs.defaultFS"; +const HDFS_DEFAULT_PORT: u16 = 8020; +/// PyIceberg keys with no equivalent in opendal's config. +const HDFS_UNSUPPORTED_KEYS: [&str; 2] = ["hdfs.user", "hdfs.kerberos_ticket"]; +/// Hadoop's keys declaring an HA nameservice: `hdfs.name-node.<nameservice>` +/// expands to them, and they are honored as well when passed through `hadoop.`. +const HA_NAMENODES_PREFIX: &str = "dfs.ha.namenodes"; +const HA_NAMENODE_RPC_ADDRESS_PREFIX: &str = "dfs.namenode.rpc-address"; + +/// Normalizes one NameNode spelling to `hdfs://host:port`, the form path +/// authorities take, so every source shares cache keys. `hdfs-native` dials +/// a socket address and has no default port, so anything else is `None`: +/// another scheme, a logical name, port 0, userinfo, a path. +fn hdfs_native_name_node(entry: &str) -> Option<String> { + let rest = entry.trim().trim_end_matches('/'); + let rest = rest.strip_prefix("hdfs://").unwrap_or(rest); + if rest.is_empty() || rest.contains("://") { + return None; + } + let url = Url::parse(&format!("hdfs://{rest}")).ok()?; + let (host, port) = (url.host_str()?, url.port()?); + let plain = host.is_empty() + || port == 0 + || !url.username().is_empty() + || url.password().is_some() + || !matches!(url.path(), "" | "/") + || url.query().is_some() + || url.fragment().is_some(); + (!plain).then(|| format!("hdfs://{host}:{port}")) +} + +/// Parses a comma-separated NameNode list, each entry normalized; an empty +/// list is `Ok` and empty. +fn hdfs_native_name_node_list(property: &str, value: &str) -> Result<Vec<String>> { + value + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(|entry| { + hdfs_native_name_node(entry).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{property}` entry: {}, expected host:port (hdfs:// optional)", + hdfs_native_redact(entry) + ), + ) + }) + }) + .collect() +} + +/// Parse iceberg properties to [`HdfsNativeConfig`]; dropped properties are +/// logged as warnings. +pub(crate) fn hdfs_native_config_parse(m: HashMap<String, String>) -> Result<HdfsNativeConfig> { + hdfs_native_config_parse_with(m, |warning| tracing::warn!("{warning}")) +} + +/// The parser proper, with the warning sink injected so tests can see what +/// was dropped without a tracing subscriber. +fn hdfs_native_config_parse_with( + mut m: HashMap<String, String>, + mut warn: impl FnMut(String), +) -> Result<HdfsNativeConfig> { + let mut cfg = HdfsNativeConfig::default(); + + // Entries are trimmed one by one: opendal splits the list on `,` as is, + // so a space after a comma would break failover to that NameNode. An + // empty result is dropped because `Operator::from_config` bypasses the + // builder's empty-string guard and `Some("")` would shadow the + // path-authority fallback below. + if let Some(name_node) = m.remove(HDFS_NAME_NODE) { + let entries = hdfs_native_name_node_list(HDFS_NAME_NODE, &name_node)?; + if !entries.is_empty() { + cfg.name_node = Some(entries.join(",")); + } + } + + // `hdfs.name-node.<nameservice>` is sugar for Hadoop's own declaration of + // an HA nameservice, which the resolver reads back from the options. + let nameservice_prefix = format!("{HDFS_NAME_NODE}."); + let declared_keys: Vec<String> = m + .keys() + .filter(|key| key.starts_with(&nameservice_prefix)) + .cloned() + .collect(); + let mut declared = Vec::new(); + for key in declared_keys { + let value = m.remove(&key).unwrap_or_default(); + let nameservice = key[nameservice_prefix.len()..].to_string(); + if nameservice.is_empty() || nameservice.chars().any(char::is_whitespace) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid property `{key}`: a nameservice name must follow `{nameservice_prefix}`" + ), + )); + } + let entries = hdfs_native_name_node_list(&key, &value)?; + if entries.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid property `{key}`: no NameNodes"), + )); + } + declared.push((nameservice, entries)); + } + + // A config carried over from PyIceberg would otherwise change identity + // silently; the client reads `HADOOP_USER_NAME` and the Kerberos cache. + for key in HDFS_UNSUPPORTED_KEYS { + if m.remove(key).is_some() { + warn(format!( + "`{key}` is not supported by the hdfs-native backend and is ignored" + )); + } + } + if m.contains_key(HDFS_HADOOP_CONF_PREFIX) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid property `{HDFS_HADOOP_CONF_PREFIX}`: a Hadoop key must follow the prefix" + ), + )); + } + + let host = m + .remove(HDFS_HOST) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + let port = m Review Comment: **Nit (iterator chains):** cheats.rs says "Functional style is often easiest to follow, but don't hesitate to use `for` if your `.iter()` chain turns messy." Three chains in this file fit that. - Here, `remove().map(trim.to_string).filter(!empty).map(parse.map_err(Error::new(..))).transpose()?` nests a fallible closure inside an `Option` chain. The `hdfs.host` lookup above repeats the same `map.filter`. A small helper such as `fn trimmed_nonempty(m: &mut HashMap<String, String>, key: &str) -> Option<String>` plus a `match` on the parse would read better. - `hdfs_native_nameservice` (L366) chains `split.map.filter.map(|id| options.get.and_then.ok_or_else(..)).collect::<Result<Vec<_>>>()?`. A `for` loop with `?` is simpler. - `hdfs_native_create_operator` (L538) ends in `.map_err(..)??`. Two separate `?` lines would show which error is the join error and which is the build error. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
