This is an automated email from the ASF dual-hosted git repository. numinnex pushed a commit to branch fix_diagnostics_for_ulimit in repository https://gitbox.apache.org/repos/asf/iggy.git
commit acec75d09dadd269a25b7b2f243d97477620a5ae Author: Grzegorz Koszyk <[email protected]> AuthorDate: Wed Jul 29 11:41:23 2026 +0200 fix(server-ng): fold io_uring remediation into runtime-creation errors --- Cargo.toml | 5 +- core/server-ng/src/bootstrap.rs | 11 +++- core/server-ng/src/main.rs | 13 +--- core/server-ng/src/server_error.rs | 6 +- core/server/src/main.rs | 40 ++++--------- core/server_common/src/diagnostics.rs | 108 +++++++++++++++++++++++++++++++++- 6 files changed, 137 insertions(+), 46 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 153461028..0147c62ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,7 +85,10 @@ aligned-vec = "0.6.4" anyhow = "1.0.104" apache-avro = "0.21.0" apple-native-keyring-store = { version = "1.0.1", features = ["keychain"] } -argon2 = "0.5.3" +# "std" is load-bearing: it cascades to rand_core/getrandom, which the +# crypto module's `OsRng` import needs even when no sibling crate in the +# build graph happens to enable it via feature unification. +argon2 = { version = "0.5.3", features = ["std"] } # Pinned to 57 because iceberg 0.9.1 still requires arrow/parquet 57. arrow = "57.3.1" arrow-array = "57.3.1" diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs index aed10ffb0..db31f2903 100644 --- a/core/server-ng/src/bootstrap.rs +++ b/core/server-ng/src/bootstrap.rs @@ -797,8 +797,15 @@ fn run_shard_thread( .bind_memory() .map_err(|source| ServerNgError::MemoryAffinityFailed { shard_id, source })?; - let runtime = create_shard_executor() - .map_err(|source| ServerNgError::ShardRuntimeCreateFailed { shard_id, source })?; + // `enrich_runtime_create_error` folds the io_uring remediation (raise + // `ulimit -l`, unblock seccomp, kernel-flag floor) into the error, so the + // guidance survives into the shard-join failure report instead of only + // stderr. Multi-shard boxes exhaust RLIMIT_MEMLOCK on per-shard rings + // before the bootstrap runtime does, so this path needs it most. + let runtime = create_shard_executor().map_err(|source| { + let source = server_common::diagnostics::enrich_runtime_create_error(source); + ServerNgError::ShardRuntimeCreateFailed { shard_id, source } + })?; let result = runtime.block_on(async move { // `shard_main`'s future grows past clippy's `large_futures` cap diff --git a/core/server-ng/src/main.rs b/core/server-ng/src/main.rs index a3e11fd64..404a4496f 100644 --- a/core/server-ng/src/main.rs +++ b/core/server-ng/src/main.rs @@ -33,18 +33,7 @@ fn main() -> Result<(), ServerNgError> { let bootstrap_runtime = match server_common::create_shard_executor() { Ok(rt) => rt, Err(e) => { - match e.kind() { - std::io::ErrorKind::InvalidInput => { - server_common::diagnostics::print_invalid_io_uring_args_info(); - } - std::io::ErrorKind::OutOfMemory => { - server_common::diagnostics::print_locked_memory_limit_info(); - } - std::io::ErrorKind::PermissionDenied => { - server_common::diagnostics::print_io_uring_permission_info(); - } - _ => {} - } + let e = server_common::diagnostics::enrich_runtime_create_error(e); panic!("Cannot create server-ng bootstrap executor: {e}"); } }; diff --git a/core/server-ng/src/server_error.rs b/core/server-ng/src/server_error.rs index b19f26778..61cd1adf4 100644 --- a/core/server-ng/src/server_error.rs +++ b/core/server-ng/src/server_error.rs @@ -48,7 +48,11 @@ pub enum ServerNgError { #[source] source: std::io::Error, }, - #[error("failed to create io_uring runtime for shard {shard_id}")] + // `{source}` is deliberately part of the Display text: the shard-join + // failure report and `%error` log fields print Display only, and the + // source carries the io_uring remediation folded in by + // `server_common::diagnostics::enrich_runtime_create_error`. + #[error("failed to create io_uring runtime for shard {shard_id}: {source}")] ShardRuntimeCreateFailed { shard_id: u16, #[source] diff --git a/core/server/src/main.rs b/core/server/src/main.rs index b5c0110f7..1f0ac18d9 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -29,11 +29,7 @@ use server::bootstrap::{ create_directories, create_shard_connections, create_shard_executor, load_config, load_metadata, resolve_persister, update_system_info, }; -use server::diagnostics::{ - ASYNCIFY_POOL_DISABLED_PANIC_MSG, print_incomplete_io_uring_ops_info, - print_invalid_io_uring_args_info, print_io_uring_permission_info, - print_locked_memory_limit_info, -}; +use server::diagnostics::{ASYNCIFY_POOL_DISABLED_PANIC_MSG, print_incomplete_io_uring_ops_info}; use server::io::fs_utils; use server::log::logger::Logging; use server::metadata::{Metadata, create_metadata_handles}; @@ -65,10 +61,10 @@ const SHARDS_TABLE_CAPACITY: usize = 16384; static SHUTDOWN_START_TIME: AtomicU64 = AtomicU64::new(0); static SHUTDOWN_INITIATED: AtomicBool = AtomicBool::new(false); -static SHARD_EXECUTOR_DIAGNOSTIC: std::sync::Once = std::sync::Once::new(); -// Separate latch from SHARD_EXECUTOR_DIAGNOSTIC: a shard that fails ring setup -// (e.g. partial ENOMEM under a tight RLIMIT_MEMLOCK) must not consume the latch -// and suppress the unsupported-opcode diagnostic from a sibling shard that did +// Separate latch from the ring-setup one inside +// `enrich_runtime_create_error`: a shard that fails ring setup (e.g. partial +// ENOMEM under a tight RLIMIT_MEMLOCK) must not consume the latch and +// suppress the unsupported-opcode diagnostic from a sibling shard that did // start. Setup vs runtime io_uring failures can co-occur across shards. static SHARD_RUNTIME_DIAGNOSTIC: std::sync::Once = std::sync::Once::new(); @@ -123,11 +119,7 @@ fn main() -> Result<(), ServerError> { let rt = match compio::runtime::Runtime::new() { Ok(rt) => rt, Err(e) => { - match e.kind() { - std::io::ErrorKind::OutOfMemory => print_locked_memory_limit_info(), - std::io::ErrorKind::PermissionDenied => print_io_uring_permission_info(), - _ => {} - } + let e = server_common::diagnostics::enrich_runtime_create_error(e); panic!("Cannot create runtime: {e}"); } }; @@ -417,21 +409,11 @@ fn main() -> Result<(), ServerError> { let rt = match create_shard_executor() { Ok(rt) => rt, Err(e) => { - match e.kind() { - std::io::ErrorKind::InvalidInput => { - SHARD_EXECUTOR_DIAGNOSTIC - .call_once(print_invalid_io_uring_args_info); - } - std::io::ErrorKind::OutOfMemory => { - SHARD_EXECUTOR_DIAGNOSTIC - .call_once(print_locked_memory_limit_info); - } - std::io::ErrorKind::PermissionDenied => { - SHARD_EXECUTOR_DIAGNOSTIC - .call_once(print_io_uring_permission_info); - } - _ => {} - } + // Prints the verbose remediation once across + // all shard threads; the panic message itself + // carries the one-line fix. + let e = + server_common::diagnostics::enrich_runtime_create_error(e); panic!("Cannot create shard-{id} executor: {e}"); } }; diff --git a/core/server_common/src/diagnostics.rs b/core/server_common/src/diagnostics.rs index 10059423f..2b745d7e4 100644 --- a/core/server_common/src/diagnostics.rs +++ b/core/server_common/src/diagnostics.rs @@ -31,6 +31,79 @@ pub const ASYNCIFY_POOL_DISABLED_PANIC_MSG: &str = #[cfg(target_os = "linux")] const DISCORD_SUPPORT_URL: &str = "https://discord.gg/apache-iggy";; +/// Classify a failed shard-executor creation and fold the matching +/// remediation into the error itself. +/// +/// The verbose remediation block (current limits, per-environment fix +/// steps) is printed to stderr once per process; the returned error +/// carries a one-line summary of the cause and the primary fix, so every +/// propagated copy - the shard-join failure list, a panic message, a log +/// line hours later in a collector - documents the remediation instead +/// of only the stderr captured at the moment of failure. +/// +/// Errors of a kind this module has no diagnosis for pass through +/// unchanged. +#[cfg(target_os = "linux")] +pub fn enrich_runtime_create_error(error: std::io::Error) -> std::io::Error { + static RUNTIME_CREATE_DIAGNOSTIC: std::sync::Once = std::sync::Once::new(); + + let kind = error.kind(); + let hint = match kind { + std::io::ErrorKind::OutOfMemory => { + RUNTIME_CREATE_DIAGNOSTIC.call_once(print_locked_memory_limit_info); + locked_memory_limit_hint() + } + std::io::ErrorKind::PermissionDenied => { + RUNTIME_CREATE_DIAGNOSTIC.call_once(print_io_uring_permission_info); + "io_uring syscalls are blocked, typically by a container seccomp \ + profile: allow io_uring_setup/io_uring_enter/io_uring_register, \ + or run with `--security-opt seccomp=unconfined` (Docker) / \ + `seccompProfile: {type: Unconfined}` (Kubernetes)" + .to_owned() + } + std::io::ErrorKind::InvalidInput => { + RUNTIME_CREATE_DIAGNOSTIC.call_once(print_invalid_io_uring_args_info); + format!( + "the kernel rejected io_uring setup flags shard executors require \ + (IORING_SETUP_COOP_TASKRUN + IORING_SETUP_TASKRUN_FLAG need Linux \ + >= {MIN_KERNEL_MAJOR}.{MIN_KERNEL_MINOR} with full io_uring support; \ + WSL2 kernels are often incomplete)" + ) + } + _ => return error, + }; + std::io::Error::new(kind, format!("{error}: {hint}")) +} + +#[cfg(not(target_os = "linux"))] +pub fn enrich_runtime_create_error(error: std::io::Error) -> std::io::Error { + error +} + +/// One-line remediation for an io_uring ring allocation denied by +/// `RLIMIT_MEMLOCK`, with the live limits baked in so a log line is +/// self-sufficient evidence of the misconfiguration. +#[cfg(target_os = "linux")] +fn locked_memory_limit_hint() -> String { + use nix::sys::resource::{Resource, getrlimit}; + + let limits = getrlimit(Resource::RLIMIT_MEMLOCK).map_or_else( + |_| "RLIMIT_MEMLOCK could not be read".to_owned(), + |(soft, hard)| { + format!( + "RLIMIT_MEMLOCK soft={}, hard={}", + format_limit(soft), + format_limit(hard) + ) + }, + ); + format!( + "io_uring was denied locked memory for its rings ({limits}): raise the \ + limit with `ulimit -l unlimited` (shell), `LimitMEMLOCK=infinity` \ + (systemd), or `--ulimit memlock=-1:-1` (Docker)" + ) +} + #[cfg(target_os = "linux")] fn print_discord_link() { eprintln!(" Need help? Join our Discord: {DISCORD_SUPPORT_URL}"); @@ -359,7 +432,40 @@ pub const fn print_incomplete_io_uring_ops_info() {} #[cfg(all(test, target_os = "linux"))] mod tests { - use super::{format_limit, parse_kernel_version}; + use super::{enrich_runtime_create_error, format_limit, parse_kernel_version}; + + #[test] + fn enrich_folds_memlock_remediation_into_the_error() { + let raw = std::io::Error::new(std::io::ErrorKind::OutOfMemory, "io_uring setup: ENOMEM"); + let enriched = enrich_runtime_create_error(raw); + let message = enriched.to_string(); + assert_eq!(enriched.kind(), std::io::ErrorKind::OutOfMemory); + assert!(message.contains("io_uring setup: ENOMEM"), "{message}"); + assert!(message.contains("ulimit -l unlimited"), "{message}"); + assert!(message.contains("RLIMIT_MEMLOCK"), "{message}"); + } + + #[test] + fn enrich_folds_seccomp_remediation_into_the_error() { + let raw = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "EPERM"); + let message = enrich_runtime_create_error(raw).to_string(); + assert!(message.contains("seccomp"), "{message}"); + } + + #[test] + fn enrich_folds_kernel_flag_remediation_into_the_error() { + let raw = std::io::Error::new(std::io::ErrorKind::InvalidInput, "EINVAL"); + let message = enrich_runtime_create_error(raw).to_string(); + assert!(message.contains("IORING_SETUP_COOP_TASKRUN"), "{message}"); + } + + #[test] + fn enrich_passes_undiagnosed_kinds_through_unchanged() { + let raw = std::io::Error::new(std::io::ErrorKind::Interrupted, "EINTR"); + let enriched = enrich_runtime_create_error(raw); + assert_eq!(enriched.kind(), std::io::ErrorKind::Interrupted); + assert_eq!(enriched.to_string(), "EINTR"); + } #[test] fn parse_kernel_version_standard() {
