This is an automated email from the ASF dual-hosted git repository.
spetz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new f67f03873 fix(server-ng): fold io_uring remediation into
runtime-creation errors (#3775)
f67f03873 is described below
commit f67f038731f5c074ee14ed23182c0637d2f6f156
Author: Grzegorz Koszyk <[email protected]>
AuthorDate: Wed Jul 29 16:22:51 2026 +0200
fix(server-ng): fold io_uring remediation into runtime-creation errors
(#3775)
---
Cargo.toml | 5 +-
core/server-ng/src/bootstrap.rs | 11 +++-
core/server-ng/src/main.rs | 13 +---
core/server-ng/src/server_error.rs | 6 +-
core/server/src/main.rs | 40 ++++---------
core/server_common/src/diagnostics.rs | 108 +++++++++++++++++++++++++++++++++-
6 files changed, 137 insertions(+), 46 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index 153461028..0147c62ed 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -85,7 +85,10 @@ aligned-vec = "0.6.4"
anyhow = "1.0.104"
apache-avro = "0.21.0"
apple-native-keyring-store = { version = "1.0.1", features = ["keychain"] }
-argon2 = "0.5.3"
+# "std" is load-bearing: it cascades to rand_core/getrandom, which the
+# crypto module's `OsRng` import needs even when no sibling crate in the
+# build graph happens to enable it via feature unification.
+argon2 = { version = "0.5.3", features = ["std"] }
# Pinned to 57 because iceberg 0.9.1 still requires arrow/parquet 57.
arrow = "57.3.1"
arrow-array = "57.3.1"
diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs
index aed10ffb0..db31f2903 100644
--- a/core/server-ng/src/bootstrap.rs
+++ b/core/server-ng/src/bootstrap.rs
@@ -797,8 +797,15 @@ fn run_shard_thread(
.bind_memory()
.map_err(|source| ServerNgError::MemoryAffinityFailed { shard_id,
source })?;
- let runtime = create_shard_executor()
- .map_err(|source| ServerNgError::ShardRuntimeCreateFailed { shard_id,
source })?;
+ // `enrich_runtime_create_error` folds the io_uring remediation (raise
+ // `ulimit -l`, unblock seccomp, kernel-flag floor) into the error, so the
+ // guidance survives into the shard-join failure report instead of only
+ // stderr. Multi-shard boxes exhaust RLIMIT_MEMLOCK on per-shard rings
+ // before the bootstrap runtime does, so this path needs it most.
+ let runtime = create_shard_executor().map_err(|source| {
+ let source =
server_common::diagnostics::enrich_runtime_create_error(source);
+ ServerNgError::ShardRuntimeCreateFailed { shard_id, source }
+ })?;
let result = runtime.block_on(async move {
// `shard_main`'s future grows past clippy's `large_futures` cap
diff --git a/core/server-ng/src/main.rs b/core/server-ng/src/main.rs
index a3e11fd64..404a4496f 100644
--- a/core/server-ng/src/main.rs
+++ b/core/server-ng/src/main.rs
@@ -33,18 +33,7 @@ fn main() -> Result<(), ServerNgError> {
let bootstrap_runtime = match server_common::create_shard_executor() {
Ok(rt) => rt,
Err(e) => {
- match e.kind() {
- std::io::ErrorKind::InvalidInput => {
-
server_common::diagnostics::print_invalid_io_uring_args_info();
- }
- std::io::ErrorKind::OutOfMemory => {
-
server_common::diagnostics::print_locked_memory_limit_info();
- }
- std::io::ErrorKind::PermissionDenied => {
-
server_common::diagnostics::print_io_uring_permission_info();
- }
- _ => {}
- }
+ let e = server_common::diagnostics::enrich_runtime_create_error(e);
panic!("Cannot create server-ng bootstrap executor: {e}");
}
};
diff --git a/core/server-ng/src/server_error.rs
b/core/server-ng/src/server_error.rs
index b19f26778..61cd1adf4 100644
--- a/core/server-ng/src/server_error.rs
+++ b/core/server-ng/src/server_error.rs
@@ -48,7 +48,11 @@ pub enum ServerNgError {
#[source]
source: std::io::Error,
},
- #[error("failed to create io_uring runtime for shard {shard_id}")]
+ // `{source}` is deliberately part of the Display text: the shard-join
+ // failure report and `%error` log fields print Display only, and the
+ // source carries the io_uring remediation folded in by
+ // `server_common::diagnostics::enrich_runtime_create_error`.
+ #[error("failed to create io_uring runtime for shard {shard_id}:
{source}")]
ShardRuntimeCreateFailed {
shard_id: u16,
#[source]
diff --git a/core/server/src/main.rs b/core/server/src/main.rs
index b5c0110f7..1f0ac18d9 100644
--- a/core/server/src/main.rs
+++ b/core/server/src/main.rs
@@ -29,11 +29,7 @@ use server::bootstrap::{
create_directories, create_shard_connections, create_shard_executor,
load_config,
load_metadata, resolve_persister, update_system_info,
};
-use server::diagnostics::{
- ASYNCIFY_POOL_DISABLED_PANIC_MSG, print_incomplete_io_uring_ops_info,
- print_invalid_io_uring_args_info, print_io_uring_permission_info,
- print_locked_memory_limit_info,
-};
+use server::diagnostics::{ASYNCIFY_POOL_DISABLED_PANIC_MSG,
print_incomplete_io_uring_ops_info};
use server::io::fs_utils;
use server::log::logger::Logging;
use server::metadata::{Metadata, create_metadata_handles};
@@ -65,10 +61,10 @@ const SHARDS_TABLE_CAPACITY: usize = 16384;
static SHUTDOWN_START_TIME: AtomicU64 = AtomicU64::new(0);
static SHUTDOWN_INITIATED: AtomicBool = AtomicBool::new(false);
-static SHARD_EXECUTOR_DIAGNOSTIC: std::sync::Once = std::sync::Once::new();
-// Separate latch from SHARD_EXECUTOR_DIAGNOSTIC: a shard that fails ring setup
-// (e.g. partial ENOMEM under a tight RLIMIT_MEMLOCK) must not consume the
latch
-// and suppress the unsupported-opcode diagnostic from a sibling shard that did
+// Separate latch from the ring-setup one inside
+// `enrich_runtime_create_error`: a shard that fails ring setup (e.g. partial
+// ENOMEM under a tight RLIMIT_MEMLOCK) must not consume the latch and
+// suppress the unsupported-opcode diagnostic from a sibling shard that did
// start. Setup vs runtime io_uring failures can co-occur across shards.
static SHARD_RUNTIME_DIAGNOSTIC: std::sync::Once = std::sync::Once::new();
@@ -123,11 +119,7 @@ fn main() -> Result<(), ServerError> {
let rt = match compio::runtime::Runtime::new() {
Ok(rt) => rt,
Err(e) => {
- match e.kind() {
- std::io::ErrorKind::OutOfMemory =>
print_locked_memory_limit_info(),
- std::io::ErrorKind::PermissionDenied =>
print_io_uring_permission_info(),
- _ => {}
- }
+ let e = server_common::diagnostics::enrich_runtime_create_error(e);
panic!("Cannot create runtime: {e}");
}
};
@@ -417,21 +409,11 @@ fn main() -> Result<(), ServerError> {
let rt = match create_shard_executor() {
Ok(rt) => rt,
Err(e) => {
- match e.kind() {
- std::io::ErrorKind::InvalidInput => {
- SHARD_EXECUTOR_DIAGNOSTIC
-
.call_once(print_invalid_io_uring_args_info);
- }
- std::io::ErrorKind::OutOfMemory => {
- SHARD_EXECUTOR_DIAGNOSTIC
-
.call_once(print_locked_memory_limit_info);
- }
- std::io::ErrorKind::PermissionDenied => {
- SHARD_EXECUTOR_DIAGNOSTIC
-
.call_once(print_io_uring_permission_info);
- }
- _ => {}
- }
+ // Prints the verbose remediation once across
+ // all shard threads; the panic message itself
+ // carries the one-line fix.
+ let e =
+
server_common::diagnostics::enrich_runtime_create_error(e);
panic!("Cannot create shard-{id} executor:
{e}");
}
};
diff --git a/core/server_common/src/diagnostics.rs
b/core/server_common/src/diagnostics.rs
index 10059423f..2b745d7e4 100644
--- a/core/server_common/src/diagnostics.rs
+++ b/core/server_common/src/diagnostics.rs
@@ -31,6 +31,79 @@ pub const ASYNCIFY_POOL_DISABLED_PANIC_MSG: &str =
#[cfg(target_os = "linux")]
const DISCORD_SUPPORT_URL: &str = "https://discord.gg/apache-iggy";;
+/// Classify a failed shard-executor creation and fold the matching
+/// remediation into the error itself.
+///
+/// The verbose remediation block (current limits, per-environment fix
+/// steps) is printed to stderr once per process; the returned error
+/// carries a one-line summary of the cause and the primary fix, so every
+/// propagated copy - the shard-join failure list, a panic message, a log
+/// line hours later in a collector - documents the remediation instead
+/// of only the stderr captured at the moment of failure.
+///
+/// Errors of a kind this module has no diagnosis for pass through
+/// unchanged.
+#[cfg(target_os = "linux")]
+pub fn enrich_runtime_create_error(error: std::io::Error) -> std::io::Error {
+ static RUNTIME_CREATE_DIAGNOSTIC: std::sync::Once = std::sync::Once::new();
+
+ let kind = error.kind();
+ let hint = match kind {
+ std::io::ErrorKind::OutOfMemory => {
+
RUNTIME_CREATE_DIAGNOSTIC.call_once(print_locked_memory_limit_info);
+ locked_memory_limit_hint()
+ }
+ std::io::ErrorKind::PermissionDenied => {
+
RUNTIME_CREATE_DIAGNOSTIC.call_once(print_io_uring_permission_info);
+ "io_uring syscalls are blocked, typically by a container seccomp \
+ profile: allow io_uring_setup/io_uring_enter/io_uring_register, \
+ or run with `--security-opt seccomp=unconfined` (Docker) / \
+ `seccompProfile: {type: Unconfined}` (Kubernetes)"
+ .to_owned()
+ }
+ std::io::ErrorKind::InvalidInput => {
+
RUNTIME_CREATE_DIAGNOSTIC.call_once(print_invalid_io_uring_args_info);
+ format!(
+ "the kernel rejected io_uring setup flags shard executors
require \
+ (IORING_SETUP_COOP_TASKRUN + IORING_SETUP_TASKRUN_FLAG need
Linux \
+ >= {MIN_KERNEL_MAJOR}.{MIN_KERNEL_MINOR} with full io_uring
support; \
+ WSL2 kernels are often incomplete)"
+ )
+ }
+ _ => return error,
+ };
+ std::io::Error::new(kind, format!("{error}: {hint}"))
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn enrich_runtime_create_error(error: std::io::Error) -> std::io::Error {
+ error
+}
+
+/// One-line remediation for an io_uring ring allocation denied by
+/// `RLIMIT_MEMLOCK`, with the live limits baked in so a log line is
+/// self-sufficient evidence of the misconfiguration.
+#[cfg(target_os = "linux")]
+fn locked_memory_limit_hint() -> String {
+ use nix::sys::resource::{Resource, getrlimit};
+
+ let limits = getrlimit(Resource::RLIMIT_MEMLOCK).map_or_else(
+ |_| "RLIMIT_MEMLOCK could not be read".to_owned(),
+ |(soft, hard)| {
+ format!(
+ "RLIMIT_MEMLOCK soft={}, hard={}",
+ format_limit(soft),
+ format_limit(hard)
+ )
+ },
+ );
+ format!(
+ "io_uring was denied locked memory for its rings ({limits}): raise the
\
+ limit with `ulimit -l unlimited` (shell), `LimitMEMLOCK=infinity` \
+ (systemd), or `--ulimit memlock=-1:-1` (Docker)"
+ )
+}
+
#[cfg(target_os = "linux")]
fn print_discord_link() {
eprintln!(" Need help? Join our Discord: {DISCORD_SUPPORT_URL}");
@@ -359,7 +432,40 @@ pub const fn print_incomplete_io_uring_ops_info() {}
#[cfg(all(test, target_os = "linux"))]
mod tests {
- use super::{format_limit, parse_kernel_version};
+ use super::{enrich_runtime_create_error, format_limit,
parse_kernel_version};
+
+ #[test]
+ fn enrich_folds_memlock_remediation_into_the_error() {
+ let raw = std::io::Error::new(std::io::ErrorKind::OutOfMemory,
"io_uring setup: ENOMEM");
+ let enriched = enrich_runtime_create_error(raw);
+ let message = enriched.to_string();
+ assert_eq!(enriched.kind(), std::io::ErrorKind::OutOfMemory);
+ assert!(message.contains("io_uring setup: ENOMEM"), "{message}");
+ assert!(message.contains("ulimit -l unlimited"), "{message}");
+ assert!(message.contains("RLIMIT_MEMLOCK"), "{message}");
+ }
+
+ #[test]
+ fn enrich_folds_seccomp_remediation_into_the_error() {
+ let raw = std::io::Error::new(std::io::ErrorKind::PermissionDenied,
"EPERM");
+ let message = enrich_runtime_create_error(raw).to_string();
+ assert!(message.contains("seccomp"), "{message}");
+ }
+
+ #[test]
+ fn enrich_folds_kernel_flag_remediation_into_the_error() {
+ let raw = std::io::Error::new(std::io::ErrorKind::InvalidInput,
"EINVAL");
+ let message = enrich_runtime_create_error(raw).to_string();
+ assert!(message.contains("IORING_SETUP_COOP_TASKRUN"), "{message}");
+ }
+
+ #[test]
+ fn enrich_passes_undiagnosed_kinds_through_unchanged() {
+ let raw = std::io::Error::new(std::io::ErrorKind::Interrupted,
"EINTR");
+ let enriched = enrich_runtime_create_error(raw);
+ assert_eq!(enriched.kind(), std::io::ErrorKind::Interrupted);
+ assert_eq!(enriched.to_string(), "EINTR");
+ }
#[test]
fn parse_kernel_version_standard() {
