This is an automated email from the ASF dual-hosted git repository. hubcio pushed a commit to branch feat/iggy-server-hardening in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 29be858a70974ebbf8439a2830d87c4fa2414b04 Author: Hubert Gruszecki <[email protected]> AuthorDate: Fri Jul 24 17:19:01 2026 +0200 feat(server-ng): make VSR clients table capacity configurable The consensus clients table was pinned at 8192 slots. When full, new registrations evict the oldest client, losing its dedup state and risking double-apply on retry; operators with larger fleets had no way to size the table, and the HTTP session cap derived from the same constant at compile time. Surface [metadata] clients_table_max (floor 2, ceiling 65536) and apply it through a boot-time setter while the table is provably empty, so the simulator and tests keep compile-time defaults and no constructor signatures change. The HTTP session cap now derives as half the configured table at bootstrap, preserving the headroom invariant that HTTP sessions can never starve non-HTTP clients. --- core/configs/src/server_ng_config/defaults.rs | 1 + core/configs/src/server_ng_config/displays.rs | 4 +- core/configs/src/server_ng_config/metadata.rs | 89 ++++++++++++++++++++++++++- core/consensus/src/client_table.rs | 54 ++++++++++++++++ core/metadata/src/impls/metadata.rs | 8 +++ core/server-ng/config.toml | 7 +++ core/server-ng/src/bootstrap.rs | 8 +++ core/server-ng/src/http.rs | 3 + core/server-ng/src/http/error.rs | 6 +- core/server-ng/src/http/session.rs | 65 ++++++++++++------- core/server-ng/src/http/state.rs | 11 +++- 11 files changed, 224 insertions(+), 32 deletions(-) diff --git a/core/configs/src/server_ng_config/defaults.rs b/core/configs/src/server_ng_config/defaults.rs index a0a49b3bd..4c3768c24 100644 --- a/core/configs/src/server_ng_config/defaults.rs +++ b/core/configs/src/server_ng_config/defaults.rs @@ -156,6 +156,7 @@ impl Default for MetadataConfig { MetadataConfig { prepare_queue_depth: metadata.prepare_queue_depth as usize, journal_slots: metadata.journal_slots as usize, + clients_table_max: metadata.clients_table_max as usize, } } } diff --git a/core/configs/src/server_ng_config/displays.rs b/core/configs/src/server_ng_config/displays.rs index 022e4e311..14f8e2497 100644 --- a/core/configs/src/server_ng_config/displays.rs +++ b/core/configs/src/server_ng_config/displays.rs @@ -56,8 +56,8 @@ impl Display for MetadataConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ prepare_queue_depth: {}, journal_slots: {} }}", - self.prepare_queue_depth, self.journal_slots, + "{{ prepare_queue_depth: {}, journal_slots: {}, clients_table_max: {} }}", + self.prepare_queue_depth, self.journal_slots, self.clients_table_max, ) } } diff --git a/core/configs/src/server_ng_config/metadata.rs b/core/configs/src/server_ng_config/metadata.rs index bfcb25336..f38840f89 100644 --- a/core/configs/src/server_ng_config/metadata.rs +++ b/core/configs/src/server_ng_config/metadata.rs @@ -18,7 +18,7 @@ //! On-disk schema for the metadata consensus plane (shard 0's VSR //! replica: users, streams, topics, sessions). //! -//! Two capacity knobs previously hardcoded in the runtime crates: +//! Three capacity knobs previously hardcoded in the runtime crates: //! //! - `prepare_queue_depth` -> `consensus::PIPELINE_PREPARE_QUEUE_MAX` //! (the pipeline's in-flight prepare bound; submits beyond it bounce @@ -26,8 +26,11 @@ //! - `journal_slots` -> `journal::prepare_journal::DEFAULT_SLOT_COUNT` //! (the WAL's in-memory index; committed-but-unsnapshotted headroom //! between forced checkpoints) +//! - `clients_table_max` -> `consensus::CLIENTS_TABLE_MAX` (the VSR +//! client-table slot count; independent of the two above). The +//! server-ng HTTP session cap tracks it at half. //! -//! The two interlock through the forced-checkpoint margin +//! The first two interlock through the forced-checkpoint margin //! (`max(64, prepare_queue_depth)` at bootstrap): while a checkpoint //! runs, up to a full prepare queue of already-pipelined ops appends //! into that margin, and `validate` keeps `journal_slots` far enough @@ -37,7 +40,7 @@ //! `core/configs` does not grow build-time edges onto `core/consensus` //! and `core/journal` (the runtime crates are the consumers of this //! config, mirroring the `IOV_MAX_LIMIT_NG` precedent in -//! [`super::message_bus`]). `core/server-ng`'s bootstrap pins both +//! [`super::message_bus`]). `core/server-ng`'s bootstrap pins these //! literals against the runtime constants with static asserts. use super::COMPONENT_NG; @@ -67,6 +70,20 @@ pub const MAX_METADATA_PREPARE_QUEUE_DEPTH: usize = 4096; /// ceiling, not a tuning target. pub const MAX_METADATA_JOURNAL_SLOTS: usize = 1 << 20; +/// Mirrors `consensus::CLIENTS_TABLE_MAX`, the VSR client-table slot count. +pub const DEFAULT_METADATA_CLIENTS_TABLE_MAX: usize = 8192; + +/// Floor on `clients_table_max`. The server-ng HTTP session cap derives as +/// `clients_table_max / 2`; below two that floors to zero and HTTP could +/// register no sessions at all. +pub const MIN_METADATA_CLIENTS_TABLE_MAX: usize = 2; + +/// Upper bound on `clients_table_max`. Every slot is preallocated for the +/// table's whole lifetime whether or not it holds a live client, so this caps +/// fixed per-shard table memory; 8x the default is far past any real client +/// population and a likely unit typo. +pub const MAX_METADATA_CLIENTS_TABLE_MAX: usize = 1 << 16; + /// Capacity tunables for the metadata consensus plane. #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] pub struct MetadataConfig { @@ -80,6 +97,13 @@ pub struct MetadataConfig { /// checkpoints; more slots = rarer checkpoints, more memory, larger /// per-checkpoint WAL rewrites. pub journal_slots: usize, + + /// Slot count of the VSR client table: how many distinct clients + /// (TCP/QUIC/WS virtual clients and HTTP sessions together) hold live + /// session state before the oldest-committed entry is evicted. The + /// server-ng HTTP session cap tracks this at half, so raising it lifts + /// both. + pub clients_table_max: usize, } impl MetadataConfig { @@ -127,6 +151,20 @@ impl Validatable<ConfigurationError> for MetadataConfig { ); return Err(ConfigurationError::InvalidConfigurationValue); } + if self.clients_table_max < MIN_METADATA_CLIENTS_TABLE_MAX { + eprintln!( + "{COMPONENT_NG} metadata.clients_table_max ({}) must be >= {MIN_METADATA_CLIENTS_TABLE_MAX}", + self.clients_table_max + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.clients_table_max > MAX_METADATA_CLIENTS_TABLE_MAX { + eprintln!( + "{COMPONENT_NG} metadata.clients_table_max ({}) exceeds the maximum ({MAX_METADATA_CLIENTS_TABLE_MAX})", + self.clients_table_max + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } Ok(()) } } @@ -140,6 +178,7 @@ mod tests { let config = MetadataConfig { prepare_queue_depth: DEFAULT_METADATA_PREPARE_QUEUE_DEPTH, journal_slots: DEFAULT_METADATA_JOURNAL_SLOTS, + clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(config.validate().is_ok()); assert_eq!(config.checkpoint_margin(), METADATA_CHECKPOINT_MARGIN_FLOOR); @@ -150,6 +189,7 @@ mod tests { let config = MetadataConfig { prepare_queue_depth: 256, journal_slots: 4096, + clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(config.validate().is_ok()); assert_eq!(config.checkpoint_margin(), 256); @@ -162,12 +202,14 @@ mod tests { let boundary = MetadataConfig { prepare_queue_depth: 256, journal_slots: 1024, + clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(boundary.validate().is_ok()); // ...one slot fewer is refused. let starved = MetadataConfig { prepare_queue_depth: 256, journal_slots: 1023, + clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(starved.validate().is_err()); } @@ -177,6 +219,47 @@ mod tests { let config = MetadataConfig { prepare_queue_depth: 0, journal_slots: DEFAULT_METADATA_JOURNAL_SLOTS, + clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, + }; + assert!(config.validate().is_err()); + } + + // The shipped config.toml default is the canonical slot count, so a + // pristine deployment sizes the table exactly as the consensus constant. + #[test] + fn embedded_default_matches_canonical_clients_table_max() { + assert_eq!( + MetadataConfig::default().clients_table_max, + DEFAULT_METADATA_CLIENTS_TABLE_MAX + ); + } + + #[test] + fn clients_table_max_below_floor_is_refused() { + let config = MetadataConfig { + prepare_queue_depth: DEFAULT_METADATA_PREPARE_QUEUE_DEPTH, + journal_slots: DEFAULT_METADATA_JOURNAL_SLOTS, + clients_table_max: MIN_METADATA_CLIENTS_TABLE_MAX - 1, + }; + assert!(config.validate().is_err()); + } + + #[test] + fn clients_table_max_at_floor_is_accepted() { + let config = MetadataConfig { + prepare_queue_depth: DEFAULT_METADATA_PREPARE_QUEUE_DEPTH, + journal_slots: DEFAULT_METADATA_JOURNAL_SLOTS, + clients_table_max: MIN_METADATA_CLIENTS_TABLE_MAX, + }; + assert!(config.validate().is_ok()); + } + + #[test] + fn clients_table_max_above_ceiling_is_refused() { + let config = MetadataConfig { + prepare_queue_depth: DEFAULT_METADATA_PREPARE_QUEUE_DEPTH, + journal_slots: DEFAULT_METADATA_JOURNAL_SLOTS, + clients_table_max: MAX_METADATA_CLIENTS_TABLE_MAX + 1, }; assert!(config.validate().is_err()); } diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 6b0a8526f..aed6fc576 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -156,6 +156,21 @@ impl ClientTable { } } + /// Resize the table to `max_clients` slots. Boot-only: reallocating a + /// populated table would silently drop live sessions, so this must run + /// before any client registers (server-ng bootstrap applies the configured + /// `[metadata] clients_table_max` here). + /// + /// # Panics + /// If the table already holds a client. + pub fn set_capacity(&mut self, max_clients: usize) { + assert!( + self.index.is_empty(), + "set_capacity must run before any client registers" + ); + *self = Self::new(max_clients); + } + /// Check request against table. Session first, then request progression. /// For Register, use [`check_register`]. /// @@ -870,6 +885,45 @@ mod tests { assert!(table.get_reply(300).is_some()); } + // Capacity resize (boot-only) + + // Resizing an empty table swaps its slot count in: a smaller cap then + // evicts once the new bound is reached. + #[test] + fn set_capacity_resizes_empty_table() { + let mut table = ClientTable::new(10); + table.set_capacity(2); + table.commit_register( + 100, + TEST_USER_ID, + make_register_reply(100, 10), + no_in_flight(), + ); + table.commit_register( + 200, + TEST_USER_ID, + make_register_reply(200, 20), + no_in_flight(), + ); + table.commit_register( + 300, + TEST_USER_ID, + make_register_reply(300, 30), + no_in_flight(), + ); + assert_eq!(table.count(), 2, "resized cap of 2 evicts the oldest"); + assert!(table.get_reply(100).is_none()); + } + + // The empty-table contract is asserted, not silently honored: resizing a + // populated table would drop live sessions, so it must panic. + #[test] + #[should_panic(expected = "before any client registers")] + fn set_capacity_rejects_a_populated_table() { + let (mut table, _session) = table_with_client(); + table.set_capacity(2); + } + // Edge cases #[test] diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 7ba69922b..40cba9857 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -563,6 +563,14 @@ impl<C, J, S, M> IggyMetadata<C, J, S, M> { } } + /// Size the VSR client table to `[metadata] clients_table_max` + /// (see [`ClientTable::set_capacity`]). Boot-only, before any client + /// registers; server-ng bootstrap applies it alongside + /// [`Self::set_checkpoint_margin`]. + pub fn set_clients_table_max(&self, max_clients: usize) { + self.client_table.borrow_mut().set_capacity(max_clients); + } + /// Resolved byte value for `MaxTopicSize::ServerDefault`. #[must_use] pub const fn default_max_topic_size(&self) -> u64 { diff --git a/core/server-ng/config.toml b/core/server-ng/config.toml index 2cd08855e..eb5510271 100644 --- a/core/server-ng/config.toml +++ b/core/server-ng/config.toml @@ -772,6 +772,13 @@ prepare_queue_depth = 32 # rewrites per checkpoint. journal_slots = 1024 +# Slot count of the VSR client table: how many distinct clients (TCP/QUIC/WS +# virtual clients and HTTP sessions together) hold live session state at once. +# When full, the client whose last commit is oldest is evicted and its next +# request re-registers. The HTTP session cap tracks this at half, so raising +# it lifts both. Must be between 2 and 65536. +clients_table_max = 8192 + # Per-partition consensus plane tunables. Unlike [metadata] (one shard-0 # plane), a pipeline exists per partition, so raising this multiplies pinned # request-buffer memory by the partition count. Keep it modest. diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs index 61cb08351..a78cec889 100644 --- a/core/server-ng/src/bootstrap.rs +++ b/core/server-ng/src/bootstrap.rs @@ -988,6 +988,10 @@ async fn shard_main( // depth: ops already pipelined while a checkpoint runs append into that // margin (config validation keeps journal_slots >= 4x this). metadata.set_checkpoint_margin(config.metadata.checkpoint_margin()); + // Size the VSR client table before listeners bind and any client registers. + // The table is empty here on both fresh boot and restart (its slots are not + // restored from snapshot), which the setter's empty-table contract requires. + metadata.set_clients_table_max(config.metadata.clients_table_max); let shard_metrics = ShardMetrics::for_shard(); // Notifier install deferred until after tick handler wires below. @@ -1704,6 +1708,9 @@ const _: () = assert!( configs::ng_partition::DEFAULT_PARTITION_PREPARE_QUEUE_DEPTH == consensus::PIPELINE_PREPARE_QUEUE_MAX ); +const _: () = assert!( + configs::ng_metadata::DEFAULT_METADATA_CLIENTS_TABLE_MAX == consensus::CLIENTS_TABLE_MAX +); const _: () = assert!(configs::ng_cluster::DEFAULT_VIEW_PROBE_ATTEMPTS_MAX == consensus::PROBE_ATTEMPTS_MAX); /// Convert a consensus-timer interval to whole ticks, floored at one tick so a @@ -2349,6 +2356,7 @@ async fn start_tcp_runtime( http_addr, &config.http, &config.http_admission, + config.metadata.clients_table_max, &config.cluster, Arc::clone(&config.system), self_ports, diff --git a/core/server-ng/src/http.rs b/core/server-ng/src/http.rs index f884006dc..423b5d58d 100644 --- a/core/server-ng/src/http.rs +++ b/core/server-ng/src/http.rs @@ -90,11 +90,13 @@ use crate::server_error::ServerNgError; /// Returns [`ServerNgError`] if the JWT manager cannot be built from /// `http_config.jwt`, the `[http.cors]` config is invalid, the `[http.tls]` /// credentials cannot be loaded, or the listener cannot bind to `addr`. +#[allow(clippy::too_many_arguments)] pub async fn start( shard: &Rc<ServerNgShard>, addr: SocketAddr, http_config: &HttpConfig, admission: &HttpAdmissionConfig, + clients_table_max: usize, cluster: &ClusterConfig, system_config: Arc<NgSystemConfig>, self_ports: TransportPorts, @@ -131,6 +133,7 @@ pub async fn start( // never consulted here. metadata_view: Arc::new(AtomicU64::new(crate::cluster_meta::METADATA_VIEW_UNKNOWN)), }, + max_http_sessions: crate::http::session::max_http_sessions(clients_table_max), max_in_flight_writes: admission.max_in_flight_writes, max_in_flight_writes_per_session: admission.max_in_flight_writes_per_session, in_flight_writes: Cell::new(0), diff --git a/core/server-ng/src/http/error.rs b/core/server-ng/src/http/error.rs index 998e03238..edf8c90f5 100644 --- a/core/server-ng/src/http/error.rs +++ b/core/server-ng/src/http/error.rs @@ -160,9 +160,9 @@ impl IntoResponse for AuthError { Self::Unauthenticated(error) => CustomError::from(error).into_response(), // A fresh session could not be established: the Register did not // commit (no caught-up primary, pipeline full, or a view-change - // cancel), or the session table is at `MAX_HTTP_SESSIONS` and - // refused the fresh registration. Transient server condition -> 503, - // retryable. + // cancel), or the session table is at its cap (half `[metadata] + // clients_table_max`) and refused the fresh registration. Transient + // server condition -> 503, retryable. Self::SessionUnavailable => service_unavailable(), } } diff --git a/core/server-ng/src/http/session.rs b/core/server-ng/src/http/session.rs index 14ac9347b..1b3bc4263 100644 --- a/core/server-ng/src/http/session.rs +++ b/core/server-ng/src/http/session.rs @@ -29,21 +29,38 @@ use futures::channel::oneshot; use message_bus::InstanceToken; use tokio::sync::Mutex; -/// Hard cap on live per-credential sessions. A leak-guard, not a tuning knob: -/// reaching it means this many distinct live tokens are in flight at once. New -/// sessions past the cap are refused with a transient 503 rather than evicting -/// a live one; the client retries. Expired entries are dropped first, so the -/// cap only bites on live oversubscription. +/// HTTP's slice of the shared VSR client table: half the configured +/// `[metadata] clients_table_max`. A leak-guard, not a tuning knob: reaching +/// the cap means that many distinct live tokens are in flight at once. New +/// sessions past it are refused with a transient 503 rather than evicting a +/// live one; the client retries. Expired entries are dropped first, so the cap +/// only bites on live oversubscription. /// /// Bounded by the shared VSR client table: HTTP sessions and the TCP/QUIC/WS -/// virtual clients all Register into the one [`CLIENTS_TABLE_MAX`]-slot table, -/// which LRU-evicts the oldest client when full. Capping HTTP at half that -/// bound keeps this plane from crowding the others out and keeps the combined -/// steady state under the shared bound, so a live idle HTTP session is not -/// routinely evicted consensus-side. The residual eviction race (both planes -/// busy) degrades gracefully: an evicted session's next control write is -/// classified as an eviction and re-registers (see [`HttpInner::forget_session`]). -pub(in crate::http) const MAX_HTTP_SESSIONS: usize = CLIENTS_TABLE_MAX / 2; +/// virtual clients all Register into the one client table, which LRU-evicts the +/// oldest client when full. Capping HTTP at half that bound keeps this plane +/// from crowding the others out and keeps the combined steady state under the +/// shared bound, so a live idle HTTP session is not routinely evicted +/// consensus-side. The residual eviction race (both planes busy) degrades +/// gracefully: an evicted session's next control write is classified as an +/// eviction and re-registers (see [`HttpInner::forget_session`]). +/// +/// The half rule lives here so a change to the ratio flows to both the runtime +/// value (threaded through `HttpInner` at boot) and the pinned default. +pub(in crate::http) const fn max_http_sessions(clients_table_max: usize) -> usize { + clients_table_max / 2 +} + +/// HTTP session cap at the shipped-default client table ([`CLIENTS_TABLE_MAX`]). +/// The runtime value ([`max_http_sessions`] of the configured +/// `clients_table_max`) equals this on a default deployment; the tests pin it. +pub(in crate::http) const DEFAULT_MAX_HTTP_SESSIONS: usize = max_http_sessions(CLIENTS_TABLE_MAX); + +// HTTP must never claim the whole shared VSR client table, or a login storm +// could evict every TCP/QUIC/WS virtual client. Compile-time pin of the +// headroom the half-cap guarantees (config validation floors the table at 2, so +// the runtime cap keeps the same headroom). +const _: () = assert!(DEFAULT_MAX_HTTP_SESSIONS < CLIENTS_TABLE_MAX); /// First per-session request id the write path hands out. VSR request numbers /// are 1-based and strictly increasing within a session. @@ -346,15 +363,21 @@ mod tests { assert_eq!(table.len(), 8, "no live session is evicted to make room"); } + // Pins the specific half ratio (not just headroom, which the module-level + // compile assert covers): narrowing the split would silently starve HTTP. #[test] - fn http_session_cap_leaves_headroom_below_the_shared_client_table_bound() { - const { - assert!( - MAX_HTTP_SESSIONS < CLIENTS_TABLE_MAX, - "HTTP must not claim the whole shared VSR client table" - ); - } - assert_eq!(MAX_HTTP_SESSIONS, CLIENTS_TABLE_MAX / 2); + fn http_session_cap_is_half_the_shared_client_table_bound() { + assert_eq!(DEFAULT_MAX_HTTP_SESSIONS, CLIENTS_TABLE_MAX / 2); + } + + // Zero behavior change at defaults: the cap computed at boot from the + // shipped `[metadata] clients_table_max` equals the historical compile-time + // default, so surfacing the table size as config leaves a default + // deployment untouched. + #[test] + fn runtime_cap_at_default_config_equals_pinned_default() { + let configured = configs::ng_metadata::MetadataConfig::default().clients_table_max; + assert_eq!(max_http_sessions(configured), DEFAULT_MAX_HTTP_SESSIONS); } // Eviction recovery: forgetting the evicted session drops exactly its diff --git a/core/server-ng/src/http/state.rs b/core/server-ng/src/http/state.rs index f3f5bfaaa..949f2f47b 100644 --- a/core/server-ng/src/http/state.rs +++ b/core/server-ng/src/http/state.rs @@ -41,8 +41,8 @@ use crate::dispatch::submit_register_on_owner; use crate::http::error::{AuthError, ReadError, primary_redirect_location}; use crate::http::jwt::JwtManager; use crate::http::session::{ - BarrierEntry, FIRST_REQUEST_ID, HttpSession, MAX_HTTP_SESSIONS, RegistrationBarrier, - forget_if_same, live_entry, sweep_expired, + BarrierEntry, FIRST_REQUEST_ID, HttpSession, RegistrationBarrier, forget_if_same, live_entry, + sweep_expired, }; /// Response header carrying the current VSR view number. Stamped by @@ -78,6 +78,11 @@ pub(in crate::http) struct HttpInner { /// requests for one credential from each running its own `Register`. pub(in crate::http) registrations: RegistrationBarrier, pub(in crate::http) roster: ClusterRoster, + /// Cap on live per-credential sessions: half the configured `[metadata] + /// clients_table_max`, so HTTP sessions cannot crowd the TCP/QUIC/WS virtual + /// clients out of the shared VSR client table. Read by `resolve_session` + /// when admitting a fresh session. + pub(in crate::http) max_http_sessions: usize, /// Shard-0-global cap on concurrently awaited partition writes, from /// `[http_admission] max_in_flight_writes`. Passed to `admit_partition_write` /// alongside the per-session cap. @@ -166,7 +171,7 @@ impl HttpInner { let (admitted, torn) = { let mut table = self.sessions.borrow_mut(); let torn = sweep_expired(&mut table, now); - if table.len() >= MAX_HTTP_SESSIONS { + if table.len() >= self.max_http_sessions { // Still full after dropping expired entries: too many // genuinely live sessions. Refuse rather than evict a // live one (its `fresh` client id is orphaned on the
