This is an automated email from the ASF dual-hosted git repository.
spetz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new ba541bb9c fix(server-ng): stop restart panic when WAL replay recovers
sessions (#3768)
ba541bb9c is described below
commit ba541bb9c9da8d37694b77cbc34fc70be64df128
Author: Hubert Gruszecki <[email protected]>
AuthorDate: Tue Jul 28 21:29:32 2026 +0200
fix(server-ng): stop restart panic when WAL replay recovers sessions (#3768)
A node that had ever served a client could no longer boot. WAL
replay rebuilds the VSR client table and bootstrap reinstalls it,
but the `set_clients_table_max` call added alongside the config
knob ran afterwards and hard-asserts an empty table, so shard 0
panicked at startup and the process exited. Fresh boots were
unaffected, which is why only restarts broke.
Apply the configured capacity before the recovered table is
installed rather than relaxing the assert. The setter rebuilds
the table from scratch, so keeping it after the install would
have traded the panic for silently dropping every resumed
session.
Recovery built its table from the compile-time default, which
the install then propagated, leaving `[metadata]
clients_table_max` inert across restarts. It now takes the
configured value the same way it already takes `journal_slots`.
---
Cargo.lock | 6 +++---
Cargo.toml | 6 +++---
core/binary_protocol/Cargo.toml | 2 +-
core/common/Cargo.toml | 2 +-
core/metadata/src/impls/metadata.rs | 4 ++--
core/metadata/src/impls/recovery.rs | 19 +++++++++++++++++--
core/sdk/Cargo.toml | 2 +-
core/server-ng/src/bootstrap.rs | 13 ++++++++-----
foreign/python/Cargo.toml | 2 +-
9 files changed, 37 insertions(+), 19 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 7591cdbf3..50939ed39 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6611,7 +6611,7 @@ checksum =
"cd62e6b5e86ea8eeeb8db1de02880a6abc01a397b2ebb64b5d74ac255318f5cb"
[[package]]
name = "iggy"
-version = "0.10.3-edge.2"
+version = "0.10.3-edge.3"
dependencies = [
"async-broadcast",
"async-dropper",
@@ -6821,7 +6821,7 @@ dependencies = [
[[package]]
name = "iggy_binary_protocol"
-version = "0.10.3-edge.2"
+version = "0.10.3-edge.3"
dependencies = [
"aligned-vec",
"bytemuck",
@@ -6833,7 +6833,7 @@ dependencies = [
[[package]]
name = "iggy_common"
-version = "0.10.3-edge.2"
+version = "0.10.3-edge.3"
dependencies = [
"aes-gcm",
"async-broadcast",
diff --git a/Cargo.toml b/Cargo.toml
index b03900ab1..153461028 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -196,10 +196,10 @@ hyper-util = { version = "0.1.20", features =
["server-auto", "service"] }
iceberg = "0.9.1"
iceberg-catalog-rest = "0.9.1"
iceberg-storage-opendal = "0.9.1"
-iggy = { path = "core/sdk", version = "0.10.3-edge.2" }
+iggy = { path = "core/sdk", version = "0.10.3-edge.3" }
iggy-cli = { path = "core/cli", version = "0.13.1-edge.1" }
-iggy_binary_protocol = { path = "core/binary_protocol", version =
"0.10.3-edge.2" }
-iggy_common = { path = "core/common", version = "0.10.3-edge.2" }
+iggy_binary_protocol = { path = "core/binary_protocol", version =
"0.10.3-edge.3" }
+iggy_common = { path = "core/common", version = "0.10.3-edge.3" }
iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.3.1-edge.1" }
indexmap = "2.14.0"
integration = { path = "core/integration" }
diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml
index 1155dcdd0..ce36c19b1 100644
--- a/core/binary_protocol/Cargo.toml
+++ b/core/binary_protocol/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "iggy_binary_protocol"
-version = "0.10.3-edge.2"
+version = "0.10.3-edge.3"
description = "Wire protocol types and codec for the Iggy binary protocol.
Shared between server and SDK."
edition = "2024"
rust-version.workspace = true
diff --git a/core/common/Cargo.toml b/core/common/Cargo.toml
index d5c80224d..3934e5bbd 100644
--- a/core/common/Cargo.toml
+++ b/core/common/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "iggy_common"
-version = "0.10.3-edge.2"
+version = "0.10.3-edge.3"
description = "Iggy is the persistent message streaming platform written in
Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing
millions of messages per second."
edition = "2024"
rust-version.workspace = true
diff --git a/core/metadata/src/impls/metadata.rs
b/core/metadata/src/impls/metadata.rs
index 5d0ae0679..cc2d263db 100644
--- a/core/metadata/src/impls/metadata.rs
+++ b/core/metadata/src/impls/metadata.rs
@@ -637,8 +637,8 @@ impl<C, J, S, M> IggyMetadata<C, J, S, M> {
/// Size the VSR client table to `[metadata] clients_table_max`
/// (see [`ClientTable::set_capacity`]). Boot-only, before any client
- /// registers; server-ng bootstrap applies it alongside
- /// [`Self::set_checkpoint_margin`].
+ /// registers and before [`Self::install_client_table`]: the resize
+ /// rebuilds the table, so a recovered one installed first would be lost.
pub fn set_clients_table_max(&self, max_clients: usize) {
self.client_table.borrow_mut().set_capacity(max_clients);
}
diff --git a/core/metadata/src/impls/recovery.rs
b/core/metadata/src/impls/recovery.rs
index 00ea8b800..35d26669f 100644
--- a/core/metadata/src/impls/recovery.rs
+++ b/core/metadata/src/impls/recovery.rs
@@ -19,7 +19,7 @@ use crate::impls::metadata::IggySnapshot;
use crate::stm::StateMachine;
use crate::stm::authz::GatedApply;
use crate::stm::snapshot::{MetadataSnapshot, RestoreSnapshot, Snapshot,
SnapshotError};
-use consensus::{CLIENTS_TABLE_MAX, ClientTable, build_reply_message,
build_reply_message_with};
+use consensus::{ClientTable, build_reply_message, build_reply_message_with};
use iggy_binary_protocol::consensus::{Operation, PrepareHeader};
use iggy_common::IggyError;
use journal::prepare_journal::{JournalError, PrepareJournal};
@@ -129,6 +129,11 @@ pub struct RecoveredMetadata<M> {
/// replayed ops land on the same baseline (and the same slab ids) they were
/// originally applied over. A snapshot already contains that baseline.
///
+/// `clients_table_max` sizes the rebuilt client table. It comes from
+/// `[metadata] clients_table_max`: the recovered table replaces the one the
+/// caller built, so reading the compile-time default here would make the knob
+/// inert on every restart.
+///
/// `solo` marks a single-replica cluster: the quorum is 1/1, so every
/// journaled op was committed the moment it was written and replay runs to
/// the journal head. The embedded `commit` stamps cannot be used there: each
@@ -140,6 +145,7 @@ pub async fn recover<M>(
data_dir: &Path,
solo: bool,
journal_slots: usize,
+ clients_table_max: usize,
seed_baseline: impl FnOnce(&M),
) -> Result<RecoveredMetadata<M>, RecoveryError>
where
@@ -198,7 +204,7 @@ where
.fold(snapshot_floor, u64::max)
};
- let mut client_table = ClientTable::new(CLIENTS_TABLE_MAX);
+ let mut client_table = ClientTable::new(clients_table_max);
let mut last_applied_op: Option<u64> = None;
let mut last_journaled_op: Option<u64> = None;
for header in &headers_to_replay {
@@ -300,6 +306,7 @@ where
#[allow(clippy::cast_possible_truncation)]
mod tests {
use super::*;
+ use consensus::CLIENTS_TABLE_MAX;
use iggy_binary_protocol::consensus::{Command2, Operation};
use journal::Journal;
use server_common::iobuf::Owned;
@@ -363,6 +370,7 @@ mod tests {
dir.path(),
false,
journal::prepare_journal::DEFAULT_SLOT_COUNT,
+ CLIENTS_TABLE_MAX,
|_| {},
)
.await
@@ -387,6 +395,7 @@ mod tests {
dir.path(),
false,
journal::prepare_journal::DEFAULT_SLOT_COUNT,
+ CLIENTS_TABLE_MAX,
|_| {},
)
.await
@@ -421,6 +430,7 @@ mod tests {
dir.path(),
false,
journal::prepare_journal::DEFAULT_SLOT_COUNT,
+ CLIENTS_TABLE_MAX,
|_| {},
)
.await
@@ -462,6 +472,7 @@ mod tests {
dir.path(),
false,
journal::prepare_journal::DEFAULT_SLOT_COUNT,
+ CLIENTS_TABLE_MAX,
|_| {},
)
.await
@@ -498,6 +509,7 @@ mod tests {
dir.path(),
false,
journal::prepare_journal::DEFAULT_SLOT_COUNT,
+ CLIENTS_TABLE_MAX,
|_| {},
)
.await
@@ -564,6 +576,7 @@ mod tests {
dir.path(),
true,
journal::prepare_journal::DEFAULT_SLOT_COUNT,
+ CLIENTS_TABLE_MAX,
|_| {},
)
.await
@@ -624,6 +637,7 @@ mod tests {
dir.path(),
true,
journal::prepare_journal::DEFAULT_SLOT_COUNT,
+ CLIENTS_TABLE_MAX,
|_| {},
)
.await
@@ -678,6 +692,7 @@ mod tests {
dir.path(),
true,
journal::prepare_journal::DEFAULT_SLOT_COUNT,
+ CLIENTS_TABLE_MAX,
|_| {},
)
.await
diff --git a/core/sdk/Cargo.toml b/core/sdk/Cargo.toml
index d308dc363..5f8d91aa9 100644
--- a/core/sdk/Cargo.toml
+++ b/core/sdk/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "iggy"
-version = "0.10.3-edge.2"
+version = "0.10.3-edge.3"
description = "Iggy is the persistent message streaming platform written in
Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing
millions of messages per second."
edition = "2024"
rust-version.workspace = true
diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs
index f30a186f0..aed10ffb0 100644
--- a/core/server-ng/src/bootstrap.rs
+++ b/core/server-ng/src/bootstrap.rs
@@ -886,6 +886,7 @@ async fn shard_main(
data_dir,
topology.replica_count == 1,
config.metadata.journal_slots,
+ config.metadata.clients_table_max,
|mux_stm| {
ensure_default_root_user(mux_stm);
},
@@ -984,9 +985,15 @@ async fn shard_main(
mux_stm,
Some(PathBuf::from(&config.system.path)),
);
+ // Size the VSR client table before listeners bind and any client
registers.
+ // Must precede the recovered-table install below: the setter rebuilds the
+ // table from scratch, so running it afterwards would drop every resumed
+ // session (and trip its empty-table assert).
+ metadata.set_clients_table_max(config.metadata.clients_table_max);
// Reinstall the sessions the WAL replay rebuilt, so a rebooted node
// dedups retries and admits continuations from clients that kept their
- // identity across the restart (IGGY-137).
+ // identity across the restart (IGGY-137). Recovery sized this table from
+ // the same config value, so the install preserves the configured cap.
if let Some(client_table) = recovered_client_table {
// Refusal (a client registered before this ran) keeps the live table
// and is logged by the callee; boot continues either way.
@@ -1000,10 +1007,6 @@ async fn shard_main(
// depth: ops already pipelined while a checkpoint runs append into that
// margin (config validation keeps journal_slots >= 4x this).
metadata.set_checkpoint_margin(config.metadata.checkpoint_margin());
- // Size the VSR client table before listeners bind and any client
registers.
- // The table is empty here on both fresh boot and restart (its slots are
not
- // restored from snapshot), which the setter's empty-table contract
requires.
- metadata.set_clients_table_max(config.metadata.clients_table_max);
let shard_metrics = ShardMetrics::for_shard();
// Notifier install deferred until after tick handler wires below.
diff --git a/foreign/python/Cargo.toml b/foreign/python/Cargo.toml
index 2dec15c20..612cec38a 100644
--- a/foreign/python/Cargo.toml
+++ b/foreign/python/Cargo.toml
@@ -37,7 +37,7 @@ doc = false
[dependencies]
bytes = "1.12.1"
futures = "0.3.33"
-iggy = { path = "../../core/sdk", version = "0.10.3-edge.2" }
+iggy = { path = "../../core/sdk", version = "0.10.3-edge.3" }
pyo3 = "0.29.0"
pyo3-async-runtimes = { version = "0.29.0", features = [
"attributes",
