This is an automated email from the ASF dual-hosted git repository. krishvishal pushed a commit to branch sim-workload-faults in repository https://gitbox.apache.org/repos/asf/iggy.git
commit f6e5db6a1f17cc68856c5894550f6ba139fea719 Author: Krishna Vishal <[email protected]> AuthorDate: Fri Aug 14 14:32:37 2026 +0530 feat(simulator): expose the network fault space to the fuzzer `PacketSimulatorOptions` has carried loss, duplication, partition and clog knobs since the packet simulator landed, all defaulting to zero, and the fuzzer offered no way to set any of them. The fault space the harness exists to explore was therefore reachable only from hand-written tests. `--faults` names a whole profile so a run does not have to spell out eleven probabilities, and the individual flags override single fields of the chosen profile for when the question is about one axis. The run prints the resulting network so a reproduction carries what was injected rather than what was asked for. Default stays a perfect network. Every profile past `none` currently stalls, and not from a cluster failure: `SimClient` has no request timeout, so the first dropped request or reply strands its only in-flight slot for the rest of the run. A 2% loss rate drains four replies where a perfect network drains 439. The profiles are wired anyway, since the fault space belongs in one place, and the client resend path is the next commit. --- core/simulator/src/bin/workload-fuzz.rs | 238 ++++++++++++++++++++++++++++++-- 1 file changed, 229 insertions(+), 9 deletions(-) diff --git a/core/simulator/src/bin/workload-fuzz.rs b/core/simulator/src/bin/workload-fuzz.rs index e3eebfb50..52fe0eeae 100644 --- a/core/simulator/src/bin/workload-fuzz.rs +++ b/core/simulator/src/bin/workload-fuzz.rs @@ -25,12 +25,19 @@ //! ```text //! workload-fuzz [--seed N] [--ticks N] [--clients N] [--replicas N] //! [--plane partition|metadata|mixed|uniform] -//! [--crash-prob F] [--no-quiesce] +//! [--faults none|light|heavy] [--crash-prob F] [--no-quiesce] +//! [network overrides: --packet-loss, --replay, --partition-mode, +//! --partition-prob, --unpartition-prob, --clog-prob, ...] //! ``` //! //! `--plane` selects the op mix (see [`ActionWeights`]). Partition-plane runs //! drain and converge most readily; `uniform` is the widest per-tick op //! coverage. +//! +//! `--faults` picks a whole network fault profile; the individual network flags +//! override single fields of the chosen profile, so exploring one axis does not +//! mean spelling out the other ten. The default profile is `none`, a perfect +//! network, so a run says what it injects rather than inheriting it. use clap::{Parser, ValueEnum}; use iggy_common::IggyByteSize; @@ -38,7 +45,7 @@ use server_common::sharding::IggyNamespace; use server_common::{MemoryPool, MemoryPoolConfigOther}; use simulator::Simulator; use simulator::client::SimClient; -use simulator::packet::PacketSimulatorOptions; +use simulator::packet::{PacketSimulatorOptions, PartitionMode, PartitionSymmetry}; use simulator::workload::actions::Action; use simulator::workload::options::{ActionWeights, WorkloadOptions}; use simulator::workload::{Workload, oracle, run}; @@ -67,6 +74,147 @@ struct Args { crash_prob: f32, #[arg(long)] no_quiesce: bool, + + /// Network fault profile. Individual network flags below override single + /// fields of the profile. + #[arg(long, value_enum, default_value_t = Faults::None)] + faults: Faults, + /// Chance a packet is dropped at delivery time. + #[arg(long, value_parser = parse_unit_interval_f64)] + packet_loss: Option<f64>, + /// Chance a packet is duplicated at delivery time. + #[arg(long, value_parser = parse_unit_interval_f64)] + replay: Option<f64>, + /// Minimum one-way delay, in ticks. + #[arg(long)] + one_way_delay_min: Option<u64>, + /// Mean one-way delay, in ticks (exponentially distributed). + #[arg(long)] + one_way_delay_mean: Option<u64>, + /// Maximum packets queued on a single link; beyond it the link drops. + #[arg(long)] + link_capacity: Option<u8>, + /// How an automatic partition picks its sides. + #[arg(long, value_enum)] + partition_mode: Option<PartitionModeArg>, + /// Whether a partition blocks both directions or just one. + #[arg(long, value_enum)] + partition_symmetry: Option<PartitionSymmetryArg>, + /// Per-tick chance a partition forms while connectivity is whole. + #[arg(long, value_parser = parse_unit_interval_f64)] + partition_prob: Option<f64>, + /// Per-tick chance a standing partition heals. + #[arg(long, value_parser = parse_unit_interval_f64)] + unpartition_prob: Option<f64>, + /// Minimum ticks a partition lasts once formed. + #[arg(long)] + partition_stability: Option<u32>, + /// Minimum ticks of whole connectivity before another partition may form. + #[arg(long)] + unpartition_stability: Option<u32>, + /// Per-tick chance any one path clogs (stops delivering, keeps queueing). + #[arg(long, value_parser = parse_unit_interval_f64)] + clog_prob: Option<f64>, + /// Mean clog duration, in ticks (exponentially distributed). + #[arg(long)] + clog_duration_mean: Option<u64>, +} + +/// Named network fault profile, in the spirit of TigerBeetle's VOPR modes: one +/// flag for "how hostile is the network", rather than eleven. +/// +/// Anything but [`Faults::None`] currently stalls the run, and not because the +/// cluster fails to make progress: `SimClient` has no request timeout, so a +/// client holds its single in-flight slot forever once the request or its reply +/// is dropped. Every profile below is therefore write-once-read-later until the +/// client grows a resend path; they are wired now so the fault space is +/// described in one place rather than rediscovered later. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum Faults { + /// Perfect network. Delays only, no loss and no partitions. + None, + /// Occasional loss, duplication and short one-sided partitions. Meant to + /// stay inside the range where a healthy cluster still drains. + Light, + /// Frequent loss, long partitions and clogged paths. Expected to stall + /// progress for stretches; use with a generous tick budget. + Heavy, +} + +/// Clap mirror of [`PartitionMode`], so the library type stays free of a clap +/// derive. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum PartitionModeArg { + None, + UniformSize, + UniformPartition, + IsolateSingle, +} + +impl From<PartitionModeArg> for PartitionMode { + fn from(value: PartitionModeArg) -> Self { + match value { + PartitionModeArg::None => Self::None, + PartitionModeArg::UniformSize => Self::UniformSize, + PartitionModeArg::UniformPartition => Self::UniformPartition, + PartitionModeArg::IsolateSingle => Self::IsolateSingle, + } + } +} + +/// Clap mirror of [`PartitionSymmetry`]; see [`PartitionModeArg`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum PartitionSymmetryArg { + Symmetric, + Asymmetric, +} + +impl From<PartitionSymmetryArg> for PartitionSymmetry { + fn from(value: PartitionSymmetryArg) -> Self { + match value { + PartitionSymmetryArg::Symmetric => Self::Symmetric, + PartitionSymmetryArg::Asymmetric => Self::Asymmetric, + } + } +} + +impl Faults { + /// Base network options for this profile. `node_count`, `client_count` and + /// `seed` are filled by the caller. + fn options(self) -> PacketSimulatorOptions { + match self { + // `PacketSimulatorOptions::default` is already a perfect network: + // delay only, every probability zero. + Self::None => PacketSimulatorOptions::default(), + Self::Light => PacketSimulatorOptions { + packet_loss_probability: 0.02, + replay_probability: 0.01, + partition_probability: 0.005, + unpartition_probability: 0.05, + partition_stability: 20, + unpartition_stability: 40, + partition_mode: PartitionMode::IsolateSingle, + partition_symmetry: PartitionSymmetry::Asymmetric, + path_clog_probability: 0.002, + path_clog_duration_mean: 10, + ..PacketSimulatorOptions::default() + }, + Self::Heavy => PacketSimulatorOptions { + packet_loss_probability: 0.10, + replay_probability: 0.03, + one_way_delay_mean: 8, + partition_probability: 0.02, + unpartition_probability: 0.02, + partition_stability: 50, + unpartition_stability: 50, + partition_mode: PartitionMode::UniformSize, + partition_symmetry: PartitionSymmetry::Asymmetric, + path_clog_probability: 0.01, + path_clog_duration_mean: 25, + ..PacketSimulatorOptions::default() + }, + } + } } /// Which plane the sampled ops target. Maps onto an [`ActionWeights`] preset. @@ -105,6 +253,68 @@ fn parse_unit_interval(raw: &str) -> Result<f32, String> { } } +/// [`parse_unit_interval`] for the network knobs, which are `f64`. +fn parse_unit_interval_f64(raw: &str) -> Result<f64, String> { + let value: f64 = raw + .parse() + .map_err(|_| format!("`{raw}` is not a number"))?; + if (0.0..=1.0).contains(&value) { + Ok(value) + } else { + Err(format!("must be within [0.0, 1.0], got {value}")) + } +} + +/// The chosen fault profile with any individually-set network flag applied over +/// it, plus the cluster shape and seed. +fn network_options(args: &Args, replicas: u8, clients: u8, seed: u64) -> PacketSimulatorOptions { + let mut options = args.faults.options(); + options.node_count = replicas; + options.client_count = clients; + options.seed = seed; + + if let Some(value) = args.packet_loss { + options.packet_loss_probability = value; + } + if let Some(value) = args.replay { + options.replay_probability = value; + } + if let Some(value) = args.one_way_delay_min { + options.one_way_delay_min = value; + } + if let Some(value) = args.one_way_delay_mean { + options.one_way_delay_mean = value; + } + if let Some(value) = args.link_capacity { + options.link_capacity = value; + } + if let Some(value) = args.partition_mode { + options.partition_mode = value.into(); + } + if let Some(value) = args.partition_symmetry { + options.partition_symmetry = value.into(); + } + if let Some(value) = args.partition_prob { + options.partition_probability = value; + } + if let Some(value) = args.unpartition_prob { + options.unpartition_probability = value; + } + if let Some(value) = args.partition_stability { + options.partition_stability = value; + } + if let Some(value) = args.unpartition_stability { + options.unpartition_stability = value; + } + if let Some(value) = args.clog_prob { + options.path_clog_probability = value; + } + if let Some(value) = args.clog_duration_mean { + options.path_clog_duration_mean = value; + } + options +} + fn main() { let args = Args::parse(); @@ -134,9 +344,25 @@ fn main() { eprintln!("workload-fuzz FAILED — reproduce with --seed {seed}\n{info}"); })); + let network_opts = network_options(&args, replicas, clients, seed); println!( "workload-fuzz: seed={seed} ticks={ticks} clients={clients} replicas={replicas} \ - plane={plane:?} crash_prob={crash_prob} quiesce={quiesce}" + plane={plane:?} faults={:?} crash_prob={crash_prob} quiesce={quiesce}", + args.faults, + ); + println!( + "network: loss={} replay={} delay={}..{} partition={:?}/{:?} \ + p_partition={} p_unpartition={} clog={} link_capacity={}", + network_opts.packet_loss_probability, + network_opts.replay_probability, + network_opts.one_way_delay_min, + network_opts.one_way_delay_mean, + network_opts.partition_mode, + network_opts.partition_symmetry, + network_opts.partition_probability, + network_opts.unpartition_probability, + network_opts.path_clog_probability, + network_opts.link_capacity, ); // poll_messages / reply paths panic without an initialized pool; disabled @@ -148,12 +374,6 @@ fn main() { }); let client_ids: Vec<u128> = (1..=u128::from(clients)).collect(); - let network_opts = PacketSimulatorOptions { - node_count: replicas, - client_count: clients, - seed, - ..PacketSimulatorOptions::default() - }; let mut sim = Simulator::new( usize::from(replicas), client_ids.iter().copied(),
