This is an automated email from the ASF dual-hosted git repository. krishvishal pushed a commit to branch simulator-liveness in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 20014f66895e0b955ca0eb065ee7544d1fdbd7d1 Author: Krishna Vishal <[email protected]> AuthorDate: Tue Sep 15 12:06:52 2026 +0530 feat(simulator): separate a vacuous run and assert through the drain --- core/simulator/src/bin/workload-fuzz.rs | 137 +++++++++++++++++++++----------- core/simulator/src/lib.rs | 119 +++++++++++++++++++-------- core/simulator/src/workload/mod.rs | 18 ++++- core/simulator/src/workload/oracle.rs | 27 ++++++- 4 files changed, 218 insertions(+), 83 deletions(-) diff --git a/core/simulator/src/bin/workload-fuzz.rs b/core/simulator/src/bin/workload-fuzz.rs index fb6ab966a..5f5e247ce 100644 --- a/core/simulator/src/bin/workload-fuzz.rs +++ b/core/simulator/src/bin/workload-fuzz.rs @@ -44,6 +44,11 @@ //! CI campaign wants: `none`/`light`/`heavy` are three points in parameter space, //! so a thousand seeds against `heavy` is the same network a thousand times. The //! drawn values print on the `network:` line and `--seed` replays them exactly. +//! +//! Exit codes: `0` passed, `2` the configuration is unusable, `3` the run proved +//! nothing because a vacuity floor went unmet, and `101` an invariant or an oracle +//! failed. Only `101` is a bug. A campaign that cannot tell `3` from `101` reports +//! its own thin seeds as failures, which is what splitting them apart is for. use clap::{Parser, ValueEnum}; use iggy_common::IggyByteSize; @@ -53,10 +58,27 @@ use simulator::Simulator; use simulator::client::SimClient; use simulator::packet::{COMMAND_LABELS, PacketSimulatorOptions, PartitionMode, PartitionSymmetry}; use simulator::workload::actions::Action; +use simulator::workload::invariants::Invariants; use simulator::workload::options::{ActionWeights, WorkloadOptions}; use simulator::workload::{FaultInjector, Workload, oracle, run_with_faults}; use strum::IntoEnumIterator; +/// Exit code for a run that proved nothing. +/// +/// Apart from the `101` a panic exits with, because a run under a vacuity floor is +/// not a failure: every oracle held and none of them had anything to compare. +const EXIT_VACUOUS: i32 = 3; + +/// Report that the run proved nothing, then exit with [`EXIT_VACUOUS`]. +/// +/// On stdout beside the coverage numbers rather than on stderr, because this is an +/// outcome of the run and not a diagnostic of one. No panic, so the reproduce line +/// the panic hook prints stays reserved for failures worth reproducing. +fn vacuous(reason: std::fmt::Arguments<'_>) -> ! { + println!("vacuous: {reason}"); + std::process::exit(EXIT_VACUOUS) +} + #[derive(Parser)] #[command(about = "Deterministic workload fuzzer for the Iggy simulator")] #[allow(clippy::struct_excessive_bools)] @@ -119,14 +141,14 @@ struct Args { /// than Iggy is. #[arg(long)] restore_partition_frontier: bool, - /// Fail the run if the entity oracle did not hold at quiesce. + /// End the run as vacuous if the entity oracle did not hold at quiesce. /// /// An eviction disarms it (the forgotten request's fate is unknown) and it /// re-arms only once the shadow is proven equal to committed state again. Without /// this flag a run whose oracle stayed disarmed still exits 0. #[arg(long)] require_entity_oracle: bool, - /// Committed workload operations this run must produce, or it fails. + /// Committed workload operations this run must produce, or it ends as vacuous. /// /// A run that commits nothing proved nothing: every oracle downstream compares an /// empty shadow against empty committed state and agrees. `0` opts out. @@ -135,7 +157,7 @@ struct Args { /// Committed ops, on EITHER plane, that must have been witnessed by more than /// one live replica, i.e. that exercised cross-replica agreement. Ignored below /// two live replicas, where the property is untestable rather than untested. - /// `0` opts out. + /// `0` opts out. An unmet floor exits [`EXIT_VACUOUS`], like every floor here. #[arg(long, default_value_t = 1)] min_ops_compared: usize, /// As `--min-ops-compared`, but METADATA ops only. @@ -147,7 +169,8 @@ struct Args { /// with `0`. #[arg(long, default_value_t = 1)] min_metadata_ops_compared: usize, - /// Fail the run if crash or restart injection was requested but never happened. + /// End the run as vacuous if crash or restart injection was requested but never + /// happened. /// Off by default, since a short run at low probability may legitimately draw /// none; on for a campaign where such a seed is silently wasted. #[arg(long)] @@ -518,13 +541,16 @@ fn validate_network_options(options: &PacketSimulatorOptions) -> Result<(), Stri /// compare the replicas against each other and against the oracle. /// /// Split out of `main` only for length. Every assert here is a hard failure by -/// design; see the individual comments for why each one is not a warning. +/// design, and the individual comments say why each one is not a warning. The +/// vacuity floors are the exception: they end the run at [`EXIT_VACUOUS`], since a +/// run that compared nothing has disproved nothing either. fn run_quiesce_phase( args: &Args, sim: &mut Simulator, workload: &mut Workload, seed: u64, replicas: u8, + invariants: &mut Invariants, ) { // Liveness phase, opt-in: a drain against a handicapped cluster has no // verdict, but healing unconditionally resolves the wedges worth reporting. @@ -546,7 +572,7 @@ fn run_quiesce_phase( // unactionable; with the client resending, a request unanswered inside the // budget is either a wedge or a liveness bug. assert!( - oracle::drive_to_quiesce(sim, workload, 50_000), + oracle::drive_to_quiesce(sim, workload, 50_000, invariants), "{}", oracle::quiesce_failure_report(sim, workload), ); @@ -554,7 +580,7 @@ fn run_quiesce_phase( // the leader as whichever live replica claims to be primary, so asserting // mid-view-change finds none or finds a deposed one, both false failures. assert!( - oracle::settle_to_stable_view(sim, workload, 50_000), + oracle::settle_to_stable_view(sim, workload, 50_000, invariants), "metadata views never converged after the drain\n{}", oracle::quiesce_failure_report(sim, workload), ); @@ -580,36 +606,37 @@ fn run_quiesce_phase( convergence.replicas_compared, convergence.namespaces_checked, ); - assert!( - !args.require_entity_oracle || workload.strict_outcome_oracle(), - "--require-entity-oracle: the entity oracle was {entity_oracle}, so this run \ - proved nothing about entity state (seed={seed:#x})" - ); + if args.require_entity_oracle && !workload.strict_outcome_oracle() { + vacuous(format_args!( + "--require-entity-oracle: the entity oracle was {entity_oracle}, so this run \ + proved nothing about entity state (seed={seed:#x})" + )); + } let live = usize::from(replicas) - sim.crashed.len(); // Either plane satisfies it: a partition-plane run commits almost no metadata, // so the metadata count alone called every such run vacuous. let compared = convergence.ops_compared + convergence.partition_ops_compared; - assert!( - args.min_ops_compared == 0 || live < 2 || compared >= args.min_ops_compared, - "--min-ops-compared {}: {live} replicas live but only {compared} op(s) witnessed \ - by more than one ({} metadata, {} partition), so cross-replica agreement went \ - untested (seed={seed:#x})", - args.min_ops_compared, - convergence.ops_compared, - convergence.partition_ops_compared, - ); + if args.min_ops_compared > 0 && live >= 2 && compared < args.min_ops_compared { + vacuous(format_args!( + "--min-ops-compared {}: {live} replicas live but only {compared} op(s) witnessed \ + by more than one ({} metadata, {} partition), so cross-replica agreement went \ + untested (seed={seed:#x})", + args.min_ops_compared, convergence.ops_compared, convergence.partition_ops_compared, + )); + } // The metadata half on its own: summing the planes above lets a partition-only // run clear that floor while the metadata oracle compares nothing. - assert!( - args.min_metadata_ops_compared == 0 - || live < 2 - || convergence.ops_compared >= args.min_metadata_ops_compared, - "--min-metadata-ops-compared {}: {live} replicas live but only {} committed metadata \ - op(s) witnessed by more than one, so the metadata oracle compared an empty chain \ - (seed={seed:#x})", - args.min_metadata_ops_compared, - convergence.ops_compared, - ); + if args.min_metadata_ops_compared > 0 + && live >= 2 + && convergence.ops_compared < args.min_metadata_ops_compared + { + vacuous(format_args!( + "--min-metadata-ops-compared {}: {live} replicas live but only {} committed metadata \ + op(s) witnessed by more than one, so the metadata oracle compared an empty chain \ + (seed={seed:#x})", + args.min_metadata_ops_compared, convergence.ops_compared, + )); + } // Again after the drain: the drain both answers outstanding requests and // issues its own resends, so the pre-drain numbers are not the final ones. print_coverage(workload); @@ -679,6 +706,9 @@ fn main() { let mut workload = Workload::new(options); let mut injector = FaultInjector::new(seed, replicas); + // One checker for the whole run: the drain in `run_quiesce_phase` continues with + // it, so a mark set during the active phase still holds the drain to account. + let mut invariants = Invariants::new(); let replies = run_with_faults( &mut sim, &mut workload, @@ -686,6 +716,7 @@ fn main() { ticks, u64::MAX, &mut injector, + &mut invariants, ); println!( "ran {ticks} ticks; {replies} replies; crashes={} restarts={} still down: {}", @@ -700,7 +731,14 @@ fn main() { print_coverage(&workload); if quiesce { - run_quiesce_phase(&args, &mut sim, &mut workload, seed, replicas); + run_quiesce_phase( + &args, + &mut sim, + &mut workload, + seed, + replicas, + &mut invariants, + ); } // After the quiesce block, so the drain's own commits count. Rejections are added @@ -709,23 +747,26 @@ fn main() { // full of them exercised the plane. let stats = workload.auditor.stats(); let commits: u64 = stats.commits_per_action.iter().sum::<u64>() + stats.committed_rejections; - assert!( - commits >= args.min_commits, - "--min-commits {}: the run committed {commits} operation(s) on the {plane:?} \ - plane, so every oracle above compared empty against empty (seed={seed:#x})", - args.min_commits, - ); + if commits < args.min_commits { + vacuous(format_args!( + "--min-commits {}: the run committed {commits} operation(s) on the {plane:?} \ + plane, so every oracle above compared empty against empty (seed={seed:#x})", + args.min_commits, + )); + } if args.require_faults { - assert!( - crash_prob <= 0.0 || injector.crashes() > 0, - "--require-faults: --crash-prob {crash_prob} crashed nothing \ - (seed={seed:#x})" - ); - assert!( - args.restart_prob <= 0.0 || injector.restarts() > 0, - "--require-faults: --restart-prob {} restarted nothing (seed={seed:#x})", - args.restart_prob, - ); + if crash_prob > 0.0 && injector.crashes() == 0 { + vacuous(format_args!( + "--require-faults: --crash-prob {crash_prob} crashed nothing \ + (seed={seed:#x})" + )); + } + if args.restart_prob > 0.0 && injector.restarts() == 0 { + vacuous(format_args!( + "--require-faults: --restart-prob {} restarted nothing (seed={seed:#x})", + args.restart_prob, + )); + } } print_command_coverage(&sim); diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 31527f6ea..08719b44b 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs @@ -2605,11 +2605,19 @@ mod tests { let mut wl = Workload::new(options); let clients = [client]; - let replies = workload::run(&mut sim, &mut wl, &clients, 2_000, u64::MAX); + let mut invariants = crate::workload::invariants::Invariants::new(); + let replies = workload::run( + &mut sim, + &mut wl, + &clients, + 2_000, + u64::MAX, + &mut invariants, + ); assert!(replies > 0, "workload produced no replies"); assert!( - oracle::drive_to_quiesce(&mut sim, &mut wl, 5_000), + oracle::drive_to_quiesce(&mut sim, &mut wl, 5_000, &mut invariants), "system did not drain within the tick budget" ); // Cross-replica agreement + entity oracle (single client => strict). @@ -2661,7 +2669,15 @@ mod tests { let mut wl = Workload::new(options); let clients = [client]; - let replies = workload::run(&mut sim, &mut wl, &clients, 3_000, u64::MAX); + let mut invariants = crate::workload::invariants::Invariants::new(); + let replies = workload::run( + &mut sim, + &mut wl, + &clients, + 3_000, + u64::MAX, + &mut invariants, + ); assert!(replies > 0, "workload produced no replies"); assert!( !sim.crashed.is_empty(), @@ -2669,7 +2685,7 @@ mod tests { ); assert!( - oracle::drive_to_quiesce(&mut sim, &mut wl, 5_000), + oracle::drive_to_quiesce(&mut sim, &mut wl, 5_000, &mut invariants), "surviving quorum did not drain within the tick budget" ); oracle::assert_converged(&sim, &mut wl); @@ -2722,7 +2738,15 @@ mod tests { let mut wl = Workload::new(options); let clients = [client]; - let replies = workload::run(&mut sim, &mut wl, &clients, 3_000, u64::MAX); + let mut invariants = crate::workload::invariants::Invariants::new(); + let replies = workload::run( + &mut sim, + &mut wl, + &clients, + 3_000, + u64::MAX, + &mut invariants, + ); assert!(replies > 0, "lossy workload produced no replies"); assert!( wl.resends() > 0, @@ -2731,7 +2755,7 @@ mod tests { ); assert!( - oracle::drive_to_quiesce(&mut sim, &mut wl, 20_000), + oracle::drive_to_quiesce(&mut sim, &mut wl, 20_000, &mut invariants), "{}", oracle::quiesce_failure_report(&sim, &wl), ); @@ -3822,7 +3846,15 @@ mod tests { let mut wl = Workload::new(options); let clients = [client]; // run() asserts the per-tick invariants every tick under injected crashes. - let replies = workload::run(&mut sim, &mut wl, &clients, 3_000, u64::MAX); + let mut invariants = crate::workload::invariants::Invariants::new(); + let replies = workload::run( + &mut sim, + &mut wl, + &clients, + 3_000, + u64::MAX, + &mut invariants, + ); let crashed = sim.crashed.len(); assert!( @@ -4050,6 +4082,7 @@ mod tests { let mut workload = Workload::new(options); let mut injector = FaultInjector::new(seed, replica_count); let clients = [client]; + let mut invariants = crate::workload::invariants::Invariants::new(); let replies = run_with_faults( &mut sim, &mut workload, @@ -4057,6 +4090,7 @@ mod tests { 3_000, u64::MAX, &mut injector, + &mut invariants, ); ( replies, @@ -4564,7 +4598,15 @@ mod tests { let mut wl = Workload::new(options); let clients = [client]; - let replies = workload::run(&mut sim, &mut wl, &clients, 4_000, u64::MAX); + let mut invariants = crate::workload::invariants::Invariants::new(); + let replies = workload::run( + &mut sim, + &mut wl, + &clients, + 4_000, + u64::MAX, + &mut invariants, + ); assert!(replies > 0, "shell workload produced no replies"); let stats = wl.auditor.stats(); @@ -4580,7 +4622,7 @@ mod tests { ); assert!( - oracle::drive_to_quiesce(&mut sim, &mut wl, 20_000), + oracle::drive_to_quiesce(&mut sim, &mut wl, 20_000, &mut invariants), "{}", oracle::quiesce_failure_report(&sim, &wl), ); @@ -4662,7 +4704,16 @@ mod tests { let clients = [client]; let mut injector = FaultInjector::new(seed, replica_count); - run_with_faults(&mut sim, &mut wl, &clients, 1_500, u64::MAX, &mut injector); + let mut invariants = crate::workload::invariants::Invariants::new(); + run_with_faults( + &mut sim, + &mut wl, + &clients, + 1_500, + u64::MAX, + &mut injector, + &mut invariants, + ); assert!( wl.auditor.stats().transient_rejections > 0, @@ -4671,7 +4722,7 @@ mod tests { ); assert!( - oracle::drive_to_quiesce(&mut sim, &mut wl, 50_000), + oracle::drive_to_quiesce(&mut sim, &mut wl, 50_000, &mut invariants), "{}", oracle::quiesce_failure_report(&sim, &wl), ); @@ -4742,6 +4793,7 @@ mod tests { let clients = [client]; let mut injector = FaultInjector::new(seed, replica_count); + let mut invariants = crate::workload::invariants::Invariants::new(); let _ = workload::run_with_faults( &mut sim, &mut workload, @@ -4749,6 +4801,7 @@ mod tests { 4_000, u64::MAX, &mut injector, + &mut invariants, ); assert!( @@ -4756,7 +4809,7 @@ mod tests { "no replica crashed, so no view change ran and this proves nothing" ); assert!( - oracle::drive_to_quiesce(&mut sim, &mut workload, 50_000), + oracle::drive_to_quiesce(&mut sim, &mut workload, 50_000, &mut invariants), "{}", oracle::quiesce_failure_report(&sim, &workload), ); @@ -4806,22 +4859,18 @@ mod tests { let clients = [client]; let mut injector = FaultInjector::new(seed, replica_count); + // The checker is read afterwards, so it is declared here rather than left to + // the driver: `chain` below is the accumulated canonical commit chain. let mut invariants = Invariants::new(); - // Driven here rather than through `workload::run` so the accumulated - // chain is readable afterwards; `run` builds its own `Invariants`. - for _ in 0..4_000u32 { - wl.tick(); - injector.step(&mut sim, &wl); - workload::resubmit_due(&mut sim, &mut wl); - if let Some((target, msg)) = wl.build_request(&clients[0]) { - sim.submit_request(clients[0].client_id(), target, msg.into_generic()); - } - for reply in sim.step() { - let cmds = wl.on_reply(&reply); - workload::apply_sim_commands(&mut sim, &cmds); - } - invariants.check(&sim, &wl); - } + workload::run_with_faults( + &mut sim, + &mut wl, + &clients, + 4_000, + u64::MAX, + &mut injector, + &mut invariants, + ); assert!( injector.restarts() > 0, @@ -4839,12 +4888,12 @@ mod tests { ); assert!( - oracle::drive_to_quiesce(&mut sim, &mut wl, 50_000), + oracle::drive_to_quiesce(&mut sim, &mut wl, 50_000, &mut invariants), "{}", oracle::quiesce_failure_report(&sim, &wl), ); assert!( - oracle::settle_to_stable_view(&mut sim, &mut wl, 50_000), + oracle::settle_to_stable_view(&mut sim, &mut wl, 50_000, &mut invariants), "metadata views never converged after the drain" ); oracle::assert_converged(&sim, &mut wl); @@ -5222,11 +5271,19 @@ mod tests { options.weights = ActionWeights::new(&[(Action::SendMessages, 100)]); let mut wl = Workload::new(options); let clients = [client]; - let replies = workload::run(&mut sim, &mut wl, &clients, 2_000, u64::MAX); + let mut invariants = crate::workload::invariants::Invariants::new(); + let replies = workload::run( + &mut sim, + &mut wl, + &clients, + 2_000, + u64::MAX, + &mut invariants, + ); assert!(replies > 0, "workload produced no replies"); assert!( - oracle::drive_to_quiesce(&mut sim, &mut wl, 5_000), + oracle::drive_to_quiesce(&mut sim, &mut wl, 5_000, &mut invariants), "system did not drain within the tick budget" ); oracle::assert_converged(&sim, &mut wl); diff --git a/core/simulator/src/workload/mod.rs b/core/simulator/src/workload/mod.rs index 0995eb7d7..06a356b70 100644 --- a/core/simulator/src/workload/mod.rs +++ b/core/simulator/src/workload/mod.rs @@ -709,6 +709,7 @@ pub fn run( clients: &[SimClient], tick_budget: u64, replies_target: u64, + invariants: &mut Invariants, ) -> u64 { let mut injector = FaultInjector::new(workload.options.seed, sim.replica_count); run_with_faults( @@ -718,11 +719,16 @@ pub fn run( tick_budget, replies_target, &mut injector, + invariants, ) } /// [`run`] against a caller-owned [`FaultInjector`], so a test can assert what /// was actually injected instead of trusting the probabilities to have fired. +/// +/// [`Invariants`] is caller-owned for a second reason: the drain that follows this +/// call carries on with the same checker, and its high-water marks and canonical +/// commit chain are what let a regression spanning the two phases be seen at all. /// # Panics /// If `injector` was built for a different replica count than `sim` has. pub fn run_with_faults( @@ -732,6 +738,7 @@ pub fn run_with_faults( tick_budget: u64, replies_target: u64, injector: &mut FaultInjector, + invariants: &mut Invariants, ) -> u64 { // The injector is caller-owned, and it sized `last_transition` from a count // nobody has checked against this simulator. Left unchecked the mismatch @@ -747,7 +754,6 @@ pub fn run_with_faults( sim.replica_count, workload.options.seed, ); - let mut invariants = Invariants::new(); let mut replies_seen = 0u64; for _ in 0..tick_budget { workload.tick(); @@ -1109,7 +1115,15 @@ mod tests { // The recovery has to leave a usable session behind. Without a fresh // registration the next request is refused with another eviction and // nothing commits. - let replies = run(&mut sim, &mut workload, &clients, 400, u64::MAX); + let mut invariants = Invariants::new(); + let replies = run( + &mut sim, + &mut workload, + &clients, + 400, + u64::MAX, + &mut invariants, + ); assert!(replies > 0, "the recovered client got no replies"); assert!( workload diff --git a/core/simulator/src/workload/oracle.rs b/core/simulator/src/workload/oracle.rs index 18b4b714d..e3d1763ec 100644 --- a/core/simulator/src/workload/oracle.rs +++ b/core/simulator/src/workload/oracle.rs @@ -36,6 +36,7 @@ use crate::Simulator; use crate::replica::Replica; +use crate::workload::invariants::Invariants; use crate::workload::shadow::Shadow; use crate::workload::{Workload, apply_sim_commands, resubmit_due, state_checker}; use consensus::{Consensus, MetadataHandle, Status}; @@ -102,8 +103,19 @@ impl CommittedMetadata { /// /// Returns `true` once drained, `false` if `max_ticks` elapses with requests /// still outstanding (a liveness failure the caller should surface). +/// +/// `invariants` is the checker the active phase ran, carried in rather than built +/// here. A drain is up to 50,000 ticks of a cluster still repairing itself, so a +/// wedge that forms during it used to surface as nothing more than "did not drain", +/// and a fresh checker would start with an empty commit chain, which is the memory +/// that names a divergence. #[must_use] -pub fn drive_to_quiesce(sim: &mut Simulator, workload: &mut Workload, max_ticks: u64) -> bool { +pub fn drive_to_quiesce( + sim: &mut Simulator, + workload: &mut Workload, + max_ticks: u64, + invariants: &mut Invariants, +) -> bool { let mut drained = false; for _ in 0..max_ticks { // The drain keeps resending: a request lost on the way out is never @@ -122,6 +134,7 @@ pub fn drive_to_quiesce(sim: &mut Simulator, workload: &mut Workload, max_ticks: for client_id in sim.take_evictions() { workload.forget_evicted_client(client_id); } + invariants.check(sim, workload); if workload.total_in_flight() == 0 { drained = true; break; @@ -135,6 +148,7 @@ pub fn drive_to_quiesce(sim: &mut Simulator, workload: &mut Workload, max_ticks: let cmds = workload.on_reply(&reply); apply_sim_commands(sim, &cmds); } + invariants.check(sim, workload); } true } @@ -251,8 +265,16 @@ pub fn quiesce_failure_report(sim: &Simulator, workload: &Workload) -> String { /// /// Returns `false` if the views never converge, which is a real liveness failure /// the caller should report rather than assert against an unsettled cluster. +/// +/// Runs `invariants` per tick for the same reason [`drive_to_quiesce`] does: this is +/// another 50,000-tick window, and it is the one a view change wedges in. #[must_use] -pub fn settle_to_stable_view(sim: &mut Simulator, workload: &mut Workload, max_ticks: u64) -> bool { +pub fn settle_to_stable_view( + sim: &mut Simulator, + workload: &mut Workload, + max_ticks: u64, + invariants: &mut Invariants, +) -> bool { for _ in 0..max_ticks { if views_are_settled(sim, workload) { return true; @@ -263,6 +285,7 @@ pub fn settle_to_stable_view(sim: &mut Simulator, workload: &mut Workload, max_t let cmds = workload.on_reply(&reply); apply_sim_commands(sim, &cmds); } + invariants.check(sim, workload); } views_are_settled(sim, workload) }
