From 19f4fe24ca3072e8e26fa1a186609005aa36d4a0 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Wed, 23 Sep 2026 16:50:14 -0700 Subject: [PATCH 1/9] MinBlockTime: search whole-second close times only Sub-second close times hit a known rounding issue in close-time handling, so the search must never propose one. Search the whole seconds in [--min-block-time-ms, --max-block-time-ms], bounds included, for the smallest passing one. The default [4000, 5000] range evaluates 4000 and, only if that fails, 5000. A range with no whole second fails with a clear error instead of evaluating nothing. Co-Authored-By: Claude Opus 5.5 (1M context) --- doc/measuring-minimum-block-time.md | 4 +- src/FSLibrary.Tests/Tests.fs | 40 +++++++++++++++++ src/FSLibrary/MinBlockTimeTest.fs | 69 +++++++++++++++++++++-------- 3 files changed, 93 insertions(+), 20 deletions(-) diff --git a/doc/measuring-minimum-block-time.md b/doc/measuring-minimum-block-time.md index f57c90fc..526e5aec 100644 --- a/doc/measuring-minimum-block-time.md +++ b/doc/measuring-minimum-block-time.md @@ -7,7 +7,7 @@ Supercluster provides two missions that measure the minimum ledger target close Other than the type of load generated, these two missions are identical. They both spin up a configurable network of stellar-core nodes, then search for the smallest `ledgerTargetCloseTimeMilliseconds` value that the network can sustain, using a binary search over the range `[--min-block-time-ms, --max-block-time-ms]`. For each candidate close time `T`, the missions upgrade the network's SCP timing settings to `T` (with proportionally scaled ballot and nomination timeouts), run ~5 minutes of load at the fixed transaction rate, then check the `ledger.age.closed-histogram` metric on every node against the SLA. If the SLA is met, the missions try again with a smaller `T`; otherwise, they try with a larger `T`. -The missions perform the binary search to find the minimum sustainable block time. Upon completion, the missions emit a log line of the form `Minimum sustainable block time: 4500 ms (fixed TPS 1000, image ...)`. +The missions perform the binary search to find the minimum sustainable block time. Upon completion, the missions emit a log line of the form `Minimum sustainable block time: 4000 ms (fixed TPS 1000, image ...)`. ## SLA: pass/fail criteria @@ -20,7 +20,7 @@ A candidate close time `T` is considered a **pass** if and only if, **on every n If any node violates any of these bounds, `T` is considered a **fail** and the binary search raises its lower bound. The same is true if the load run itself errors (e.g., stellar-core's internal `loadgen-run-failed` counter increments, nodes fall out of sync, or peers report inconsistent ledger hashes) — in that case the mission treats the iteration as a fail and the search continues upward. -The search terminates when the upper and lower bounds are within 100 ms of each other. If no candidate in the range satisfied the SLA, the mission fails with `"No block time in [lo, hi] ms satisfied the SLA at TPS N"`. +Candidate close times are always whole seconds: the search runs over the whole seconds in `[--min-block-time-ms, --max-block-time-ms]`, bounds included, so the default range evaluates `4000` ms and, only if that fails, `5000` ms. If no candidate in the range satisfied the SLA, the mission fails with `"No block time in [lo, hi] ms satisfied the SLA at TPS N"`. ## Docker images with performance tests enabled diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index f19d06a3..eaedf63c 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -1235,3 +1235,43 @@ let ``core container env is unchanged when no extra variables are given`` () = [ "STELLAR_CORE_PEER_SHORT_NAME"; "ASAN_OPTIONS" ], c.Env |> Seq.map (fun e -> e.Name) |> List.ofSeq ) + +[] +let ``Min block time candidates are the whole seconds in the range, bounds included`` () = + Assert.Equal([ 4000; 5000 ], MinBlockTimeTest.wholeSecondCandidates 4000 5000) + Assert.Equal([ 2000; 3000; 4000 ], MinBlockTimeTest.wholeSecondCandidates 1500 4999) + Assert.Equal([ 1000 ], MinBlockTimeTest.wholeSecondCandidates 0 1000) + Assert.Empty(MinBlockTimeTest.wholeSecondCandidates 1100 1900) + +[] +let ``Min block time search finds the smallest passing candidate`` () = + let candidates = [ 1000 .. 1000 .. 5000 ] + + for threshold in candidates do + let evaluated = System.Collections.Generic.List() + + let result = + MinBlockTimeTest.searchMinPassing + candidates + (fun t -> + evaluated.Add t + t >= threshold) + + Assert.Equal(Some threshold, result) + Assert.InRange(evaluated.Count, 1, 3) + + Assert.Equal(None, MinBlockTimeTest.searchMinPassing candidates (fun _ -> false)) + Assert.Equal(None, MinBlockTimeTest.searchMinPassing [] (fun _ -> true)) + + // The default [4000, 5000] range evaluates 4000 first, then 5000 only if 4000 fails. + let evaluated = System.Collections.Generic.List() + + let result = + MinBlockTimeTest.searchMinPassing + [ 4000; 5000 ] + (fun t -> + evaluated.Add t + t >= 5000) + + Assert.Equal(Some 5000, result) + Assert.Equal([ 4000; 5000 ], List.ofSeq evaluated) diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index 0ec8f8e2..1a019d6f 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -22,7 +22,30 @@ open StellarSupercluster let private smallNetworkSize = 10 -let private searchThresholdMs = 100 +// Candidate close times are always whole seconds: sub-second values hit a +// known rounding issue in close-time handling, so the search must never +// propose one. The candidates are the whole seconds in [minMs, maxMs], bounds +// included, so the default [4000, 5000] range evaluates 4000 and, only if that +// fails, 5000. Exposed for unit tests. +let wholeSecondCandidates (minMs: int) (maxMs: int) : int list = + let first = max 1000 (((minMs + 999) / 1000) * 1000) + let last = (maxMs / 1000) * 1000 + [ first .. 1000 .. last ] + +// Binary search over ascending candidates for the smallest one that passes, +// assuming every candidate above a passing one passes too. Returns None when +// none passes. Exposed for unit tests. +let searchMinPassing (candidates: int list) (passes: int -> bool) : int option = + let arr = Array.ofList candidates + let mutable failIdx = -1 + let mutable passIdx = arr.Length + + while passIdx - failIdx > 1 do + let mid = (failIdx + passIdx) / 2 + + if passes arr.[mid] then passIdx <- mid else failIdx <- mid + + if passIdx < arr.Length then Some arr.[passIdx] else None // For the purposes of min block test, use high value to avoid noise from SCP timeouts let private timeout = 2000 @@ -540,11 +563,20 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: context.minBlockTimeMs context.maxBlockTimeMs - let mutable lo = context.minBlockTimeMs - let mutable hi = context.maxBlockTimeMs - let mutable bestPassing = None + let candidates = wholeSecondCandidates context.minBlockTimeMs context.maxBlockTimeMs - LogInfo "Starting min block time search: T in [%d, %d] ms, fixed TPS = %d" lo hi fixedTxRate + if List.isEmpty candidates then + failwithf + "No whole-second close time in [%d, %d] ms: --min-block-time-ms and --max-block-time-ms must include at least one whole second" + context.minBlockTimeMs + context.maxBlockTimeMs + + LogInfo + "Starting min block time search: T in [%d, %d] ms (whole-second candidates: %s), fixed TPS = %d" + context.minBlockTimeMs + context.maxBlockTimeMs + (candidates |> List.map string |> String.concat ", ") + fixedTxRate // Restart-or-sleep between iterations. Pre-generated modes require // a full restart because the pregenerated txs have baked-in @@ -572,22 +604,23 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: System.Threading.Thread.Sleep(5 * 60 * 1000) formation.EnsureAllNodesInSync allNodes - let mutable needsRecovery = false - - while hi - lo > searchThresholdMs do - if needsRecovery then restartCoreSetsOrWait () + let needsRecovery = ref false - let mid = lo + (hi - lo) / 2 + // One search step: recover from the previous candidate if needed, + // then evaluate T. + let evaluateCandidate (t: int) : bool = + if needsRecovery.Value then restartCoreSetsOrWait () - if evaluateAt mid then - LogInfo "SLA met at T=%dms; lowering upper bound" mid - hi <- mid - bestPassing <- Some mid - needsRecovery <- false + if evaluateAt t then + LogInfo "SLA met at T=%dms; lowering upper bound" t + needsRecovery.Value <- false + true else - LogInfo "SLA not met at T=%dms; raising lower bound" mid - lo <- mid - needsRecovery <- true + LogInfo "SLA not met at T=%dms; raising lower bound" t + needsRecovery.Value <- true + false + + let bestPassing = searchMinPassing candidates evaluateCandidate match bestPassing with | Some t -> From abc6c47a9f8df8d19dc7b2e769181609d70aa02f Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Wed, 23 Sep 2026 16:50:37 -0700 Subject: [PATCH 2/9] MinBlockTime: evaluate a single close time when min == max --min-block-time-ms == --max-block-time-ms == T evaluates exactly T once, with no rounding and no search, through the same evaluation path. Previously min == max was rejected. Fixed-target runs (e.g. T = 1000 ms) no longer need a range and a binary search. Co-Authored-By: Claude Opus 5.5 (1M context) --- doc/measuring-minimum-block-time.md | 4 ++-- src/FSLibrary/MinBlockTimeTest.fs | 29 +++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/doc/measuring-minimum-block-time.md b/doc/measuring-minimum-block-time.md index 526e5aec..be428170 100644 --- a/doc/measuring-minimum-block-time.md +++ b/doc/measuring-minimum-block-time.md @@ -20,7 +20,7 @@ A candidate close time `T` is considered a **pass** if and only if, **on every n If any node violates any of these bounds, `T` is considered a **fail** and the binary search raises its lower bound. The same is true if the load run itself errors (e.g., stellar-core's internal `loadgen-run-failed` counter increments, nodes fall out of sync, or peers report inconsistent ledger hashes) — in that case the mission treats the iteration as a fail and the search continues upward. -Candidate close times are always whole seconds: the search runs over the whole seconds in `[--min-block-time-ms, --max-block-time-ms]`, bounds included, so the default range evaluates `4000` ms and, only if that fails, `5000` ms. If no candidate in the range satisfied the SLA, the mission fails with `"No block time in [lo, hi] ms satisfied the SLA at TPS N"`. +Candidate close times are always whole seconds: the search runs over the whole seconds in `[--min-block-time-ms, --max-block-time-ms]`, bounds included, so the default range evaluates `4000` ms and, only if that fails, `5000` ms. Equal bounds evaluate exactly that close time once, without rounding. If no candidate in the range satisfied the SLA, the mission fails with `"No block time in [lo, hi] ms satisfied the SLA at TPS N"`. ## Docker images with performance tests enabled @@ -38,7 +38,7 @@ These parameters affect both `MinBlockTimeClassic` and `MinBlockTimeMixed` missi * `--tx-rate`: The fixed transaction rate (TPS) used for every iteration of the search. For `MinBlockTimeMixed`, this is used only when neither `--classic-tx-rate` nor `--soroban-tx-rate` is set, in which case the mission splits it evenly between classic and Soroban streams. The mission answers the question "what is the smallest block time the network can sustain at this TPS?" so choosing a TPS the network clearly cannot sustain (e.g., above the network's max TPS at default block time) will result in the mission failing with no block time satisfying the SLA. * `--min-block-time-ms`: Binary search lower bound, in milliseconds. Defaults to `4000`. -* `--max-block-time-ms`: Binary search upper bound, in milliseconds. Defaults to `5000`, which is also the protocol's maximum allowed ledger target close time — setting this higher will cause the mission to fail at startup, since validators reject upgrades above the protocol cap. Must be strictly greater than `--min-block-time-ms`. +* `--max-block-time-ms`: Binary search upper bound, in milliseconds. Defaults to `5000`, which is also the protocol's maximum allowed ledger target close time — setting this higher will cause the mission to fail at startup, since validators reject upgrades above the protocol cap. Must not be less than `--min-block-time-ms`; when the two are equal the mission evaluates exactly that close time once, with no search. * `--num-pregenerated-txs`: Number of pre-generated signed classic transactions to create per loadgen node. `MinBlockTimeClassic` uses these on small networks (≤30 nodes) when it automatically switches classic payment load to `PayPregenerated`; `MinBlockTimeMixed` always uses them for the classic stream in its `MIXED_PREGEN_*` mode. Defaults to `2500000` * `--pubnet-data`: Network topology to use. Defaults to a topology of tier 1 validators. See [Specifying network topologies](#specifying-network-topologies) for details on how to specify a custom topology. * `--netdelay-image`: Helper image providing simulated network delay for latency simulation. SDF provides a public image on dockerhub at `stellar/sdf-netdelay`. diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index 1a019d6f..f9acb1f9 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -557,15 +557,24 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: formation.EnsureAllNodesInSync allNodes checkLedgerAgeSLA ledgerAgePercentiles targetMs - if context.minBlockTimeMs >= context.maxBlockTimeMs then + // An explicit single-candidate target: --min-block-time-ms == + // --max-block-time-ms == T evaluates exactly that T once, in + // milliseconds, with no whole-second rounding and no search (upstream + // rejects min == max). The whole-second search below is unchanged for + // min < max. Everything downstream (capacity, ledger target close + // time, SCP timeouts, verdict and the final "No block time" failure) + // is the same code path evaluateAt already uses. + let singleCandidate = context.minBlockTimeMs = context.maxBlockTimeMs + + if context.minBlockTimeMs > context.maxBlockTimeMs then failwithf - "--min-block-time-ms=%d must be strictly less than --max-block-time-ms=%d" + "--min-block-time-ms=%d must not exceed --max-block-time-ms=%d" context.minBlockTimeMs context.maxBlockTimeMs let candidates = wholeSecondCandidates context.minBlockTimeMs context.maxBlockTimeMs - if List.isEmpty candidates then + if not singleCandidate && List.isEmpty candidates then failwithf "No whole-second close time in [%d, %d] ms: --min-block-time-ms and --max-block-time-ms must include at least one whole second" context.minBlockTimeMs @@ -620,7 +629,19 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: needsRecovery.Value <- true false - let bestPassing = searchMinPassing candidates evaluateCandidate + let bestPassing = + if singleCandidate then + let t = context.minBlockTimeMs + LogInfo "Explicit single-candidate target T=%dms (no whole-second rounding, no search)" t + + if evaluateAt t then + LogInfo "SLA met at T=%dms (single candidate)" t + Some t + else + LogInfo "SLA not met at T=%dms (single candidate)" t + None + else + searchMinPassing candidates evaluateCandidate match bestPassing with | Some t -> From 96b480949aa067a5f3f9b615f0614fa5e40ead67 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Thu, 24 Sep 2026 11:03:34 -0700 Subject: [PATCH 3/9] Add --tier1-org-count with diverse synthetic organizations The synthetic Tier1 topology used by the MinBlockTime* and MaxTPS* missions has 10 organizations. --tier1-org-count takes it up to 40 by adding up to 30 synthetic organizations in a fixed order. At 40 they use 38 locations the base topology does not (33 of them new), across North America, Europe, Asia, South America, Oceania, Africa and the Middle East. The simulated delay grows with distance, so larger networks also get a realistic latency floor. Without the flag the topology is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- doc/measuring-minimum-block-time.md | 1 + src/App/Program.fs | 19 ++++- src/FSLibrary.Tests/Tests.fs | 59 ++++++++++++++ src/FSLibrary/MaxTPSTest.fs | 3 +- src/FSLibrary/MinBlockTimeTest.fs | 3 +- src/FSLibrary/StellarMissionContext.fs | 3 + src/FSLibrary/StellarNetworkData.fs | 104 ++++++++++++++++++++++++- 7 files changed, 187 insertions(+), 5 deletions(-) diff --git a/doc/measuring-minimum-block-time.md b/doc/measuring-minimum-block-time.md index be428170..2f628acf 100644 --- a/doc/measuring-minimum-block-time.md +++ b/doc/measuring-minimum-block-time.md @@ -41,6 +41,7 @@ These parameters affect both `MinBlockTimeClassic` and `MinBlockTimeMixed` missi * `--max-block-time-ms`: Binary search upper bound, in milliseconds. Defaults to `5000`, which is also the protocol's maximum allowed ledger target close time — setting this higher will cause the mission to fail at startup, since validators reject upgrades above the protocol cap. Must not be less than `--min-block-time-ms`; when the two are equal the mission evaluates exactly that close time once, with no search. * `--num-pregenerated-txs`: Number of pre-generated signed classic transactions to create per loadgen node. `MinBlockTimeClassic` uses these on small networks (≤30 nodes) when it automatically switches classic payment load to `PayPregenerated`; `MinBlockTimeMixed` always uses them for the classic stream in its `MIXED_PREGEN_*` mode. Defaults to `2500000` * `--pubnet-data`: Network topology to use. Defaults to a topology of tier 1 validators. See [Specifying network topologies](#specifying-network-topologies) for details on how to specify a custom topology. +* `--tier1-org-count`: Organizations (three validators each) in that default tier 1 topology, from `10` (the default) to `40`. Beyond 10, synthetic organizations are added in a fixed order (`x01`, `x02`, ...), spread over further cloud regions in North America, Europe, Asia, South America, Oceania, Africa and the Middle East; the simulated network delay between two validators grows with their distance, so larger counts also raise the network's latency floor. * `--netdelay-image`: Helper image providing simulated network delay for latency simulation. SDF provides a public image on dockerhub at `stellar/sdf-netdelay`. ### Additional options for mixed pre-generated classic and synthetic Soroban traffic diff --git a/src/App/Program.fs b/src/App/Program.fs index 037ca63b..7762b4ad 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -149,7 +149,8 @@ type MissionOptions bimodalDrift: seq, driftPct: int, ledgerCloseTimeMs: int option, - forceOldStyleTriggerTimer: bool option + forceOldStyleTriggerTimer: bool option, + tier1OrgCount: int option ) = [] @@ -709,6 +710,11 @@ type MissionOptions Required = false)>] member self.ForceOldStyleTriggerTimer = forceOldStyleTriggerTimer + [] + member self.Tier1OrgCount = tier1OrgCount + let splitLabel (lab: string) : (string * string option) = match lab.Split ':' |> Array.toList with | [ x ] -> x, None @@ -773,6 +779,16 @@ let main argv = if mission.MeasureE2eLatency && mission.LoadgenKeys.IsNone then failwith "Error: --measure-e2e-latency requires --loadgen-keys" + match mission.Tier1OrgCount with + | Some n when + n < StellarNetworkData.tier1BaseOrgCount + || n > StellarNetworkData.tier1MaxOrgCount -> + failwithf + "Error: --tier1-org-count must be between %d and %d" + StellarNetworkData.tier1BaseOrgCount + StellarNetworkData.tier1MaxOrgCount + | _ -> () + let _ = logToConsoleAndFile (sprintf "%s/stellar-supercluster.log" mission.Destination) let ll = @@ -965,6 +981,7 @@ let main argv = minBlockTimeMixedMode = mission.MinBlockTimeMixedMode minBlockTimeMixedClassicTxRate = mission.MinBlockTimeMixedClassicTxRate minBlockTimeMixedSorobanTxRate = mission.MinBlockTimeMixedSorobanTxRate + tier1OrgCount = mission.Tier1OrgCount runForMinBlockTime = false forceOldStyleTriggerTimerPct = mission.ForceOldStyleTriggerTimerPct uniformDrift = List.ofSeq mission.UniformDrift diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index eaedf63c..cf55664a 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -152,6 +152,7 @@ let ctx : MissionContext = minBlockTimeMixedMode = "mixed_pregen_sac_payment" minBlockTimeMixedClassicTxRate = None minBlockTimeMixedSorobanTxRate = None + tier1OrgCount = None runForMinBlockTime = false forceOldStyleTriggerTimerPct = 0 uniformDrift = [] @@ -1275,3 +1276,61 @@ let ``Min block time search finds the smallest passing candidate`` () = Assert.Equal(Some 5000, result) Assert.Equal([ 4000; 5000 ], List.ofSeq evaluated) + +[] +let ``Tier1 topology keeps its 10 organizations unless --tier1-org-count adds diverse ones`` () = + let orgs (sets: CoreSet list) = sets |> List.map (fun cs -> cs.name.StringName) |> List.sort + let upstream = StableApproximateTier1CoreSets "img" false + Assert.Equal(10, upstream.Length) + Assert.Equal(orgs upstream, orgs (StableApproximateTier1CoreSetsWithOrgCount "img" false None)) + Assert.Equal(orgs upstream, orgs (StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 10))) + let twelve = StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 12) + Assert.Equal(List.sort (orgs upstream @ [ "x01"; "x02" ]), orgs twelve) + // Adding organizations never changes the base ones. + let locsOf (sets: CoreSet list) name = (sets |> List.find (fun cs -> cs.name.StringName = name)).options.nodeLocs + + for org in orgs upstream do + Assert.Equal(locsOf upstream org, locsOf twelve org) + + Assert.Equal(40, tier1MaxOrgCount) + let all = StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 40) + Assert.Equal(40, all.Length) + Assert.Equal(120, all |> List.sumBy (fun cs -> cs.options.nodeCount)) + Assert.All(all, (fun cs -> Assert.Equal(3, cs.options.nodeLocs.Value.Length))) + // Far more locations than the base topology's 13, on every inhabited + // continent (southern-hemisphere and eastern-hemisphere sites included). + let distinctLocs (sets: CoreSet list) = sets |> List.collect (fun cs -> cs.options.nodeLocs.Value) |> List.distinct + + let locs = distinctLocs all + Assert.Equal(13, (distinctLocs upstream).Length) + + Assert.Equal( + 37, + (distinctLocs (StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 19))) + .Length + ) + + Assert.Equal(51, locs.Length) + Assert.Contains(locs, (fun l -> l.lat < -30.0 && l.lon > 140.0)) // Oceania + Assert.Contains(locs, (fun l -> l.lat < -20.0 && l.lon < -40.0)) // South America + Assert.Contains(locs, (fun l -> l.lat < -20.0 && l.lon > 10.0 && l.lon < 40.0)) // southern Africa + // Each extra organization is distinct: no two share the same three locations. + let extraLocSets = + all + |> List.filter (fun cs -> cs.name.StringName.StartsWith "x") + |> List.map (fun cs -> List.sort cs.options.nodeLocs.Value) + + Assert.Equal(30, extraLocSets.Length) + Assert.Equal(30, extraLocSets |> List.distinct |> List.length) + // The quorum grows with the organizations: 67% of 40 inner sets. + match all.Head.options.quorumSet with + | ExplicitQuorum q -> Assert.Equal(40, q.innerQuorumSets.Length) + | _ -> failwith "Expected explicit organization quorum" + + Assert.ThrowsAny + (fun () -> StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 9) |> ignore) + |> ignore + + Assert.ThrowsAny + (fun () -> StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 41) |> ignore) + |> ignore diff --git a/src/FSLibrary/MaxTPSTest.fs b/src/FSLibrary/MaxTPSTest.fs index 5dbb101e..b884ecbb 100644 --- a/src/FSLibrary/MaxTPSTest.fs +++ b/src/FSLibrary/MaxTPSTest.fs @@ -129,9 +129,10 @@ let maxTPSTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadG if context.pubnetData.IsSome then FullPubnetCoreSets context true false else - StableApproximateTier1CoreSets + StableApproximateTier1CoreSetsWithOrgCount context.image (if context.flatQuorum.IsSome then context.flatQuorum.Value else false) + context.tier1OrgCount // PayPregenerated requires node restart between failed iterations to ensure validity of the pregenerated transactions // However, large-scale simulation restarts can be slow, so for now only use the new mode on small networks diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index f9acb1f9..6f12e4ae 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -362,9 +362,10 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: if context.pubnetData.IsSome then FullPubnetCoreSets context true false else - StableApproximateTier1CoreSets + StableApproximateTier1CoreSetsWithOrgCount context.image (if context.flatQuorum.IsSome then context.flatQuorum.Value else false) + context.tier1OrgCount // Mirrors MaxTPSTest: on small networks, GeneratePaymentLoad runs out of // source accounts at high TPS, so switch to PayPregenerated which uses diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index f5565d52..9f7042b5 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -165,6 +165,9 @@ type MissionContext = minBlockTimeMixedMode: string minBlockTimeMixedClassicTxRate: int option minBlockTimeMixedSorobanTxRate: int option + // --tier1-org-count: organizations in StableApproximateTier1CoreSets + // (None = its 10; up to 40 adds StellarNetworkData.tier1ExtraOrgs). + tier1OrgCount: int option runForMinBlockTime: bool forceOldStyleTriggerTimerPct: int uniformDrift: int list diff --git a/src/FSLibrary/StellarNetworkData.fs b/src/FSLibrary/StellarNetworkData.fs index cffd4617..28243e76 100644 --- a/src/FSLibrary/StellarNetworkData.fs +++ b/src/FSLibrary/StellarNetworkData.fs @@ -147,6 +147,44 @@ let Taipei = { lat = 25.0329; lon = 121.5654 } // of nodes. let Tokyo = { lat = 35.6895; lon = 139.69171 } +// Further cloud and hosting regions, used only by the synthetic organizations +// StableApproximateTier1CoreSetsWithOrgCount adds beyond its base 10 (not by +// pubnet simulations, which draw from `locations` below). +let Amsterdam = { lat = 52.3676; lon = 4.9041 } // Azure westeurope, many DCs + +let Atlanta = { lat = 33.749; lon = -84.388 } +let Auckland = { lat = -36.8485; lon = 174.7633 } // AWS ap-southeast-6 +let Bahrain = { lat = 26.0667; lon = 50.5577 } // AWS me-south-1 +let Bogota = { lat = 4.711; lon = -74.0721 } +let BuenosAires = { lat = -34.6037; lon = -58.3816 } +let CapeTown = { lat = -33.9249; lon = 18.4241 } // AWS af-south-1 +let Chicago = { lat = 41.8781; lon = -87.6298 } +let Dallas = { lat = 32.7767; lon = -96.797 } // GCP us-south1 +let Dubai = { lat = 25.2048; lon = 55.2708 } // Azure uaenorth +let Dublin = { lat = 53.3498; lon = -6.2603 } // AWS eu-west-1 +let Jakarta = { lat = -6.2088; lon = 106.8456 } // AWS ap-southeast-3 +let Johannesburg = { lat = -26.2041; lon = 28.0473 } // Azure southafricanorth +let Lagos = { lat = 6.5244; lon = 3.3792 } +let LosAngeles = { lat = 34.0522; lon = -118.2437 } // GCP us-west2 +let Madrid = { lat = 40.4168; lon = -3.7038 } // GCP europe-southwest1 +let Melbourne = { lat = -37.8136; lon = 144.9631 } // GCP australia-southeast2 +let MexicoCity = { lat = 19.4326; lon = -99.1332 } +let Miami = { lat = 25.7617; lon = -80.1918 } +let Milan = { lat = 45.4642; lon = 9.19 } // AWS eu-south-1 +let Mumbai = { lat = 19.076; lon = 72.8777 } // AWS ap-south-1 +let Nuremberg = { lat = 49.4521; lon = 11.0767 } // Hetzner +let Osaka = { lat = 34.6937; lon = 135.5023 } // AWS ap-northeast-3 +let Paris = { lat = 48.8566; lon = 2.3522 } // AWS eu-west-3 +let Santiago = { lat = -33.4489; lon = -70.6693 } // GCP southamerica-west1 +let SanJose = { lat = 37.3382; lon = -121.8863 } +let Seoul = { lat = 37.5665; lon = 126.978 } // AWS ap-northeast-2 +let Stockholm = { lat = 59.3293; lon = 18.0686 } // AWS eu-north-1 +let Sydney = { lat = -33.8688; lon = 151.2093 } // AWS ap-southeast-2 +let TelAviv = { lat = 32.0853; lon = 34.7818 } // GCP me-west1 +let Toronto = { lat = 43.6532; lon = -79.3832 } // GCP northamerica-northeast2 +let Warsaw = { lat = 52.2297; lon = 21.0122 } // GCP europe-central2, OVH +let Zurich = { lat = 47.3769; lon = 8.5417 } // GCP europe-west6 + let locations = [ Ashburn Brussels @@ -955,9 +993,57 @@ let TestnetCoreSetOptions (image: string) = initialization = { CoreSetInitialization.Default with waitForConsensus = true } dumpDatabase = false } +// Synthetic organizations, in the order they are added, that extend +// StableApproximateTier1CoreSets past its 10 organizations (--tier1-org-count). +// Most spread their three validators over two regions, like real operators. +// Every block of additions keeps the mix at roughly one third North America, +// one third Europe and one third elsewhere (Asia, South America, Oceania, +// Africa, Middle East). The base 10 use 13 distinct locations; with 19 +// organizations there are 37, with all 40 there are 51. +let tier1ExtraOrgs : (string * GeoLoc list) list = + [ ("x01", [ Dublin; Amsterdam; Toronto ]) + ("x02", [ SaoPaulo; Miami; Madrid ]) + ("x03", [ Tokyo; Seoul; SanJose ]) + ("x04", [ Mumbai; Bahrain; Frankfurt ]) + ("x05", [ Sydney; Singapore; LosAngeles ]) + ("x06", [ Paris; Stockholm; Chicago ]) + ("x07", [ Dallas; Atlanta; Warsaw ]) + ("x08", [ CapeTown; Johannesburg; Purfleet ]) + ("x09", [ Zurich; Milan; Ashburn ]) + ("x10", [ MexicoCity; Dallas; Bogota ]) + ("x11", [ Osaka; Taipei; Portland ]) + ("x12", [ Nuremberg; Falkenstein; Helsinki ]) + ("x13", [ Jakarta; Singapore; HongKong ]) + ("x14", [ Toronto; Beauharnois; Dublin ]) + ("x15", [ Santiago; BuenosAires; Miami ]) + ("x16", [ TelAviv; Dubai; Milan ]) + ("x17", [ Melbourne; Sydney; Auckland ]) + ("x18", [ Chicago; CouncilBluffs; Frankfurt ]) + ("x19", [ Lagos; Paris; Ashburn ]) + ("x20", [ SanJose; LosAngeles; Tokyo ]) + ("x21", [ Amsterdam; Warsaw; Stockholm ]) + ("x22", [ Atlanta; Clifton; Madrid ]) + ("x23", [ Chennai; Mumbai; Singapore ]) + ("x24", [ Columbus; Chicago; Brussels ]) + ("x25", [ Zurich; Frankfurt; Seoul ]) + ("x26", [ Portland; SanJose; Dublin ]) + ("x27", [ Purfleet; Paris; SaoPaulo ]) + ("x28", [ Dallas; MexicoCity; Toronto ]) + ("x29", [ Helsinki; Stockholm; Osaka ]) + ("x30", [ Miami; Ashburn; Nuremberg ]) ] + +let tier1BaseOrgCount = 10 + +let tier1MaxOrgCount = tier1BaseOrgCount + List.length tier1ExtraOrgs + // This coreset is a synthetic approximation of the Tier1 group, intended -// to be used in stable benchmarks rather than experiments. -let StableApproximateTier1CoreSets (image: string) (flatQuorum: bool) : CoreSet list = +// to be used in stable benchmarks rather than experiments. orgCount (None = +// the 10 base organizations) adds tier1ExtraOrgs, up to tier1MaxOrgCount. +let StableApproximateTier1CoreSetsWithOrgCount + (image: string) + (flatQuorum: bool) + (orgCount: int option) + : CoreSet list = let allOrgs : Map = Map.ofList [ ("bd", [ Brussels; CouncilBluffs; Taipei ]) ("ct", [ Frankfurt; Purfleet; Brussels ]) @@ -970,6 +1056,17 @@ let StableApproximateTier1CoreSets (image: string) (flatQuorum: bool) : CoreSet ("rg", [ Brussels; Frankfurt; Brussels ]) ("sdf", [ Ashburn; Ashburn; Ashburn ]) ] + let extra = + match orgCount with + | None -> 0 + | Some n when n >= tier1BaseOrgCount && n <= tier1MaxOrgCount -> n - tier1BaseOrgCount + | Some n -> + failwithf "tier1 organization count must be between %d and %d, got %d" tier1BaseOrgCount tier1MaxOrgCount n + + let extraOrgs = tier1ExtraOrgs |> List.truncate extra + + let allOrgs = Map.ofList (Map.toList allOrgs @ extraOrgs) + let allOrgPairs = Map.toList allOrgs let orgKeys _ nodes = List.map (fun _ -> KeyPair.Random()) nodes let allKeys = Map.map orgKeys allOrgs @@ -1028,3 +1125,6 @@ let StableApproximateTier1CoreSets (image: string) (flatQuorum: bool) : CoreSet options = coreSetOpts } List.map orgCoreSet allOrgPairs + +let StableApproximateTier1CoreSets (image: string) (flatQuorum: bool) : CoreSet list = + StableApproximateTier1CoreSetsWithOrgCount image flatQuorum None From dc834b21d3201575e261e69d55f929f8b48a6fdd Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Thu, 24 Sep 2026 11:03:59 -0700 Subject: [PATCH 4/9] MinBlockTimeMixed: run MIXED_PREGEN load on every validator The overlay-only MIXED_PREGEN_* load ran on node 0 of each load-generating organization. Run it on every validator of those organizations instead, each with its own disjoint slice of the genesis accounts and pre-generated transactions and an equal share of the offered rate for the whole load window. Rate remainders go to the first peers, and each peer's transaction budget follows its own rate so all peers submit for the same duration. The run uses at most one generator per requested TPS, counting every validator of an organization, so no validator gets a zero share; only the organizations it uses get pre-generated transactions. Per-validator pre-generation is keyed on the mission's actual load mode, so MinBlockTimeClassic and other missions keep one shared pre-generation command. Other loadgen paths keep the existing split. Co-Authored-By: Claude Opus 5.5 (1M context) --- doc/measuring-minimum-block-time.md | 2 + src/App/Program.fs | 1 + src/FSLibrary.Tests/Tests.fs | 184 +++++++++++++++++++++++++ src/FSLibrary/MinBlockTimeTest.fs | 49 +++++-- src/FSLibrary/StellarKubeSpecs.fs | 39 +++++- src/FSLibrary/StellarMissionContext.fs | 5 + src/FSLibrary/StellarStatefulSets.fs | 104 ++++++++------ 7 files changed, 332 insertions(+), 52 deletions(-) diff --git a/doc/measuring-minimum-block-time.md b/doc/measuring-minimum-block-time.md index 2f628acf..0c66ca3d 100644 --- a/doc/measuring-minimum-block-time.md +++ b/doc/measuring-minimum-block-time.md @@ -52,6 +52,8 @@ In addition to the parameters in the previous section, `MinBlockTimeMixed` suppo * `--classic-tx-rate`: Classic payment TPS for the pre-generated classic stream. * `--soroban-tx-rate`: Soroban TPS for the selected synthetic Soroban stream. +The load runs on every validator of the load-generating organizations, each with its own slice of the genesis accounts, at an equal share of the offered rate for the whole load window. + If neither stream-specific TPS is set, `--tx-rate` is split evenly between classic and Soroban traffic. If either stream-specific TPS is set, any omitted stream defaults to `0`, and the fixed TPS for the mission is the sum of `--classic-tx-rate` and `--soroban-tx-rate`. Before enabling overlay-only mode, the mission upgrades classic max tx set size from `--classic-tx-rate * 15`, and Soroban network limits from the selected transaction type's per-transaction resources multiplied by `--soroban-tx-rate * 15` (~15 seconds of throughput as leeway). diff --git a/src/App/Program.fs b/src/App/Program.fs index 7762b4ad..6d169ae0 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -982,6 +982,7 @@ let main argv = minBlockTimeMixedClassicTxRate = mission.MinBlockTimeMixedClassicTxRate minBlockTimeMixedSorobanTxRate = mission.MinBlockTimeMixedSorobanTxRate tier1OrgCount = mission.Tier1OrgCount + pregenerateTxsPerValidator = false runForMinBlockTime = false forceOldStyleTriggerTimerPct = mission.ForceOldStyleTriggerTimerPct uniformDrift = List.ofSeq mission.UniformDrift diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index cf55664a..f83a60b8 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -153,6 +153,7 @@ let ctx : MissionContext = minBlockTimeMixedClassicTxRate = None minBlockTimeMixedSorobanTxRate = None tier1OrgCount = None + pregenerateTxsPerValidator = false runForMinBlockTime = false forceOldStyleTriggerTimerPct = 0 uniformDrift = [] @@ -1277,6 +1278,124 @@ let ``Min block time search finds the smallest passing candidate`` () = Assert.Equal(Some 5000, result) Assert.Equal([ 4000; 5000 ], List.ofSeq evaluated) +[] +[] +[] +[] +[] +let ``All validator partitions preserve rate duration and disjoint account slices`` (rate: int) = + let full = + { LoadGen.GetDefault() with + mode = MixedPregenSACPayment + accounts = 1200000 + txrate = rate + txs = rate * 960 + classicTxRate = Some 0 + sorobanTxRate = Some rate } + + let shares = StellarStatefulSets.PartitionValidatorLoad 57 true full + Assert.Equal(57, shares.Length) + Assert.Equal(rate, shares |> List.sumBy (fun s -> s.txrate)) + Assert.Equal(rate * 960, shares |> List.sumBy (fun s -> s.txs)) + Assert.Equal(1199964, shares |> List.sumBy (fun s -> s.accounts)) + let rates = shares |> List.map (fun s -> s.txrate) + Assert.True(List.max rates - List.min rates <= 1) + + shares + |> List.iteri + (fun i s -> + Assert.Equal(21052, s.accounts) + Assert.Equal(i * 21052, s.offset) + Assert.Equal(s.txrate * 960, s.txs) + Assert.Equal(Some s.txrate, s.sorobanTxRate) + Assert.Equal(Some 0, s.classicTxRate) + Assert.True(s.offset + s.accounts <= full.accounts)) + + for a, b in List.pairwise shares do + Assert.True(a.offset + a.accounts <= b.offset) + +[] +let ``57 validator topology and initialization match every generator partition`` () = + let sets = StableApproximateTier1CoreSetsWithOrgCount "frozen-image" false (Some 19) + Assert.Equal(19, sets.Length) + Assert.Equal(57, sets |> List.sumBy (fun s -> s.options.nodeCount)) + + let full = + { LoadGen.GetDefault() with + mode = MixedPregenSACPayment + accounts = 1200000 + txrate = 5000 + txs = 4800000 + classicTxRate = Some 0 + sorobanTxRate = Some 5000 } + + let shares = StellarStatefulSets.PartitionValidatorLoad 57 true full + let keys = sets |> List.collect (fun s -> s.keys |> Array.toList) + Assert.Equal(57, keys |> List.map (fun k -> k.AccountId) |> Set.ofList |> Set.count) + + sets + |> List.iteri + (fun orgIndex cs -> + Assert.Equal(Some true, cs.options.tier1) + Assert.Equal(3, cs.options.nodeCount) + Assert.Equal(57, cs.options.preferredPeersMap.Value.Count) + + for peers in cs.options.preferredPeersMap.Value.Values do + Assert.Equal(56, peers.Length) + + match cs.options.quorumSet with + | ExplicitQuorum q -> + Assert.Equal(Some 67, q.thresholdPercent) + Assert.Equal(19, q.innerQuorumSets.Length) + + for inner in q.innerQuorumSets do + Assert.Equal(Some 51, inner.thresholdPercent) + Assert.Equal(3, inner.validators.Count) + | _ -> failwith "Expected explicit organization quorum" + + let options = + { cs.options with + initialization = + { cs.options.initialization with + pregenerateTxs = Some(10000, 21052, orgIndex * 3 * 21052) } } + + for i in 0 .. 2 do + let actual = PregenerationOptionsForPeer options i + let share = shares.[orgIndex * 3 + i] + Assert.Equal(Some(10000, share.accounts, share.offset), actual.initialization.pregenerateTxs)) + +[] +let ``Fixed duration partition rejects a partial second transaction budget`` () = + let full = { LoadGen.GetDefault() with accounts = 1200000; txrate = 5000; txs = 4800001 } + + Assert.Throws(fun () -> StellarStatefulSets.PartitionValidatorLoad 57 true full |> ignore) + |> ignore + +[] +let ``Submission accounting includes all started validators and preserves legacy selection`` () = + let sets = StableApproximateTier1CoreSetsWithOrgCount "frozen-image" false (Some 19) + let selected = StellarStatefulSets.LoadgenPeerIndices true sets + Assert.Equal(57, selected.Length) + + Assert.Equal( + 57, + selected + |> List.map (fun (cs, i) -> cs.keys.[i].AccountId) + |> Set.ofList + |> Set.count + ) + + Assert.Equal( + [ 19; 19; 19 ], + [ for i in 0 .. 2 -> selected |> List.filter (fun (_, j) -> i = j) |> List.length ] + ) + // A one-peer-per-organization tally would incorrectly report only one third. + let submitted = selected |> List.sumBy (fun (_, i) -> if i = 0 then 34000 else 33000) + Assert.Equal(1900000, submitted) + let legacy = StellarStatefulSets.LoadgenPeerIndices false sets + Assert.Equal(19, legacy.Length) + Assert.All(legacy, (fun (_, i) -> Assert.Equal(0, i))) + [] let ``Tier1 topology keeps its 10 organizations unless --tier1-org-count adds diverse ones`` () = let orgs (sets: CoreSet list) = sets |> List.map (fun cs -> cs.name.StringName) |> List.sort @@ -1334,3 +1453,68 @@ let ``Tier1 topology keeps its 10 organizations unless --tier1-org-count adds di Assert.ThrowsAny (fun () -> StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 41) |> ignore) |> ignore + +[] +let ``MIXED_PREGEN runs use at most one generator per requested TPS`` () = + let sets = StableApproximateTier1CoreSets "img" false + let count everyValidator tps = (MinBlockTimeTest.activeLoadGenCoreSets everyValidator tps sets).Length + // With load on every validator each organization brings its 3 validators. + Assert.Equal(10, count true 5000) + Assert.Equal(10, count true 30) + Assert.Equal(3, count true 10) + Assert.Equal(1, count true 2) + Assert.Equal(1, count true 0) + // One generator per organization otherwise, as upstream. + Assert.Equal(5, count false 5) + Assert.Equal(10, count false 5000) + +[] +let ``MIXED_PREGEN load runs on every validator in MinBlockTime runs`` () = + let minBlock = { ctx with runForMinBlockTime = true } + Assert.True(LoadOnEveryValidator minBlock MixedPregenSACPayment) + Assert.False(LoadOnEveryValidator minBlock GeneratePaymentLoad) + Assert.False(LoadOnEveryValidator ctx MixedPregenSACPayment) + + // Upstream's per-node split: equal accounts and txs, offsets by slice. + let full = + { LoadGen.GetDefault() with + accounts = 1000 + txrate = 100 + txs = 30000 + spikesize = 10 } + + let shares = StellarStatefulSets.PartitionValidatorLoad 3 false full + Assert.Equal([ 333; 333; 333 ], shares |> List.map (fun s -> s.accounts)) + Assert.Equal([ 0; 333; 666 ], shares |> List.map (fun s -> s.offset)) + Assert.Equal([ 10000; 10000; 10000 ], shares |> List.map (fun s -> s.txs)) + Assert.Equal([ 34; 33; 33 ], shares |> List.map (fun s -> s.txrate)) + +[] +let ``Validators pregenerate their own account slices only when the MinBlockTime load runs on every validator`` () = + let pregenOpts = + { coreSetOptions with + nodeCount = 3 + initialization = { coreSetOptions.initialization with pregenerateTxs = Some(1000, 100, 0) } } + + let pregenSet = MakeLiveCoreSet "pregen" pregenOpts + + let script (c: MissionContext) = + let pod = + (MakeNetworkCfg { c with installNetworkDelay = Some false } [ pregenSet ] passOpt) + .ToPodTemplateSpec pregenSet + + let core = + pod.Spec.Containers + |> Seq.find (fun k -> k.Name = CfgVal.stellarCoreContainerName "run") + + String.concat " " core.Args + + // A MinBlockTime context whose mixed mode is only the option default (e.g. + // MinBlockTimeClassic) keeps one shared pregeneration command. + let shared = script { ctx with runForMinBlockTime = true } + Assert.Contains("'--offset 0'", shared) + Assert.DoesNotContain("'--offset 100'", shared) + let perValidator = script { ctx with runForMinBlockTime = true; pregenerateTxsPerValidator = true } + + for offset in [ 0; 100; 200 ] do + Assert.Contains(sprintf "'--offset %d'" offset, perValidator) diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index 6f12e4ae..d2c6a0ae 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -357,6 +357,23 @@ let private checkLedgerAgeSLA (percentiles: (Peer * float * float) list) (target ok +// The load-generating core sets a MIXED_PREGEN_* run uses: at most one +// generator per requested TPS, so every generator gets a non-zero share, and +// always at least one set. With load on every validator +// (StellarKubeSpecs.LoadOnEveryValidator) a core set brings all of its +// validators as generators. Exposed for unit tests. +let activeLoadGenCoreSets (everyValidator: bool) (requestedTps: int) (loadGenNodes: CoreSet list) : CoreSet list = + let generators (cs: CoreSet) = if everyValidator then cs.options.nodeCount else 1 + + let fitting = + loadGenNodes + |> List.scan (fun total cs -> total + generators cs) 0 + |> List.tail + |> List.takeWhile (fun total -> total <= max 1 requestedTps) + |> List.length + + List.truncate (max 1 fitting) loadGenNodes + let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadGen option) = let allNodes = if context.pubnetData.IsSome then @@ -397,6 +414,10 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: let isLoadGenNode cs = List.exists (fun (cs': CoreSet) -> cs' = cs) loadGenNodes + // MIXED_PREGEN_* load runs on every validator of the active core sets, + // each with its own account slice. + let everyValidator = StellarKubeSpecs.LoadOnEveryValidator context baseLoadGen.mode + let activeLoadGenNodes = if isMixedPregenMode baseLoadGen.mode then let requestedCount = @@ -404,21 +425,28 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: (baseLoadGen.classicTxRate |> Option.defaultValue 0) (baseLoadGen.sorobanTxRate |> Option.defaultValue 0) - loadGenNodes - |> List.truncate (min (List.length loadGenNodes) (max 1 requestedCount)) + activeLoadGenCoreSets everyValidator requestedCount loadGenNodes else loadGenNodes + let isActiveLoadGenNode cs = List.exists (fun (cs': CoreSet) -> cs' = cs) activeLoadGenNodes + + let context = { context with pregenerateTxsPerValidator = everyValidator } + // For pre-generated modes, partition genesis accounts evenly across // loadgen nodes and assign offsets so each active node signs txs against - // its own slice. Mixed pregen keeps every tier1 core set initialized, but - // partitions accounts by the active loadgen count so low-TPS runs still - // have enough local accounts on the node that generates load. + // its own slice. Mixed pregen partitions accounts over the active loadgen + // nodes only, so low-TPS runs still have enough local accounts on the + // nodes that generate load; the other core sets pregenerate nothing. With load + // on every validator, each core set's slice covers all of its validators + // (StellarKubeSpecs.PregenerationOptionsForPeer splits it per pod). let allNodes = match context.numPregeneratedTxs, context.genesisTestAccountCount, baseLoadGen.mode with | Some txs, Some accounts, mode when usesPregeneratedTxs mode -> let partitionCount = - if isMixedPregenMode mode then + if isMixedPregenMode mode && everyValidator then + List.sumBy (fun (cs: CoreSet) -> cs.options.nodeCount) activeLoadGenNodes + elif isMixedPregenMode mode then List.length activeLoadGenNodes else List.length loadGenNodes @@ -429,10 +457,13 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: List.map (fun (cs: CoreSet) -> let pregenerateTxs = - if isLoadGenNode cs then - let i = if isMixedPregenMode mode then j % partitionCount else j + if isLoadGenNode cs && (not (isMixedPregenMode mode) || isActiveLoadGenNode cs) then + let i = j + + j <- + j + + (if isMixedPregenMode mode && everyValidator then cs.options.nodeCount else 1) - j <- j + 1 Some(txs, accountsPerNode, accountsPerNode * i) else Some(0, 1, 0) diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 800ad348..3fa9efc0 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -5,6 +5,7 @@ module StellarKubeSpecs open StellarCoreCfg +open StellarCoreHTTP open k8s.Models open StellarMissionContext open StellarNetworkCfg @@ -34,6 +35,27 @@ type ConfigOption = // container picks up a peer-specific config. | PeerSpecificConfigFile +// MinBlockTimeMixed's MIXED_PREGEN_* load runs on every validator rather than +// one node per load-generating core set, each with its own account slice (see +// PregenerationOptionsForPeer and StellarStatefulSets.LoadgenPeerIndices). +let LoadOnEveryValidator (ctx: MissionContext) (mode: LoadGenMode) : bool = + ctx.runForMinBlockTime && isMixedPregenMode mode + +// For LoadOnEveryValidator runs (MissionContext.pregenerateTxsPerValidator): +// the organization's options store its first account offset; each validator +// gets the following disjoint slice. +let PregenerationOptionsForPeer (opts: CoreSetOptions) (index: int) = + if index < 0 || index >= opts.nodeCount then + invalidArg "index" "Invalid validator index" + + let init = opts.initialization + + let perNode = + init.pregenerateTxs + |> Option.map (fun (txs, accounts, offset) -> txs, accounts, offset + accounts * index) + + { opts with initialization = { init with pregenerateTxs = perNode } } + let CoreContainerVolumeMounts (peerOrJobNames: string array) (configOpt: ConfigOption) : V1VolumeMount array = let arr = [| V1VolumeMount(name = CfgVal.dataVolumeName, mountPath = CfgVal.dataVolumePath) |] @@ -920,7 +942,22 @@ type NetworkCfg with let cfgOpt = PeerSpecificConfigFile let volumes = Array.append peerCfgVolumes [| dataVol; historyCfgVolume |] - let initCommands = self.getInitCommands cfgOpt coreSet.options + let initCommands = + if self.missionContext.pregenerateTxsPerValidator then + peerNames + |> Array.mapi + (fun i name -> + let commands = self.getInitCommands cfgOpt (PregenerationOptionsForPeer coreSet.options i) + + let test = + ShCmd [| ShWord.OfStr "test" + ShWord.Var CfgVal.peerNameEnvVarName + ShWord.OfStr "=" + ShWord.OfStr name |] + + ShCmd.ShIf(test, ShCmd.ShSeq commands, [||], None)) + else + self.getInitCommands cfgOpt coreSet.options let runCmd = [| "run" |] diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 9f7042b5..d019b5cd 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -168,6 +168,11 @@ type MissionContext = // --tier1-org-count: organizations in StableApproximateTier1CoreSets // (None = its 10; up to 40 adds StellarNetworkData.tier1ExtraOrgs). tier1OrgCount: int option + // Set by MinBlockTimeTest when its load runs on every validator + // (StellarKubeSpecs.LoadOnEveryValidator for the mission's actual load + // mode): each validator then pregenerates transactions for its own + // account slice (StellarKubeSpecs.PregenerationOptionsForPeer). + pregenerateTxsPerValidator: bool runForMinBlockTime: bool forceOldStyleTriggerTimerPct: int uniformDrift: int list diff --git a/src/FSLibrary/StellarStatefulSets.fs b/src/FSLibrary/StellarStatefulSets.fs index 4127b00d..d14b8ea7 100644 --- a/src/FSLibrary/StellarStatefulSets.fs +++ b/src/FSLibrary/StellarStatefulSets.fs @@ -70,6 +70,49 @@ let private getAveragePeerCount (topology: Map) : float = let nodeCount = Map.count topology if nodeCount > 0 then float total / float nodeCount else 0.0 +// Divide aggregate rate, account space and duration across actual generators. +// In fixed-duration mode (LoadOnEveryValidator runs), derive each transaction +// budget from its assigned rate; equal transaction budgets would make +// remainder-rate peers finish early. Otherwise this is upstream's even split. +let PartitionValidatorLoad (n: int) (fixedDuration: bool) (full: LoadGen) = + if n <= 0 then invalidArg "n" "Need at least one generator" + + if fixedDuration && full.accounts < n then + invalidArg "n" "Need accounts for every generator" + + if fixedDuration && (full.txrate <= 0 || full.txs % full.txrate <> 0) then + invalidArg "full" "Fixed-duration load must have an integral duration" + + let fraction value i = value / n + (if i < value % n then 1 else 0) + + [ for i in 0 .. n - 1 do + let classic = full.classicTxRate |> Option.map (fun v -> fraction v i) + let soroban = full.sorobanTxRate |> Option.map (fun v -> fraction v i) + + let rate = + match classic, soroban with + | Some c, Some s -> c + s + | Some c, None -> c + | None, Some s -> s + | None, None -> fraction full.txrate i + + yield + { full with + accounts = full.accounts / n + offset = (full.accounts / n) * i + txs = if fixedDuration then rate * (full.txs / full.txrate) else full.txs / n + spikesize = fraction full.spikesize i + txrate = rate + classicTxRate = classic + sorobanTxRate = soroban } ] + +// A single generator selection is shared by launch and final submission +// accounting: every validator of each core set when everyValidator +// (StellarKubeSpecs.LoadOnEveryValidator), else node 0 of each, as upstream. +let LoadgenPeerIndices (everyValidator: bool) (coreSets: CoreSet list) = + coreSets + |> List.collect (fun cs -> [ for i in 0 .. (if everyValidator then cs.options.nodeCount - 1 else 0) -> cs, i ]) + // Pure check of an observed stellar-core pod -> worker-node mapping for // --one-stellar-core-per-host: every pod in mustBeScheduled is listed exactly // once and on a node, and no two scheduled pods share a node. Pods not yet @@ -280,6 +323,10 @@ let autoscalerView type StellarFormation with + member self.LoadGenPeers (coreSets: CoreSet list) (loadGen: LoadGen) = + LoadgenPeerIndices(LoadOnEveryValidator self.NetworkCfg.missionContext loadGen.mode) coreSets + |> List.map (fun (cs, i) -> self.NetworkCfg.GetPeer cs i) + member self.GetCoreSetForStatefulSet(ss: V1StatefulSet) = List.find (fun cs -> (self.NetworkCfg.StatefulSetName cs).StringName = ss.Name()) self.NetworkCfg.CoreSetList @@ -782,40 +829,12 @@ type StellarFormation with |> ignore // This is similar to RunLoadgen but runs a 1/N fractional portion of a - // given LoadGen on node 0 of each of N CoreSets. + // given LoadGen on each of N generators (LoadGenPeers: node 0 of each + // CoreSet, or every validator of each for LoadOnEveryValidator runs). member self.RunMultiLoadgen (coreSets: CoreSet list) (fullLoadGen: LoadGen) = - let n = List.length coreSets - - let fractionalLoadGen (i: int) : LoadGen = - // Spread remainder across the first r nodes instead of dumping it - // entirely on the last one, so per-node load stays even. - let getFraction attr = - let q = attr / n - let r = attr % n - if i < r then q + 1 else q - - let getOptionalFraction attr = Option.map getFraction attr - - let classicShare = getOptionalFraction fullLoadGen.classicTxRate - let sorobanShare = getOptionalFraction fullLoadGen.sorobanTxRate - - // In mixed mode derive txrate from the component rates so that - // txrate, classicTxRate, and sorobanTxRate stay consistent on - // every peer (independent splits would diverge by 1 per slice). - let txrateShare = - match classicShare, sorobanShare with - | Some c, Some s -> c + s - | Some c, None -> c - | None, Some s -> s - | None, None -> getFraction fullLoadGen.txrate - - { fullLoadGen with - accounts = fullLoadGen.accounts / n - txs = fullLoadGen.txs / n - spikesize = getFraction fullLoadGen.spikesize - txrate = txrateShare - classicTxRate = classicShare - sorobanTxRate = sorobanShare } + let everyValidator = LoadOnEveryValidator self.NetworkCfg.missionContext fullLoadGen.mode + let loadGenPeers = self.LoadGenPeers coreSets fullLoadGen + let shares = PartitionValidatorLoad loadGenPeers.Length everyValidator fullLoadGen let hasNonZeroRate (loadGen: LoadGen) = match loadGen.classicTxRate, loadGen.sorobanTxRate with @@ -824,22 +843,23 @@ type StellarFormation with | None, Some sorobanRate -> sorobanRate <> 0 | None, None -> true - let loadGenPeers = List.map (fun cs -> self.NetworkCfg.GetPeer cs 0) coreSets - let peerLoadGens = - loadGenPeers - |> List.indexed - |> List.map - (fun (i, peer) -> - let loadGen = fractionalLoadGen i - let offset = loadGen.accounts * i - (peer, { loadGen with offset = offset }, offset)) + List.zip loadGenPeers shares + |> List.map (fun (peer, loadGen) -> peer, loadGen, loadGen.offset) |> List.filter (fun (_, loadGen, _) -> hasNonZeroRate loadGen) if List.isEmpty peerLoadGens then failwith "Loadgen failed: no peer has a non-zero tx rate" for (peer, peerSpecificLoadgen, offset) in peerLoadGens do + LogInfo + "LOAD_SHARE peer=%s rate=%d txs=%d accounts=%d offset=%d" + peer.ShortName.StringName + peerSpecificLoadgen.txrate + peerSpecificLoadgen.txs + peerSpecificLoadgen.accounts + offset + LogInfo "Loadgen: %s with offset %d" (peer.GenerateLoad peerSpecificLoadgen) offset while List.exists From 80f30e4758f918eee4969c1257764c0acd052ba4 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Thu, 24 Sep 2026 11:27:17 -0700 Subject: [PATCH 5/9] MinBlockTimeMixed: judge overlay-only candidates in mode A loadgen failure now fails the candidate in every mode and the search continues, as the doc already says, instead of aborting the mission. Overlay-only candidates never apply their transactions, but loadgen still completes only once every transaction its node submitted has been included in a closed ledger: stellar-core counts inclusion in this mode (the herder's self-delay timers on master; the self-count counters on the Rust-overlay core since 9ec523e43f, 2026-07-31). A failure, such as load left out of ledgers, therefore fails the candidate. Everything else is judged while still in overlay-only mode: the close-time SLA, the consistency and sync checks, and, as a cross-check from the ledgers' own counts, at least 95% of the offered transactions reaching ledgers (the ledger.transaction.count total over the window). --measure-e2e-latency output is logged in this mode too. Apply is not re-enabled after the window: turning it back on handed the nodes whatever was left in their queues to apply at once, which pushed them out of sync. The nodes are restarted before the next candidate, after a pass as well as a failure. The SLA thresholds are logged per candidate. Co-Authored-By: Claude Opus 5.5 (1M context) --- doc/measuring-minimum-block-time.md | 2 + src/FSLibrary/MinBlockTimeTest.fs | 194 ++++++++++++++++++++++++---- 2 files changed, 174 insertions(+), 22 deletions(-) diff --git a/doc/measuring-minimum-block-time.md b/doc/measuring-minimum-block-time.md index 0c66ca3d..30f3e116 100644 --- a/doc/measuring-minimum-block-time.md +++ b/doc/measuring-minimum-block-time.md @@ -20,6 +20,8 @@ A candidate close time `T` is considered a **pass** if and only if, **on every n If any node violates any of these bounds, `T` is considered a **fail** and the binary search raises its lower bound. The same is true if the load run itself errors (e.g., stellar-core's internal `loadgen-run-failed` counter increments, nodes fall out of sync, or peers report inconsistent ledger hashes) — in that case the mission treats the iteration as a fail and the search continues upward. +Overlay-only candidates (`MinBlockTimeMixed`) never apply their transactions, but stellar-core's load generator still completes only once every transaction it submitted has been included in a closed ledger, so a load generator failure (such as load left out of ledgers) fails the candidate. That needs stellar-core master, or a Rust-overlay image built from 2026-07-31 on (earlier ones never count the transactions as included, so every overlay-only candidate fails). These candidates are judged while still in overlay-only mode: in addition to the close-time SLA above, the consistency and sync checks must pass and, as a cross-check, at least 95% of the offered transactions must reach ledgers (`ledger.transaction.count`). Apply is not re-enabled; the nodes are restarted, discarding what is left in their queues, before the next candidate. + Candidate close times are always whole seconds: the search runs over the whole seconds in `[--min-block-time-ms, --max-block-time-ms]`, bounds included, so the default range evaluates `4000` ms and, only if that fails, `5000` ms. Equal bounds evaluate exactly that close time once, without rounding. If no candidate in the range satisfied the SLA, the mission fails with `"No block time in [lo, hi] ms satisfied the SLA at TPS N"`. ## Docker images with performance tests enabled diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index d2c6a0ae..acce6d7c 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -268,6 +268,60 @@ let private readLedgerAgePercentiles (peer: Peer) : Peer * float * float = let h = peer.GetMetrics().LedgerAgeClosedHistogram peer, float h.``75``, float h.``99`` +// Transactions included in closed ledgers since the candidate cleared the +// metrics. ledger.transaction.count is Updated by LedgerManagerImpl even in +// overlay-only mode (it is marked before the apply-skip branch), so its sum +// counts every transaction that made it into a tx set with apply disabled. +let private readLedgerTxsIncluded (peer: Peer) : Peer * float = + peer, float (peer.GetMetrics().LedgerTransactionCount.Sum) + +let private collectLedgerTxsIncluded (formation: StellarFormation) (coreSets: CoreSet list) : (Peer * float) list = + formation.NetworkCfg.PeersInSets(List.toArray coreSets) + |> List.map (fun peer -> async { return readLedgerTxsIncluded peer }) + |> Async.Parallel + |> Async.RunSynchronously + |> Array.toList + +// Fraction of the offered transactions that must reach ledgers for an +// overlay-only candidate to count. Comparing totals over the window, rather +// than txs per ledger against TPS x T, is not skewed by the idle ledgers +// around the load or by long ledgers carrying more. +let private minInclusionFraction = 0.95 + +// Cross-check for overlay-only candidates. Loadgen completes only once every +// transaction its node submitted has been included in a closed ledger, so a +// completed run includes essentially everything; this confirms it from the +// ledgers' own transaction counts, which checkLedgerAgeSLA does not look at +// (closing near-empty ledgers on schedule passes it trivially). +let private checkInclusionSLA (included: (Peer * float) list) (targetMs: int) (offered: int) : bool = + if List.isEmpty included || offered <= 0 then + true + else + let floorTxs = float offered * minInclusionFraction + let observed = included |> List.averageBy snd + let worstPeer, worstTxs = included |> List.minBy snd + let ok = observed >= floorTxs + + LogInfo + "Inclusion at T=%dms: %.0f of %d offered transactions reached ledgers (%.1f%%), worst peer=%s %.0f -> %s" + targetMs + observed + offered + (100.0 * observed / float offered) + worstPeer.ShortName.StringName + worstTxs + (if ok then "PASS" else "FAIL") + + if not ok then + LogError + "Only %.1f%% of the %d offered transactions reached ledgers at T=%dms (need >= %.0f%%). The network paced ledgers but was not carrying the load, so the close-time result is meaningless." + (100.0 * observed / float offered) + offered + targetMs + (100.0 * minInclusionFraction) + + ok + let private collectLedgerAgePercentiles (formation: StellarFormation) (coreSets: CoreSet list) @@ -322,6 +376,13 @@ let private checkLedgerAgeSLA (percentiles: (Peer * float * float) list) (target let p99Max = tf * 2.0 let mutable ok = true + LogInfo + "SLA thresholds at T=%dms: every peer's ledger.age.closed-histogram P75 in [%.0f, %.0f) ms and P99 <= %.0f ms (the P75 band is the temporarily widened +/-20%%; the target is a close-time target, not a P99 guarantee)" + targetMs + tLo + tHi + p99Max + let formatDeviation value = let deviation = (value - tf) / tf * 100.0 @@ -564,30 +625,114 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: upgradeSorobanMaxTxSetSize targetMs formation.clearMetrics allNodes - // A loadgen failure is not an SLA signal — it usually means the - // requested TPS is too high for the network, or that loadgen - // itself lost a tx in the pipeline. Treating it as "SLA missed" - // would mislead the binary search, so fail the mission loudly. - try - if isMixedPregenMode baseLoadGen.mode then - withOverlayOnlyMode - formation - allNodes - (fun () -> formation.RunMultiLoadgen activeLoadGenNodes loadGen) + // Per doc/measuring-minimum-block-time.md, a loadgen failure + // (application lagging the offered load, a node dropping out + // mid-window, etc.) counts as a failed iteration: the search + // raises its lower bound and continues. Killing the mission + // here would let one bad window discard the whole search. + if isMixedPregenMode baseLoadGen.mode then + // Overlay-only path. Core skips apply by design, but loadgen + // still completes only once every transaction its node + // submitted has been included in a closed ledger (core + // counts inclusion in this mode), so a loadgen failure, + // such as load left out of ledgers, fails the candidate. + // Everything else is measured while apply is still + // disabled, describing the state we measured: + // * percentiles, the inclusion cross-check and the + // pairwise consistency and sync checks; + // * apply is never re-enabled, and the between-iteration + // restart discards whatever is left in the nodes' + // queues rather than making the network apply it at + // once, which previously pushed nodes out of sync + // minutes after the window. + toggleOverlayOnlyMode formation allNodes + + let mutable failureReason = + try + formation.RunMultiLoadgen activeLoadGenNodes loadGen + None + with e -> + LogError "Load generation FAILED at T=%dms: %s" targetMs e.Message + Some(sprintf "load generation failed: %s" e.Message) + + let percentiles = collectLedgerAgePercentiles formation allNodes + + // Read in-mode for the same reason as the percentiles: the + // between-iteration restart resets these counters. + let included = collectLedgerTxsIncluded formation allNodes + + if not (checkInclusionSLA included targetMs loadGen.txs) && failureReason.IsNone then + failureReason <- Some "under 95% of the offered transactions reached ledgers" + + if context.measureE2eLatency then + logE2eLatencyMetrics formation activeLoadGenNodes + + // Consistency AND in-sync are both still enforced, here, + // while apply is still disabled: a node out of sync at this + // point is a real problem with the run we just measured. + try + formation.CheckNoErrorsAndPairwiseConsistency() + formation.EnsureAllNodesInSync allNodes + with e -> + LogWarn + "Health check failed at T=%dms in overlay-only mode: %s — iteration counts as a fail" + targetMs + e.Message + + if failureReason.IsNone then + failureReason <- Some(sprintf "health check: %s" e.Message) + + let slaOk = failureReason.IsNone && checkLedgerAgeSLA percentiles targetMs + + LogInfo + "Candidate T=%dms at %d TPS: %s%s" + targetMs + fixedTxRate + (if slaOk then "PASS" else "FAIL") + (match failureReason with + | _ when slaOk -> "" + | Some reason -> sprintf " (%s)" reason + | None -> " (close-time SLA not met)") + + slaOk + else + let loadgenOk = + try + formation.RunMultiLoadgen activeLoadGenNodes loadGen + true + with e -> + LogWarn "Loadgen failed at T=%dms (%s); treating iteration as SLA fail" targetMs e.Message + + (try + let pct = collectLedgerAgePercentiles formation allNodes + + if not (List.isEmpty pct) then + let avgP75 = pct |> List.averageBy (fun (_, p75, _) -> p75) + let avgP99 = pct |> List.averageBy (fun (_, _, p99) -> p99) + + LogInfo + "Post-failure ledger age at T=%dms: avg-p75=%.0f (%+.1f%% vs target) avg-p99=%.0f" + targetMs + avgP75 + (100.0 * (avgP75 - float targetMs) / float targetMs) + avgP99 + with e2 -> LogWarn "Could not collect post-failure percentiles: %s" e2.Message) + + false + + if not loadgenOk then + false else - formation.RunMultiLoadgen activeLoadGenNodes loadGen - with e -> failwithf "Loadgen failed at T=%dms; TPS might be too high (%s)" targetMs e.Message - - // Snapshot SLA metrics before consistency checks; those can take - // long enough to skew the ledger age percentiles. - let ledgerAgePercentiles = collectLedgerAgePercentiles formation allNodes + // Snapshot SLA metrics before consistency checks; those can take + // long enough to skew the ledger age percentiles. + let ledgerAgePercentiles = collectLedgerAgePercentiles formation allNodes - if context.measureE2eLatency then - logE2eLatencyMetrics formation activeLoadGenNodes + if context.measureE2eLatency then + logE2eLatencyMetrics formation activeLoadGenNodes - formation.CheckNoErrorsAndPairwiseConsistency() - formation.EnsureAllNodesInSync allNodes - checkLedgerAgeSLA ledgerAgePercentiles targetMs + formation.CheckNoErrorsAndPairwiseConsistency() + formation.EnsureAllNodesInSync allNodes + checkLedgerAgeSLA ledgerAgePercentiles targetMs // An explicit single-candidate target: --min-block-time-ms == // --max-block-time-ms == T evaluates exactly that T once, in @@ -654,7 +799,12 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: if evaluateAt t then LogInfo "SLA met at T=%dms; lowering upper bound" t - needsRecovery.Value <- false + // Overlay-only runs never apply their transactions, so the + // nodes' state and queues are not fit for the next + // iteration: leftovers starve its upgrade-contract loadgen, + // which then fails hard. Restart after a pass as well, not + // just after a failure. + needsRecovery.Value <- isMixedPregenMode baseLoadGen.mode true else LogInfo "SLA not met at T=%dms; raising lower bound" t From 4db25cede9cbe5a9cbf1aead42f21126a4c60579 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Thu, 1 Oct 2026 12:08:26 -0700 Subject: [PATCH 6/9] Keep per-validator pregeneration failures fatal With load on every validator, each pod's init commands ran inside an `if test $POD = ; then { ...; }; fi` per validator. Within the braces the commands were sequenced with `;` rather than `&&`, and the `if` of a non-matching pod returns 0, so a failing new-db or pregenerate-loadgen-txs no longer kept stellar-core from starting. Only the pregeneration step differs per validator, so getInitCommands now builds one init chain for the pod and branches just that step: an if/elif per validator, with `else false` so a pod matching none fails. The branch is part of the `&&` chain again, so any init failure stops the pod before core runs, as for every other core set, and the script holds the shared steps once instead of once per validator. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/FSLibrary.Tests/Tests.fs | 33 ++++++++++++++--- src/FSLibrary/StellarKubeSpecs.fs | 61 +++++++++++++++++++------------ 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index f83a60b8..a9062a50 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -766,7 +766,9 @@ type Tests(output: ITestOutputHelper) = let nCfgWithoutSimulateApply = MakeNetworkCfg { ctx with simulateApplyWeight = None; simulateApplyDuration = None } [ coreSet ] passOpt - let cmds = nCfgWithoutSimulateApply.getInitCommands PeerSpecificConfigFile coreSet.options + let cmds = + nCfgWithoutSimulateApply.getInitCommands PeerSpecificConfigFile coreSet.options None + let cmdStr = ShAnd(cmds).ToString() let exp = @@ -775,7 +777,7 @@ type Tests(output: ITestOutputHelper) = Assert.Equal(exp, cmdStr) - let cmds = nCfg.getInitCommands PeerSpecificConfigFile coreSet.options + let cmds = nCfg.getInitCommands PeerSpecificConfigFile coreSet.options None let cmdStr = ShAnd(cmds).ToString() Assert.Equal(exp, cmdStr) @@ -1498,10 +1500,11 @@ let ``Validators pregenerate their own account slices only when the MinBlockTime let pregenSet = MakeLiveCoreSet "pregen" pregenOpts + let nCfg (c: MissionContext) = + MakeNetworkCfg { c with installNetworkDelay = Some false } [ pregenSet ] passOpt + let script (c: MissionContext) = - let pod = - (MakeNetworkCfg { c with installNetworkDelay = Some false } [ pregenSet ] passOpt) - .ToPodTemplateSpec pregenSet + let pod = (nCfg c).ToPodTemplateSpec pregenSet let core = pod.Spec.Containers @@ -1514,7 +1517,25 @@ let ``Validators pregenerate their own account slices only when the MinBlockTime let shared = script { ctx with runForMinBlockTime = true } Assert.Contains("'--offset 0'", shared) Assert.DoesNotContain("'--offset 100'", shared) - let perValidator = script { ctx with runForMinBlockTime = true; pregenerateTxsPerValidator = true } + let perValidatorCtx = { ctx with runForMinBlockTime = true; pregenerateTxsPerValidator = true } + let perValidator = script perValidatorCtx for offset in [ 0; 100; 200 ] do Assert.Contains(sprintf "'--offset %d'" offset, perValidator) + + // One branch per validator inside the init chain, so a failed + // pregeneration still keeps stellar-core from starting, and a pod that + // matches no validator fails instead of starting without its txs. + let names = Array.init 3 (fun i -> ((nCfg perValidatorCtx).PodName pregenSet i).StringName) + + let cmds = + (nCfg perValidatorCtx).getInitCommands PeerSpecificConfigFile pregenSet.options (Some names) + + let branches = + cmds + |> Array.choose + (function + | ShIf (_, _, elifs, Some otherwise) -> Some(elifs.Length, otherwise.ToString()) + | _ -> None) + + Assert.Equal<(int * string) array>([| (2, "false") |], branches) diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 3fa9efc0..b6b81039 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -678,7 +678,13 @@ type NetworkCfg with | None -> cfgs | Some (opts) -> Array.append cfgs [| self.JobConfigMap(opts) |] - member self.getInitCommands (configOpt: ConfigOption) (opts: CoreSetOptions) : ShCmd array = + // perValidatorPeerNames: the pods of a MissionContext.pregenerateTxsPerValidator + // core set, in validator order, when each pregenerates its own account slice. + member self.getInitCommands + (configOpt: ConfigOption) + (opts: CoreSetOptions) + (perValidatorPeerNames: string array option) + : ShCmd array = let cfgWords = cfgFileArgs configOpt InitCoreContainer let runCore args = @@ -753,17 +759,35 @@ type NetworkCfg with // we want. let newHistIgnoreError = ignoreError newHist - let pregenerate = - match init.pregenerateTxs with - | None -> None - | Some (txs, accounts, offset) -> - runCoreIf - true - [| "pregenerate-loadgen-txs" + let pregenerateTxs (txs: int, accounts: int, offset: int) = + runCore [| "pregenerate-loadgen-txs" "--count " + txs.ToString() "--accounts " + accounts.ToString() "--offset " + offset.ToString() |] + // Per validator, each pod pregenerates its own slice + // (PregenerationOptionsForPeer). The branch stays in the init chain, so + // a failure still keeps stellar-core from starting, and a pod that + // matches no validator fails instead of starting without its txs. + let pregenerate = + match init.pregenerateTxs, perValidatorPeerNames with + | None, _ -> None + | Some slice, None -> Some(pregenerateTxs slice) + | Some _, Some names -> + let branch i (name: string) = + let isPeer = + ShCmd [| ShWord.OfStr "test" + ShWord.Var CfgVal.peerNameEnvVarName + ShWord.OfStr "=" + ShWord.OfStr name |] + + isPeer, pregenerateTxs (PregenerationOptionsForPeer opts i).initialization.pregenerateTxs.Value + + match Array.mapi branch names |> List.ofArray with + | [] -> None + | (isFirst, first) :: rest -> + Some(ShCmd.ShIf(isFirst, first, Array.ofList rest, Some(ShCmd.OfStr "false"))) + let initialCatchup = runCoreIf init.initialCatchup [| "catchup"; "current/0" |] let cmds = @@ -850,7 +874,7 @@ type NetworkCfg with | None -> [| CoreContainerForCommand image cfgOpt asan self.missionContext.coreEnv res command [||] [| jobName |] |] | Some (opts) -> - let initCmds = self.getInitCommands cfgOpt opts + let initCmds = self.getInitCommands cfgOpt opts None let coreContainer = CoreContainerForCommand @@ -943,21 +967,10 @@ type NetworkCfg with let volumes = Array.append peerCfgVolumes [| dataVol; historyCfgVolume |] let initCommands = - if self.missionContext.pregenerateTxsPerValidator then - peerNames - |> Array.mapi - (fun i name -> - let commands = self.getInitCommands cfgOpt (PregenerationOptionsForPeer coreSet.options i) - - let test = - ShCmd [| ShWord.OfStr "test" - ShWord.Var CfgVal.peerNameEnvVarName - ShWord.OfStr "=" - ShWord.OfStr name |] - - ShCmd.ShIf(test, ShCmd.ShSeq commands, [||], None)) - else - self.getInitCommands cfgOpt coreSet.options + self.getInitCommands + cfgOpt + coreSet.options + (if self.missionContext.pregenerateTxsPerValidator then Some peerNames else None) let runCmd = [| "run" |] From ee4b9bad95b5ad0941253e1b8bd00e545c3353b9 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Thu, 1 Oct 2026 12:09:07 -0700 Subject: [PATCH 7/9] Use --tier-1-orgs-to-add for the synthetic tier 1 topology Instead of a separate --tier1-org-count that only applies to the synthetic topology, --tier-1-orgs-to-add now means "tier 1 organizations to add" in both cases: with --pubnet-data it scales that topology as before, and otherwise MinBlockTime* and MaxTPS* add that many of the synthetic organizations (0 to 30) to StableApproximateTier1CoreSets' base 10. --tier-1-orgs-to-add 9 is the 19-organization topology that --tier1-org-count 19 used to give. The topology test now checks the added organizations directly: their order and locations, that the base organizations are unchanged, and that they reach named regions the base topology has no validator in, rather than counting distinct locations. Co-Authored-By: Claude Opus 5.5 (1M context) --- doc/measuring-minimum-block-time.md | 2 +- src/App/Program.fs | 21 +------ src/FSLibrary.Tests/Tests.fs | 82 +++++++++++++------------- src/FSLibrary/MaxTPSTest.fs | 4 +- src/FSLibrary/MinBlockTimeTest.fs | 4 +- src/FSLibrary/StellarMissionContext.fs | 3 - src/FSLibrary/StellarNetworkData.fs | 35 ++++------- 7 files changed, 60 insertions(+), 91 deletions(-) diff --git a/doc/measuring-minimum-block-time.md b/doc/measuring-minimum-block-time.md index 30f3e116..572fd3ba 100644 --- a/doc/measuring-minimum-block-time.md +++ b/doc/measuring-minimum-block-time.md @@ -43,7 +43,7 @@ These parameters affect both `MinBlockTimeClassic` and `MinBlockTimeMixed` missi * `--max-block-time-ms`: Binary search upper bound, in milliseconds. Defaults to `5000`, which is also the protocol's maximum allowed ledger target close time — setting this higher will cause the mission to fail at startup, since validators reject upgrades above the protocol cap. Must not be less than `--min-block-time-ms`; when the two are equal the mission evaluates exactly that close time once, with no search. * `--num-pregenerated-txs`: Number of pre-generated signed classic transactions to create per loadgen node. `MinBlockTimeClassic` uses these on small networks (≤30 nodes) when it automatically switches classic payment load to `PayPregenerated`; `MinBlockTimeMixed` always uses them for the classic stream in its `MIXED_PREGEN_*` mode. Defaults to `2500000` * `--pubnet-data`: Network topology to use. Defaults to a topology of tier 1 validators. See [Specifying network topologies](#specifying-network-topologies) for details on how to specify a custom topology. -* `--tier1-org-count`: Organizations (three validators each) in that default tier 1 topology, from `10` (the default) to `40`. Beyond 10, synthetic organizations are added in a fixed order (`x01`, `x02`, ...), spread over further cloud regions in North America, Europe, Asia, South America, Oceania, Africa and the Middle East; the simulated network delay between two validators grows with their distance, so larger counts also raise the network's latency floor. +* `--tier-1-orgs-to-add`: Organizations (three validators each) to add to that default tier 1 topology of 10 organizations, from `0` (the default) to `30`. They are added in a fixed order (`x01`, `x02`, ...), spread over further cloud regions in North America, Europe, Asia, South America, Oceania, Africa and the Middle East; the simulated network delay between two validators grows with their distance, so larger counts also raise the network's latency floor. With `--pubnet-data`, the flag instead adds tier 1 organizations to that topology, as in `SimulatePubnet`. * `--netdelay-image`: Helper image providing simulated network delay for latency simulation. SDF provides a public image on dockerhub at `stellar/sdf-netdelay`. ### Additional options for mixed pre-generated classic and synthetic Soroban traffic diff --git a/src/App/Program.fs b/src/App/Program.fs index 6d169ae0..1d3138ba 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -149,8 +149,7 @@ type MissionOptions bimodalDrift: seq, driftPct: int, ledgerCloseTimeMs: int option, - forceOldStyleTriggerTimer: bool option, - tier1OrgCount: int option + forceOldStyleTriggerTimer: bool option ) = [] @@ -502,7 +501,7 @@ type MissionOptions member self.SimulateApplyWeight = simulateApplyWeight [] member self.Tier1OrgsToAdd = tier1OrgsToAdd @@ -710,11 +709,6 @@ type MissionOptions Required = false)>] member self.ForceOldStyleTriggerTimer = forceOldStyleTriggerTimer - [] - member self.Tier1OrgCount = tier1OrgCount - let splitLabel (lab: string) : (string * string option) = match lab.Split ':' |> Array.toList with | [ x ] -> x, None @@ -779,16 +773,6 @@ let main argv = if mission.MeasureE2eLatency && mission.LoadgenKeys.IsNone then failwith "Error: --measure-e2e-latency requires --loadgen-keys" - match mission.Tier1OrgCount with - | Some n when - n < StellarNetworkData.tier1BaseOrgCount - || n > StellarNetworkData.tier1MaxOrgCount -> - failwithf - "Error: --tier1-org-count must be between %d and %d" - StellarNetworkData.tier1BaseOrgCount - StellarNetworkData.tier1MaxOrgCount - | _ -> () - let _ = logToConsoleAndFile (sprintf "%s/stellar-supercluster.log" mission.Destination) let ll = @@ -981,7 +965,6 @@ let main argv = minBlockTimeMixedMode = mission.MinBlockTimeMixedMode minBlockTimeMixedClassicTxRate = mission.MinBlockTimeMixedClassicTxRate minBlockTimeMixedSorobanTxRate = mission.MinBlockTimeMixedSorobanTxRate - tier1OrgCount = mission.Tier1OrgCount pregenerateTxsPerValidator = false runForMinBlockTime = false forceOldStyleTriggerTimerPct = mission.ForceOldStyleTriggerTimerPct diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index a9062a50..7bf1af93 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -152,7 +152,6 @@ let ctx : MissionContext = minBlockTimeMixedMode = "mixed_pregen_sac_payment" minBlockTimeMixedClassicTxRate = None minBlockTimeMixedSorobanTxRate = None - tier1OrgCount = None pregenerateTxsPerValidator = false runForMinBlockTime = false forceOldStyleTriggerTimerPct = 0 @@ -1318,7 +1317,7 @@ let ``All validator partitions preserve rate duration and disjoint account slice [] let ``57 validator topology and initialization match every generator partition`` () = - let sets = StableApproximateTier1CoreSetsWithOrgCount "frozen-image" false (Some 19) + let sets = StableApproximateTier1CoreSetsWithExtraOrgs "frozen-image" false 9 Assert.Equal(19, sets.Length) Assert.Equal(57, sets |> List.sumBy (fun s -> s.options.nodeCount)) @@ -1375,7 +1374,7 @@ let ``Fixed duration partition rejects a partial second transaction budget`` () [] let ``Submission accounting includes all started validators and preserves legacy selection`` () = - let sets = StableApproximateTier1CoreSetsWithOrgCount "frozen-image" false (Some 19) + let sets = StableApproximateTier1CoreSetsWithExtraOrgs "frozen-image" false 9 let selected = StellarStatefulSets.LoadgenPeerIndices true sets Assert.Equal(57, selected.Length) @@ -1399,61 +1398,62 @@ let ``Submission accounting includes all started validators and preserves legacy Assert.All(legacy, (fun (_, i) -> Assert.Equal(0, i))) [] -let ``Tier1 topology keeps its 10 organizations unless --tier1-org-count adds diverse ones`` () = +let ``--tier-1-orgs-to-add extends the synthetic tier 1 topology with diverse organizations`` () = let orgs (sets: CoreSet list) = sets |> List.map (fun cs -> cs.name.StringName) |> List.sort - let upstream = StableApproximateTier1CoreSets "img" false - Assert.Equal(10, upstream.Length) - Assert.Equal(orgs upstream, orgs (StableApproximateTier1CoreSetsWithOrgCount "img" false None)) - Assert.Equal(orgs upstream, orgs (StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 10))) - let twelve = StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 12) - Assert.Equal(List.sort (orgs upstream @ [ "x01"; "x02" ]), orgs twelve) - // Adding organizations never changes the base ones. let locsOf (sets: CoreSet list) name = (sets |> List.find (fun cs -> cs.name.StringName = name)).options.nodeLocs - - for org in orgs upstream do - Assert.Equal(locsOf upstream org, locsOf twelve org) - - Assert.Equal(40, tier1MaxOrgCount) - let all = StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 40) + let baseSets = StableApproximateTier1CoreSets "img" false + Assert.Equal(10, baseSets.Length) + Assert.Equal(orgs baseSets, orgs (StableApproximateTier1CoreSetsWithExtraOrgs "img" false 0)) + + // Organizations are added in tier1ExtraOrgs order with their own + // locations, and never change the base ones. + let twelve = StableApproximateTier1CoreSetsWithExtraOrgs "img" false 2 + Assert.Equal(List.sort (orgs baseSets @ [ "x01"; "x02" ]), orgs twelve) + Assert.Equal(Some [ Dublin; Amsterdam; Toronto ], locsOf twelve "x01") + + for org in orgs baseSets do + Assert.Equal(locsOf baseSets org, locsOf twelve org) + + // All 30: 40 organizations of 3 validators, no two extra ones at the same + // three locations. + let all = StableApproximateTier1CoreSetsWithExtraOrgs "img" false 30 Assert.Equal(40, all.Length) - Assert.Equal(120, all |> List.sumBy (fun cs -> cs.options.nodeCount)) + Assert.All(all, (fun cs -> Assert.Equal(3, cs.options.nodeCount))) Assert.All(all, (fun cs -> Assert.Equal(3, cs.options.nodeLocs.Value.Length))) - // Far more locations than the base topology's 13, on every inhabited - // continent (southern-hemisphere and eastern-hemisphere sites included). - let distinctLocs (sets: CoreSet list) = sets |> List.collect (fun cs -> cs.options.nodeLocs.Value) |> List.distinct - - let locs = distinctLocs all - Assert.Equal(13, (distinctLocs upstream).Length) - Assert.Equal( - 37, - (distinctLocs (StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 19))) - .Length - ) - - Assert.Equal(51, locs.Length) - Assert.Contains(locs, (fun l -> l.lat < -30.0 && l.lon > 140.0)) // Oceania - Assert.Contains(locs, (fun l -> l.lat < -20.0 && l.lon < -40.0)) // South America - Assert.Contains(locs, (fun l -> l.lat < -20.0 && l.lon > 10.0 && l.lon < 40.0)) // southern Africa - // Each extra organization is distinct: no two share the same three locations. let extraLocSets = all |> List.filter (fun cs -> cs.name.StringName.StartsWith "x") |> List.map (fun cs -> List.sort cs.options.nodeLocs.Value) - Assert.Equal(30, extraLocSets.Length) Assert.Equal(30, extraLocSets |> List.distinct |> List.length) - // The quorum grows with the organizations: 67% of 40 inner sets. + + // The extra organizations reach regions with no base validator: Oceania, + // South America, Africa, the Middle East and more of Asia. + let locsIn (sets: CoreSet list) = sets |> List.collect (fun cs -> cs.options.nodeLocs.Value) + + for region in [ Sydney + Auckland + SaoPaulo + BuenosAires + CapeTown + Lagos + Dubai + TelAviv + Tokyo + Mumbai ] do + Assert.DoesNotContain(region, locsIn baseSets) + Assert.Contains(region, locsIn all) + + // The quorum grows with the organizations: one inner set per organization. match all.Head.options.quorumSet with | ExplicitQuorum q -> Assert.Equal(40, q.innerQuorumSets.Length) | _ -> failwith "Expected explicit organization quorum" - Assert.ThrowsAny - (fun () -> StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 9) |> ignore) + Assert.ThrowsAny(fun () -> StableApproximateTier1CoreSetsWithExtraOrgs "img" false -1 |> ignore) |> ignore - Assert.ThrowsAny - (fun () -> StableApproximateTier1CoreSetsWithOrgCount "img" false (Some 41) |> ignore) + Assert.ThrowsAny(fun () -> StableApproximateTier1CoreSetsWithExtraOrgs "img" false 31 |> ignore) |> ignore [] diff --git a/src/FSLibrary/MaxTPSTest.fs b/src/FSLibrary/MaxTPSTest.fs index b884ecbb..66d0ee73 100644 --- a/src/FSLibrary/MaxTPSTest.fs +++ b/src/FSLibrary/MaxTPSTest.fs @@ -129,10 +129,10 @@ let maxTPSTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadG if context.pubnetData.IsSome then FullPubnetCoreSets context true false else - StableApproximateTier1CoreSetsWithOrgCount + StableApproximateTier1CoreSetsWithExtraOrgs context.image (if context.flatQuorum.IsSome then context.flatQuorum.Value else false) - context.tier1OrgCount + context.tier1OrgsToAdd // PayPregenerated requires node restart between failed iterations to ensure validity of the pregenerated transactions // However, large-scale simulation restarts can be slow, so for now only use the new mode on small networks diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index acce6d7c..1d37a528 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -440,10 +440,10 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: if context.pubnetData.IsSome then FullPubnetCoreSets context true false else - StableApproximateTier1CoreSetsWithOrgCount + StableApproximateTier1CoreSetsWithExtraOrgs context.image (if context.flatQuorum.IsSome then context.flatQuorum.Value else false) - context.tier1OrgCount + context.tier1OrgsToAdd // Mirrors MaxTPSTest: on small networks, GeneratePaymentLoad runs out of // source accounts at high TPS, so switch to PayPregenerated which uses diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index d019b5cd..52b8ab1e 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -165,9 +165,6 @@ type MissionContext = minBlockTimeMixedMode: string minBlockTimeMixedClassicTxRate: int option minBlockTimeMixedSorobanTxRate: int option - // --tier1-org-count: organizations in StableApproximateTier1CoreSets - // (None = its 10; up to 40 adds StellarNetworkData.tier1ExtraOrgs). - tier1OrgCount: int option // Set by MinBlockTimeTest when its load runs on every validator // (StellarKubeSpecs.LoadOnEveryValidator for the mission's actual load // mode): each validator then pregenerates transactions for its own diff --git a/src/FSLibrary/StellarNetworkData.fs b/src/FSLibrary/StellarNetworkData.fs index 28243e76..f3a47beb 100644 --- a/src/FSLibrary/StellarNetworkData.fs +++ b/src/FSLibrary/StellarNetworkData.fs @@ -148,7 +148,7 @@ let Taipei = { lat = 25.0329; lon = 121.5654 } let Tokyo = { lat = 35.6895; lon = 139.69171 } // Further cloud and hosting regions, used only by the synthetic organizations -// StableApproximateTier1CoreSetsWithOrgCount adds beyond its base 10 (not by +// StableApproximateTier1CoreSetsWithExtraOrgs adds beyond its base 10 (not by // pubnet simulations, which draw from `locations` below). let Amsterdam = { lat = 52.3676; lon = 4.9041 } // Azure westeurope, many DCs @@ -994,7 +994,7 @@ let TestnetCoreSetOptions (image: string) = dumpDatabase = false } // Synthetic organizations, in the order they are added, that extend -// StableApproximateTier1CoreSets past its 10 organizations (--tier1-org-count). +// StableApproximateTier1CoreSets past its 10 organizations (--tier-1-orgs-to-add). // Most spread their three validators over two regions, like real operators. // Every block of additions keeps the mix at roughly one third North America, // one third Europe and one third elsewhere (Asia, South America, Oceania, @@ -1032,18 +1032,10 @@ let tier1ExtraOrgs : (string * GeoLoc list) list = ("x29", [ Helsinki; Stockholm; Osaka ]) ("x30", [ Miami; Ashburn; Nuremberg ]) ] -let tier1BaseOrgCount = 10 - -let tier1MaxOrgCount = tier1BaseOrgCount + List.length tier1ExtraOrgs - // This coreset is a synthetic approximation of the Tier1 group, intended -// to be used in stable benchmarks rather than experiments. orgCount (None = -// the 10 base organizations) adds tier1ExtraOrgs, up to tier1MaxOrgCount. -let StableApproximateTier1CoreSetsWithOrgCount - (image: string) - (flatQuorum: bool) - (orgCount: int option) - : CoreSet list = +// to be used in stable benchmarks rather than experiments. Its 10 +// organizations are followed by the first extraOrgs of tier1ExtraOrgs. +let StableApproximateTier1CoreSetsWithExtraOrgs (image: string) (flatQuorum: bool) (extraOrgs: int) : CoreSet list = let allOrgs : Map = Map.ofList [ ("bd", [ Brussels; CouncilBluffs; Taipei ]) ("ct", [ Frankfurt; Purfleet; Brussels ]) @@ -1056,16 +1048,13 @@ let StableApproximateTier1CoreSetsWithOrgCount ("rg", [ Brussels; Frankfurt; Brussels ]) ("sdf", [ Ashburn; Ashburn; Ashburn ]) ] - let extra = - match orgCount with - | None -> 0 - | Some n when n >= tier1BaseOrgCount && n <= tier1MaxOrgCount -> n - tier1BaseOrgCount - | Some n -> - failwithf "tier1 organization count must be between %d and %d, got %d" tier1BaseOrgCount tier1MaxOrgCount n - - let extraOrgs = tier1ExtraOrgs |> List.truncate extra + if extraOrgs < 0 || extraOrgs > List.length tier1ExtraOrgs then + failwithf + "--tier-1-orgs-to-add must be between 0 and %d for the synthetic tier 1 topology, got %d" + (List.length tier1ExtraOrgs) + extraOrgs - let allOrgs = Map.ofList (Map.toList allOrgs @ extraOrgs) + let allOrgs = Map.ofList (Map.toList allOrgs @ List.truncate extraOrgs tier1ExtraOrgs) let allOrgPairs = Map.toList allOrgs let orgKeys _ nodes = List.map (fun _ -> KeyPair.Random()) nodes @@ -1127,4 +1116,4 @@ let StableApproximateTier1CoreSetsWithOrgCount List.map orgCoreSet allOrgPairs let StableApproximateTier1CoreSets (image: string) (flatQuorum: bool) : CoreSet list = - StableApproximateTier1CoreSetsWithOrgCount image flatQuorum None + StableApproximateTier1CoreSetsWithExtraOrgs image flatQuorum 0 From e99e328a44dd25bd69a9d0df90a34445dd59f812 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Thu, 1 Oct 2026 12:09:47 -0700 Subject: [PATCH 8/9] MinBlockTime: one evaluation path for every candidate evaluateAt ran overlay-only (MIXED_PREGEN_*) and apply-mode candidates down two branches that had drifted apart: only apply mode logged the SLA after a loadgen failure, the metric snapshot was explained in one branch, and a failed health check failed an overlay-only candidate but aborted the whole mission in apply mode, although the doc says it fails the candidate. Both now take the same steps: overlay-only mode (if any), loadgen, one metric snapshot read before the health checks, the inclusion cross-check (overlay-only only), the health checks, and one verdict line. Every failure fails the candidate, as documented, and the close-time SLA is always logged. The inclusion check returns its own failure reason, so its message follows minInclusionFraction instead of a hard-coded 95%. Equal --min/--max-block-time-ms bounds now go through the same search as a one-candidate list (closeTimeCandidates), still evaluated exactly as given, instead of a separate branch. MinBlockTimeTest now owns the every-validator decision: it is simply MIXED_PREGEN_* load, so StellarKubeSpecs.LoadOnEveryValidator is gone, RunMultiLoadgen and LoadGenPeers read the context's pregenerateTxsPerValidator flag, and the account partitioning uses that one condition and the active load-generating sets. withOverlayOnlyMode moves to MissionTriggerTimerMixConsensus, its only user. Also: return types on PartitionValidatorLoad, LoadgenPeerIndices and PregenerationOptionsForPeer, no more "upstream" in comments, and a note on why needsRecovery is a ref cell. Tests: the partition tests derive their numbers (57 = 19 organizations x 3 validators; 21,052 = 1,200,000 / 57, floored) instead of hard-coding them, explain why a partial-second budget is rejected, drop a tally that only re-checked the selection count, cover all three MIXED_PREGEN modes, and add cases for which partition values are floored and which are spread. The search test records evaluations in a ref cell instead of a System.Collections.Generic.List. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/FSLibrary.Tests/Tests.fs | 245 +++++++++------ src/FSLibrary/MinBlockTimeTest.fs | 288 +++++++----------- .../MissionTriggerTimerMixConsensus.fs | 10 + src/FSLibrary/StellarKubeSpecs.fs | 15 +- src/FSLibrary/StellarMissionContext.fs | 8 +- src/FSLibrary/StellarStatefulSets.fs | 22 +- 6 files changed, 301 insertions(+), 287 deletions(-) diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 7bf1af93..e33a7302 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -1246,38 +1246,56 @@ let ``Min block time candidates are the whole seconds in the range, bounds inclu Assert.Equal([ 1000 ], MinBlockTimeTest.wholeSecondCandidates 0 1000) Assert.Empty(MinBlockTimeTest.wholeSecondCandidates 1100 1900) +[] +let ``Equal min block time bounds evaluate exactly that close time`` () = + Assert.Equal([ 4000 ], MinBlockTimeTest.closeTimeCandidates 4000 4000) + // An explicit target is not rounded to a whole second. + Assert.Equal([ 1500 ], MinBlockTimeTest.closeTimeCandidates 1500 1500) + Assert.Equal([ 4000; 5000 ], MinBlockTimeTest.closeTimeCandidates 4000 5000) + Assert.Empty(MinBlockTimeTest.closeTimeCandidates 1100 1900) + +// The search result and the candidates it evaluated, in order. +let private searchMinPassingTrace (candidates: int list) (passes: int -> bool) : int option * int list = + let evaluated = ref [] + + let result = + MinBlockTimeTest.searchMinPassing + candidates + (fun t -> + evaluated.Value <- evaluated.Value @ [ t ] + passes t) + + result, evaluated.Value + [] let ``Min block time search finds the smallest passing candidate`` () = let candidates = [ 1000 .. 1000 .. 5000 ] for threshold in candidates do - let evaluated = System.Collections.Generic.List() - - let result = - MinBlockTimeTest.searchMinPassing - candidates - (fun t -> - evaluated.Add t - t >= threshold) - + let result, evaluated = searchMinPassingTrace candidates (fun t -> t >= threshold) Assert.Equal(Some threshold, result) - Assert.InRange(evaluated.Count, 1, 3) + // Binary search over 5 candidates: at most ceil(log2 6) = 3 evaluations. + Assert.InRange(evaluated.Length, 1, 3) - Assert.Equal(None, MinBlockTimeTest.searchMinPassing candidates (fun _ -> false)) - Assert.Equal(None, MinBlockTimeTest.searchMinPassing [] (fun _ -> true)) + Assert.Equal(None, fst (searchMinPassingTrace candidates (fun _ -> false))) + Assert.Equal(None, fst (searchMinPassingTrace [] (fun _ -> true))) // The default [4000, 5000] range evaluates 4000 first, then 5000 only if 4000 fails. - let evaluated = System.Collections.Generic.List() - - let result = - MinBlockTimeTest.searchMinPassing - [ 4000; 5000 ] - (fun t -> - evaluated.Add t - t >= 5000) - - Assert.Equal(Some 5000, result) - Assert.Equal([ 4000; 5000 ], List.ofSeq evaluated) + Assert.Equal((Some 5000, [ 4000; 5000 ]), searchMinPassingTrace [ 4000; 5000 ] (fun t -> t >= 5000)) + Assert.Equal((Some 4000, [ 4000 ]), searchMinPassingTrace [ 4000; 5000 ] (fun t -> t >= 4000)) + // A single candidate is evaluated once. + Assert.Equal((None, [ 1500 ]), searchMinPassingTrace [ 1500 ] (fun _ -> false)) + +// The shape of a MinBlockTimeMixed milestone run: --tier-1-orgs-to-add 9 gives +// 19 organizations of 3 validators, so 57 load generators. 1,200,000 genesis +// accounts do not divide evenly among 57 (1,200,000 = 57 * 21,052 + 36), so +// the tests below see both floored and spread values. +let private milestoneOrgsToAdd = 9 + +let private milestoneOrgs = 10 + milestoneOrgsToAdd +let private milestoneValidators = milestoneOrgs * 3 +let private milestoneAccounts = 1200000 +let private milestoneAccountsPerValidator = milestoneAccounts / milestoneValidators [] [] @@ -1285,29 +1303,34 @@ let ``Min block time search finds the smallest passing candidate`` () = [] [] let ``All validator partitions preserve rate duration and disjoint account slices`` (rate: int) = + let durationSec = 960 + let full = { LoadGen.GetDefault() with mode = MixedPregenSACPayment - accounts = 1200000 + accounts = milestoneAccounts txrate = rate - txs = rate * 960 + txs = rate * durationSec classicTxRate = Some 0 sorobanTxRate = Some rate } - let shares = StellarStatefulSets.PartitionValidatorLoad 57 true full - Assert.Equal(57, shares.Length) + let shares = StellarStatefulSets.PartitionValidatorLoad milestoneValidators true full + Assert.Equal(milestoneValidators, shares.Length) + // Rates are spread, at most one apart, so they add up to the offered rate; + // each transaction budget is its rate for the whole duration. Assert.Equal(rate, shares |> List.sumBy (fun s -> s.txrate)) - Assert.Equal(rate * 960, shares |> List.sumBy (fun s -> s.txs)) - Assert.Equal(1199964, shares |> List.sumBy (fun s -> s.accounts)) + Assert.Equal(rate * durationSec, shares |> List.sumBy (fun s -> s.txs)) let rates = shares |> List.map (fun s -> s.txrate) Assert.True(List.max rates - List.min rates <= 1) + // Accounts are floored: 57 * 21,052 = 1,199,964, leaving 36 unused. + Assert.Equal(milestoneValidators * milestoneAccountsPerValidator, shares |> List.sumBy (fun s -> s.accounts)) shares |> List.iteri (fun i s -> - Assert.Equal(21052, s.accounts) - Assert.Equal(i * 21052, s.offset) - Assert.Equal(s.txrate * 960, s.txs) + Assert.Equal(milestoneAccountsPerValidator, s.accounts) + Assert.Equal(i * milestoneAccountsPerValidator, s.offset) + Assert.Equal(s.txrate * durationSec, s.txs) Assert.Equal(Some s.txrate, s.sorobanTxRate) Assert.Equal(Some 0, s.classicTxRate) Assert.True(s.offset + s.accounts <= full.accounts)) @@ -1316,49 +1339,104 @@ let ``All validator partitions preserve rate duration and disjoint account slice Assert.True(a.offset + a.accounts <= b.offset) [] -let ``57 validator topology and initialization match every generator partition`` () = - let sets = StableApproximateTier1CoreSetsWithExtraOrgs "frozen-image" false 9 - Assert.Equal(19, sets.Length) - Assert.Equal(57, sets |> List.sumBy (fun s -> s.options.nodeCount)) +let ``Partition spreads rates but floors accounts and even-split txs`` () = + // 3 generators, and every value leaves a remainder. + let full = + { LoadGen.GetDefault() with + accounts = 1000 + txrate = 100 + txs = 30001 + spikesize = 10 } + + let shares = StellarStatefulSets.PartitionValidatorLoad 3 false full + let field f = shares |> List.map f + // Rates and spike sizes are spread: the first generators each take one of + // the remainder, so the shares still add up (100 = 34 + 33 + 33). + Assert.Equal([ 34; 33; 33 ], field (fun s -> s.txrate)) + Assert.Equal([ 4; 3; 3 ], field (fun s -> s.spikesize)) + // Accounts are floored to 1000 / 3 = 333 per generator, offset by slice, + // so the last account is unused. + Assert.Equal([ 333; 333; 333 ], field (fun s -> s.accounts)) + Assert.Equal([ 0; 333; 666 ], field (fun s -> s.offset)) + // Without a fixed duration each generator gets 30001 / 3 = 10000 txs, + // floored too. + Assert.Equal([ 10000; 10000; 10000 ], field (fun s -> s.txs)) + +[] +let ``Fixed duration partition spreads classic and Soroban rates separately`` () = + let full = + { LoadGen.GetDefault() with + accounts = 900 + txrate = 15 + txs = 15 * 300 + classicTxRate = Some 10 + sorobanTxRate = Some 5 } + + let shares = StellarStatefulSets.PartitionValidatorLoad 3 true full + let field f = shares |> List.map f + Assert.Equal([ Some 4; Some 3; Some 3 ], field (fun s -> s.classicTxRate)) + Assert.Equal([ Some 2; Some 2; Some 1 ], field (fun s -> s.sorobanTxRate)) + // Each generator's rate is the sum of its two streams, and its budget is + // that rate for the full 300 s, so the budgets add up exactly. + Assert.Equal([ 6; 5; 4 ], field (fun s -> s.txrate)) + Assert.Equal([ 1800; 1500; 1200 ], field (fun s -> s.txs)) + Assert.Equal(full.txs, shares |> List.sumBy (fun s -> s.txs)) + +[] +let ``Milestone topology pregenerates each validator's slice of its load partition`` () = + let sets = + StableApproximateTier1CoreSetsWithExtraOrgs "frozen-image" false milestoneOrgsToAdd + + Assert.Equal(milestoneOrgs, sets.Length) + Assert.Equal(milestoneValidators, sets |> List.sumBy (fun s -> s.options.nodeCount)) let full = { LoadGen.GetDefault() with mode = MixedPregenSACPayment - accounts = 1200000 + accounts = milestoneAccounts txrate = 5000 - txs = 4800000 + txs = 5000 * 960 classicTxRate = Some 0 sorobanTxRate = Some 5000 } - let shares = StellarStatefulSets.PartitionValidatorLoad 57 true full + let shares = StellarStatefulSets.PartitionValidatorLoad milestoneValidators true full let keys = sets |> List.collect (fun s -> s.keys |> Array.toList) - Assert.Equal(57, keys |> List.map (fun k -> k.AccountId) |> Set.ofList |> Set.count) + Assert.Equal(milestoneValidators, keys |> List.map (fun k -> k.AccountId) |> Set.ofList |> Set.count) sets |> List.iteri (fun orgIndex cs -> Assert.Equal(Some true, cs.options.tier1) Assert.Equal(3, cs.options.nodeCount) - Assert.Equal(57, cs.options.preferredPeersMap.Value.Count) + // Full mesh: every validator prefers the other 56. + Assert.Equal(milestoneValidators, cs.options.preferredPeersMap.Value.Count) for peers in cs.options.preferredPeersMap.Value.Values do - Assert.Equal(56, peers.Length) + Assert.Equal(milestoneValidators - 1, peers.Length) match cs.options.quorumSet with | ExplicitQuorum q -> Assert.Equal(Some 67, q.thresholdPercent) - Assert.Equal(19, q.innerQuorumSets.Length) + Assert.Equal(milestoneOrgs, q.innerQuorumSets.Length) for inner in q.innerQuorumSets do Assert.Equal(Some 51, inner.thresholdPercent) Assert.Equal(3, inner.validators.Count) | _ -> failwith "Expected explicit organization quorum" + // MinBlockTimeTest gives each organization the slice starting at + // its first validator's offset; PregenerationOptionsForPeer must + // then hand each validator the slice its load share uses. let options = { cs.options with initialization = { cs.options.initialization with - pregenerateTxs = Some(10000, 21052, orgIndex * 3 * 21052) } } + pregenerateTxs = + Some( + 10000, + milestoneAccountsPerValidator, + orgIndex * 3 * milestoneAccountsPerValidator + ) } } for i in 0 .. 2 do let actual = PregenerationOptionsForPeer options i @@ -1367,19 +1445,31 @@ let ``57 validator topology and initialization match every generator partition`` [] let ``Fixed duration partition rejects a partial second transaction budget`` () = - let full = { LoadGen.GetDefault() with accounts = 1200000; txrate = 5000; txs = 4800001 } + // Each validator's budget is its rate times the duration in whole seconds. + // 4,800,001 txs at 5,000 TPS is 960.0002 s, which no such budgets add up + // to, so the partition refuses it rather than dropping or adding a tx. + let full = + { LoadGen.GetDefault() with + accounts = milestoneAccounts + txrate = 5000 + txs = 4800001 } - Assert.Throws(fun () -> StellarStatefulSets.PartitionValidatorLoad 57 true full |> ignore) + Assert.Throws + (fun () -> + StellarStatefulSets.PartitionValidatorLoad milestoneValidators true full + |> ignore) |> ignore [] -let ``Submission accounting includes all started validators and preserves legacy selection`` () = - let sets = StableApproximateTier1CoreSetsWithExtraOrgs "frozen-image" false 9 +let ``Load runs on every validator of each core set or on node 0 of each`` () = + let sets = + StableApproximateTier1CoreSetsWithExtraOrgs "frozen-image" false milestoneOrgsToAdd + let selected = StellarStatefulSets.LoadgenPeerIndices true sets - Assert.Equal(57, selected.Length) + Assert.Equal(milestoneValidators, selected.Length) Assert.Equal( - 57, + milestoneValidators, selected |> List.map (fun (cs, i) -> cs.keys.[i].AccountId) |> Set.ofList @@ -1387,15 +1477,13 @@ let ``Submission accounting includes all started validators and preserves legacy ) Assert.Equal( - [ 19; 19; 19 ], + [ milestoneOrgs; milestoneOrgs; milestoneOrgs ], [ for i in 0 .. 2 -> selected |> List.filter (fun (_, j) -> i = j) |> List.length ] ) - // A one-peer-per-organization tally would incorrectly report only one third. - let submitted = selected |> List.sumBy (fun (_, i) -> if i = 0 then 34000 else 33000) - Assert.Equal(1900000, submitted) - let legacy = StellarStatefulSets.LoadgenPeerIndices false sets - Assert.Equal(19, legacy.Length) - Assert.All(legacy, (fun (_, i) -> Assert.Equal(0, i))) + + let nodeZero = StellarStatefulSets.LoadgenPeerIndices false sets + Assert.Equal(milestoneOrgs, nodeZero.Length) + Assert.All(nodeZero, (fun (_, i) -> Assert.Equal(0, i))) [] let ``--tier-1-orgs-to-add extends the synthetic tier 1 topology with diverse organizations`` () = @@ -1459,37 +1547,24 @@ let ``--tier-1-orgs-to-add extends the synthetic tier 1 topology with diverse or [] let ``MIXED_PREGEN runs use at most one generator per requested TPS`` () = let sets = StableApproximateTier1CoreSets "img" false - let count everyValidator tps = (MinBlockTimeTest.activeLoadGenCoreSets everyValidator tps sets).Length - // With load on every validator each organization brings its 3 validators. - Assert.Equal(10, count true 5000) - Assert.Equal(10, count true 30) - Assert.Equal(3, count true 10) - Assert.Equal(1, count true 2) - Assert.Equal(1, count true 0) - // One generator per organization otherwise, as upstream. - Assert.Equal(5, count false 5) - Assert.Equal(10, count false 5000) + let count tps = (MinBlockTimeTest.activeLoadGenCoreSets tps sets).Length + // Load runs on every validator, so each organization brings 3 generators: + // 10 TPS fits 3 organizations (9 generators), not 4 (12). + Assert.Equal(10, count 5000) + Assert.Equal(10, count 30) + Assert.Equal(3, count 10) + // Always at least one organization, even below its 3 generators. + Assert.Equal(1, count 2) + Assert.Equal(1, count 0) [] -let ``MIXED_PREGEN load runs on every validator in MinBlockTime runs`` () = - let minBlock = { ctx with runForMinBlockTime = true } - Assert.True(LoadOnEveryValidator minBlock MixedPregenSACPayment) - Assert.False(LoadOnEveryValidator minBlock GeneratePaymentLoad) - Assert.False(LoadOnEveryValidator ctx MixedPregenSACPayment) +let ``MinBlockTime runs every MIXED_PREGEN mode, and only those, on every validator`` () = + // MinBlockTimeTest loads every validator exactly when isMixedPregenMode. + for mode in [ MixedPregenSACPayment; MixedPregenOZTokenTransfer; MixedPregenSoroswapSwap ] do + Assert.True(isMixedPregenMode mode) - // Upstream's per-node split: equal accounts and txs, offsets by slice. - let full = - { LoadGen.GetDefault() with - accounts = 1000 - txrate = 100 - txs = 30000 - spikesize = 10 } - - let shares = StellarStatefulSets.PartitionValidatorLoad 3 false full - Assert.Equal([ 333; 333; 333 ], shares |> List.map (fun s -> s.accounts)) - Assert.Equal([ 0; 333; 666 ], shares |> List.map (fun s -> s.offset)) - Assert.Equal([ 10000; 10000; 10000 ], shares |> List.map (fun s -> s.txs)) - Assert.Equal([ 34; 33; 33 ], shares |> List.map (fun s -> s.txrate)) + for mode in [ GeneratePaymentLoad; PayPregenerated ] do + Assert.False(isMixedPregenMode mode) [] let ``Validators pregenerate their own account slices only when the MinBlockTime load runs on every validator`` () = diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index 1d37a528..7460592d 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -32,6 +32,13 @@ let wholeSecondCandidates (minMs: int) (maxMs: int) : int list = let last = (maxMs / 1000) * 1000 [ first .. 1000 .. last ] +// The close times a run evaluates. Equal bounds name one explicit target, such +// as a milestone latency, which is evaluated exactly once as given, in +// milliseconds and without whole-second rounding; otherwise the whole seconds +// in the range. Exposed for unit tests. +let closeTimeCandidates (minMs: int) (maxMs: int) : int list = + if minMs = maxMs then [ minMs ] else wholeSecondCandidates minMs maxMs + // Binary search over ascending candidates for the smallest one that passes, // assuming every candidate above a passing one passes too. Returns None when // none passes. Exposed for unit tests. @@ -246,24 +253,14 @@ let upgradeMixedPregenSorobanLimits waitForMixedPregenSorobanLimits peer limits -let private toggleOverlayOnlyMode (formation: StellarFormation) (coreSets: CoreSet list) = +// Exposed for reuse by MissionTriggerTimerMixConsensus. +let toggleOverlayOnlyMode (formation: StellarFormation) (coreSets: CoreSet list) = formation.NetworkCfg.EachPeerInSets (List.toArray coreSets) (fun peer -> let res = peer.ToggleOverlayOnlyMode() LogInfo "Toggled overlay-only mode on %s: %s" peer.ShortName.StringName res) -// Exposed for reuse by MissionTriggerTimerMixConsensus. -let withOverlayOnlyMode (formation: StellarFormation) (coreSets: CoreSet list) (f: unit -> unit) = - LogInfo "Enabling overlay-only mode" - toggleOverlayOnlyMode formation coreSets - - try - f () - finally - LogInfo "Disabling overlay-only mode" - toggleOverlayOnlyMode formation coreSets - let private readLedgerAgePercentiles (peer: Peer) : Peer * float * float = let h = peer.GetMetrics().LedgerAgeClosedHistogram peer, float h.``75``, float h.``99`` @@ -292,10 +289,11 @@ let private minInclusionFraction = 0.95 // transaction its node submitted has been included in a closed ledger, so a // completed run includes essentially everything; this confirms it from the // ledgers' own transaction counts, which checkLedgerAgeSLA does not look at -// (closing near-empty ledgers on schedule passes it trivially). -let private checkInclusionSLA (included: (Peer * float) list) (targetMs: int) (offered: int) : bool = +// (closing near-empty ledgers on schedule passes it trivially). Returns why +// the candidate fails, or None. +let private inclusionFailure (included: (Peer * float) list) (targetMs: int) (offered: int) : string option = if List.isEmpty included || offered <= 0 then - true + None else let floorTxs = float offered * minInclusionFraction let observed = included |> List.averageBy snd @@ -312,7 +310,9 @@ let private checkInclusionSLA (included: (Peer * float) list) (targetMs: int) (o worstTxs (if ok then "PASS" else "FAIL") - if not ok then + if ok then + None + else LogError "Only %.1f%% of the %d offered transactions reached ledgers at T=%dms (need >= %.0f%%). The network paced ledgers but was not carrying the load, so the close-time result is meaningless." (100.0 * observed / float offered) @@ -320,7 +320,7 @@ let private checkInclusionSLA (included: (Peer * float) list) (targetMs: int) (o targetMs (100.0 * minInclusionFraction) - ok + Some(sprintf "under %.0f%% of the offered transactions reached ledgers" (100.0 * minInclusionFraction)) let private collectLedgerAgePercentiles (formation: StellarFormation) @@ -418,17 +418,14 @@ let private checkLedgerAgeSLA (percentiles: (Peer * float * float) list) (target ok -// The load-generating core sets a MIXED_PREGEN_* run uses: at most one -// generator per requested TPS, so every generator gets a non-zero share, and -// always at least one set. With load on every validator -// (StellarKubeSpecs.LoadOnEveryValidator) a core set brings all of its -// validators as generators. Exposed for unit tests. -let activeLoadGenCoreSets (everyValidator: bool) (requestedTps: int) (loadGenNodes: CoreSet list) : CoreSet list = - let generators (cs: CoreSet) = if everyValidator then cs.options.nodeCount else 1 - +// The load-generating core sets a MIXED_PREGEN_* run uses. Its load runs on +// every validator of these sets, so they hold at most one validator per +// requested TPS (every generator gets a non-zero share), and always at least +// one set. Exposed for unit tests. +let activeLoadGenCoreSets (requestedTps: int) (loadGenNodes: CoreSet list) : CoreSet list = let fitting = loadGenNodes - |> List.scan (fun total cs -> total + generators cs) 0 + |> List.scan (fun total (cs: CoreSet) -> total + cs.options.nodeCount) 0 |> List.tail |> List.takeWhile (fun total -> total <= max 1 requestedTps) |> List.length @@ -473,20 +470,19 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: else loadGenNodes - let isLoadGenNode cs = List.exists (fun (cs': CoreSet) -> cs' = cs) loadGenNodes - // MIXED_PREGEN_* load runs on every validator of the active core sets, - // each with its own account slice. - let everyValidator = StellarKubeSpecs.LoadOnEveryValidator context baseLoadGen.mode + // each with its own account slice; other modes run it on node 0 of each + // load-generating core set. + let everyValidator = isMixedPregenMode baseLoadGen.mode let activeLoadGenNodes = - if isMixedPregenMode baseLoadGen.mode then + if everyValidator then let requestedCount = max (baseLoadGen.classicTxRate |> Option.defaultValue 0) (baseLoadGen.sorobanTxRate |> Option.defaultValue 0) - activeLoadGenCoreSets everyValidator requestedCount loadGenNodes + activeLoadGenCoreSets requestedCount loadGenNodes else loadGenNodes @@ -494,37 +490,27 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: let context = { context with pregenerateTxsPerValidator = everyValidator } - // For pre-generated modes, partition genesis accounts evenly across - // loadgen nodes and assign offsets so each active node signs txs against - // its own slice. Mixed pregen partitions accounts over the active loadgen - // nodes only, so low-TPS runs still have enough local accounts on the - // nodes that generate load; the other core sets pregenerate nothing. With load - // on every validator, each core set's slice covers all of its validators - // (StellarKubeSpecs.PregenerationOptionsForPeer splits it per pod). + // For pre-generated modes, partition genesis accounts evenly across the + // active loadgen nodes and assign offsets so each signs txs against its own + // slice; the other core sets pregenerate nothing, so low-TPS mixed runs + // still have enough local accounts on the nodes that generate load. With + // load on every validator, each core set's slice covers all of its + // validators (StellarKubeSpecs.PregenerationOptionsForPeer splits it per + // pod). let allNodes = match context.numPregeneratedTxs, context.genesisTestAccountCount, baseLoadGen.mode with | Some txs, Some accounts, mode when usesPregeneratedTxs mode -> - let partitionCount = - if isMixedPregenMode mode && everyValidator then - List.sumBy (fun (cs: CoreSet) -> cs.options.nodeCount) activeLoadGenNodes - elif isMixedPregenMode mode then - List.length activeLoadGenNodes - else - List.length loadGenNodes - + let generators (cs: CoreSet) = if everyValidator then cs.options.nodeCount else 1 + let partitionCount = List.sumBy generators activeLoadGenNodes let accountsPerNode = accounts / partitionCount let mutable j = 0 List.map (fun (cs: CoreSet) -> let pregenerateTxs = - if isLoadGenNode cs && (not (isMixedPregenMode mode) || isActiveLoadGenNode cs) then + if isActiveLoadGenNode cs then let i = j - - j <- - j - + (if isMixedPregenMode mode && everyValidator then cs.options.nodeCount else 1) - + j <- j + generators cs Some(txs, accountsPerNode, accountsPerNode * i) else Some(0, 1, 0) @@ -625,123 +611,81 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: upgradeSorobanMaxTxSetSize targetMs formation.clearMetrics allNodes - // Per doc/measuring-minimum-block-time.md, a loadgen failure - // (application lagging the offered load, a node dropping out - // mid-window, etc.) counts as a failed iteration: the search - // raises its lower bound and continues. Killing the mission - // here would let one bad window discard the whole search. - if isMixedPregenMode baseLoadGen.mode then - // Overlay-only path. Core skips apply by design, but loadgen - // still completes only once every transaction its node - // submitted has been included in a closed ledger (core - // counts inclusion in this mode), so a loadgen failure, - // such as load left out of ledgers, fails the candidate. - // Everything else is measured while apply is still - // disabled, describing the state we measured: - // * percentiles, the inclusion cross-check and the - // pairwise consistency and sync checks; - // * apply is never re-enabled, and the between-iteration - // restart discards whatever is left in the nodes' - // queues rather than making the network apply it at - // once, which previously pushed nodes out of sync - // minutes after the window. - toggleOverlayOnlyMode formation allNodes - - let mutable failureReason = - try - formation.RunMultiLoadgen activeLoadGenNodes loadGen - None - with e -> - LogError "Load generation FAILED at T=%dms: %s" targetMs e.Message - Some(sprintf "load generation failed: %s" e.Message) - - let percentiles = collectLedgerAgePercentiles formation allNodes - - // Read in-mode for the same reason as the percentiles: the - // between-iteration restart resets these counters. - let included = collectLedgerTxsIncluded formation allNodes - - if not (checkInclusionSLA included targetMs loadGen.txs) && failureReason.IsNone then - failureReason <- Some "under 95% of the offered transactions reached ledgers" - - if context.measureE2eLatency then - logE2eLatencyMetrics formation activeLoadGenNodes - - // Consistency AND in-sync are both still enforced, here, - // while apply is still disabled: a node out of sync at this - // point is a real problem with the run we just measured. + // MIXED_PREGEN_* candidates run in overlay-only mode. Core skips + // apply, but loadgen still completes only once every transaction + // its node submitted has been included in a closed ledger (core + // counts inclusion in this mode). Apply is never re-enabled: the + // restart before the next candidate discards whatever is left in + // the nodes' queues rather than making the network apply it at + // once, which previously pushed nodes out of sync minutes after + // the window. Everything below is therefore read in-mode. + let overlayOnly = isMixedPregenMode baseLoadGen.mode + + if overlayOnly then toggleOverlayOnlyMode formation allNodes + + // Per doc/measuring-minimum-block-time.md, each failure below (a + // loadgen failure, such as application lagging the offered load + // or a node dropping out mid-window, unreadable metrics, missing + // inclusion, or nodes out of sync or inconsistent) fails the + // candidate, not the mission: the search raises its lower bound + // and continues. Killing the mission would let one bad window + // discard the whole search. + let loadgenFailure = try - formation.CheckNoErrorsAndPairwiseConsistency() - formation.EnsureAllNodesInSync allNodes + formation.RunMultiLoadgen activeLoadGenNodes loadGen + None with e -> - LogWarn - "Health check failed at T=%dms in overlay-only mode: %s — iteration counts as a fail" - targetMs - e.Message - - if failureReason.IsNone then - failureReason <- Some(sprintf "health check: %s" e.Message) - - let slaOk = failureReason.IsNone && checkLedgerAgeSLA percentiles targetMs - - LogInfo - "Candidate T=%dms at %d TPS: %s%s" - targetMs - fixedTxRate - (if slaOk then "PASS" else "FAIL") - (match failureReason with - | _ when slaOk -> "" - | Some reason -> sprintf " (%s)" reason - | None -> " (close-time SLA not met)") - - slaOk - else - let loadgenOk = - try - formation.RunMultiLoadgen activeLoadGenNodes loadGen - true - with e -> - LogWarn "Loadgen failed at T=%dms (%s); treating iteration as SLA fail" targetMs e.Message - - (try - let pct = collectLedgerAgePercentiles formation allNodes - - if not (List.isEmpty pct) then - let avgP75 = pct |> List.averageBy (fun (_, p75, _) -> p75) - let avgP99 = pct |> List.averageBy (fun (_, _, p99) -> p99) - - LogInfo - "Post-failure ledger age at T=%dms: avg-p75=%.0f (%+.1f%% vs target) avg-p99=%.0f" - targetMs - avgP75 - (100.0 * (avgP75 - float targetMs) / float targetMs) - avgP99 - with e2 -> LogWarn "Could not collect post-failure percentiles: %s" e2.Message) - - false - - if not loadgenOk then - false - else - // Snapshot SLA metrics before consistency checks; those can take - // long enough to skew the ledger age percentiles. - let ledgerAgePercentiles = collectLedgerAgePercentiles formation allNodes + LogWarn "Loadgen failed at T=%dms: %s" targetMs e.Message + Some(sprintf "load generation failed: %s" e.Message) + + // Snapshot the window's metrics before the health checks: core + // keeps closing ledgers after loadgen exits, and the checks can + // take long enough to skew the ledger age percentiles. + let snapshot = + try + let percentiles = collectLedgerAgePercentiles formation allNodes + let included = if overlayOnly then collectLedgerTxsIncluded formation allNodes else [] if context.measureE2eLatency then logE2eLatencyMetrics formation activeLoadGenNodes + Ok(percentiles, included) + with e -> + LogWarn "Could not read metrics at T=%dms: %s" targetMs e.Message + Error(sprintf "could not read metrics: %s" e.Message) + + let healthFailure = + try formation.CheckNoErrorsAndPairwiseConsistency() formation.EnsureAllNodesInSync allNodes - checkLedgerAgeSLA ledgerAgePercentiles targetMs + None + with e -> + LogWarn "Health check failed at T=%dms: %s" targetMs e.Message + Some(sprintf "health check: %s" e.Message) + + // The close-time SLA is logged even when the candidate already + // failed, so every window reports where its close times landed. + let slaOk, metricsFailure = + match snapshot with + | Ok (percentiles, included) -> + checkLedgerAgeSLA percentiles targetMs, inclusionFailure included targetMs loadGen.txs + | Error reason -> false, Some reason - // An explicit single-candidate target: --min-block-time-ms == - // --max-block-time-ms == T evaluates exactly that T once, in - // milliseconds, with no whole-second rounding and no search (upstream - // rejects min == max). The whole-second search below is unchanged for - // min < max. Everything downstream (capacity, ledger target close - // time, SCP timeouts, verdict and the final "No block time" failure) - // is the same code path evaluateAt already uses. - let singleCandidate = context.minBlockTimeMs = context.maxBlockTimeMs + let failure = List.tryPick id [ loadgenFailure; metricsFailure; healthFailure ] + + let pass = failure.IsNone && slaOk + + LogInfo + "Candidate T=%dms at %d TPS: %s%s" + targetMs + fixedTxRate + (if pass then "PASS" else "FAIL") + (match failure with + | Some reason -> sprintf " (%s)" reason + | None when not slaOk -> " (close-time SLA not met)" + | None -> "") + + pass if context.minBlockTimeMs > context.maxBlockTimeMs then failwithf @@ -749,16 +693,16 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: context.minBlockTimeMs context.maxBlockTimeMs - let candidates = wholeSecondCandidates context.minBlockTimeMs context.maxBlockTimeMs + let candidates = closeTimeCandidates context.minBlockTimeMs context.maxBlockTimeMs - if not singleCandidate && List.isEmpty candidates then + if List.isEmpty candidates then failwithf "No whole-second close time in [%d, %d] ms: --min-block-time-ms and --max-block-time-ms must include at least one whole second" context.minBlockTimeMs context.maxBlockTimeMs LogInfo - "Starting min block time search: T in [%d, %d] ms (whole-second candidates: %s), fixed TPS = %d" + "Starting min block time search: T in [%d, %d] ms (candidates: %s), fixed TPS = %d" context.minBlockTimeMs context.maxBlockTimeMs (candidates |> List.map string |> String.concat ", ") @@ -790,6 +734,8 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: System.Threading.Thread.Sleep(5 * 60 * 1000) formation.EnsureAllNodesInSync allNodes + // A ref cell rather than a mutable: evaluateCandidate is a closure, + // and F# closures cannot capture mutable locals. let needsRecovery = ref false // One search step: recover from the previous candidate if needed, @@ -811,19 +757,7 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: needsRecovery.Value <- true false - let bestPassing = - if singleCandidate then - let t = context.minBlockTimeMs - LogInfo "Explicit single-candidate target T=%dms (no whole-second rounding, no search)" t - - if evaluateAt t then - LogInfo "SLA met at T=%dms (single candidate)" t - Some t - else - LogInfo "SLA not met at T=%dms (single candidate)" t - None - else - searchMinPassing candidates evaluateCandidate + let bestPassing = searchMinPassing candidates evaluateCandidate match bestPassing with | Some t -> diff --git a/src/FSLibrary/MissionTriggerTimerMixConsensus.fs b/src/FSLibrary/MissionTriggerTimerMixConsensus.fs index dda69d29..366ad507 100644 --- a/src/FSLibrary/MissionTriggerTimerMixConsensus.fs +++ b/src/FSLibrary/MissionTriggerTimerMixConsensus.fs @@ -112,6 +112,16 @@ let private parseDrift (context: MissionContext) : ClockDriftDistribution = | u, [] -> failwith (sprintf "--uniform-drift requires exactly 2 values (lower,upper), got %d" u.Length) | [], b -> failwith (sprintf "--bimodal-drift requires exactly 4 values (min1,max1,min2,max2), got %d" b.Length) +let private withOverlayOnlyMode (formation: StellarFormation) (coreSets: CoreSet list) (f: unit -> unit) = + LogInfo "Enabling overlay-only mode" + toggleOverlayOnlyMode formation coreSets + + try + f () + finally + LogInfo "Disabling overlay-only mode" + toggleOverlayOnlyMode formation coreSets + let triggerTimerMixConsensus (baseContext: MissionContext) = // This mission assigns the trigger timer per node; a blanket setting for // all nodes would defeat its purpose. diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index b6b81039..81f82975 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -5,7 +5,6 @@ module StellarKubeSpecs open StellarCoreCfg -open StellarCoreHTTP open k8s.Models open StellarMissionContext open StellarNetworkCfg @@ -35,16 +34,10 @@ type ConfigOption = // container picks up a peer-specific config. | PeerSpecificConfigFile -// MinBlockTimeMixed's MIXED_PREGEN_* load runs on every validator rather than -// one node per load-generating core set, each with its own account slice (see -// PregenerationOptionsForPeer and StellarStatefulSets.LoadgenPeerIndices). -let LoadOnEveryValidator (ctx: MissionContext) (mode: LoadGenMode) : bool = - ctx.runForMinBlockTime && isMixedPregenMode mode - -// For LoadOnEveryValidator runs (MissionContext.pregenerateTxsPerValidator): -// the organization's options store its first account offset; each validator -// gets the following disjoint slice. -let PregenerationOptionsForPeer (opts: CoreSetOptions) (index: int) = +// For MissionContext.pregenerateTxsPerValidator runs, whose load runs on every +// validator: the organization's options store its first account offset; each +// validator gets the following disjoint slice. +let PregenerationOptionsForPeer (opts: CoreSetOptions) (index: int) : CoreSetOptions = if index < 0 || index >= opts.nodeCount then invalidArg "index" "Invalid validator index" diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 52b8ab1e..a994f078 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -165,10 +165,10 @@ type MissionContext = minBlockTimeMixedMode: string minBlockTimeMixedClassicTxRate: int option minBlockTimeMixedSorobanTxRate: int option - // Set by MinBlockTimeTest when its load runs on every validator - // (StellarKubeSpecs.LoadOnEveryValidator for the mission's actual load - // mode): each validator then pregenerates transactions for its own - // account slice (StellarKubeSpecs.PregenerationOptionsForPeer). + // Set by MinBlockTimeTest for MIXED_PREGEN_* load: each validator + // pregenerates transactions for its own account slice + // (StellarKubeSpecs.PregenerationOptionsForPeer), and RunMultiLoadgen + // runs the load on every validator rather than node 0 of each core set. pregenerateTxsPerValidator: bool runForMinBlockTime: bool forceOldStyleTriggerTimerPct: int diff --git a/src/FSLibrary/StellarStatefulSets.fs b/src/FSLibrary/StellarStatefulSets.fs index d14b8ea7..eebed04f 100644 --- a/src/FSLibrary/StellarStatefulSets.fs +++ b/src/FSLibrary/StellarStatefulSets.fs @@ -71,10 +71,11 @@ let private getAveragePeerCount (topology: Map) : float = if nodeCount > 0 then float total / float nodeCount else 0.0 // Divide aggregate rate, account space and duration across actual generators. -// In fixed-duration mode (LoadOnEveryValidator runs), derive each transaction +// In fixed-duration mode (load on every validator), derive each transaction // budget from its assigned rate; equal transaction budgets would make -// remainder-rate peers finish early. Otherwise this is upstream's even split. -let PartitionValidatorLoad (n: int) (fixedDuration: bool) (full: LoadGen) = +// remainder-rate peers finish early. Otherwise every generator gets an equal +// share of the transactions. +let PartitionValidatorLoad (n: int) (fixedDuration: bool) (full: LoadGen) : LoadGen list = if n <= 0 then invalidArg "n" "Need at least one generator" if fixedDuration && full.accounts < n then @@ -108,8 +109,8 @@ let PartitionValidatorLoad (n: int) (fixedDuration: bool) (full: LoadGen) = // A single generator selection is shared by launch and final submission // accounting: every validator of each core set when everyValidator -// (StellarKubeSpecs.LoadOnEveryValidator), else node 0 of each, as upstream. -let LoadgenPeerIndices (everyValidator: bool) (coreSets: CoreSet list) = +// (MissionContext.pregenerateTxsPerValidator), else node 0 of each. +let LoadgenPeerIndices (everyValidator: bool) (coreSets: CoreSet list) : (CoreSet * int) list = coreSets |> List.collect (fun cs -> [ for i in 0 .. (if everyValidator then cs.options.nodeCount - 1 else 0) -> cs, i ]) @@ -323,8 +324,8 @@ let autoscalerView type StellarFormation with - member self.LoadGenPeers (coreSets: CoreSet list) (loadGen: LoadGen) = - LoadgenPeerIndices(LoadOnEveryValidator self.NetworkCfg.missionContext loadGen.mode) coreSets + member self.LoadGenPeers(coreSets: CoreSet list) : Peer list = + LoadgenPeerIndices self.NetworkCfg.missionContext.pregenerateTxsPerValidator coreSets |> List.map (fun (cs, i) -> self.NetworkCfg.GetPeer cs i) member self.GetCoreSetForStatefulSet(ss: V1StatefulSet) = @@ -830,10 +831,11 @@ type StellarFormation with // This is similar to RunLoadgen but runs a 1/N fractional portion of a // given LoadGen on each of N generators (LoadGenPeers: node 0 of each - // CoreSet, or every validator of each for LoadOnEveryValidator runs). + // CoreSet, or every validator of each when each pregenerated its own + // account slice). member self.RunMultiLoadgen (coreSets: CoreSet list) (fullLoadGen: LoadGen) = - let everyValidator = LoadOnEveryValidator self.NetworkCfg.missionContext fullLoadGen.mode - let loadGenPeers = self.LoadGenPeers coreSets fullLoadGen + let everyValidator = self.NetworkCfg.missionContext.pregenerateTxsPerValidator + let loadGenPeers = self.LoadGenPeers coreSets let shares = PartitionValidatorLoad loadGenPeers.Length everyValidator fullLoadGen let hasNonZeroRate (loadGen: LoadGen) = From b3761af19af57304a0823303f763e36a37a64c64 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Thu, 1 Oct 2026 14:00:50 -0700 Subject: [PATCH 9/9] MinBlockTime: exact inclusion check, and abort when a load goes unmeasured Two changes to how a candidate is judged after its load. Inclusion (overlay-only candidates). The cross-check passed when the nodes' average count of transactions in ledgers reached 95% of the offered load. But loadgen reports success only once every transaction it submitted has been included in a closed ledger, and nodes that close the same ledgers count the same transactions, so after a load the network carried every node reports the same count, covering the offered load. The 5% slack could only hide the discrepancy the check exists to catch, and the average let one node's excess hide another's shortfall. After a completed load every node must now report the same count, covering the offered load; a node still closing the last loaded ledger is re-read for up to two close times. Otherwise the candidate fails, with each node's count logged: a node that fell further behind means the network did not sustain the close time on every node, and a shortfall on every node means the load was not carried. After a failed load the candidate has already failed, so the counts are not read. Unreadable metrics. The single evaluation path counted unreadable metrics as a failed candidate. After a completed load that is a harness or network problem, not a verdict: the candidate was never measured, and failing it would let the search raise its lower bound on no evidence and report a close time that is too high. As on main, the mission aborts instead. After a failed load, usually a node dropping out, which also leaves its metrics unreadable, the read failure is only logged and the search continues. Co-Authored-By: Claude Opus 5.5 (1M context) --- doc/measuring-minimum-block-time.md | 4 +- src/FSLibrary.Tests/Tests.fs | 30 ++++++ src/FSLibrary/MinBlockTimeTest.fs | 156 ++++++++++++++++++---------- 3 files changed, 134 insertions(+), 56 deletions(-) diff --git a/doc/measuring-minimum-block-time.md b/doc/measuring-minimum-block-time.md index 572fd3ba..2138e677 100644 --- a/doc/measuring-minimum-block-time.md +++ b/doc/measuring-minimum-block-time.md @@ -18,9 +18,9 @@ A candidate close time `T` is considered a **pass** if and only if, **on every n > **FIXME (P75 tolerance):** the intended P75 band is `[0.95·T, 1.05·T)` (±5%), but stellar-core currently has performance regressions that prevent the stricter band from being achievable under load. The tolerance has been temporarily widened to ±20% so the test can exercise the rest of the pipeline; tighten it back to ±5% (or narrower) once those regressions are fixed. -If any node violates any of these bounds, `T` is considered a **fail** and the binary search raises its lower bound. The same is true if the load run itself errors (e.g., stellar-core's internal `loadgen-run-failed` counter increments, nodes fall out of sync, or peers report inconsistent ledger hashes) — in that case the mission treats the iteration as a fail and the search continues upward. +If any node violates any of these bounds, `T` is considered a **fail** and the binary search raises its lower bound. The same is true if the load run itself errors (e.g., stellar-core's internal `loadgen-run-failed` counter increments, nodes fall out of sync, or peers report inconsistent ledger hashes) — in that case the mission treats the iteration as a fail and the search continues upward. If instead the load completes but its metrics cannot be read, the candidate was not measured, so the mission aborts rather than count it as a fail. -Overlay-only candidates (`MinBlockTimeMixed`) never apply their transactions, but stellar-core's load generator still completes only once every transaction it submitted has been included in a closed ledger, so a load generator failure (such as load left out of ledgers) fails the candidate. That needs stellar-core master, or a Rust-overlay image built from 2026-07-31 on (earlier ones never count the transactions as included, so every overlay-only candidate fails). These candidates are judged while still in overlay-only mode: in addition to the close-time SLA above, the consistency and sync checks must pass and, as a cross-check, at least 95% of the offered transactions must reach ledgers (`ledger.transaction.count`). Apply is not re-enabled; the nodes are restarted, discarding what is left in their queues, before the next candidate. +Overlay-only candidates (`MinBlockTimeMixed`) never apply their transactions, but stellar-core's load generator still completes only once every transaction it submitted has been included in a closed ledger, so a load generator failure (such as load left out of ledgers) fails the candidate. That needs stellar-core master, or a Rust-overlay image built from 2026-07-31 on (earlier ones never count the transactions as included, so every overlay-only candidate fails). These candidates are judged while still in overlay-only mode: in addition to the close-time SLA above, the consistency and sync checks must pass and, as a cross-check after a completed load, every node must report the same `ledger.transaction.count` total, covering every offered transaction (a node still closing the last loaded ledger gets up to two close times to catch up). Otherwise, for example when a node fell further behind, the candidate fails. Apply is not re-enabled; the nodes are restarted, discarding what is left in their queues, before the next candidate. Candidate close times are always whole seconds: the search runs over the whole seconds in `[--min-block-time-ms, --max-block-time-ms]`, bounds included, so the default range evaluates `4000` ms and, only if that fails, `5000` ms. Equal bounds evaluate exactly that close time once, without rounding. If no candidate in the range satisfied the SLA, the mission fails with `"No block time in [lo, hi] ms satisfied the SLA at TPS N"`. diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index e33a7302..41ee961c 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -1267,6 +1267,36 @@ let private searchMinPassingTrace (candidates: int list) (passes: int -> bool) : result, evaluated.Value +[] +let ``Overlay-only inclusion agrees only when every node holds the same count covering the offered load`` () = + let agrees = MinBlockTimeTest.inclusionAgrees 1000 + + Assert.True( + agrees [ "a", 1000.0 + "b", 1000.0 + "c", 1000.0 ] + ) + // A node that has not closed the last loaded ledger yet. + Assert.False( + agrees [ "a", 1000.0 + "b", 1000.0 + "c", 880.0 ] + ) + // A node counting transactions the others did not. + Assert.False( + agrees [ "a", 1000.0 + "b", 1000.0 + "c", 1003.0 ] + ) + // Every node agrees, but on fewer than were offered. + Assert.False( + agrees [ "a", 900.0 + "b", 900.0 + "c", 900.0 ] + ) + + Assert.False(agrees []) + [] let ``Min block time search finds the smallest passing candidate`` () = let candidates = [ 1000 .. 1000 .. 5000 ] diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index 7460592d..14cc501b 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -272,55 +272,83 @@ let private readLedgerAgePercentiles (peer: Peer) : Peer * float * float = let private readLedgerTxsIncluded (peer: Peer) : Peer * float = peer, float (peer.GetMetrics().LedgerTransactionCount.Sum) -let private collectLedgerTxsIncluded (formation: StellarFormation) (coreSets: CoreSet list) : (Peer * float) list = - formation.NetworkCfg.PeersInSets(List.toArray coreSets) - |> List.map (fun peer -> async { return readLedgerTxsIncluded peer }) - |> Async.Parallel - |> Async.RunSynchronously - |> Array.toList - -// Fraction of the offered transactions that must reach ledgers for an -// overlay-only candidate to count. Comparing totals over the window, rather -// than txs per ledger against TPS x T, is not skewed by the idle ledgers -// around the load or by long ledgers carrying more. -let private minInclusionFraction = 0.95 - -// Cross-check for overlay-only candidates. Loadgen completes only once every -// transaction its node submitted has been included in a closed ledger, so a -// completed run includes essentially everything; this confirms it from the -// ledgers' own transaction counts, which checkLedgerAgeSLA does not look at -// (closing near-empty ledgers on schedule passes it trivially). Returns why -// the candidate fails, or None. +// Whether every node reports the same count of transactions in its ledgers, +// covering the `offered` load. Exposed for unit tests. +let inclusionAgrees (offered: int) (included: ('node * float) list) : bool = + match included |> List.map snd |> List.distinct with + | [ txs ] -> txs >= float offered + | _ -> false + +// Every node's included-transaction count after a completed load, re-read for +// up to two close times until they agree (inclusionAgrees): a node that had +// not yet closed the last loaded ledger when first read catches up. After a +// completed load only empty ledgers close, so waiting cannot hide a mismatch. +let private collectLedgerTxsIncluded + (formation: StellarFormation) + (coreSets: CoreSet list) + (offered: int) + (targetMs: int) + : (Peer * float) list = + let readAll () = + formation.NetworkCfg.PeersInSets(List.toArray coreSets) + |> List.map (fun peer -> async { return readLedgerTxsIncluded peer }) + |> Async.Parallel + |> Async.RunSynchronously + |> Array.toList + + let clock = System.Diagnostics.Stopwatch.StartNew() + let mutable included = readAll () + + while not (inclusionAgrees offered included) + && clock.ElapsedMilliseconds < 2L * int64 targetMs do + System.Threading.Thread.Sleep 1000 + included <- readAll () + + included + +// Cross-check for overlay-only candidates after a completed load: every node +// must report the same count of transactions in its ledgers, covering the +// offered load. Loadgen completes only once every transaction its node +// submitted has been included in a closed ledger, and nodes that close the +// same ledgers count the same transactions, so a load the network carried +// agrees; this confirms it from the ledgers' own counts, which +// checkLedgerAgeSLA does not look at (closing near-empty ledgers on schedule +// passes it trivially). Comparing totals over the window, rather than txs per +// ledger against TPS x T, is not skewed by the idle ledgers around the load or +// by long ledgers carrying more. A node still behind after +// collectLedgerTxsIncluded's re-reads, or a shortfall on every node, fails the +// candidate. Returns why, or None. let private inclusionFailure (included: (Peer * float) list) (targetMs: int) (offered: int) : string option = - if List.isEmpty included || offered <= 0 then + if List.isEmpty included then None else - let floorTxs = float offered * minInclusionFraction - let observed = included |> List.averageBy snd - let worstPeer, worstTxs = included |> List.minBy snd - let ok = observed >= floorTxs + match included |> List.map snd |> List.distinct with + | [ txs ] when txs >= float offered -> + LogInfo + "Inclusion at T=%dms: every node's ledgers hold %.0f transactions for the %d offered -> PASS" + targetMs + txs + offered - LogInfo - "Inclusion at T=%dms: %.0f of %d offered transactions reached ledgers (%.1f%%), worst peer=%s %.0f -> %s" - targetMs - observed - offered - (100.0 * observed / float offered) - worstPeer.ShortName.StringName - worstTxs - (if ok then "PASS" else "FAIL") - - if ok then None - else + | [ txs ] -> LogError - "Only %.1f%% of the %d offered transactions reached ledgers at T=%dms (need >= %.0f%%). The network paced ledgers but was not carrying the load, so the close-time result is meaningless." - (100.0 * observed / float offered) + "Every node's ledgers hold %.0f of the %d offered transactions at T=%dms, although loadgen reported all of them included." + txs offered targetMs - (100.0 * minInclusionFraction) - Some(sprintf "under %.0f%% of the offered transactions reached ledgers" (100.0 * minInclusionFraction)) + Some(sprintf "every node's ledgers hold %.0f of the %d offered transactions" txs offered) + | _ -> + LogError + "Nodes disagree on how many transactions their ledgers hold at T=%dms (%s; %d offered): a node fell behind, forked or did not close every ledger itself." + targetMs + (included + |> List.map (fun (peer, txs) -> sprintf "%s: %.0f" peer.ShortName.StringName txs) + |> String.concat ", ") + offered + + Some "nodes disagree on how many transactions their ledgers hold" let private collectLedgerAgePercentiles (formation: StellarFormation) @@ -625,11 +653,11 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: // Per doc/measuring-minimum-block-time.md, each failure below (a // loadgen failure, such as application lagging the offered load - // or a node dropping out mid-window, unreadable metrics, missing - // inclusion, or nodes out of sync or inconsistent) fails the - // candidate, not the mission: the search raises its lower bound - // and continues. Killing the mission would let one bad window - // discard the whole search. + // or a node dropping out mid-window, nodes disagreeing on the + // transactions in their ledgers, or nodes out of sync or + // inconsistent) fails the candidate, not the mission: the search + // raises its lower bound and continues. Killing the mission would + // let one bad window discard the whole search. let loadgenFailure = try formation.RunMultiLoadgen activeLoadGenNodes loadGen @@ -641,18 +669,38 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: // Snapshot the window's metrics before the health checks: core // keeps closing ledgers after loadgen exits, and the checks can // take long enough to skew the ledger age percentiles. + // + // Metrics that cannot be read after a completed load leave the + // candidate unmeasured, a harness or network problem rather than + // a verdict, so the mission aborts instead of letting the search + // move on as if it had failed. After a failed load the candidate + // has already failed, usually because a node dropped out, which + // also makes its metrics unreadable, so that is only logged. let snapshot = try let percentiles = collectLedgerAgePercentiles formation allNodes - let included = if overlayOnly then collectLedgerTxsIncluded formation allNodes else [] + + // Only a completed load is cross-checked: a failed one has + // already failed the candidate. + let included = + if overlayOnly && loadgenFailure.IsNone then + collectLedgerTxsIncluded formation allNodes loadGen.txs targetMs + else + [] if context.measureE2eLatency then logE2eLatencyMetrics formation activeLoadGenNodes - Ok(percentiles, included) - with e -> - LogWarn "Could not read metrics at T=%dms: %s" targetMs e.Message - Error(sprintf "could not read metrics: %s" e.Message) + Some(percentiles, included) + with + | e when loadgenFailure.IsNone -> + failwithf + "Could not read metrics at T=%dms after a completed load, so the candidate cannot be judged: %s" + targetMs + e.Message + | e -> + LogWarn "Could not read metrics at T=%dms after the failed load: %s" targetMs e.Message + None let healthFailure = try @@ -665,13 +713,13 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: // The close-time SLA is logged even when the candidate already // failed, so every window reports where its close times landed. - let slaOk, metricsFailure = + let slaOk, missingInclusion = match snapshot with - | Ok (percentiles, included) -> + | Some (percentiles, included) -> checkLedgerAgeSLA percentiles targetMs, inclusionFailure included targetMs loadGen.txs - | Error reason -> false, Some reason + | None -> false, None - let failure = List.tryPick id [ loadgenFailure; metricsFailure; healthFailure ] + let failure = List.tryPick id [ loadgenFailure; missingInclusion; healthFailure ] let pass = failure.IsNone && slaOk