Skip to main content

agentmux_launcher/saga/
mod.rs

1// Copyright 2026, AgentMux Corp.
2// SPDX-License-Identifier: Apache-2.0
3//
4// Phase E.1a — saga coordinator infrastructure.
5// Phase F.5 — first concrete saga consumer (`pool_respawn`).
6//
7// NOT the same framework as `agentmux_srv::sagas`, despite the shared name.
8// This one is a `Saga` TRAIT driven by a bus subscription, built for the
9// cross-process case; srv's is plain async functions with in-process
10// dispatch, because every saga it runs mutates only srv state. The two
11// share exactly one method name (`new`) and no types. That split is
12// deliberate, not drift — see docs/architecture/DECISION_SAGA_REDUCER_TWO_FRAMEWORKS_2026_09_07.md
13// and `docs/retro/saga-coordinator-location-analysis-2026-04-30.md`.
14// Do not "unify" them without reading those first.
15//
16// A saga is a state machine that orchestrates a multi-step,
17// multi-reducer flow (e.g. tear-off touches host pool + srv
18// workspace + launcher window registration). Sagas exist where
19// per-flow correctness needs explicit coordination — distributed
20// subscriber callbacks aren't enough.
21//
22// E.1a shipped this module as framework-only (no real sagas).
23// **F.5 lights up the first concrete saga (`pool_respawn`) plus the
24// minimal coordinator wiring needed to drive it**: the bus-
25// subscription loop now starts sagas in response to trigger events,
26// routes subsequent bus events into in-flight sagas via
27// `Saga::on_event`, dispatches `SagaAction` results, and emits
28// `SagaStarted` / `SagaCompleted` / `SagaFailed` brackets.
29//
30// Design (per `docs/specs/SPEC_PHASE_E_SRV_REDUCER_2026_04_29.md` §7
31// + `docs/specs/SPEC_PHASE_F_HOST_REDUCER_2026-05-01.md` §7.1):
32//
33//   trait Saga {
34//       fn start(&mut self, ctx: &SagaCtx) -> SagaAction;
35//       fn on_event(&mut self, event: &Event, ctx: &SagaCtx) -> SagaAction;
36//       fn name(&self) -> &'static str;
37//   }
38//
39//   enum SagaAction {
40//       IssueCmd { target: PipeTarget, cmd: Command },
41//       Done,
42//       Failed { reason: String },
43//       Wait,
44//   }
45//
46// The `SagaCoordinator` task subscribes to the broadcast bus,
47// routes events to in-flight sagas, dispatches IssueCmd actions to
48// the appropriate pipe, and emits `SagaStarted` / `SagaCompleted`
49// / `SagaFailed` so subscribers (renderer) can buffer-until-complete.
50//
51// **Cross-process dispatch — CPD-3 LIVE.** The launcher → host
52// command pipe is now wired via `HostPipe::send_command()` (CPD-2
53// + CPD-3). F.5's `pool_respawn` and F.6's `window_cleanup_cascade`
54// sagas issue `IssueCmd::Host` actions; the coordinator dispatches
55// them through the wire instead of merely logging. Per-saga
56// timeouts (`Saga::timeout()`, default 5s, F.6 overrides 30s)
57// + per-saga deadline tasks fire `SagaFailed` if a host action
58// stays unresolved past its budget. Host-emitted
59// `Command::ReportSagaActionFailed` translates into
60// `Event::SagaActionFailed` and terminates the matching saga.
61//
62// Per-variant `saga_id` tagging on existing Command/Event variants
63// is deferred; F.5's coordinator doesn't yet need it because:
64//   - only one `pool_respawn` saga can be in flight per promote
65//     (sagas are dispatched on `PoolWindowPromoted` 1:1);
66//   - `Event::PoolWindowAdded` from the *implicit* refill is the
67//     unique terminal signal — no foreign event type collides.
68// When concurrent promotes can happen (and produce overlapping
69// `PoolWindowAdded` streams), a future PR adds saga_id correlation.
70//
71// Saga state is in-memory. Launcher restart abandons in-flight
72// sagas; renderer-side timeouts cover the visible consequence.
73
74use std::sync::Arc;
75use std::time::Duration;
76
77use agentmux_common::ipc::{Command, Event};
78
79use crate::host_pipe::HostPipe;
80
81pub mod pool_respawn;
82pub mod window_cleanup;
83
84#[cfg(test)]
85mod integration_tests;
86
87// LSD-1 (PR LSD-1) — durable launcher saga log + API. Foundations
88// only: the coordinator does NOT call any of these methods yet.
89// Pillar 1 Step 6 (SPEC_PILLAR1_STEP6_SAGA_COLLAPSE_2026_07_16) — the
90// registry is IN-MEMORY now. The durable SQLite log
91// (SPEC_LAUNCHER_SAGA_DURABILITY_2026-05-01, LSD-1..4) and its startup
92// recovery walker were deleted: recovery never compensated anything (it
93// only wrote `failed_compensation` tombstones for `--diag sagas`), and
94// with srv authoritative + crash-reproject there is nothing durable left
95// for an interrupted launcher saga to protect. The coordinator's live
96// write-through below is unchanged — it now feeds in-process diagnostics
97// instead of a WAL.
98pub(crate) mod log;
99pub use log::LauncherSagaLog;
100use log::SagaOutcome;
101
102/// Where a `SagaAction::IssueCmd` should be dispatched.
103///
104/// `LauncherSelf` means "feed this command to the launcher's own
105/// reducer" (in-process); `Host` and `Srv` mean "forward to the
106/// peer's pipe."
107///
108/// **CPD-3 status:** `Host` is now LIVE — the saga coordinator's
109/// `apply_action` dispatches `IssueCmd::Host` actions through
110/// `HostPipe::send_command()` over the launcher → host wire.
111/// `LauncherSelf` and `Srv` remain reserved for class-D/E sagas.
112///
113/// F.7 cleanup audit: only `Host` is constructed today (F.5/F.6 saga
114/// IssueCmds). `LauncherSelf` and `Srv` are framework slots reserved
115/// for the cross-process dispatch follow-up phase per
116/// `SPEC_PHASE_F_HOST_REDUCER_2026-05-01.md` §4.3. Allow stays.
117#[derive(Debug, Clone, Copy, PartialEq, Eq)]
118#[allow(dead_code)] // LauncherSelf + Srv reserved for future cross-process sagas
119pub enum PipeTarget {
120    LauncherSelf,
121    Host,
122    Srv,
123}
124
125/// What a saga decides to do next, returned from `start` /
126/// `on_event`. The coordinator drives the saga forward by reacting
127/// to this enum.
128///
129/// F.7 cleanup audit: `IssueCmd`, `Done`, `Wait` are all constructed
130/// by F.5 + F.6 sagas. `Failed` is consumed in `apply_action` (and
131/// its corresponding `emit_failed` is now actively called by the
132/// evict-and-replace path), but no shipped saga *constructs*
133/// `Failed` yet — sagas today only succeed or wait. Variant-level
134/// allow scopes the dead-code suppression precisely to that one
135/// reserved variant.
136#[derive(Debug)]
137pub enum SagaAction {
138    /// Dispatch a command on the target pipe; saga remains in flight.
139    IssueCmd {
140        target: PipeTarget,
141        cmd: Command,
142    },
143    /// Saga succeeded. Coordinator removes it from `in_flight`
144    /// and emits `Event::SagaCompleted`.
145    Done,
146    /// Saga failed irrecoverably. Coordinator emits `Event::SagaFailed`
147    /// after dispatching any compensation IssueCmds the saga issued
148    /// before returning Failed. Reserved for sagas with explicit
149    /// failure conditions (e.g. cross-process dispatch follow-up
150    /// + saga timeouts).
151    #[allow(dead_code)] // reserved for sagas that explicitly fail; F.5/F.6 only succeed or wait
152    Failed { reason: String },
153    /// Saga is waiting for an event it hasn't seen yet. No-op for
154    /// the coordinator until the next bus event.
155    Wait,
156}
157
158/// Read-only context passed to saga callbacks. Currently carries
159/// only the saga_id; kept as a struct (rather than a bare u64) so
160/// future fields can be added without touching every `Saga` impl.
161///
162/// F.7 cleanup audit: `saga_id` is unread by the shipped sagas
163/// (F.5/F.6 don't need it — coordinator already routes events
164/// per-registry). Reserved for the per-event saga_id correlation
165/// follow-up; allow stays.
166#[allow(dead_code)] // saga_id reserved for per-event correlation follow-up
167#[derive(Debug, Clone, Copy)]
168pub struct SagaCtx {
169    pub saga_id: u64,
170}
171
172/// A multi-step, multi-reducer state machine. Implementations
173/// describe one logical operation (tear-off, pool-respawn, etc.).
174///
175/// Lifecycle: coordinator calls `start` once when the saga is
176/// added to `in_flight`. After that, every event on the bus is
177/// passed to `on_event`. The saga inspects the event, advances
178/// its internal state, returns the next action.
179///
180/// **Identification:** sagas know which events belong to them by
181/// inspecting saga_id in lifecycle events or by matching patterns
182/// in event payloads (e.g. specific labels). F.5 sagas correlate
183/// by event type only (`pool_respawn` matches any
184/// `PoolWindowAdded` after start). Per-variant `saga_id` tagging
185/// on commands/events is deferred until concurrent same-type
186/// sagas are needed.
187pub trait Saga: Send {
188    fn start(&mut self, ctx: &SagaCtx) -> SagaAction;
189    fn on_event(&mut self, event: &Event, ctx: &SagaCtx) -> SagaAction;
190    fn name(&self) -> &'static str;
191    /// LSD-2 — saga's input arguments serialized for the durable log's
192    /// `input_json` column. The coordinator calls this once at
193    /// `spawn_saga` time (before `start`) and writes the result via
194    /// `LauncherSagaLog::start_saga`. Operators see this in
195    /// `--diag sagas` output (e.g. `{"closed_label":"win-3"}`) so they
196    /// can tell which window's cleanup a recovered-failed saga
197    /// belonged to.
198    ///
199    /// Default `Value::Null` — sagas with no input fields can ignore
200    /// it. Concrete sagas should override with a `serde_json::json!`
201    /// of their constructor args.
202    fn input_snapshot(&self) -> serde_json::Value {
203        serde_json::Value::Null
204    }
205
206    /// CPD-3 — per-saga deadline budget for completing all
207    /// `IssueCmd`+wait cycles. Coordinator arms a timer when the saga
208    /// registers; if the saga is still in_flight when `timeout()`
209    /// elapses, it is force-failed (`SagaFailed { reason: "saga
210    /// timeout" }`) and removed from the registry.
211    ///
212    /// Default 5s — fits class-C single-step host dispatch (e.g.
213    /// pool respawn). `WindowCleanupCascade` overrides to 30s
214    /// because pane drain on a workspace with many panes can
215    /// legitimately take that long. Per spec §3.10.
216    fn timeout(&self) -> Duration {
217        Duration::from_secs(5)
218    }
219}
220
221/// Saga coordinator task.
222///
223/// Owns the registry of in-flight sagas, allocates saga ids,
224/// routes events to sagas, dispatches IssueCmd actions to the
225/// appropriate pipes, and emits lifecycle events on the broadcast
226/// bus.
227///
228/// F.5 adds the first real consumer (`pool_respawn`). The
229/// `in_flight` registry is now actually populated; the bus loop
230/// dispatches events into it.
231/// LSD-2 — coordinator's per-saga book-keeping. Wraps the boxed saga
232/// with the durable-log step bookkeeping the coordinator needs to call
233/// `LauncherSagaLog::finish_step` when the awaited bus event lands.
234///
235/// `awaiting_step` is `Some(idx)` when the saga is parked on a
236/// `Wait`-then-event pivot (the most recent `IssueCmd` allocated step
237/// `idx`); `None` between dispatches and after termination. Tracking
238/// it on the in-flight record (rather than inside each saga impl) keeps
239/// the durability concern out of saga authors' hands — they only deal
240/// with `SagaAction`.
241///
242/// `next_step_index` is the monotonic counter the coordinator
243/// `fetch_add(1)`s on every `IssueCmd` for this saga. Mirrors srv's
244/// `SagaCtx::step_index`. Lives per-saga (not coordinator-global) so
245/// concurrent sagas don't interleave indices in the log.
246struct InFlightSaga {
247    saga: Box<dyn Saga>,
248    awaiting_step: Option<u32>,
249    next_step_index: u32,
250}
251
252pub struct SagaCoordinator {
253    /// Monotonic saga-id allocator.
254    next_saga_id: std::sync::atomic::AtomicU64,
255    /// In-flight sagas keyed by saga_id, each wrapped with the
256    /// durable-log bookkeeping (LSD-2: `awaiting_step` +
257    /// `next_step_index`).
258    in_flight: tokio::sync::Mutex<std::collections::HashMap<u64, InFlightSaga>>,
259    /// Reference to the broadcast bus so the coordinator can emit
260    /// `SagaStarted` / `SagaCompleted` / `SagaFailed`.
261    events_tx: tokio::sync::broadcast::Sender<Event>,
262    /// Reference to the launcher's reducer state for `bump_version`
263    /// when emitting saga lifecycle events.
264    state: Arc<tokio::sync::Mutex<crate::state::State>>,
265    /// LSD-2 — durable saga log. `None` in tests that don't exercise
266    /// the durability path (the saga then logs + remains in flight,
267    /// preserving the pre-LSD-2 behavior tests rely on for end-to-end
268    /// bracket assertions). When `Some`, every saga lifecycle
269    /// transition (`spawn_saga` → `start_saga`, `IssueCmd` →
270    /// `start_step`, awaited-event consumed → `finish_step`,
271    /// `Done` → `terminate_saga(Completed)`, `Failed` / evicted →
272    /// `terminate_saga(Failed)`) writes to this log.
273    log: Option<Arc<LauncherSagaLog>>,
274    /// CPD-3 — launcher → host pipe wrapper. `apply_action` for
275    /// `IssueCmd::Host` dispatches via `host_pipe.send_command()` when
276    /// installed. `None` in tests that don't exercise host dispatch
277    /// (those drive sagas via synthetic terminal events on the bus).
278    host_pipe: Option<Arc<HostPipe>>,
279}
280
281impl SagaCoordinator {
282    pub fn new(
283        events_tx: tokio::sync::broadcast::Sender<Event>,
284        state: Arc<tokio::sync::Mutex<crate::state::State>>,
285    ) -> Self {
286        Self {
287            next_saga_id: std::sync::atomic::AtomicU64::new(1),
288            in_flight: tokio::sync::Mutex::new(std::collections::HashMap::new()),
289            events_tx,
290            state,
291            log: None,
292            host_pipe: None,
293        }
294    }
295
296    /// CPD-3 — install the host pipe so `IssueCmd::Host` actions are
297    /// dispatched live (instead of log-only). Builder-style setter so
298    /// existing tests + the `next_id_is_monotonic` smoke don't have to
299    /// construct a HostPipe. Production wiring (`main.rs`) calls this
300    /// once before `run_coordinator` is spawned.
301    pub fn with_host_pipe(mut self, host_pipe: Arc<HostPipe>) -> Self {
302        self.host_pipe = Some(host_pipe);
303        self
304    }
305
306    /// LSD-2 — install the durable saga log so saga lifecycle
307    /// transitions are persisted to `<data-dir>/db/launcher-sagas.db`.
308    /// Builder-style setter rather than a constructor parameter so
309    /// existing tests + the `next_id_is_monotonic` smoke don't have
310    /// to construct an in-memory log. Production wiring (in
311    /// `main.rs`) calls this once before `run_coordinator` is spawned.
312    pub fn with_log(mut self, log: Arc<LauncherSagaLog>) -> Result<Self, crate::saga::log::LogError> {
313        // Seed the saga_id allocator from the highest persisted id
314        // so a launcher restart cannot reuse an id that already
315        // exists in `launcher_saga`. Reusing would (a) fail new
316        // INSERTs on duplicate-PK, and (b) silently mutate prior
317        // saga rows via `terminate_saga` / `finish_step` UPDATEs
318        // keyed by saga_id — corrupting saga history + recovery
319        // diagnostics. (codex P1 PR #645 round 1.)
320        //
321        // On `max_saga_id()` error: return Err so caller (main.rs)
322        // can fail launcher startup loudly. Continuing with a default
323        // next_saga_id=1 while the log is still attached would leave
324        // the coordinator in the exact failure mode this seed is
325        // meant to prevent. (codex P1 PR #645 round 2.)
326        let max = log.max_saga_id()?;
327        self.next_saga_id
328            .store(max + 1, std::sync::atomic::Ordering::Relaxed);
329        if max > 0 {
330            crate::log(&format!(
331                "[saga] seeded next_saga_id={} from launcher_saga.max(saga_id)={}",
332                max + 1,
333                max
334            ));
335        }
336        self.log = Some(log);
337        Ok(self)
338    }
339
340    /// Allocate the next saga_id. Monotonic per launcher run.
341    pub fn next_id(&self) -> u64 {
342        self.next_saga_id
343            .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
344    }
345
346    /// Bump the launcher's `event_version` for a coordinator-emitted
347    /// lifecycle event. Brief mutex hold — no I/O between lock and
348    /// drop.
349    async fn next_event_version(&self) -> u64 {
350        let mut state = self.state.lock().await;
351        state.bump_version()
352    }
353
354    /// Emit `Event::SagaStarted` after registering a saga.
355    async fn emit_started(&self, saga_id: u64, name: &'static str) {
356        let v = self.next_event_version().await;
357        let _ = self.events_tx.send(Event::SagaStarted {
358            saga_id,
359            name: name.to_string(),
360            version: v,
361        });
362    }
363
364    /// Emit `Event::SagaCompleted` after a saga returns `Done`.
365    async fn emit_completed(&self, saga_id: u64) {
366        let v = self.next_event_version().await;
367        let _ = self.events_tx.send(Event::SagaCompleted {
368            saga_id,
369            version: v,
370        });
371    }
372
373    /// Emit `Event::SagaFailed` after a saga returns `Failed` or
374    /// after evict-and-replace cancels a same-kind in-flight saga.
375    /// F.7 cleanup audit: prior `#[allow(dead_code)]` removed — F.6
376    /// evict-and-replace policy now actively dispatches this on
377    /// every concurrent same-kind retrigger.
378    async fn emit_failed(&self, saga_id: u64, reason: String) {
379        let v = self.next_event_version().await;
380        let _ = self.events_tx.send(Event::SagaFailed {
381            saga_id,
382            reason,
383            version: v,
384        });
385    }
386
387    /// Emit `SagaFailed` for every in-flight saga and clear the registry.
388    /// Called on clean launcher shutdown so no saga bracket is left open
389    /// in the durable log (open brackets would otherwise trigger spurious
390    /// LSD-3 compensation on next startup).
391    ///
392    /// Writes directly to the durable log when installed — more reliable
393    /// than relying on bus-subscriber delivery during shutdown teardown.
394    pub async fn cancel_all_in_flight(&self, reason: &str) {
395        let ids: Vec<u64> = {
396            let mut registry = self.in_flight.lock().await;
397            registry.drain().map(|(id, _)| id).collect()
398        };
399        for saga_id in ids {
400            // Cancel any pending host-pipe frames so a reconnecting host
401            // doesn't drain them as orphan side-effects.
402            if let Some(pipe) = self.host_pipe.as_ref() {
403                pipe.cancel_saga(saga_id).await;
404            }
405            // Emit on the broadcast bus for live subscribers.
406            self.emit_failed(saga_id, reason.to_string()).await;
407            // Write directly to the durable log — bypasses task scheduling
408            // and is guaranteed to reach SQLite even during shutdown.
409            if let Some(log) = self.log.as_ref() {
410                let _ = log.terminate_saga(
411                    saga_id,
412                    crate::saga::log::SagaOutcome::Failed { reason: reason.to_string() },
413                );
414            }
415        }
416    }
417
418    /// Apply a `SagaAction` returned by `start` or `on_event`. Returns
419    /// an `ApplyOutcome` carrying:
420    ///   - `in_flight`: true → caller keeps the saga in `in_flight`,
421    ///     false → caller removes it.
422    ///   - `awaiting_step`: `Some(idx)` if the action was an
423    ///     `IssueCmd` that allocated step `idx` and the saga is now
424    ///     parked waiting for the echo event. Caller stores this on
425    ///     the InFlightSaga so the next non-`Wait` `on_event` return
426    ///     can call `LauncherSagaLog::finish_step(saga_id, idx, ev)`.
427    ///
428    /// CPD-3 — `IssueCmd::Host` dispatches live through `HostPipe`
429    /// when one is installed. Tests without a host_pipe (the existing
430    /// F.5/F.6 unit + integration tests) fall back to log-only so
431    /// synthetic-terminal-event drivers continue to work.
432    /// `LauncherSelf` and `Srv` targets remain log-only — reserved
433    /// for class-D/E sagas, no consumer today.
434    ///
435    /// LSD-2 — every state transition writes to `LauncherSagaLog`
436    /// when one is installed: `IssueCmd` → `start_step`, `Done` →
437    /// `terminate_saga(Completed)`, `Failed` →
438    /// `terminate_saga(Failed)`.
439    ///
440    /// `step_index` is the caller-allocated index for THIS dispatch.
441    /// On the first dispatch in `spawn_saga`, the caller passes 0;
442    /// on subsequent dispatches from the event loop, the caller pulls
443    /// + bumps the per-InFlightSaga counter.
444    ///
445    /// Terminal paths (`Done`, `Failed`, host-send-error) all go
446    /// through `claim_terminal` so only one of {normal Done/Failed,
447    /// timeout-task, SagaActionFailed listener, host-send-error}
448    /// can win the terminal-event race for a given saga_id.
449    /// (reagent P1 PR #644 round 1 + 2.)
450    async fn apply_action(
451        &self,
452        saga_id: u64,
453        name: &'static str,
454        action: SagaAction,
455        step_index: u32,
456    ) -> ApplyOutcome {
457        match action {
458            SagaAction::IssueCmd { target, cmd } => {
459                // LSD-2 — write the durable `pending` step row BEFORE
460                // dispatch so a crash mid-dispatch leaves a recoverable
461                // breadcrumb (LSD-3's walker upgrades this saga to
462                // `failed_compensation` on next launcher startup).
463                let step_name = derive_step_name(&cmd, target);
464                if let Some(log) = self.log.as_ref() {
465                    if let Err(e) =
466                        log.start_step(saga_id, step_index, &step_name, target, &cmd)
467                    {
468                        crate::log(&format!(
469                            "[saga] saga_id={} name={} start_step log write failed: {} — continuing (in-memory path remains authoritative for this run)",
470                            saga_id, name, e
471                        ));
472                    }
473                }
474
475                match target {
476                    PipeTarget::Host => {
477                        // CPD-3 — dispatch live through the host pipe
478                        // when installed. Tests without a host_pipe
479                        // (most existing F.5/F.6 unit + integration
480                        // tests) fall back to log-only so synthetic-
481                        // terminal-event drivers continue to work.
482                        let Some(host_pipe) = self.host_pipe.as_ref() else {
483                            crate::log(&format!(
484                                "[saga] saga_id={} name={} IssueCmd target=Host cmd={:?} (no host_pipe installed — log-only)",
485                                saga_id, name, cmd
486                            ));
487                            return ApplyOutcome::in_flight_awaiting(step_index);
488                        };
489                        let cmd_with_id = inject_saga_id(cmd, saga_id);
490                        match host_pipe.send_command(&cmd_with_id).await {
491                            Ok(()) => {
492                                crate::log(&format!(
493                                    "[saga] saga_id={} name={} IssueCmd::Host dispatched cmd={:?}",
494                                    saga_id, name, cmd_with_id
495                                ));
496                                ApplyOutcome::in_flight_awaiting(step_index)
497                            }
498                            Err(e) => {
499                                // CRITICAL: do NOT terminate the saga
500                                // here. send_command's send_frame already
501                                // re-buffered the failed frame for retry
502                                // on reconnect (with the real saga_id).
503                                // If we ALSO emit SagaFailed now:
504                                //   - 30s later expire_pending_if_timed_out
505                                //     drains the buffer + emit_drop_failure
506                                //     emits a SECOND SagaFailed for the
507                                //     same saga_id (double-emit), OR
508                                //   - host reconnects in <30s and the
509                                //     buffered command executes as an
510                                //     orphaned side effect after the saga
511                                //     bracket is already closed.
512                                // (reagent P1 PR #644 round 6.)
513                                //
514                                // Round 7 design: keep the saga in_flight.
515                                // The retry path resolves it (Report*
516                                // arrives → on_event drives forward) OR
517                                // the saga's own timeout (5s default,
518                                // 30s for F.6) fires + claim_terminal'd
519                                // SagaFailed. Either way, exactly one
520                                // terminal event is emitted.
521                                crate::log(&format!(
522                                    "[saga] saga_id={} name={} host pipe send transient err: {} — frame buffered for retry; saga stays in-flight (round 7 fix)",
523                                    saga_id, name, e
524                                ));
525                                ApplyOutcome::in_flight_awaiting(step_index)
526                            }
527                        }
528                    }
529                    PipeTarget::LauncherSelf | PipeTarget::Srv => {
530                        // Reserved for class-D/E sagas; out of scope
531                        // per SPEC_CROSS_PROCESS_DISPATCH §3.6. Log-
532                        // only retains framework shape until a
533                        // consumer arrives.
534                        crate::log(&format!(
535                            "[saga] saga_id={} name={} IssueCmd target={:?} cmd={:?} (target not yet wired; log-only)",
536                            saga_id, name, target, cmd
537                        ));
538                        ApplyOutcome::in_flight_awaiting(step_index)
539                    }
540                }
541            }
542            SagaAction::Wait => ApplyOutcome::in_flight_no_change(),
543            SagaAction::Done => {
544                // Atomic claim: only one of {normal Done, timeout-task,
545                // SagaActionFailed listener, host-send-error} can win
546                // the terminal-event race. Without this guard a Done
547                // could fire SagaCompleted while the timeout task
548                // simultaneously fires SagaFailed, producing duplicate
549                // terminal events for the same saga (reagent P1 PR
550                // #644 round 1).
551                if !self.claim_terminal(saga_id).await {
552                    return ApplyOutcome::terminated();
553                }
554                crate::log(&format!(
555                    "[saga] saga_id={} name={} Done — emitting SagaCompleted",
556                    saga_id, name
557                ));
558                if let Some(log) = self.log.as_ref() {
559                    if let Err(e) = log.terminate_saga(saga_id, SagaOutcome::Completed) {
560                        crate::log(&format!(
561                            "[saga] saga_id={} terminate_saga(Completed) log write failed: {}",
562                            saga_id, e
563                        ));
564                    }
565                }
566                self.emit_completed(saga_id).await;
567                ApplyOutcome::terminated()
568            }
569            SagaAction::Failed { reason } => {
570                if !self.claim_terminal(saga_id).await {
571                    return ApplyOutcome::terminated();
572                }
573                crate::log(&format!(
574                    "[saga] saga_id={} name={} Failed reason={} — emitting SagaFailed",
575                    saga_id, name, reason
576                ));
577                if let Some(log) = self.log.as_ref() {
578                    if let Err(e) = log.terminate_saga(
579                        saga_id,
580                        SagaOutcome::Failed {
581                            reason: reason.clone(),
582                        },
583                    ) {
584                        crate::log(&format!(
585                            "[saga] saga_id={} terminate_saga(Failed) log write failed: {}",
586                            saga_id, e
587                        ));
588                    }
589                }
590                self.emit_failed(saga_id, reason).await;
591                ApplyOutcome::terminated()
592            }
593        }
594    }
595
596    /// Atomically remove `saga_id` from `in_flight` and return whether
597    /// the caller was the one who removed it (i.e. has the right to
598    /// emit the terminal `SagaCompleted` / `SagaFailed`). Idempotent:
599    /// a second caller for the same saga_id sees `false`. Used by the
600    /// `SagaAction::Done` and `SagaAction::Failed` paths in
601    /// `apply_action`, by the per-saga timeout task in `spawn_saga`,
602    /// by the `Event::SagaActionFailed` listener in `run_coordinator`,
603    /// by the host-send-failure path in `apply_action::IssueCmd::Host`,
604    /// and by the eviction path in `run_coordinator`.
605    /// (reagent P1 PR #644 round 1.)
606    async fn claim_terminal(&self, saga_id: u64) -> bool {
607        let claimed = {
608            let mut registry = self.in_flight.lock().await;
609            registry.remove(&saga_id).is_some()
610        };
611        // Purge any host-pipe pending frames tagged with this saga_id
612        // so a host reconnect post-terminal doesn't drain them as
613        // orphan side effects, and the 30s expiry path doesn't emit
614        // a second SagaFailed for the same saga_id.
615        // (codex + reagent P1 PR #644 round 7.)
616        if claimed {
617            if let Some(pipe) = self.host_pipe.as_ref() {
618                pipe.cancel_saga(saga_id).await;
619            }
620        }
621        claimed
622    }
623
624    /// Register a fresh saga, calling `start` and applying its first
625    /// action. The caller has already determined that the saga
626    /// should fire (e.g. matched a trigger event). Returns the saga's
627    /// allocated id (logged + bracketed in `SagaStarted`).
628    ///
629    /// LSD-2 — also writes a `running` row to the durable saga log
630    /// before invoking `saga.start()`, and (via `apply_action`)
631    /// records the saga's first dispatched step.
632    ///
633    /// CPD-3 — also arms a per-saga deadline timer. If the saga is
634    /// still in_flight when `saga.timeout()` elapses, a background
635    /// task force-fails it (`SagaFailed { reason: "saga timeout" }`)
636    /// and removes it from the registry. Per spec §3.10. Insert-then-
637    /// start ordering is required so `apply_action`'s `claim_terminal`
638    /// guard can succeed for immediate-completion sagas (codex P2 PR
639    /// #644 round 2).
640    async fn spawn_saga(self: &Arc<Self>, saga: Box<dyn Saga>) -> u64 {
641        let saga_id = self.next_id();
642        let name = saga.name();
643        let input = saga.input_snapshot();
644        let saga_timeout = saga.timeout();
645        crate::log(&format!(
646            "[saga] starting saga_id={} name={}",
647            saga_id, name
648        ));
649        // LSD-2 — write the durable `running` row BEFORE
650        // `emit_started` so a crash between this point and
651        // `apply_action` still leaves a recoverable breadcrumb
652        // (LSD-3's walker will upgrade it to `failed_compensation` on
653        // next launcher startup). Mirrors srv's `emit_saga_started`
654        // ordering: durable log before bus.
655        if let Some(log) = self.log.as_ref() {
656            if let Err(e) = log.start_saga(saga_id, name, &input) {
657                crate::log(&format!(
658                    "[saga] saga_id={} start_saga log write failed: {} — continuing (in-memory path remains authoritative for this run)",
659                    saga_id, e
660                ));
661            }
662        }
663        // Emit SagaStarted FIRST so any subscriber buffering by
664        // saga_id sees the bracket open before any per-step events.
665        // Mirrors `agentmux-srv::sagas::emit_saga_started` ordering.
666        self.emit_started(saga_id, name).await;
667
668        // CPD-3 — insert the saga into in_flight BEFORE calling
669        // start(), so that if start() returns SagaAction::Done or
670        // SagaAction::Failed immediately, apply_action's terminal
671        // paths can claim_terminal successfully and emit the matching
672        // SagaCompleted/SagaFailed bracket. Pre-round-3 we started
673        // saga.start() before insertion, which broke immediate-
674        // completion sagas (claim_terminal returned false → no
675        // terminal event → dangling SagaStarted bracket).
676        // (codex P2 PR #644 round 2.)
677        let action = {
678            let mut registry = self.in_flight.lock().await;
679            // CPD-4 — concurrent same-kind sagas now correlate by
680            // saga_id (`Saga::on_event` filters on
681            // `ctx.saga_id == event.saga_id`). The pre-CPD-4
682            // observability warning (PR #634 codex P1) is retained
683            // at info level for operators tracking concurrency
684            // density via `--diag wrr`, but the underlying
685            // correctness issue is closed.
686            let same_kind_count = registry
687                .values()
688                .filter(|s| s.saga.name() == name)
689                .count();
690            if same_kind_count >= 1 {
691                crate::log(&format!(
692                    "[saga] starting {} saga_id={} while {} other(s) of same kind in flight (CPD-4 per-saga correlation — sagas coexist cleanly)",
693                    name, saga_id, same_kind_count,
694                ));
695            }
696            registry.insert(
697                saga_id,
698                InFlightSaga {
699                    saga,
700                    awaiting_step: None,
701                    next_step_index: 0,
702                },
703            );
704            // Drive start() while still under the registry lock so
705            // we have a mutable reference to the just-inserted saga.
706            // start() is non-async and does no I/O — bounded hold time.
707            let in_flight_saga = registry.get_mut(&saga_id).expect("just inserted");
708            let ctx = SagaCtx { saga_id };
709            in_flight_saga.saga.start(&ctx)
710        };
711
712        // First step on a fresh saga is index 0. apply_action may
713        // remove the saga via claim_terminal (Done/Failed/host-send-
714        // error) without re-locking the registry from here.
715        let outcome = self.apply_action(saga_id, name, action, 0).await;
716        if outcome.in_flight {
717            // Update the bookkeeping for the freshly-issued step.
718            // If the first action allocated step 0, the saga is now
719            // parked awaiting that step's echo event; next allocation
720            // is index 1. Otherwise (Wait — saga has no first
721            // dispatch yet), keep the counter at 0.
722            let mut registry = self.in_flight.lock().await;
723            if let Some(in_flight_saga) = registry.get_mut(&saga_id) {
724                in_flight_saga.awaiting_step = outcome.awaiting_step;
725                in_flight_saga.next_step_index =
726                    if outcome.awaiting_step.is_some() { 1 } else { 0 };
727            }
728            drop(registry);
729
730            // CPD-3 — arm the per-saga deadline timer. If the saga
731            // is still in_flight when `saga_timeout` elapses, fail
732            // it out. Spawned task captures an Arc clone of the
733            // coordinator so it can outlive the saga's deadline.
734            // Uses `claim_terminal` so it doesn't race with the
735            // normal Done/Failed path. (reagent P1 PR #644 round 1.)
736            let coord_for_timeout = Arc::clone(self);
737            tokio::spawn(async move {
738                tokio::time::sleep(saga_timeout).await;
739                if coord_for_timeout.claim_terminal(saga_id).await {
740                    crate::log(&format!(
741                        "[saga] saga_id={} name={} timed out after {:?} — emitting SagaFailed",
742                        saga_id, name, saga_timeout
743                    ));
744                    let reason = "saga timeout".to_string();
745                    if let Some(log) = coord_for_timeout.log.as_ref() {
746                        if let Err(e) = log.terminate_saga(
747                            saga_id,
748                            SagaOutcome::Failed {
749                                reason: reason.clone(),
750                            },
751                        ) {
752                            crate::log(&format!(
753                                "[saga] saga_id={} timeout terminate_saga(Failed) log write failed: {}",
754                                saga_id, e
755                            ));
756                        }
757                    }
758                    coord_for_timeout
759                        .emit_failed(saga_id, reason)
760                        .await;
761                }
762            });
763        }
764        saga_id
765    }
766}
767
768/// LSD-2 — outcome of `apply_action`. Combines the prior `bool` (is
769/// the saga still in flight?) with the new `awaiting_step` book-
770/// keeping the coordinator needs to call `LauncherSagaLog::finish_step`
771/// when the awaited bus event lands.
772#[derive(Debug, Clone, Copy)]
773struct ApplyOutcome {
774    in_flight: bool,
775    /// If the action was an `IssueCmd`, the step index it allocated;
776    /// the coordinator parks this on the InFlightSaga. `None` for
777    /// `Wait` / `Done` / `Failed`.
778    awaiting_step: Option<u32>,
779}
780
781impl ApplyOutcome {
782    fn in_flight_awaiting(step_index: u32) -> Self {
783        Self {
784            in_flight: true,
785            awaiting_step: Some(step_index),
786        }
787    }
788    fn in_flight_no_change() -> Self {
789        Self {
790            in_flight: true,
791            awaiting_step: None,
792        }
793    }
794    fn terminated() -> Self {
795        Self {
796            in_flight: false,
797            awaiting_step: None,
798        }
799    }
800}
801
802/// LSD-2 — short, greppable name for a `Command` dispatched as part
803/// of a saga step. Mirrors srv's `command_discriminant_name` in spirit
804/// (snake_case strings rather than `Debug` formatting) but prefixes
805/// with `issue_cmd_<target>_<discriminant>` so `--diag sagas` output
806/// makes the dispatch target obvious without a separate column lookup.
807///
808/// Falls back to `issue_cmd_<target>_unknown` for variants serde can't
809/// stringify (shouldn't happen for the snake_case-tagged Command enum;
810/// defensive default).
811fn derive_step_name(cmd: &Command, target: PipeTarget) -> String {
812    let target_str = match target {
813        PipeTarget::LauncherSelf => "launcher_self",
814        PipeTarget::Host => "host",
815        PipeTarget::Srv => "srv",
816    };
817    let discriminant = match serde_json::to_value(cmd) {
818        Ok(serde_json::Value::Object(map)) => map
819            .get("cmd")
820            .and_then(|v| v.as_str())
821            .map(|s| s.to_string())
822            .unwrap_or_else(|| "unknown".to_string()),
823        _ => "unknown".to_string(),
824    };
825    format!("issue_cmd_{}_{}", target_str, discriminant)
826}
827
828/// CPD-3 — fill in the `saga_id` field on a host-bound `Command`
829/// before dispatch. Sagas construct their `IssueCmd` actions with a
830/// placeholder `saga_id: 0` (they don't know their coordinator-
831/// allocated id at action-construction time); the coordinator
832/// rewrites the field at dispatch time so the host can echo it back
833/// on the matching `Report*`.
834///
835/// Exhaustive match: any host-bound Command variant added later that
836/// forgets to plumb `saga_id` will refuse to compile here. Non-host-
837/// bound variants (Report*, Identify, etc.) panic loudly because
838/// sagas only emit `IssueCmd::Host` for the three host-bound
839/// command kinds covered below.
840fn inject_saga_id(cmd: Command, saga_id: u64) -> Command {
841    match cmd {
842        Command::SpawnPoolWindow { .. } => Command::SpawnPoolWindow { saga_id },
843        Command::ReapPanes { label, .. } => Command::ReapPanes { label, saga_id },
844        Command::DrainPoolIfLast { label, .. } => {
845            Command::DrainPoolIfLast { label, saga_id }
846        }
847        // Defense-in-depth: every non-host-dispatched Command variant
848        // (Report*, Identify, etc.) is a coding bug if it reaches
849        // this point — the F.5/F.6 sagas only emit IssueCmd::Host
850        // for the three command kinds above.
851        other => panic!(
852            "inject_saga_id called on non-host-bound Command variant: {:?}",
853            other
854        ),
855    }
856}
857
858/// Inspect a bus event for "should this start a fresh saga?" Returns
859/// the constructed saga (boxed) on a hit; `None` otherwise.
860///
861/// Triggers wired to date:
862/// * F.5: `Event::PoolWindowPromoted` → `pool_respawn::PoolRespawn`
863/// * F.6: `Event::WindowClosed` →
864///   `window_cleanup::WindowCleanupCascade`
865///
866/// Future sagas extend this match.
867///
868/// CPD-4 — the coordinator no longer evicts same-kind in-flight
869/// sagas before `spawn_saga`; per-saga event correlation
870/// (`Saga::on_event` filters by `ctx.saga_id`) lets concurrent
871/// same-kind sagas coexist cleanly. The pre-CPD-4 evict-and-replace
872/// gate (PR #634 round 3) is removed in `run_coordinator`.
873fn match_trigger(event: &Event) -> Option<Box<dyn Saga>> {
874    match event {
875        Event::PoolWindowPromoted { label, .. } => {
876            Some(Box::new(pool_respawn::PoolRespawn::new(label.clone())))
877        }
878        // (codex P1 PR #637.) Only fire the cleanup cascade on
879        // CLEAN closes. `Event::WindowClosed { crash_detected: true }`
880        // comes from `wrr::apply_hwnd_destroyed` after a host/renderer
881        // crash; the host never sent `ReportPanesReaped` /
882        // `ReportPoolDrainDecision`, so the saga would stay
883        // in-flight indefinitely (only cleared by a later same-kind
884        // eviction or launcher restart) leaving a SagaStarted
885        // bracket dangling for subscribers that buffer on lifecycle.
886        Event::WindowClosed { label, crash_detected: false, .. } => Some(Box::new(
887            window_cleanup::WindowCleanupCascade::new(label.clone()),
888        )),
889        _ => None,
890    }
891}
892
893/// Run the coordinator's bus-subscription loop.
894///
895/// **Receiver is passed in, not subscribed inside.** Subscribing
896/// before `tokio::spawn` (in `main.rs`) ensures events emitted
897/// between coordinator construction and the first `recv()` aren't
898/// lost to the race window. Same pattern as `event_log::run_disk_writer`.
899/// (reagent P2 PR #609.)
900///
901/// Dispatch order on each event:
902///   1. Match against trigger table → start any new sagas via
903///      `spawn_saga` (which emits `SagaStarted` and applies the
904///      saga's initial action).
905///   2. Feed the same event into every in-flight saga's `on_event`.
906///      Sagas returning `Done` / `Failed` are removed from
907///      `in_flight` after their lifecycle event is emitted.
908///
909/// **Self-emitted events (`SagaStarted` / `SagaCompleted` /
910/// `SagaFailed`) are NOT re-fed into sagas.** A saga reacting to its
911/// own start event would loop. Filtered at the top of the dispatch
912/// path.
913pub async fn run_coordinator(
914    coord: Arc<SagaCoordinator>,
915    mut events_rx: tokio::sync::broadcast::Receiver<Event>,
916) {
917    crate::log("[saga] coordinator started");
918
919    loop {
920        match events_rx.recv().await {
921            Ok(event) => {
922                // Skip our own lifecycle events to avoid loops.
923                if matches!(
924                    event,
925                    Event::SagaStarted { .. } | Event::SagaCompleted { .. } | Event::SagaFailed { .. }
926                ) {
927                    continue;
928                }
929
930                // CPD-3 — host reported via `Command::ReportSagaActionFailed`
931                // that a saga-issued action failed. Launcher reducer
932                // translates this into `Event::SagaActionFailed`
933                // (CPD-1 schema). Terminate the matching saga
934                // immediately rather than waiting for a Report* that
935                // won't come.
936                if let Event::SagaActionFailed { saga_id, reason, .. } = &event {
937                    let saga_id = *saga_id;
938                    let reason = reason.clone();
939                    // claim_terminal: atomic take-ownership of the
940                    // terminal-event slot, preventing duplicate-emission
941                    // races with the saga's own Done/Failed path or its
942                    // timeout task. (reagent P1 PR #644 round 1.)
943                    if coord.claim_terminal(saga_id).await {
944                        crate::log(&format!(
945                            "[saga] saga_id={} terminating from host SagaActionFailed reason={}",
946                            saga_id, reason
947                        ));
948                        let full_reason = format!("host action failed: {}", reason);
949                        if let Some(log) = coord.log.as_ref() {
950                            if let Err(e) = log.terminate_saga(
951                                saga_id,
952                                SagaOutcome::Failed {
953                                    reason: full_reason.clone(),
954                                },
955                            ) {
956                                crate::log(&format!(
957                                    "[saga] saga_id={} SagaActionFailed terminate_saga(Failed) log write failed: {}",
958                                    saga_id, e
959                                ));
960                            }
961                        }
962                        coord.emit_failed(saga_id, full_reason).await;
963                    }
964                    continue;
965                }
966
967                // Step 1 — start any new sagas this event triggers.
968                //
969                // **CPD-4 — evict-and-replace retired.** Pre-CPD-4
970                // (codex P1 PR #634 round 3) the coordinator forced
971                // at-most-one same-kind saga in flight by emitting
972                // `SagaFailed { reason: "evicted" }` on the prior
973                // saga whenever a fresh trigger landed. That was a
974                // workaround for the broadcast-routing limitation:
975                // every `PoolWindowAdded` / `PanesReaped` /
976                // `PoolDrained` / `PoolNotLast` advanced *every*
977                // matching in-flight saga, so two concurrent sagas
978                // would cross-talk and complete each other's bracket.
979                //
980                // CPD-4 closes that limitation. Saga `on_event` now
981                // filters by `event.saga_id == ctx.saga_id`
982                // (`pool_respawn::on_event`,
983                // `window_cleanup::on_event`). Concurrent same-kind
984                // sagas coexist; each consumes only events tagged
985                // with its own coordinator-allocated id. Eviction is
986                // no longer needed — and would actively harm
987                // correctness (a healthy saga gets `SagaFailed`d the
988                // moment a sibling fires).
989                //
990                // Per spec §3.7 + execution-plan §2 batch 3.
991                if let Some(saga) = match_trigger(&event) {
992                    coord.spawn_saga(saga).await;
993                }
994
995                // Step 2 — feed the event into every in-flight saga.
996                // Two-pass to avoid holding the registry lock across
997                // `apply_action` (which itself locks state to bump
998                // version when emitting SagaCompleted/Failed).
999                //
1000                // LSD-2 — when a saga's `on_event` returns anything
1001                // OTHER than `Wait`, it has consumed its awaited bus
1002                // event; record the step's success in the durable log
1003                // (`finish_step(saga_id, awaiting_step, &event)`) before
1004                // dispatching the next action. The new action then
1005                // either parks the saga on a fresh step (next index
1006                // pulled from `next_step_index`) or terminates it.
1007                struct PendingAction {
1008                    saga_id: u64,
1009                    name: &'static str,
1010                    action: SagaAction,
1011                    /// Awaited step index that this on_event return
1012                    /// "consumed" — `Some` only when on_event returned
1013                    /// non-`Wait`. Caller will `finish_step` on it
1014                    /// before invoking `apply_action` for the new
1015                    /// action.
1016                    consumed_step: Option<u32>,
1017                    /// Next free step index for this saga; if the new
1018                    /// action is `IssueCmd`, `apply_action` writes to
1019                    /// this index.
1020                    next_idx: u32,
1021                }
1022                let actions: Vec<PendingAction> = {
1023                    let mut in_flight = coord.in_flight.lock().await;
1024                    let mut out = Vec::new();
1025                    for (saga_id, in_flight_saga) in in_flight.iter_mut() {
1026                        let ctx = SagaCtx { saga_id: *saga_id };
1027                        let action = in_flight_saga.saga.on_event(&event, &ctx);
1028                        // If the saga consumed its awaited event,
1029                        // capture the awaited index now and clear it.
1030                        // `Wait` keeps awaiting_step unchanged.
1031                        let consumed_step = if matches!(action, SagaAction::Wait) {
1032                            None
1033                        } else {
1034                            in_flight_saga.awaiting_step.take()
1035                        };
1036                        let next_idx = in_flight_saga.next_step_index;
1037                        out.push(PendingAction {
1038                            saga_id: *saga_id,
1039                            name: in_flight_saga.saga.name(),
1040                            action,
1041                            consumed_step,
1042                            next_idx,
1043                        });
1044                    }
1045                    out
1046                };
1047                for pending in actions {
1048                    let PendingAction {
1049                        saga_id,
1050                        name,
1051                        action,
1052                        consumed_step,
1053                        next_idx,
1054                    } = pending;
1055                    // LSD-2 — record the awaited step's success in
1056                    // the durable log. `event` is the event that
1057                    // caused the saga to advance.
1058                    if let Some(idx) = consumed_step {
1059                        if let Some(log) = coord.log.as_ref() {
1060                            if let Err(e) = log.finish_step(saga_id, idx, &event) {
1061                                crate::log(&format!(
1062                                    "[saga] saga_id={} finish_step log write failed: {}",
1063                                    saga_id, e
1064                                ));
1065                            }
1066                        }
1067                    }
1068                    let issued_cmd = matches!(action, SagaAction::IssueCmd { .. });
1069                    let outcome = coord.apply_action(saga_id, name, action, next_idx).await;
1070                    if !outcome.in_flight {
1071                        coord.in_flight.lock().await.remove(&saga_id);
1072                    } else if let Some(awaited) = outcome.awaiting_step {
1073                        // Update the saga's bookkeeping to reflect
1074                        // the freshly-issued step.
1075                        let mut registry = coord.in_flight.lock().await;
1076                        if let Some(in_flight_saga) = registry.get_mut(&saga_id) {
1077                            in_flight_saga.awaiting_step = Some(awaited);
1078                            // Bump only if this dispatch consumed the
1079                            // pre-allocated index (always true for
1080                            // IssueCmd; defensive guard).
1081                            if issued_cmd {
1082                                in_flight_saga.next_step_index = awaited + 1;
1083                            }
1084                        }
1085                    }
1086                }
1087            }
1088            Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1089                crate::log(&format!("[saga] coordinator lagged, missed {} events", n));
1090            }
1091            Err(tokio::sync::broadcast::error::RecvError::Closed) => {
1092                crate::log("[saga] coordinator stopping (bus closed)");
1093                return;
1094            }
1095        }
1096    }
1097}
1098
1099#[cfg(test)]
1100mod tests {
1101    use super::*;
1102    use agentmux_common::ipc::{ClientKind, LifecyclePhase};
1103
1104    /// Smoke test from E.1a, retained: coordinator drains unrelated
1105    /// events without panicking. F.5 expanded the loop body but the
1106    /// invariant — non-trigger events are no-ops — still holds.
1107    #[tokio::test]
1108    async fn coordinator_drains_unrelated_events_without_panic() {
1109        let (events_tx, _rx) = tokio::sync::broadcast::channel::<Event>(64);
1110        let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1111        let coord = Arc::new(SagaCoordinator::new(events_tx.clone(), Arc::clone(&state)));
1112        let coord_rx = events_tx.subscribe();
1113        let handle = tokio::spawn(run_coordinator(Arc::clone(&coord), coord_rx));
1114
1115        for v in 1..=5 {
1116            let _ = events_tx.send(Event::LifecyclePhaseChanged {
1117                from: LifecyclePhase::Starting,
1118                to: LifecyclePhase::Running,
1119                version: v,
1120            });
1121        }
1122        let _ = events_tx.send(Event::ProcessSpawned {
1123            pid: 42,
1124            kind: ClientKind::Tool,
1125            client_version: "test".into(),
1126            version: 6,
1127        });
1128
1129        tokio::time::sleep(std::time::Duration::from_millis(10)).await;
1130        // No sagas should be in flight (no trigger events).
1131        assert!(coord.in_flight.lock().await.is_empty());
1132        drop(events_tx);
1133        let _ = tokio::time::timeout(std::time::Duration::from_secs(1), handle).await;
1134    }
1135
1136    #[test]
1137    fn next_id_is_monotonic() {
1138        let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(64);
1139        let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1140        let coord = SagaCoordinator::new(events_tx, state);
1141        let a = coord.next_id();
1142        let b = coord.next_id();
1143        let c = coord.next_id();
1144        assert!(a < b);
1145        assert!(b < c);
1146    }
1147
1148    /// F.5 — verify the trigger table picks up `PoolWindowPromoted`.
1149    /// The end-to-end coordinator test lives in
1150    /// `pool_respawn::tests::coordinator_brackets_promote_with_saga_lifecycle_events`
1151    /// where the pool-respawn saga's expected behavior is asserted.
1152    #[test]
1153    fn match_trigger_pool_window_promoted_starts_pool_respawn() {
1154        let event = Event::PoolWindowPromoted {
1155            label: "window-pool-abc".into(),
1156            version: 1,
1157        };
1158        let saga = match_trigger(&event).expect("PoolWindowPromoted should trigger a saga");
1159        assert_eq!(saga.name(), "pool_respawn_on_promote");
1160    }
1161
1162    /// F.6 — verify the trigger table picks up `WindowClosed`. The
1163    /// end-to-end coordinator test lives in
1164    /// `window_cleanup::tests::coordinator_brackets_close_with_saga_lifecycle_events`.
1165    #[test]
1166    fn match_trigger_window_closed_starts_window_cleanup_cascade() {
1167        let event = Event::WindowClosed {
1168            label: "main".into(),
1169            version: 1,
1170            crash_detected: false,
1171        };
1172        let saga = match_trigger(&event).expect("WindowClosed should trigger a saga");
1173        assert_eq!(saga.name(), "window_cleanup_cascade");
1174    }
1175
1176    /// (codex P1 PR #637 round 2.) Crash-detected closes (originating
1177    /// from `wrr::apply_hwnd_destroyed`) must NOT trigger the saga —
1178    /// the host never sent the cleanup reports, so the saga would
1179    /// stay in-flight forever.
1180    #[test]
1181    fn match_trigger_skips_crash_detected_window_closed() {
1182        let event = Event::WindowClosed {
1183            label: "crashed-window".into(),
1184            version: 1,
1185            crash_detected: true,
1186        };
1187        assert!(
1188            match_trigger(&event).is_none(),
1189            "crash-detected close should NOT spawn a saga",
1190        );
1191    }
1192
1193    #[test]
1194    fn match_trigger_returns_none_for_non_trigger_events() {
1195        let cases = vec![
1196            Event::PoolWindowAdded {
1197                label: "window-pool-abc".into(),
1198                version: 1,
1199                saga_id: None,
1200            },
1201            Event::PoolWindowRemoved {
1202                label: "window-pool-abc".into(),
1203                version: 1,
1204            },
1205            Event::WindowOpened {
1206                label: "main".into(),
1207                kind: agentmux_common::ipc::WindowKind::FullInstance,
1208                parent_label: None,
1209                version: 1,
1210            },
1211            Event::LifecyclePhaseChanged {
1212                from: LifecyclePhase::Starting,
1213                to: LifecyclePhase::Running,
1214                version: 1,
1215            },
1216            // F.6 step-1/step-2 terminal events are NOT triggers
1217            // themselves — the saga consumes them, but receiving one
1218            // when no saga is in flight should be a no-op.
1219            Event::PanesReaped {
1220                label: "main".into(),
1221                version: 1,
1222                saga_id: None,
1223            },
1224            Event::PoolDrained {
1225                label: "main".into(),
1226                version: 1,
1227                saga_id: None,
1228            },
1229            Event::PoolNotLast {
1230                label: "main".into(),
1231                version: 1,
1232                saga_id: None,
1233            },
1234        ];
1235        for event in cases {
1236            assert!(
1237                match_trigger(&event).is_none(),
1238                "non-trigger event spawned a saga: {:?}",
1239                event
1240            );
1241        }
1242    }
1243
1244    /// Saga lifecycle events on the bus must not feed back into
1245    /// sagas (would cause loops). Sanity-check the filter directly.
1246    #[tokio::test]
1247    async fn coordinator_does_not_self_trigger_on_lifecycle_events() {
1248        let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(64);
1249        let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1250        let coord = Arc::new(SagaCoordinator::new(events_tx.clone(), Arc::clone(&state)));
1251        let coord_rx = events_tx.subscribe();
1252        let handle = tokio::spawn(run_coordinator(Arc::clone(&coord), coord_rx));
1253        tokio::task::yield_now().await;
1254
1255        // Push a SagaStarted-shaped event; if the coordinator
1256        // re-spawned, it would eventually exhaust the saga_id atomic
1257        // by looping. We verify simpler: in_flight stays empty.
1258        let _ = events_tx.send(Event::SagaStarted {
1259            saga_id: 999,
1260            name: "foreign_saga".into(),
1261            version: 1,
1262        });
1263        let _ = events_tx.send(Event::SagaCompleted {
1264            saga_id: 999,
1265            version: 2,
1266        });
1267        tokio::time::sleep(std::time::Duration::from_millis(10)).await;
1268        assert!(coord.in_flight.lock().await.is_empty());
1269        drop(events_tx);
1270        let _ = tokio::time::timeout(std::time::Duration::from_secs(1), handle).await;
1271    }
1272
1273    // ---------- Phase F.7 — saga-lifecycle property tests ----------
1274    //
1275    // Random sequences of `WindowClosed` / `PoolWindowPromoted`
1276    // events fed through `match_trigger`. Asserts:
1277    //   1. `WindowClosed { crash_detected: true }` NEVER produces a
1278    //      saga (F.6 round-2 gate, codex P1 #637).
1279    //   2. `WindowClosed { crash_detected: false }` ALWAYS produces
1280    //      exactly one window_cleanup_cascade saga.
1281    //   3. `PoolWindowPromoted` ALWAYS produces exactly one
1282    //      pool_respawn_on_promote saga.
1283    //   4. The saga's name (the (kind, key) tuple's "kind" component)
1284    //      is one of the two known names — no foreign saga slips
1285    //      through the trigger table.
1286    //
1287    // The end-to-end coordinator-level "exactly one terminal lifecycle
1288    // event per saga" assertion lives in
1289    // `pool_respawn::tests::coordinator_brackets_promote_with_saga_lifecycle_events`
1290    // and `window_cleanup::tests::coordinator_brackets_close_with_saga_lifecycle_events`
1291    // (already shipped in F.5 + F.6). The proptests here add the
1292    // randomized-input dimension those happy-path tests don't cover.
1293    //
1294    // Cases capped at 64 (default 1024 too slow for CI). Same bound
1295    // the srv reducer's E.7 proptest uses.
1296
1297    use proptest::prelude::*;
1298
1299    /// Trigger event variants the F.5/F.6 sagas can be triggered by,
1300    /// plus crash-detected close (the negative case).
1301    #[derive(Debug, Clone)]
1302    enum F7TriggerEvent {
1303        /// Should spawn `window_cleanup_cascade`.
1304        WindowClosedClean { label: String },
1305        /// Should NOT spawn anything (F.6 round-2 gate).
1306        WindowClosedCrashed { label: String },
1307        /// Should spawn `pool_respawn_on_promote`.
1308        PoolWindowPromoted { label: String },
1309        /// A non-trigger event — must produce no saga.
1310        Unrelated,
1311    }
1312
1313    fn f7_trigger_strategy() -> impl Strategy<Value = F7TriggerEvent> {
1314        prop_oneof![
1315            3 => "[a-c]{1,3}".prop_map(|label| F7TriggerEvent::WindowClosedClean { label }),
1316            1 => "[a-c]{1,3}".prop_map(|label| F7TriggerEvent::WindowClosedCrashed { label }),
1317            3 => "[a-c]{1,3}".prop_map(|label| F7TriggerEvent::PoolWindowPromoted { label }),
1318            2 => Just(F7TriggerEvent::Unrelated),
1319        ]
1320    }
1321
1322    fn make_event(t: F7TriggerEvent, version: u64) -> Event {
1323        match t {
1324            F7TriggerEvent::WindowClosedClean { label } => Event::WindowClosed {
1325                label,
1326                version,
1327                crash_detected: false,
1328            },
1329            F7TriggerEvent::WindowClosedCrashed { label } => Event::WindowClosed {
1330                label,
1331                version,
1332                crash_detected: true,
1333            },
1334            F7TriggerEvent::PoolWindowPromoted { label } => {
1335                Event::PoolWindowPromoted { label, version }
1336            }
1337            F7TriggerEvent::Unrelated => Event::Pong { nonce: 0, version },
1338        }
1339    }
1340
1341    proptest! {
1342        #![proptest_config(ProptestConfig {
1343            cases: 64,
1344            ..ProptestConfig::default()
1345        })]
1346
1347        /// For every event in a random sequence, `match_trigger`
1348        /// returns the expected saga (or None for crash / unrelated).
1349        /// Exhaustive shape check — catches accidental regressions
1350        /// in the trigger table.
1351        #[test]
1352        fn f7_match_trigger_matches_expected_saga_kind(
1353            triggers in prop::collection::vec(f7_trigger_strategy(), 0..40)
1354        ) {
1355            for (i, t) in triggers.into_iter().enumerate() {
1356                let event = make_event(t.clone(), (i + 1) as u64);
1357                let saga = match_trigger(&event);
1358                match (&t, &saga) {
1359                    (F7TriggerEvent::WindowClosedClean { .. }, Some(s)) => {
1360                        prop_assert_eq!(s.name(), "window_cleanup_cascade");
1361                    }
1362                    (F7TriggerEvent::PoolWindowPromoted { .. }, Some(s)) => {
1363                        prop_assert_eq!(s.name(), "pool_respawn_on_promote");
1364                    }
1365                    (F7TriggerEvent::WindowClosedCrashed { .. }, None) => {
1366                        // Expected — crash-detected close must never
1367                        // spawn a saga (F.6 codex P1 #637).
1368                    }
1369                    (F7TriggerEvent::Unrelated, None) => {
1370                        // Expected — non-trigger event.
1371                    }
1372                    (other_t, other_s) => {
1373                        prop_assert!(
1374                            false,
1375                            "trigger {:?} produced unexpected saga match: {:?}",
1376                            other_t,
1377                            other_s.as_ref().map(|s| s.name()),
1378                        );
1379                    }
1380                }
1381            }
1382        }
1383
1384        /// Crash-detected `WindowClosed` events NEVER produce a
1385        /// saga, regardless of label. Reinforces invariant #1
1386        /// independently — a stricter shrinker target.
1387        #[test]
1388        fn f7_crash_detected_close_never_spawns_saga(
1389            label in "[a-c]{1,3}",
1390            version in 1u64..1_000_000,
1391        ) {
1392            let event = Event::WindowClosed { label, version, crash_detected: true };
1393            prop_assert!(
1394                match_trigger(&event).is_none(),
1395                "crash-detected WindowClosed must not spawn a saga",
1396            );
1397        }
1398
1399        /// Clean-close `WindowClosed` ALWAYS produces a
1400        /// window_cleanup_cascade saga, regardless of label.
1401        #[test]
1402        fn f7_clean_close_always_spawns_window_cleanup_cascade(
1403            label in "[a-c]{1,3}",
1404            version in 1u64..1_000_000,
1405        ) {
1406            let event = Event::WindowClosed { label, version, crash_detected: false };
1407            let saga = match_trigger(&event)
1408                .expect("clean-close should spawn a saga");
1409            prop_assert_eq!(saga.name(), "window_cleanup_cascade");
1410        }
1411    }
1412
1413    /// **CPD-4 — concurrent same-kind sagas coexist without
1414    /// cross-talk.** Random sequences of trigger events drive the
1415    /// coordinator; the registry is allowed to hold multiple sagas of
1416    /// the same kind simultaneously (the pre-CPD-4 evict-and-replace
1417    /// gate is retired). Invariant under test: NO `SagaFailed` event
1418    /// fires with reason `"evicted"` — sagas only terminate via
1419    /// `Done` (their tagged echo lands) or via the saga timeout
1420    /// (~5-30s, far longer than this test's 50ms settling window).
1421    ///
1422    /// Replaces `f7_evict_and_replace_keeps_one_saga_per_kind`. The
1423    /// behavioral inversion is the core of CPD-4: pre-CPD-4 we
1424    /// asserted ≤1 in-flight per kind (and accepted premature
1425    /// `SagaFailed { reason: "evicted" }`); CPD-4 asserts no
1426    /// premature evictions.
1427    #[tokio::test]
1428    async fn cpd4_concurrent_sagas_no_eviction() {
1429        // Tiny LCG for deterministic per-seed sequences. Glibc's
1430        // constants — quality irrelevant; we just need reproducible
1431        // bit patterns across CI runs.
1432        fn lcg_next(state: &mut u64) -> u64 {
1433            *state = state
1434                .wrapping_mul(1_103_515_245)
1435                .wrapping_add(12_345);
1436            *state
1437        }
1438
1439        let seeds: &[u64] = &[1, 7, 42, 99, 1234, 5678, 0xDEADBEEF, 0xC0FFEE];
1440        for seed in seeds {
1441            let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(256);
1442            let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1443            let coord = Arc::new(SagaCoordinator::new(events_tx.clone(), Arc::clone(&state)));
1444            let mut witness = events_tx.subscribe();
1445            let coord_rx = events_tx.subscribe();
1446            let _handle = tokio::spawn(run_coordinator(Arc::clone(&coord), coord_rx));
1447            tokio::task::yield_now().await;
1448
1449            // Build a deterministic sequence from the seed.
1450            let mut rng = *seed;
1451            let labels = ["a", "b", "c"];
1452            for v in 1u64..=20 {
1453                let label_idx = (lcg_next(&mut rng) % labels.len() as u64) as usize;
1454                let label = labels[label_idx].to_string();
1455                let pick = lcg_next(&mut rng) % 3;
1456                let event = match pick {
1457                    0 => Event::WindowClosed {
1458                        label,
1459                        version: v,
1460                        crash_detected: false,
1461                    },
1462                    1 => Event::PoolWindowPromoted { label, version: v },
1463                    _ => Event::Pong { nonce: v, version: v },
1464                };
1465                let _ = events_tx.send(event);
1466                // Yield so the coordinator processes one event before
1467                // we send the next. Tightens the concurrent-overlap
1468                // window.
1469                tokio::task::yield_now().await;
1470            }
1471            // Brief settling — give the coordinator time to process
1472            // the last few events + emit terminal lifecycles.
1473            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1474
1475            // Invariant: NO SagaFailed with "evicted" reason.
1476            // We don't assert the registry size — concurrent sagas
1477            // are allowed to pile up under per-saga correlation
1478            // because we never feed them their tagged terminal
1479            // events in this test. They'll reach their timeout
1480            // eventually, but not within 50ms.
1481            let mut evictions = 0u32;
1482            while let Ok(Ok(ev)) = tokio::time::timeout(
1483                std::time::Duration::from_millis(5),
1484                witness.recv(),
1485            )
1486            .await
1487            {
1488                if let Event::SagaFailed { reason, .. } = ev {
1489                    if reason.contains("evicted") {
1490                        evictions += 1;
1491                    }
1492                }
1493            }
1494            assert_eq!(
1495                evictions, 0,
1496                "seed {} — CPD-4 invariant violated: observed {} SagaFailed events with reason='evicted'; \
1497                 evict-and-replace should be retired so concurrent same-kind sagas coexist",
1498                seed, evictions,
1499            );
1500        }
1501    }
1502
1503    /// Saga forward-path emission contract: every saga that
1504    /// terminates emits exactly ONE terminal lifecycle event
1505    /// (`SagaCompleted` xor `SagaFailed`). Drives a small batch of
1506    /// triggers through the coordinator and counts emitted
1507    /// lifecycle events on the bus. Catches regressions where a
1508    /// saga's `Done` action somehow fires both `emit_completed`
1509    /// and `emit_failed`, or neither.
1510    #[tokio::test]
1511    async fn f7_each_saga_emits_exactly_one_terminal_event() {
1512        let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(256);
1513        let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1514        let coord = Arc::new(SagaCoordinator::new(events_tx.clone(), Arc::clone(&state)));
1515        let mut witness = events_tx.subscribe();
1516        let coord_rx = events_tx.subscribe();
1517        let _handle = tokio::spawn(run_coordinator(Arc::clone(&coord), coord_rx));
1518        tokio::task::yield_now().await;
1519
1520        // Drive ONE clean cleanup-cascade saga + ONE pool-respawn
1521        // saga to completion. CPD-4: route terminals by saga_id, so
1522        // we trigger each saga, capture its allocated id from
1523        // SagaStarted, then send the matching tagged terminal events.
1524        let _ = events_tx.send(Event::WindowClosed {
1525            label: "main".into(),
1526            version: 1,
1527            crash_detected: false,
1528        });
1529        let _ = events_tx.send(Event::PoolWindowPromoted {
1530            label: "pool-1".into(),
1531            version: 2,
1532        });
1533
1534        // Drain the bus with a deadline. Count one terminal event
1535        // (SagaCompleted xor SagaFailed) per saga_id. As each
1536        // SagaStarted lands, dispatch the matching tagged terminals.
1537        let mut started_ids = std::collections::HashSet::<u64>::new();
1538        let mut cleanup_saga_id: Option<u64> = None;
1539        let mut respawn_saga_id: Option<u64> = None;
1540        let mut version: u64 = 100;
1541        let mut terminal_for_id = std::collections::HashMap::<u64, &'static str>::new();
1542        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3);
1543        while std::time::Instant::now() < deadline {
1544            match tokio::time::timeout(std::time::Duration::from_millis(50), witness.recv()).await
1545            {
1546                Ok(Ok(Event::SagaStarted { saga_id, name, .. })) => {
1547                    started_ids.insert(saga_id);
1548                    if name == "window_cleanup_cascade" && cleanup_saga_id.is_none() {
1549                        cleanup_saga_id = Some(saga_id);
1550                        version += 1;
1551                        let _ = events_tx.send(Event::PanesReaped {
1552                            label: "main".into(),
1553                            version,
1554                            saga_id: Some(saga_id),
1555                        });
1556                    } else if name == "pool_respawn_on_promote" && respawn_saga_id.is_none() {
1557                        respawn_saga_id = Some(saga_id);
1558                        version += 1;
1559                        let _ = events_tx.send(Event::PoolWindowAdded {
1560                            label: "pool-2".into(),
1561                            version,
1562                            saga_id: Some(saga_id),
1563                        });
1564                    }
1565                }
1566                Ok(Ok(Event::PanesReaped { saga_id: Some(sid), .. })) => {
1567                    if Some(sid) == cleanup_saga_id {
1568                        version += 1;
1569                        let _ = events_tx.send(Event::PoolDrained {
1570                            label: "main".into(),
1571                            version,
1572                            saga_id: Some(sid),
1573                        });
1574                    }
1575                }
1576                Ok(Ok(Event::SagaCompleted { saga_id, .. })) => {
1577                    let prev = terminal_for_id.insert(saga_id, "completed");
1578                    assert!(
1579                        prev.is_none(),
1580                        "saga_id {} got two terminal events: prior={:?} now=completed",
1581                        saga_id,
1582                        prev,
1583                    );
1584                }
1585                Ok(Ok(Event::SagaFailed { saga_id, .. })) => {
1586                    let prev = terminal_for_id.insert(saga_id, "failed");
1587                    assert!(
1588                        prev.is_none(),
1589                        "saga_id {} got two terminal events: prior={:?} now=failed",
1590                        saga_id,
1591                        prev,
1592                    );
1593                }
1594                Ok(Ok(_)) => {}
1595                Ok(Err(_)) => break,
1596                Err(_) => {
1597                    // No new events for 50ms — we've drained.
1598                    if started_ids.len() == 2
1599                        && started_ids.iter().all(|id| terminal_for_id.contains_key(id))
1600                    {
1601                        break;
1602                    }
1603                }
1604            }
1605        }
1606        // Both sagas started.
1607        assert_eq!(
1608            started_ids.len(),
1609            2,
1610            "expected 2 SagaStarted events (one per saga), got {}: {:?}",
1611            started_ids.len(),
1612            started_ids,
1613        );
1614        // Each got exactly one terminal event.
1615        for id in &started_ids {
1616            assert!(
1617                terminal_for_id.contains_key(id),
1618                "saga_id {} never got a terminal lifecycle event",
1619                id,
1620            );
1621        }
1622        // No spurious terminal events for sagas we didn't observe
1623        // start.
1624        for id in terminal_for_id.keys() {
1625            assert!(
1626                started_ids.contains(id),
1627                "saga_id {} got a terminal event but no SagaStarted observed",
1628                id,
1629            );
1630        }
1631    }
1632
1633    // ---------- LSD-2 — coordinator <-> LauncherSagaLog wiring ------
1634    //
1635    // Three end-to-end tests that drive the coordinator with a real
1636    // in-memory `LauncherSagaLog` and assert lifecycle rows land in
1637    // the expected state. Mirror the test inventory pinned in
1638    // `docs/specs/SPEC_LAUNCHER_SAGA_DURABILITY_2026-05-01.md` §4 PR2.
1639    //
1640    // The tests use `pool_respawn::PoolRespawn` because its single-
1641    // step shape (one IssueCmd, one terminal echo) keeps assertions
1642    // tight; the `window_cleanup_cascade` two-step shape is exercised
1643    // implicitly by the existing F.6 coordinator tests + adds a
1644    // multi-step example in `saga_step_finish_records_event_payload`.
1645
1646    use crate::saga::log::LauncherSagaLog;
1647    use std::sync::Arc as StdArc;
1648
1649    /// Helper: spin up coordinator + in-memory log.
1650    fn spawn_coord_with_log()
1651        -> (StdArc<SagaCoordinator>, StdArc<LauncherSagaLog>, tokio::sync::broadcast::Sender<Event>)
1652    {
1653        let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(256);
1654        let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1655        let log = StdArc::new(LauncherSagaLog::open_in_memory().expect("in-memory log"));
1656        let coord = StdArc::new(
1657            SagaCoordinator::new(events_tx.clone(), Arc::clone(&state))
1658                .with_log(StdArc::clone(&log))
1659                .expect("with_log on fresh in-memory log should succeed"),
1660        );
1661        (coord, log, events_tx)
1662    }
1663
1664    /// LSD-2 — drive a saga (PoolRespawn) through to completion and
1665    /// verify the durable log records:
1666    ///   - `launcher_saga` row with state='completed', non-null
1667    ///     ended_at, no failure_reason, input_json round-trips the
1668    ///     `promoted_label`.
1669    ///   - one step row in 'succeeded' state with output_json
1670    ///     populated.
1671    #[tokio::test]
1672    async fn saga_completes_writes_lifecycle_to_log() {
1673        let (coord, log, events_tx) = spawn_coord_with_log();
1674        let mut witness = events_tx.subscribe();
1675        let coord_rx = events_tx.subscribe();
1676        let _handle = tokio::spawn(run_coordinator(StdArc::clone(&coord), coord_rx));
1677        tokio::task::yield_now().await;
1678
1679        // Trigger the saga.
1680        let _ = events_tx.send(Event::PoolWindowPromoted {
1681            label: "window-pool-abc".into(),
1682            version: 1,
1683        });
1684        // CPD-4: wait for SagaStarted to learn the saga_id, then send
1685        // the awaited terminal tagged with that id. Pre-CPD-4 we
1686        // could send `saga_id: None`; under per-saga correlation
1687        // the saga only consumes its own echo.
1688        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
1689        while std::time::Instant::now() < deadline {
1690            if let Ok(Ok(Event::SagaStarted { saga_id, .. })) =
1691                tokio::time::timeout(std::time::Duration::from_millis(50), witness.recv()).await
1692            {
1693                let _ = events_tx.send(Event::PoolWindowAdded {
1694                    label: "window-pool-xyz".into(),
1695                    version: 2,
1696                    saga_id: Some(saga_id),
1697                });
1698                break;
1699            }
1700        }
1701
1702        // Settle: coordinator runs apply_action which awaits state
1703        // mutex + bus send. 200ms is generous for the in-process loop.
1704        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1705
1706        let snapshots = log.snapshot_recent(10).expect("snapshot_recent");
1707        assert_eq!(snapshots.len(), 1, "expected exactly one saga row");
1708        let s = &snapshots[0];
1709        assert_eq!(s.name, "pool_respawn_on_promote");
1710        assert_eq!(s.state, "completed");
1711        assert!(s.ended_at.is_some(), "ended_at must be set on completion");
1712        assert!(s.failure_reason.is_none());
1713        assert_eq!(s.step_count, 1, "expected exactly one succeeded step");
1714        // Input snapshot round-trips the saga's constructor arg.
1715        let parsed: serde_json::Value = serde_json::from_str(&s.input_json).unwrap();
1716        assert_eq!(parsed["promoted_label"], "window-pool-abc");
1717    }
1718
1719    /// **CPD-4 — two concurrent same-kind sagas BOTH complete
1720    /// cleanly.** Repurposed from `saga_fails_writes_failed_state`
1721    /// (which previously verified the evict-and-replace path).
1722    /// Per-saga event correlation lets two `pool_respawn_on_promote`
1723    /// sagas coexist; each consumes its own saga_id-tagged refill
1724    /// echo, so the durable log records two `completed` rows — no
1725    /// `failed` rows from the retired eviction policy.
1726    #[tokio::test]
1727    async fn saga_concurrent_same_kind_both_complete() {
1728        let (coord, log, events_tx) = spawn_coord_with_log();
1729        let mut witness = events_tx.subscribe();
1730        let coord_rx = events_tx.subscribe();
1731        let _handle = tokio::spawn(run_coordinator(StdArc::clone(&coord), coord_rx));
1732        tokio::task::yield_now().await;
1733
1734        // First promote — kicks off saga A.
1735        let _ = events_tx.send(Event::PoolWindowPromoted {
1736            label: "window-pool-a".into(),
1737            version: 1,
1738        });
1739        // Second promote BEFORE A's terminal arrives — under CPD-4
1740        // saga B coexists with A (no eviction).
1741        let _ = events_tx.send(Event::PoolWindowPromoted {
1742            label: "window-pool-b".into(),
1743            version: 2,
1744        });
1745
1746        // Capture both saga_ids from their `SagaStarted` brackets,
1747        // then fire each saga's tagged refill so both terminate
1748        // Completed. The order doesn't matter because per-saga
1749        // correlation routes events by id, not by FIFO.
1750        let mut ids = Vec::<u64>::new();
1751        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
1752        let mut version: u64 = 100;
1753        while std::time::Instant::now() < deadline {
1754            if let Ok(Ok(ev)) =
1755                tokio::time::timeout(std::time::Duration::from_millis(50), witness.recv()).await
1756            {
1757                if let Event::SagaStarted { saga_id, name, .. } = ev {
1758                    if name == "pool_respawn_on_promote" {
1759                        ids.push(saga_id);
1760                        version += 1;
1761                        let _ = events_tx.send(Event::PoolWindowAdded {
1762                            label: format!("refill-for-{}", saga_id),
1763                            version,
1764                            saga_id: Some(saga_id),
1765                        });
1766                    }
1767                }
1768                if ids.len() == 2 {
1769                    break;
1770                }
1771            }
1772        }
1773        assert_eq!(ids.len(), 2, "expected 2 SagaStarted events");
1774
1775        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1776
1777        let snapshots = log.snapshot_recent(10).expect("snapshot_recent");
1778        assert_eq!(
1779            snapshots.len(),
1780            2,
1781            "expected two saga rows (concurrent, no eviction)"
1782        );
1783        // BOTH must be 'completed' — eviction is retired.
1784        for s in &snapshots {
1785            assert_eq!(
1786                s.state, "completed",
1787                "CPD-4: both concurrent sagas must complete cleanly, got state={:?} reason={:?}",
1788                s.state, s.failure_reason,
1789            );
1790            assert!(s.failure_reason.is_none());
1791        }
1792    }
1793
1794    /// LSD-2 — when a saga's `on_event` consumes its awaited bus
1795    /// event, the coordinator calls `LauncherSagaLog::finish_step`
1796    /// with the event payload. To inspect `output_json` directly via
1797    /// the public LSD-1 API, we drive a saga only PART of the way —
1798    /// the cleanup cascade's Step 1 lands its echo (`PanesReaped`)
1799    /// but Step 2 is left in `pending` so the saga stays unresolved.
1800    /// `unresolved_sagas` then exposes the full step list including
1801    /// Step 0's `output_json`.
1802    #[tokio::test]
1803    async fn saga_step_finish_records_event_payload() {
1804        let (coord, log, events_tx) = spawn_coord_with_log();
1805        let mut witness = events_tx.subscribe();
1806        let coord_rx = events_tx.subscribe();
1807        let _handle = tokio::spawn(run_coordinator(StdArc::clone(&coord), coord_rx));
1808        tokio::task::yield_now().await;
1809
1810        // Trigger the cascade. Window-cleanup has two steps; we feed
1811        // only the first echo (`PanesReaped`) so Step 0 transitions
1812        // pending→succeeded but Step 1 (`DrainPoolIfLast`) stays
1813        // pending — the saga remains in_flight + queryable as
1814        // unresolved.
1815        //
1816        // CPD-4: tag PanesReaped with the saga's id so the saga
1817        // actually advances (per-saga correlation).
1818        let _ = events_tx.send(Event::WindowClosed {
1819            label: "main".into(),
1820            version: 1,
1821            crash_detected: false,
1822        });
1823        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
1824        while std::time::Instant::now() < deadline {
1825            if let Ok(Ok(Event::SagaStarted { saga_id, .. })) =
1826                tokio::time::timeout(std::time::Duration::from_millis(50), witness.recv()).await
1827            {
1828                let _ = events_tx.send(Event::PanesReaped {
1829                    label: "main".into(),
1830                    version: 2,
1831                    saga_id: Some(saga_id),
1832                });
1833                break;
1834            }
1835        }
1836        // INTENTIONALLY no `PoolDrained` / `PoolNotLast`.
1837
1838        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1839
1840        let unresolved = log.unresolved_sagas().expect("unresolved_sagas");
1841        assert_eq!(unresolved.len(), 1, "expected one unresolved saga");
1842        let saga = &unresolved[0];
1843        assert_eq!(saga.name, "window_cleanup_cascade");
1844        assert_eq!(saga.state, "running");
1845        // Two step rows: Step 0 succeeded (PanesReaped echo), Step 1
1846        // pending (DrainPoolIfLast dispatched but no echo arrived).
1847        assert_eq!(saga.steps.len(), 2, "expected 2 step rows");
1848        let step0 = &saga.steps[0];
1849        assert_eq!(step0.step_index, 0);
1850        assert_eq!(step0.state, "succeeded");
1851        assert!(step0.ended_at.is_some(), "succeeded step needs ended_at");
1852        let output_json = step0
1853            .output_json
1854            .as_ref()
1855            .expect("output_json must be set after finish_step");
1856        // Round-trip — the JSON should deserialize to the awaited Event.
1857        let parsed: Event = serde_json::from_str(output_json).expect("event round-trips");
1858        match parsed {
1859            Event::PanesReaped { label, .. } => assert_eq!(label, "main"),
1860            other => panic!("expected PanesReaped, got {:?}", other),
1861        }
1862        let step1 = &saga.steps[1];
1863        assert_eq!(step1.step_index, 1);
1864        assert_eq!(step1.state, "pending");
1865        assert!(step1.output_json.is_none(), "pending step has no output");
1866    }
1867}