agentmux_launcher/saga/mod.rs
1// Copyright 2026, AgentMux Corp.
2// SPDX-License-Identifier: Apache-2.0
3//
4// Phase E.1a — saga coordinator infrastructure.
5// Phase F.5 — first concrete saga consumer (`pool_respawn`).
6//
7// NOT the same framework as `agentmux_srv::sagas`, despite the shared name.
8// This one is a `Saga` TRAIT driven by a bus subscription, built for the
9// cross-process case; srv's is plain async functions with in-process
10// dispatch, because every saga it runs mutates only srv state. The two
11// share exactly one method name (`new`) and no types. That split is
12// deliberate, not drift — see docs/architecture/DECISION_SAGA_REDUCER_TWO_FRAMEWORKS_2026_09_07.md
13// and `docs/retro/saga-coordinator-location-analysis-2026-04-30.md`.
14// Do not "unify" them without reading those first.
15//
16// A saga is a state machine that orchestrates a multi-step,
17// multi-reducer flow (e.g. tear-off touches host pool + srv
18// workspace + launcher window registration). Sagas exist where
19// per-flow correctness needs explicit coordination — distributed
20// subscriber callbacks aren't enough.
21//
22// E.1a shipped this module as framework-only (no real sagas).
23// **F.5 lights up the first concrete saga (`pool_respawn`) plus the
24// minimal coordinator wiring needed to drive it**: the bus-
25// subscription loop now starts sagas in response to trigger events,
26// routes subsequent bus events into in-flight sagas via
27// `Saga::on_event`, dispatches `SagaAction` results, and emits
28// `SagaStarted` / `SagaCompleted` / `SagaFailed` brackets.
29//
30// Design (per `docs/specs/SPEC_PHASE_E_SRV_REDUCER_2026_04_29.md` §7
31// + `docs/specs/SPEC_PHASE_F_HOST_REDUCER_2026-05-01.md` §7.1):
32//
33// trait Saga {
34// fn start(&mut self, ctx: &SagaCtx) -> SagaAction;
35// fn on_event(&mut self, event: &Event, ctx: &SagaCtx) -> SagaAction;
36// fn name(&self) -> &'static str;
37// }
38//
39// enum SagaAction {
40// IssueCmd { target: PipeTarget, cmd: Command },
41// Done,
42// Failed { reason: String },
43// Wait,
44// }
45//
46// The `SagaCoordinator` task subscribes to the broadcast bus,
47// routes events to in-flight sagas, dispatches IssueCmd actions to
48// the appropriate pipe, and emits `SagaStarted` / `SagaCompleted`
49// / `SagaFailed` so subscribers (renderer) can buffer-until-complete.
50//
51// **Cross-process dispatch — CPD-3 LIVE.** The launcher → host
52// command pipe is now wired via `HostPipe::send_command()` (CPD-2
53// + CPD-3). F.5's `pool_respawn` and F.6's `window_cleanup_cascade`
54// sagas issue `IssueCmd::Host` actions; the coordinator dispatches
55// them through the wire instead of merely logging. Per-saga
56// timeouts (`Saga::timeout()`, default 5s, F.6 overrides 30s)
57// + per-saga deadline tasks fire `SagaFailed` if a host action
58// stays unresolved past its budget. Host-emitted
59// `Command::ReportSagaActionFailed` translates into
60// `Event::SagaActionFailed` and terminates the matching saga.
61//
62// Per-variant `saga_id` tagging on existing Command/Event variants
63// is deferred; F.5's coordinator doesn't yet need it because:
64// - only one `pool_respawn` saga can be in flight per promote
65// (sagas are dispatched on `PoolWindowPromoted` 1:1);
66// - `Event::PoolWindowAdded` from the *implicit* refill is the
67// unique terminal signal — no foreign event type collides.
68// When concurrent promotes can happen (and produce overlapping
69// `PoolWindowAdded` streams), a future PR adds saga_id correlation.
70//
71// Saga state is in-memory. Launcher restart abandons in-flight
72// sagas; renderer-side timeouts cover the visible consequence.
73
74use std::sync::Arc;
75use std::time::Duration;
76
77use agentmux_common::ipc::{Command, Event};
78
79use crate::host_pipe::HostPipe;
80
81pub mod pool_respawn;
82pub mod window_cleanup;
83
84#[cfg(test)]
85mod integration_tests;
86
87// LSD-1 (PR LSD-1) — durable launcher saga log + API. Foundations
88// only: the coordinator does NOT call any of these methods yet.
89// Pillar 1 Step 6 (SPEC_PILLAR1_STEP6_SAGA_COLLAPSE_2026_07_16) — the
90// registry is IN-MEMORY now. The durable SQLite log
91// (SPEC_LAUNCHER_SAGA_DURABILITY_2026-05-01, LSD-1..4) and its startup
92// recovery walker were deleted: recovery never compensated anything (it
93// only wrote `failed_compensation` tombstones for `--diag sagas`), and
94// with srv authoritative + crash-reproject there is nothing durable left
95// for an interrupted launcher saga to protect. The coordinator's live
96// write-through below is unchanged — it now feeds in-process diagnostics
97// instead of a WAL.
98pub(crate) mod log;
99pub use log::LauncherSagaLog;
100use log::SagaOutcome;
101
102/// Where a `SagaAction::IssueCmd` should be dispatched.
103///
104/// `LauncherSelf` means "feed this command to the launcher's own
105/// reducer" (in-process); `Host` and `Srv` mean "forward to the
106/// peer's pipe."
107///
108/// **CPD-3 status:** `Host` is now LIVE — the saga coordinator's
109/// `apply_action` dispatches `IssueCmd::Host` actions through
110/// `HostPipe::send_command()` over the launcher → host wire.
111/// `LauncherSelf` and `Srv` remain reserved for class-D/E sagas.
112///
113/// F.7 cleanup audit: only `Host` is constructed today (F.5/F.6 saga
114/// IssueCmds). `LauncherSelf` and `Srv` are framework slots reserved
115/// for the cross-process dispatch follow-up phase per
116/// `SPEC_PHASE_F_HOST_REDUCER_2026-05-01.md` §4.3. Allow stays.
117#[derive(Debug, Clone, Copy, PartialEq, Eq)]
118#[allow(dead_code)] // LauncherSelf + Srv reserved for future cross-process sagas
119pub enum PipeTarget {
120 LauncherSelf,
121 Host,
122 Srv,
123}
124
125/// What a saga decides to do next, returned from `start` /
126/// `on_event`. The coordinator drives the saga forward by reacting
127/// to this enum.
128///
129/// F.7 cleanup audit: `IssueCmd`, `Done`, `Wait` are all constructed
130/// by F.5 + F.6 sagas. `Failed` is consumed in `apply_action` (and
131/// its corresponding `emit_failed` is now actively called by the
132/// evict-and-replace path), but no shipped saga *constructs*
133/// `Failed` yet — sagas today only succeed or wait. Variant-level
134/// allow scopes the dead-code suppression precisely to that one
135/// reserved variant.
136#[derive(Debug)]
137pub enum SagaAction {
138 /// Dispatch a command on the target pipe; saga remains in flight.
139 IssueCmd {
140 target: PipeTarget,
141 cmd: Command,
142 },
143 /// Saga succeeded. Coordinator removes it from `in_flight`
144 /// and emits `Event::SagaCompleted`.
145 Done,
146 /// Saga failed irrecoverably. Coordinator emits `Event::SagaFailed`
147 /// after dispatching any compensation IssueCmds the saga issued
148 /// before returning Failed. Reserved for sagas with explicit
149 /// failure conditions (e.g. cross-process dispatch follow-up
150 /// + saga timeouts).
151 #[allow(dead_code)] // reserved for sagas that explicitly fail; F.5/F.6 only succeed or wait
152 Failed { reason: String },
153 /// Saga is waiting for an event it hasn't seen yet. No-op for
154 /// the coordinator until the next bus event.
155 Wait,
156}
157
158/// Read-only context passed to saga callbacks. Currently carries
159/// only the saga_id; kept as a struct (rather than a bare u64) so
160/// future fields can be added without touching every `Saga` impl.
161///
162/// F.7 cleanup audit: `saga_id` is unread by the shipped sagas
163/// (F.5/F.6 don't need it — coordinator already routes events
164/// per-registry). Reserved for the per-event saga_id correlation
165/// follow-up; allow stays.
166#[allow(dead_code)] // saga_id reserved for per-event correlation follow-up
167#[derive(Debug, Clone, Copy)]
168pub struct SagaCtx {
169 pub saga_id: u64,
170}
171
172/// A multi-step, multi-reducer state machine. Implementations
173/// describe one logical operation (tear-off, pool-respawn, etc.).
174///
175/// Lifecycle: coordinator calls `start` once when the saga is
176/// added to `in_flight`. After that, every event on the bus is
177/// passed to `on_event`. The saga inspects the event, advances
178/// its internal state, returns the next action.
179///
180/// **Identification:** sagas know which events belong to them by
181/// inspecting saga_id in lifecycle events or by matching patterns
182/// in event payloads (e.g. specific labels). F.5 sagas correlate
183/// by event type only (`pool_respawn` matches any
184/// `PoolWindowAdded` after start). Per-variant `saga_id` tagging
185/// on commands/events is deferred until concurrent same-type
186/// sagas are needed.
187pub trait Saga: Send {
188 fn start(&mut self, ctx: &SagaCtx) -> SagaAction;
189 fn on_event(&mut self, event: &Event, ctx: &SagaCtx) -> SagaAction;
190 fn name(&self) -> &'static str;
191 /// LSD-2 — saga's input arguments serialized for the durable log's
192 /// `input_json` column. The coordinator calls this once at
193 /// `spawn_saga` time (before `start`) and writes the result via
194 /// `LauncherSagaLog::start_saga`. Operators see this in
195 /// `--diag sagas` output (e.g. `{"closed_label":"win-3"}`) so they
196 /// can tell which window's cleanup a recovered-failed saga
197 /// belonged to.
198 ///
199 /// Default `Value::Null` — sagas with no input fields can ignore
200 /// it. Concrete sagas should override with a `serde_json::json!`
201 /// of their constructor args.
202 fn input_snapshot(&self) -> serde_json::Value {
203 serde_json::Value::Null
204 }
205
206 /// CPD-3 — per-saga deadline budget for completing all
207 /// `IssueCmd`+wait cycles. Coordinator arms a timer when the saga
208 /// registers; if the saga is still in_flight when `timeout()`
209 /// elapses, it is force-failed (`SagaFailed { reason: "saga
210 /// timeout" }`) and removed from the registry.
211 ///
212 /// Default 5s — fits class-C single-step host dispatch (e.g.
213 /// pool respawn). `WindowCleanupCascade` overrides to 30s
214 /// because pane drain on a workspace with many panes can
215 /// legitimately take that long. Per spec §3.10.
216 fn timeout(&self) -> Duration {
217 Duration::from_secs(5)
218 }
219}
220
221/// Saga coordinator task.
222///
223/// Owns the registry of in-flight sagas, allocates saga ids,
224/// routes events to sagas, dispatches IssueCmd actions to the
225/// appropriate pipes, and emits lifecycle events on the broadcast
226/// bus.
227///
228/// F.5 adds the first real consumer (`pool_respawn`). The
229/// `in_flight` registry is now actually populated; the bus loop
230/// dispatches events into it.
231/// LSD-2 — coordinator's per-saga book-keeping. Wraps the boxed saga
232/// with the durable-log step bookkeeping the coordinator needs to call
233/// `LauncherSagaLog::finish_step` when the awaited bus event lands.
234///
235/// `awaiting_step` is `Some(idx)` when the saga is parked on a
236/// `Wait`-then-event pivot (the most recent `IssueCmd` allocated step
237/// `idx`); `None` between dispatches and after termination. Tracking
238/// it on the in-flight record (rather than inside each saga impl) keeps
239/// the durability concern out of saga authors' hands — they only deal
240/// with `SagaAction`.
241///
242/// `next_step_index` is the monotonic counter the coordinator
243/// `fetch_add(1)`s on every `IssueCmd` for this saga. Mirrors srv's
244/// `SagaCtx::step_index`. Lives per-saga (not coordinator-global) so
245/// concurrent sagas don't interleave indices in the log.
246struct InFlightSaga {
247 saga: Box<dyn Saga>,
248 awaiting_step: Option<u32>,
249 next_step_index: u32,
250}
251
252pub struct SagaCoordinator {
253 /// Monotonic saga-id allocator.
254 next_saga_id: std::sync::atomic::AtomicU64,
255 /// In-flight sagas keyed by saga_id, each wrapped with the
256 /// durable-log bookkeeping (LSD-2: `awaiting_step` +
257 /// `next_step_index`).
258 in_flight: tokio::sync::Mutex<std::collections::HashMap<u64, InFlightSaga>>,
259 /// Reference to the broadcast bus so the coordinator can emit
260 /// `SagaStarted` / `SagaCompleted` / `SagaFailed`.
261 events_tx: tokio::sync::broadcast::Sender<Event>,
262 /// Reference to the launcher's reducer state for `bump_version`
263 /// when emitting saga lifecycle events.
264 state: Arc<tokio::sync::Mutex<crate::state::State>>,
265 /// LSD-2 — durable saga log. `None` in tests that don't exercise
266 /// the durability path (the saga then logs + remains in flight,
267 /// preserving the pre-LSD-2 behavior tests rely on for end-to-end
268 /// bracket assertions). When `Some`, every saga lifecycle
269 /// transition (`spawn_saga` → `start_saga`, `IssueCmd` →
270 /// `start_step`, awaited-event consumed → `finish_step`,
271 /// `Done` → `terminate_saga(Completed)`, `Failed` / evicted →
272 /// `terminate_saga(Failed)`) writes to this log.
273 log: Option<Arc<LauncherSagaLog>>,
274 /// CPD-3 — launcher → host pipe wrapper. `apply_action` for
275 /// `IssueCmd::Host` dispatches via `host_pipe.send_command()` when
276 /// installed. `None` in tests that don't exercise host dispatch
277 /// (those drive sagas via synthetic terminal events on the bus).
278 host_pipe: Option<Arc<HostPipe>>,
279}
280
281impl SagaCoordinator {
282 pub fn new(
283 events_tx: tokio::sync::broadcast::Sender<Event>,
284 state: Arc<tokio::sync::Mutex<crate::state::State>>,
285 ) -> Self {
286 Self {
287 next_saga_id: std::sync::atomic::AtomicU64::new(1),
288 in_flight: tokio::sync::Mutex::new(std::collections::HashMap::new()),
289 events_tx,
290 state,
291 log: None,
292 host_pipe: None,
293 }
294 }
295
296 /// CPD-3 — install the host pipe so `IssueCmd::Host` actions are
297 /// dispatched live (instead of log-only). Builder-style setter so
298 /// existing tests + the `next_id_is_monotonic` smoke don't have to
299 /// construct a HostPipe. Production wiring (`main.rs`) calls this
300 /// once before `run_coordinator` is spawned.
301 pub fn with_host_pipe(mut self, host_pipe: Arc<HostPipe>) -> Self {
302 self.host_pipe = Some(host_pipe);
303 self
304 }
305
306 /// LSD-2 — install the durable saga log so saga lifecycle
307 /// transitions are persisted to `<data-dir>/db/launcher-sagas.db`.
308 /// Builder-style setter rather than a constructor parameter so
309 /// existing tests + the `next_id_is_monotonic` smoke don't have
310 /// to construct an in-memory log. Production wiring (in
311 /// `main.rs`) calls this once before `run_coordinator` is spawned.
312 pub fn with_log(mut self, log: Arc<LauncherSagaLog>) -> Result<Self, crate::saga::log::LogError> {
313 // Seed the saga_id allocator from the highest persisted id
314 // so a launcher restart cannot reuse an id that already
315 // exists in `launcher_saga`. Reusing would (a) fail new
316 // INSERTs on duplicate-PK, and (b) silently mutate prior
317 // saga rows via `terminate_saga` / `finish_step` UPDATEs
318 // keyed by saga_id — corrupting saga history + recovery
319 // diagnostics. (codex P1 PR #645 round 1.)
320 //
321 // On `max_saga_id()` error: return Err so caller (main.rs)
322 // can fail launcher startup loudly. Continuing with a default
323 // next_saga_id=1 while the log is still attached would leave
324 // the coordinator in the exact failure mode this seed is
325 // meant to prevent. (codex P1 PR #645 round 2.)
326 let max = log.max_saga_id()?;
327 self.next_saga_id
328 .store(max + 1, std::sync::atomic::Ordering::Relaxed);
329 if max > 0 {
330 crate::log(&format!(
331 "[saga] seeded next_saga_id={} from launcher_saga.max(saga_id)={}",
332 max + 1,
333 max
334 ));
335 }
336 self.log = Some(log);
337 Ok(self)
338 }
339
340 /// Allocate the next saga_id. Monotonic per launcher run.
341 pub fn next_id(&self) -> u64 {
342 self.next_saga_id
343 .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
344 }
345
346 /// Bump the launcher's `event_version` for a coordinator-emitted
347 /// lifecycle event. Brief mutex hold — no I/O between lock and
348 /// drop.
349 async fn next_event_version(&self) -> u64 {
350 let mut state = self.state.lock().await;
351 state.bump_version()
352 }
353
354 /// Emit `Event::SagaStarted` after registering a saga.
355 async fn emit_started(&self, saga_id: u64, name: &'static str) {
356 let v = self.next_event_version().await;
357 let _ = self.events_tx.send(Event::SagaStarted {
358 saga_id,
359 name: name.to_string(),
360 version: v,
361 });
362 }
363
364 /// Emit `Event::SagaCompleted` after a saga returns `Done`.
365 async fn emit_completed(&self, saga_id: u64) {
366 let v = self.next_event_version().await;
367 let _ = self.events_tx.send(Event::SagaCompleted {
368 saga_id,
369 version: v,
370 });
371 }
372
373 /// Emit `Event::SagaFailed` after a saga returns `Failed` or
374 /// after evict-and-replace cancels a same-kind in-flight saga.
375 /// F.7 cleanup audit: prior `#[allow(dead_code)]` removed — F.6
376 /// evict-and-replace policy now actively dispatches this on
377 /// every concurrent same-kind retrigger.
378 async fn emit_failed(&self, saga_id: u64, reason: String) {
379 let v = self.next_event_version().await;
380 let _ = self.events_tx.send(Event::SagaFailed {
381 saga_id,
382 reason,
383 version: v,
384 });
385 }
386
387 /// Emit `SagaFailed` for every in-flight saga and clear the registry.
388 /// Called on clean launcher shutdown so no saga bracket is left open
389 /// in the durable log (open brackets would otherwise trigger spurious
390 /// LSD-3 compensation on next startup).
391 ///
392 /// Writes directly to the durable log when installed — more reliable
393 /// than relying on bus-subscriber delivery during shutdown teardown.
394 pub async fn cancel_all_in_flight(&self, reason: &str) {
395 let ids: Vec<u64> = {
396 let mut registry = self.in_flight.lock().await;
397 registry.drain().map(|(id, _)| id).collect()
398 };
399 for saga_id in ids {
400 // Cancel any pending host-pipe frames so a reconnecting host
401 // doesn't drain them as orphan side-effects.
402 if let Some(pipe) = self.host_pipe.as_ref() {
403 pipe.cancel_saga(saga_id).await;
404 }
405 // Emit on the broadcast bus for live subscribers.
406 self.emit_failed(saga_id, reason.to_string()).await;
407 // Write directly to the durable log — bypasses task scheduling
408 // and is guaranteed to reach SQLite even during shutdown.
409 if let Some(log) = self.log.as_ref() {
410 let _ = log.terminate_saga(
411 saga_id,
412 crate::saga::log::SagaOutcome::Failed { reason: reason.to_string() },
413 );
414 }
415 }
416 }
417
418 /// Apply a `SagaAction` returned by `start` or `on_event`. Returns
419 /// an `ApplyOutcome` carrying:
420 /// - `in_flight`: true → caller keeps the saga in `in_flight`,
421 /// false → caller removes it.
422 /// - `awaiting_step`: `Some(idx)` if the action was an
423 /// `IssueCmd` that allocated step `idx` and the saga is now
424 /// parked waiting for the echo event. Caller stores this on
425 /// the InFlightSaga so the next non-`Wait` `on_event` return
426 /// can call `LauncherSagaLog::finish_step(saga_id, idx, ev)`.
427 ///
428 /// CPD-3 — `IssueCmd::Host` dispatches live through `HostPipe`
429 /// when one is installed. Tests without a host_pipe (the existing
430 /// F.5/F.6 unit + integration tests) fall back to log-only so
431 /// synthetic-terminal-event drivers continue to work.
432 /// `LauncherSelf` and `Srv` targets remain log-only — reserved
433 /// for class-D/E sagas, no consumer today.
434 ///
435 /// LSD-2 — every state transition writes to `LauncherSagaLog`
436 /// when one is installed: `IssueCmd` → `start_step`, `Done` →
437 /// `terminate_saga(Completed)`, `Failed` →
438 /// `terminate_saga(Failed)`.
439 ///
440 /// `step_index` is the caller-allocated index for THIS dispatch.
441 /// On the first dispatch in `spawn_saga`, the caller passes 0;
442 /// on subsequent dispatches from the event loop, the caller pulls
443 /// + bumps the per-InFlightSaga counter.
444 ///
445 /// Terminal paths (`Done`, `Failed`, host-send-error) all go
446 /// through `claim_terminal` so only one of {normal Done/Failed,
447 /// timeout-task, SagaActionFailed listener, host-send-error}
448 /// can win the terminal-event race for a given saga_id.
449 /// (reagent P1 PR #644 round 1 + 2.)
450 async fn apply_action(
451 &self,
452 saga_id: u64,
453 name: &'static str,
454 action: SagaAction,
455 step_index: u32,
456 ) -> ApplyOutcome {
457 match action {
458 SagaAction::IssueCmd { target, cmd } => {
459 // LSD-2 — write the durable `pending` step row BEFORE
460 // dispatch so a crash mid-dispatch leaves a recoverable
461 // breadcrumb (LSD-3's walker upgrades this saga to
462 // `failed_compensation` on next launcher startup).
463 let step_name = derive_step_name(&cmd, target);
464 if let Some(log) = self.log.as_ref() {
465 if let Err(e) =
466 log.start_step(saga_id, step_index, &step_name, target, &cmd)
467 {
468 crate::log(&format!(
469 "[saga] saga_id={} name={} start_step log write failed: {} — continuing (in-memory path remains authoritative for this run)",
470 saga_id, name, e
471 ));
472 }
473 }
474
475 match target {
476 PipeTarget::Host => {
477 // CPD-3 — dispatch live through the host pipe
478 // when installed. Tests without a host_pipe
479 // (most existing F.5/F.6 unit + integration
480 // tests) fall back to log-only so synthetic-
481 // terminal-event drivers continue to work.
482 let Some(host_pipe) = self.host_pipe.as_ref() else {
483 crate::log(&format!(
484 "[saga] saga_id={} name={} IssueCmd target=Host cmd={:?} (no host_pipe installed — log-only)",
485 saga_id, name, cmd
486 ));
487 return ApplyOutcome::in_flight_awaiting(step_index);
488 };
489 let cmd_with_id = inject_saga_id(cmd, saga_id);
490 match host_pipe.send_command(&cmd_with_id).await {
491 Ok(()) => {
492 crate::log(&format!(
493 "[saga] saga_id={} name={} IssueCmd::Host dispatched cmd={:?}",
494 saga_id, name, cmd_with_id
495 ));
496 ApplyOutcome::in_flight_awaiting(step_index)
497 }
498 Err(e) => {
499 // CRITICAL: do NOT terminate the saga
500 // here. send_command's send_frame already
501 // re-buffered the failed frame for retry
502 // on reconnect (with the real saga_id).
503 // If we ALSO emit SagaFailed now:
504 // - 30s later expire_pending_if_timed_out
505 // drains the buffer + emit_drop_failure
506 // emits a SECOND SagaFailed for the
507 // same saga_id (double-emit), OR
508 // - host reconnects in <30s and the
509 // buffered command executes as an
510 // orphaned side effect after the saga
511 // bracket is already closed.
512 // (reagent P1 PR #644 round 6.)
513 //
514 // Round 7 design: keep the saga in_flight.
515 // The retry path resolves it (Report*
516 // arrives → on_event drives forward) OR
517 // the saga's own timeout (5s default,
518 // 30s for F.6) fires + claim_terminal'd
519 // SagaFailed. Either way, exactly one
520 // terminal event is emitted.
521 crate::log(&format!(
522 "[saga] saga_id={} name={} host pipe send transient err: {} — frame buffered for retry; saga stays in-flight (round 7 fix)",
523 saga_id, name, e
524 ));
525 ApplyOutcome::in_flight_awaiting(step_index)
526 }
527 }
528 }
529 PipeTarget::LauncherSelf | PipeTarget::Srv => {
530 // Reserved for class-D/E sagas; out of scope
531 // per SPEC_CROSS_PROCESS_DISPATCH §3.6. Log-
532 // only retains framework shape until a
533 // consumer arrives.
534 crate::log(&format!(
535 "[saga] saga_id={} name={} IssueCmd target={:?} cmd={:?} (target not yet wired; log-only)",
536 saga_id, name, target, cmd
537 ));
538 ApplyOutcome::in_flight_awaiting(step_index)
539 }
540 }
541 }
542 SagaAction::Wait => ApplyOutcome::in_flight_no_change(),
543 SagaAction::Done => {
544 // Atomic claim: only one of {normal Done, timeout-task,
545 // SagaActionFailed listener, host-send-error} can win
546 // the terminal-event race. Without this guard a Done
547 // could fire SagaCompleted while the timeout task
548 // simultaneously fires SagaFailed, producing duplicate
549 // terminal events for the same saga (reagent P1 PR
550 // #644 round 1).
551 if !self.claim_terminal(saga_id).await {
552 return ApplyOutcome::terminated();
553 }
554 crate::log(&format!(
555 "[saga] saga_id={} name={} Done — emitting SagaCompleted",
556 saga_id, name
557 ));
558 if let Some(log) = self.log.as_ref() {
559 if let Err(e) = log.terminate_saga(saga_id, SagaOutcome::Completed) {
560 crate::log(&format!(
561 "[saga] saga_id={} terminate_saga(Completed) log write failed: {}",
562 saga_id, e
563 ));
564 }
565 }
566 self.emit_completed(saga_id).await;
567 ApplyOutcome::terminated()
568 }
569 SagaAction::Failed { reason } => {
570 if !self.claim_terminal(saga_id).await {
571 return ApplyOutcome::terminated();
572 }
573 crate::log(&format!(
574 "[saga] saga_id={} name={} Failed reason={} — emitting SagaFailed",
575 saga_id, name, reason
576 ));
577 if let Some(log) = self.log.as_ref() {
578 if let Err(e) = log.terminate_saga(
579 saga_id,
580 SagaOutcome::Failed {
581 reason: reason.clone(),
582 },
583 ) {
584 crate::log(&format!(
585 "[saga] saga_id={} terminate_saga(Failed) log write failed: {}",
586 saga_id, e
587 ));
588 }
589 }
590 self.emit_failed(saga_id, reason).await;
591 ApplyOutcome::terminated()
592 }
593 }
594 }
595
596 /// Atomically remove `saga_id` from `in_flight` and return whether
597 /// the caller was the one who removed it (i.e. has the right to
598 /// emit the terminal `SagaCompleted` / `SagaFailed`). Idempotent:
599 /// a second caller for the same saga_id sees `false`. Used by the
600 /// `SagaAction::Done` and `SagaAction::Failed` paths in
601 /// `apply_action`, by the per-saga timeout task in `spawn_saga`,
602 /// by the `Event::SagaActionFailed` listener in `run_coordinator`,
603 /// by the host-send-failure path in `apply_action::IssueCmd::Host`,
604 /// and by the eviction path in `run_coordinator`.
605 /// (reagent P1 PR #644 round 1.)
606 async fn claim_terminal(&self, saga_id: u64) -> bool {
607 let claimed = {
608 let mut registry = self.in_flight.lock().await;
609 registry.remove(&saga_id).is_some()
610 };
611 // Purge any host-pipe pending frames tagged with this saga_id
612 // so a host reconnect post-terminal doesn't drain them as
613 // orphan side effects, and the 30s expiry path doesn't emit
614 // a second SagaFailed for the same saga_id.
615 // (codex + reagent P1 PR #644 round 7.)
616 if claimed {
617 if let Some(pipe) = self.host_pipe.as_ref() {
618 pipe.cancel_saga(saga_id).await;
619 }
620 }
621 claimed
622 }
623
624 /// Register a fresh saga, calling `start` and applying its first
625 /// action. The caller has already determined that the saga
626 /// should fire (e.g. matched a trigger event). Returns the saga's
627 /// allocated id (logged + bracketed in `SagaStarted`).
628 ///
629 /// LSD-2 — also writes a `running` row to the durable saga log
630 /// before invoking `saga.start()`, and (via `apply_action`)
631 /// records the saga's first dispatched step.
632 ///
633 /// CPD-3 — also arms a per-saga deadline timer. If the saga is
634 /// still in_flight when `saga.timeout()` elapses, a background
635 /// task force-fails it (`SagaFailed { reason: "saga timeout" }`)
636 /// and removes it from the registry. Per spec §3.10. Insert-then-
637 /// start ordering is required so `apply_action`'s `claim_terminal`
638 /// guard can succeed for immediate-completion sagas (codex P2 PR
639 /// #644 round 2).
640 async fn spawn_saga(self: &Arc<Self>, saga: Box<dyn Saga>) -> u64 {
641 let saga_id = self.next_id();
642 let name = saga.name();
643 let input = saga.input_snapshot();
644 let saga_timeout = saga.timeout();
645 crate::log(&format!(
646 "[saga] starting saga_id={} name={}",
647 saga_id, name
648 ));
649 // LSD-2 — write the durable `running` row BEFORE
650 // `emit_started` so a crash between this point and
651 // `apply_action` still leaves a recoverable breadcrumb
652 // (LSD-3's walker will upgrade it to `failed_compensation` on
653 // next launcher startup). Mirrors srv's `emit_saga_started`
654 // ordering: durable log before bus.
655 if let Some(log) = self.log.as_ref() {
656 if let Err(e) = log.start_saga(saga_id, name, &input) {
657 crate::log(&format!(
658 "[saga] saga_id={} start_saga log write failed: {} — continuing (in-memory path remains authoritative for this run)",
659 saga_id, e
660 ));
661 }
662 }
663 // Emit SagaStarted FIRST so any subscriber buffering by
664 // saga_id sees the bracket open before any per-step events.
665 // Mirrors `agentmux-srv::sagas::emit_saga_started` ordering.
666 self.emit_started(saga_id, name).await;
667
668 // CPD-3 — insert the saga into in_flight BEFORE calling
669 // start(), so that if start() returns SagaAction::Done or
670 // SagaAction::Failed immediately, apply_action's terminal
671 // paths can claim_terminal successfully and emit the matching
672 // SagaCompleted/SagaFailed bracket. Pre-round-3 we started
673 // saga.start() before insertion, which broke immediate-
674 // completion sagas (claim_terminal returned false → no
675 // terminal event → dangling SagaStarted bracket).
676 // (codex P2 PR #644 round 2.)
677 let action = {
678 let mut registry = self.in_flight.lock().await;
679 // CPD-4 — concurrent same-kind sagas now correlate by
680 // saga_id (`Saga::on_event` filters on
681 // `ctx.saga_id == event.saga_id`). The pre-CPD-4
682 // observability warning (PR #634 codex P1) is retained
683 // at info level for operators tracking concurrency
684 // density via `--diag wrr`, but the underlying
685 // correctness issue is closed.
686 let same_kind_count = registry
687 .values()
688 .filter(|s| s.saga.name() == name)
689 .count();
690 if same_kind_count >= 1 {
691 crate::log(&format!(
692 "[saga] starting {} saga_id={} while {} other(s) of same kind in flight (CPD-4 per-saga correlation — sagas coexist cleanly)",
693 name, saga_id, same_kind_count,
694 ));
695 }
696 registry.insert(
697 saga_id,
698 InFlightSaga {
699 saga,
700 awaiting_step: None,
701 next_step_index: 0,
702 },
703 );
704 // Drive start() while still under the registry lock so
705 // we have a mutable reference to the just-inserted saga.
706 // start() is non-async and does no I/O — bounded hold time.
707 let in_flight_saga = registry.get_mut(&saga_id).expect("just inserted");
708 let ctx = SagaCtx { saga_id };
709 in_flight_saga.saga.start(&ctx)
710 };
711
712 // First step on a fresh saga is index 0. apply_action may
713 // remove the saga via claim_terminal (Done/Failed/host-send-
714 // error) without re-locking the registry from here.
715 let outcome = self.apply_action(saga_id, name, action, 0).await;
716 if outcome.in_flight {
717 // Update the bookkeeping for the freshly-issued step.
718 // If the first action allocated step 0, the saga is now
719 // parked awaiting that step's echo event; next allocation
720 // is index 1. Otherwise (Wait — saga has no first
721 // dispatch yet), keep the counter at 0.
722 let mut registry = self.in_flight.lock().await;
723 if let Some(in_flight_saga) = registry.get_mut(&saga_id) {
724 in_flight_saga.awaiting_step = outcome.awaiting_step;
725 in_flight_saga.next_step_index =
726 if outcome.awaiting_step.is_some() { 1 } else { 0 };
727 }
728 drop(registry);
729
730 // CPD-3 — arm the per-saga deadline timer. If the saga
731 // is still in_flight when `saga_timeout` elapses, fail
732 // it out. Spawned task captures an Arc clone of the
733 // coordinator so it can outlive the saga's deadline.
734 // Uses `claim_terminal` so it doesn't race with the
735 // normal Done/Failed path. (reagent P1 PR #644 round 1.)
736 let coord_for_timeout = Arc::clone(self);
737 tokio::spawn(async move {
738 tokio::time::sleep(saga_timeout).await;
739 if coord_for_timeout.claim_terminal(saga_id).await {
740 crate::log(&format!(
741 "[saga] saga_id={} name={} timed out after {:?} — emitting SagaFailed",
742 saga_id, name, saga_timeout
743 ));
744 let reason = "saga timeout".to_string();
745 if let Some(log) = coord_for_timeout.log.as_ref() {
746 if let Err(e) = log.terminate_saga(
747 saga_id,
748 SagaOutcome::Failed {
749 reason: reason.clone(),
750 },
751 ) {
752 crate::log(&format!(
753 "[saga] saga_id={} timeout terminate_saga(Failed) log write failed: {}",
754 saga_id, e
755 ));
756 }
757 }
758 coord_for_timeout
759 .emit_failed(saga_id, reason)
760 .await;
761 }
762 });
763 }
764 saga_id
765 }
766}
767
768/// LSD-2 — outcome of `apply_action`. Combines the prior `bool` (is
769/// the saga still in flight?) with the new `awaiting_step` book-
770/// keeping the coordinator needs to call `LauncherSagaLog::finish_step`
771/// when the awaited bus event lands.
772#[derive(Debug, Clone, Copy)]
773struct ApplyOutcome {
774 in_flight: bool,
775 /// If the action was an `IssueCmd`, the step index it allocated;
776 /// the coordinator parks this on the InFlightSaga. `None` for
777 /// `Wait` / `Done` / `Failed`.
778 awaiting_step: Option<u32>,
779}
780
781impl ApplyOutcome {
782 fn in_flight_awaiting(step_index: u32) -> Self {
783 Self {
784 in_flight: true,
785 awaiting_step: Some(step_index),
786 }
787 }
788 fn in_flight_no_change() -> Self {
789 Self {
790 in_flight: true,
791 awaiting_step: None,
792 }
793 }
794 fn terminated() -> Self {
795 Self {
796 in_flight: false,
797 awaiting_step: None,
798 }
799 }
800}
801
802/// LSD-2 — short, greppable name for a `Command` dispatched as part
803/// of a saga step. Mirrors srv's `command_discriminant_name` in spirit
804/// (snake_case strings rather than `Debug` formatting) but prefixes
805/// with `issue_cmd_<target>_<discriminant>` so `--diag sagas` output
806/// makes the dispatch target obvious without a separate column lookup.
807///
808/// Falls back to `issue_cmd_<target>_unknown` for variants serde can't
809/// stringify (shouldn't happen for the snake_case-tagged Command enum;
810/// defensive default).
811fn derive_step_name(cmd: &Command, target: PipeTarget) -> String {
812 let target_str = match target {
813 PipeTarget::LauncherSelf => "launcher_self",
814 PipeTarget::Host => "host",
815 PipeTarget::Srv => "srv",
816 };
817 let discriminant = match serde_json::to_value(cmd) {
818 Ok(serde_json::Value::Object(map)) => map
819 .get("cmd")
820 .and_then(|v| v.as_str())
821 .map(|s| s.to_string())
822 .unwrap_or_else(|| "unknown".to_string()),
823 _ => "unknown".to_string(),
824 };
825 format!("issue_cmd_{}_{}", target_str, discriminant)
826}
827
828/// CPD-3 — fill in the `saga_id` field on a host-bound `Command`
829/// before dispatch. Sagas construct their `IssueCmd` actions with a
830/// placeholder `saga_id: 0` (they don't know their coordinator-
831/// allocated id at action-construction time); the coordinator
832/// rewrites the field at dispatch time so the host can echo it back
833/// on the matching `Report*`.
834///
835/// Exhaustive match: any host-bound Command variant added later that
836/// forgets to plumb `saga_id` will refuse to compile here. Non-host-
837/// bound variants (Report*, Identify, etc.) panic loudly because
838/// sagas only emit `IssueCmd::Host` for the three host-bound
839/// command kinds covered below.
840fn inject_saga_id(cmd: Command, saga_id: u64) -> Command {
841 match cmd {
842 Command::SpawnPoolWindow { .. } => Command::SpawnPoolWindow { saga_id },
843 Command::ReapPanes { label, .. } => Command::ReapPanes { label, saga_id },
844 Command::DrainPoolIfLast { label, .. } => {
845 Command::DrainPoolIfLast { label, saga_id }
846 }
847 // Defense-in-depth: every non-host-dispatched Command variant
848 // (Report*, Identify, etc.) is a coding bug if it reaches
849 // this point — the F.5/F.6 sagas only emit IssueCmd::Host
850 // for the three command kinds above.
851 other => panic!(
852 "inject_saga_id called on non-host-bound Command variant: {:?}",
853 other
854 ),
855 }
856}
857
858/// Inspect a bus event for "should this start a fresh saga?" Returns
859/// the constructed saga (boxed) on a hit; `None` otherwise.
860///
861/// Triggers wired to date:
862/// * F.5: `Event::PoolWindowPromoted` → `pool_respawn::PoolRespawn`
863/// * F.6: `Event::WindowClosed` →
864/// `window_cleanup::WindowCleanupCascade`
865///
866/// Future sagas extend this match.
867///
868/// CPD-4 — the coordinator no longer evicts same-kind in-flight
869/// sagas before `spawn_saga`; per-saga event correlation
870/// (`Saga::on_event` filters by `ctx.saga_id`) lets concurrent
871/// same-kind sagas coexist cleanly. The pre-CPD-4 evict-and-replace
872/// gate (PR #634 round 3) is removed in `run_coordinator`.
873fn match_trigger(event: &Event) -> Option<Box<dyn Saga>> {
874 match event {
875 Event::PoolWindowPromoted { label, .. } => {
876 Some(Box::new(pool_respawn::PoolRespawn::new(label.clone())))
877 }
878 // (codex P1 PR #637.) Only fire the cleanup cascade on
879 // CLEAN closes. `Event::WindowClosed { crash_detected: true }`
880 // comes from `wrr::apply_hwnd_destroyed` after a host/renderer
881 // crash; the host never sent `ReportPanesReaped` /
882 // `ReportPoolDrainDecision`, so the saga would stay
883 // in-flight indefinitely (only cleared by a later same-kind
884 // eviction or launcher restart) leaving a SagaStarted
885 // bracket dangling for subscribers that buffer on lifecycle.
886 Event::WindowClosed { label, crash_detected: false, .. } => Some(Box::new(
887 window_cleanup::WindowCleanupCascade::new(label.clone()),
888 )),
889 _ => None,
890 }
891}
892
893/// Run the coordinator's bus-subscription loop.
894///
895/// **Receiver is passed in, not subscribed inside.** Subscribing
896/// before `tokio::spawn` (in `main.rs`) ensures events emitted
897/// between coordinator construction and the first `recv()` aren't
898/// lost to the race window. Same pattern as `event_log::run_disk_writer`.
899/// (reagent P2 PR #609.)
900///
901/// Dispatch order on each event:
902/// 1. Match against trigger table → start any new sagas via
903/// `spawn_saga` (which emits `SagaStarted` and applies the
904/// saga's initial action).
905/// 2. Feed the same event into every in-flight saga's `on_event`.
906/// Sagas returning `Done` / `Failed` are removed from
907/// `in_flight` after their lifecycle event is emitted.
908///
909/// **Self-emitted events (`SagaStarted` / `SagaCompleted` /
910/// `SagaFailed`) are NOT re-fed into sagas.** A saga reacting to its
911/// own start event would loop. Filtered at the top of the dispatch
912/// path.
913pub async fn run_coordinator(
914 coord: Arc<SagaCoordinator>,
915 mut events_rx: tokio::sync::broadcast::Receiver<Event>,
916) {
917 crate::log("[saga] coordinator started");
918
919 loop {
920 match events_rx.recv().await {
921 Ok(event) => {
922 // Skip our own lifecycle events to avoid loops.
923 if matches!(
924 event,
925 Event::SagaStarted { .. } | Event::SagaCompleted { .. } | Event::SagaFailed { .. }
926 ) {
927 continue;
928 }
929
930 // CPD-3 — host reported via `Command::ReportSagaActionFailed`
931 // that a saga-issued action failed. Launcher reducer
932 // translates this into `Event::SagaActionFailed`
933 // (CPD-1 schema). Terminate the matching saga
934 // immediately rather than waiting for a Report* that
935 // won't come.
936 if let Event::SagaActionFailed { saga_id, reason, .. } = &event {
937 let saga_id = *saga_id;
938 let reason = reason.clone();
939 // claim_terminal: atomic take-ownership of the
940 // terminal-event slot, preventing duplicate-emission
941 // races with the saga's own Done/Failed path or its
942 // timeout task. (reagent P1 PR #644 round 1.)
943 if coord.claim_terminal(saga_id).await {
944 crate::log(&format!(
945 "[saga] saga_id={} terminating from host SagaActionFailed reason={}",
946 saga_id, reason
947 ));
948 let full_reason = format!("host action failed: {}", reason);
949 if let Some(log) = coord.log.as_ref() {
950 if let Err(e) = log.terminate_saga(
951 saga_id,
952 SagaOutcome::Failed {
953 reason: full_reason.clone(),
954 },
955 ) {
956 crate::log(&format!(
957 "[saga] saga_id={} SagaActionFailed terminate_saga(Failed) log write failed: {}",
958 saga_id, e
959 ));
960 }
961 }
962 coord.emit_failed(saga_id, full_reason).await;
963 }
964 continue;
965 }
966
967 // Step 1 — start any new sagas this event triggers.
968 //
969 // **CPD-4 — evict-and-replace retired.** Pre-CPD-4
970 // (codex P1 PR #634 round 3) the coordinator forced
971 // at-most-one same-kind saga in flight by emitting
972 // `SagaFailed { reason: "evicted" }` on the prior
973 // saga whenever a fresh trigger landed. That was a
974 // workaround for the broadcast-routing limitation:
975 // every `PoolWindowAdded` / `PanesReaped` /
976 // `PoolDrained` / `PoolNotLast` advanced *every*
977 // matching in-flight saga, so two concurrent sagas
978 // would cross-talk and complete each other's bracket.
979 //
980 // CPD-4 closes that limitation. Saga `on_event` now
981 // filters by `event.saga_id == ctx.saga_id`
982 // (`pool_respawn::on_event`,
983 // `window_cleanup::on_event`). Concurrent same-kind
984 // sagas coexist; each consumes only events tagged
985 // with its own coordinator-allocated id. Eviction is
986 // no longer needed — and would actively harm
987 // correctness (a healthy saga gets `SagaFailed`d the
988 // moment a sibling fires).
989 //
990 // Per spec §3.7 + execution-plan §2 batch 3.
991 if let Some(saga) = match_trigger(&event) {
992 coord.spawn_saga(saga).await;
993 }
994
995 // Step 2 — feed the event into every in-flight saga.
996 // Two-pass to avoid holding the registry lock across
997 // `apply_action` (which itself locks state to bump
998 // version when emitting SagaCompleted/Failed).
999 //
1000 // LSD-2 — when a saga's `on_event` returns anything
1001 // OTHER than `Wait`, it has consumed its awaited bus
1002 // event; record the step's success in the durable log
1003 // (`finish_step(saga_id, awaiting_step, &event)`) before
1004 // dispatching the next action. The new action then
1005 // either parks the saga on a fresh step (next index
1006 // pulled from `next_step_index`) or terminates it.
1007 struct PendingAction {
1008 saga_id: u64,
1009 name: &'static str,
1010 action: SagaAction,
1011 /// Awaited step index that this on_event return
1012 /// "consumed" — `Some` only when on_event returned
1013 /// non-`Wait`. Caller will `finish_step` on it
1014 /// before invoking `apply_action` for the new
1015 /// action.
1016 consumed_step: Option<u32>,
1017 /// Next free step index for this saga; if the new
1018 /// action is `IssueCmd`, `apply_action` writes to
1019 /// this index.
1020 next_idx: u32,
1021 }
1022 let actions: Vec<PendingAction> = {
1023 let mut in_flight = coord.in_flight.lock().await;
1024 let mut out = Vec::new();
1025 for (saga_id, in_flight_saga) in in_flight.iter_mut() {
1026 let ctx = SagaCtx { saga_id: *saga_id };
1027 let action = in_flight_saga.saga.on_event(&event, &ctx);
1028 // If the saga consumed its awaited event,
1029 // capture the awaited index now and clear it.
1030 // `Wait` keeps awaiting_step unchanged.
1031 let consumed_step = if matches!(action, SagaAction::Wait) {
1032 None
1033 } else {
1034 in_flight_saga.awaiting_step.take()
1035 };
1036 let next_idx = in_flight_saga.next_step_index;
1037 out.push(PendingAction {
1038 saga_id: *saga_id,
1039 name: in_flight_saga.saga.name(),
1040 action,
1041 consumed_step,
1042 next_idx,
1043 });
1044 }
1045 out
1046 };
1047 for pending in actions {
1048 let PendingAction {
1049 saga_id,
1050 name,
1051 action,
1052 consumed_step,
1053 next_idx,
1054 } = pending;
1055 // LSD-2 — record the awaited step's success in
1056 // the durable log. `event` is the event that
1057 // caused the saga to advance.
1058 if let Some(idx) = consumed_step {
1059 if let Some(log) = coord.log.as_ref() {
1060 if let Err(e) = log.finish_step(saga_id, idx, &event) {
1061 crate::log(&format!(
1062 "[saga] saga_id={} finish_step log write failed: {}",
1063 saga_id, e
1064 ));
1065 }
1066 }
1067 }
1068 let issued_cmd = matches!(action, SagaAction::IssueCmd { .. });
1069 let outcome = coord.apply_action(saga_id, name, action, next_idx).await;
1070 if !outcome.in_flight {
1071 coord.in_flight.lock().await.remove(&saga_id);
1072 } else if let Some(awaited) = outcome.awaiting_step {
1073 // Update the saga's bookkeeping to reflect
1074 // the freshly-issued step.
1075 let mut registry = coord.in_flight.lock().await;
1076 if let Some(in_flight_saga) = registry.get_mut(&saga_id) {
1077 in_flight_saga.awaiting_step = Some(awaited);
1078 // Bump only if this dispatch consumed the
1079 // pre-allocated index (always true for
1080 // IssueCmd; defensive guard).
1081 if issued_cmd {
1082 in_flight_saga.next_step_index = awaited + 1;
1083 }
1084 }
1085 }
1086 }
1087 }
1088 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1089 crate::log(&format!("[saga] coordinator lagged, missed {} events", n));
1090 }
1091 Err(tokio::sync::broadcast::error::RecvError::Closed) => {
1092 crate::log("[saga] coordinator stopping (bus closed)");
1093 return;
1094 }
1095 }
1096 }
1097}
1098
1099#[cfg(test)]
1100mod tests {
1101 use super::*;
1102 use agentmux_common::ipc::{ClientKind, LifecyclePhase};
1103
1104 /// Smoke test from E.1a, retained: coordinator drains unrelated
1105 /// events without panicking. F.5 expanded the loop body but the
1106 /// invariant — non-trigger events are no-ops — still holds.
1107 #[tokio::test]
1108 async fn coordinator_drains_unrelated_events_without_panic() {
1109 let (events_tx, _rx) = tokio::sync::broadcast::channel::<Event>(64);
1110 let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1111 let coord = Arc::new(SagaCoordinator::new(events_tx.clone(), Arc::clone(&state)));
1112 let coord_rx = events_tx.subscribe();
1113 let handle = tokio::spawn(run_coordinator(Arc::clone(&coord), coord_rx));
1114
1115 for v in 1..=5 {
1116 let _ = events_tx.send(Event::LifecyclePhaseChanged {
1117 from: LifecyclePhase::Starting,
1118 to: LifecyclePhase::Running,
1119 version: v,
1120 });
1121 }
1122 let _ = events_tx.send(Event::ProcessSpawned {
1123 pid: 42,
1124 kind: ClientKind::Tool,
1125 client_version: "test".into(),
1126 version: 6,
1127 });
1128
1129 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
1130 // No sagas should be in flight (no trigger events).
1131 assert!(coord.in_flight.lock().await.is_empty());
1132 drop(events_tx);
1133 let _ = tokio::time::timeout(std::time::Duration::from_secs(1), handle).await;
1134 }
1135
1136 #[test]
1137 fn next_id_is_monotonic() {
1138 let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(64);
1139 let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1140 let coord = SagaCoordinator::new(events_tx, state);
1141 let a = coord.next_id();
1142 let b = coord.next_id();
1143 let c = coord.next_id();
1144 assert!(a < b);
1145 assert!(b < c);
1146 }
1147
1148 /// F.5 — verify the trigger table picks up `PoolWindowPromoted`.
1149 /// The end-to-end coordinator test lives in
1150 /// `pool_respawn::tests::coordinator_brackets_promote_with_saga_lifecycle_events`
1151 /// where the pool-respawn saga's expected behavior is asserted.
1152 #[test]
1153 fn match_trigger_pool_window_promoted_starts_pool_respawn() {
1154 let event = Event::PoolWindowPromoted {
1155 label: "window-pool-abc".into(),
1156 version: 1,
1157 };
1158 let saga = match_trigger(&event).expect("PoolWindowPromoted should trigger a saga");
1159 assert_eq!(saga.name(), "pool_respawn_on_promote");
1160 }
1161
1162 /// F.6 — verify the trigger table picks up `WindowClosed`. The
1163 /// end-to-end coordinator test lives in
1164 /// `window_cleanup::tests::coordinator_brackets_close_with_saga_lifecycle_events`.
1165 #[test]
1166 fn match_trigger_window_closed_starts_window_cleanup_cascade() {
1167 let event = Event::WindowClosed {
1168 label: "main".into(),
1169 version: 1,
1170 crash_detected: false,
1171 };
1172 let saga = match_trigger(&event).expect("WindowClosed should trigger a saga");
1173 assert_eq!(saga.name(), "window_cleanup_cascade");
1174 }
1175
1176 /// (codex P1 PR #637 round 2.) Crash-detected closes (originating
1177 /// from `wrr::apply_hwnd_destroyed`) must NOT trigger the saga —
1178 /// the host never sent the cleanup reports, so the saga would
1179 /// stay in-flight forever.
1180 #[test]
1181 fn match_trigger_skips_crash_detected_window_closed() {
1182 let event = Event::WindowClosed {
1183 label: "crashed-window".into(),
1184 version: 1,
1185 crash_detected: true,
1186 };
1187 assert!(
1188 match_trigger(&event).is_none(),
1189 "crash-detected close should NOT spawn a saga",
1190 );
1191 }
1192
1193 #[test]
1194 fn match_trigger_returns_none_for_non_trigger_events() {
1195 let cases = vec![
1196 Event::PoolWindowAdded {
1197 label: "window-pool-abc".into(),
1198 version: 1,
1199 saga_id: None,
1200 },
1201 Event::PoolWindowRemoved {
1202 label: "window-pool-abc".into(),
1203 version: 1,
1204 },
1205 Event::WindowOpened {
1206 label: "main".into(),
1207 kind: agentmux_common::ipc::WindowKind::FullInstance,
1208 parent_label: None,
1209 version: 1,
1210 },
1211 Event::LifecyclePhaseChanged {
1212 from: LifecyclePhase::Starting,
1213 to: LifecyclePhase::Running,
1214 version: 1,
1215 },
1216 // F.6 step-1/step-2 terminal events are NOT triggers
1217 // themselves — the saga consumes them, but receiving one
1218 // when no saga is in flight should be a no-op.
1219 Event::PanesReaped {
1220 label: "main".into(),
1221 version: 1,
1222 saga_id: None,
1223 },
1224 Event::PoolDrained {
1225 label: "main".into(),
1226 version: 1,
1227 saga_id: None,
1228 },
1229 Event::PoolNotLast {
1230 label: "main".into(),
1231 version: 1,
1232 saga_id: None,
1233 },
1234 ];
1235 for event in cases {
1236 assert!(
1237 match_trigger(&event).is_none(),
1238 "non-trigger event spawned a saga: {:?}",
1239 event
1240 );
1241 }
1242 }
1243
1244 /// Saga lifecycle events on the bus must not feed back into
1245 /// sagas (would cause loops). Sanity-check the filter directly.
1246 #[tokio::test]
1247 async fn coordinator_does_not_self_trigger_on_lifecycle_events() {
1248 let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(64);
1249 let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1250 let coord = Arc::new(SagaCoordinator::new(events_tx.clone(), Arc::clone(&state)));
1251 let coord_rx = events_tx.subscribe();
1252 let handle = tokio::spawn(run_coordinator(Arc::clone(&coord), coord_rx));
1253 tokio::task::yield_now().await;
1254
1255 // Push a SagaStarted-shaped event; if the coordinator
1256 // re-spawned, it would eventually exhaust the saga_id atomic
1257 // by looping. We verify simpler: in_flight stays empty.
1258 let _ = events_tx.send(Event::SagaStarted {
1259 saga_id: 999,
1260 name: "foreign_saga".into(),
1261 version: 1,
1262 });
1263 let _ = events_tx.send(Event::SagaCompleted {
1264 saga_id: 999,
1265 version: 2,
1266 });
1267 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
1268 assert!(coord.in_flight.lock().await.is_empty());
1269 drop(events_tx);
1270 let _ = tokio::time::timeout(std::time::Duration::from_secs(1), handle).await;
1271 }
1272
1273 // ---------- Phase F.7 — saga-lifecycle property tests ----------
1274 //
1275 // Random sequences of `WindowClosed` / `PoolWindowPromoted`
1276 // events fed through `match_trigger`. Asserts:
1277 // 1. `WindowClosed { crash_detected: true }` NEVER produces a
1278 // saga (F.6 round-2 gate, codex P1 #637).
1279 // 2. `WindowClosed { crash_detected: false }` ALWAYS produces
1280 // exactly one window_cleanup_cascade saga.
1281 // 3. `PoolWindowPromoted` ALWAYS produces exactly one
1282 // pool_respawn_on_promote saga.
1283 // 4. The saga's name (the (kind, key) tuple's "kind" component)
1284 // is one of the two known names — no foreign saga slips
1285 // through the trigger table.
1286 //
1287 // The end-to-end coordinator-level "exactly one terminal lifecycle
1288 // event per saga" assertion lives in
1289 // `pool_respawn::tests::coordinator_brackets_promote_with_saga_lifecycle_events`
1290 // and `window_cleanup::tests::coordinator_brackets_close_with_saga_lifecycle_events`
1291 // (already shipped in F.5 + F.6). The proptests here add the
1292 // randomized-input dimension those happy-path tests don't cover.
1293 //
1294 // Cases capped at 64 (default 1024 too slow for CI). Same bound
1295 // the srv reducer's E.7 proptest uses.
1296
1297 use proptest::prelude::*;
1298
1299 /// Trigger event variants the F.5/F.6 sagas can be triggered by,
1300 /// plus crash-detected close (the negative case).
1301 #[derive(Debug, Clone)]
1302 enum F7TriggerEvent {
1303 /// Should spawn `window_cleanup_cascade`.
1304 WindowClosedClean { label: String },
1305 /// Should NOT spawn anything (F.6 round-2 gate).
1306 WindowClosedCrashed { label: String },
1307 /// Should spawn `pool_respawn_on_promote`.
1308 PoolWindowPromoted { label: String },
1309 /// A non-trigger event — must produce no saga.
1310 Unrelated,
1311 }
1312
1313 fn f7_trigger_strategy() -> impl Strategy<Value = F7TriggerEvent> {
1314 prop_oneof![
1315 3 => "[a-c]{1,3}".prop_map(|label| F7TriggerEvent::WindowClosedClean { label }),
1316 1 => "[a-c]{1,3}".prop_map(|label| F7TriggerEvent::WindowClosedCrashed { label }),
1317 3 => "[a-c]{1,3}".prop_map(|label| F7TriggerEvent::PoolWindowPromoted { label }),
1318 2 => Just(F7TriggerEvent::Unrelated),
1319 ]
1320 }
1321
1322 fn make_event(t: F7TriggerEvent, version: u64) -> Event {
1323 match t {
1324 F7TriggerEvent::WindowClosedClean { label } => Event::WindowClosed {
1325 label,
1326 version,
1327 crash_detected: false,
1328 },
1329 F7TriggerEvent::WindowClosedCrashed { label } => Event::WindowClosed {
1330 label,
1331 version,
1332 crash_detected: true,
1333 },
1334 F7TriggerEvent::PoolWindowPromoted { label } => {
1335 Event::PoolWindowPromoted { label, version }
1336 }
1337 F7TriggerEvent::Unrelated => Event::Pong { nonce: 0, version },
1338 }
1339 }
1340
1341 proptest! {
1342 #![proptest_config(ProptestConfig {
1343 cases: 64,
1344 ..ProptestConfig::default()
1345 })]
1346
1347 /// For every event in a random sequence, `match_trigger`
1348 /// returns the expected saga (or None for crash / unrelated).
1349 /// Exhaustive shape check — catches accidental regressions
1350 /// in the trigger table.
1351 #[test]
1352 fn f7_match_trigger_matches_expected_saga_kind(
1353 triggers in prop::collection::vec(f7_trigger_strategy(), 0..40)
1354 ) {
1355 for (i, t) in triggers.into_iter().enumerate() {
1356 let event = make_event(t.clone(), (i + 1) as u64);
1357 let saga = match_trigger(&event);
1358 match (&t, &saga) {
1359 (F7TriggerEvent::WindowClosedClean { .. }, Some(s)) => {
1360 prop_assert_eq!(s.name(), "window_cleanup_cascade");
1361 }
1362 (F7TriggerEvent::PoolWindowPromoted { .. }, Some(s)) => {
1363 prop_assert_eq!(s.name(), "pool_respawn_on_promote");
1364 }
1365 (F7TriggerEvent::WindowClosedCrashed { .. }, None) => {
1366 // Expected — crash-detected close must never
1367 // spawn a saga (F.6 codex P1 #637).
1368 }
1369 (F7TriggerEvent::Unrelated, None) => {
1370 // Expected — non-trigger event.
1371 }
1372 (other_t, other_s) => {
1373 prop_assert!(
1374 false,
1375 "trigger {:?} produced unexpected saga match: {:?}",
1376 other_t,
1377 other_s.as_ref().map(|s| s.name()),
1378 );
1379 }
1380 }
1381 }
1382 }
1383
1384 /// Crash-detected `WindowClosed` events NEVER produce a
1385 /// saga, regardless of label. Reinforces invariant #1
1386 /// independently — a stricter shrinker target.
1387 #[test]
1388 fn f7_crash_detected_close_never_spawns_saga(
1389 label in "[a-c]{1,3}",
1390 version in 1u64..1_000_000,
1391 ) {
1392 let event = Event::WindowClosed { label, version, crash_detected: true };
1393 prop_assert!(
1394 match_trigger(&event).is_none(),
1395 "crash-detected WindowClosed must not spawn a saga",
1396 );
1397 }
1398
1399 /// Clean-close `WindowClosed` ALWAYS produces a
1400 /// window_cleanup_cascade saga, regardless of label.
1401 #[test]
1402 fn f7_clean_close_always_spawns_window_cleanup_cascade(
1403 label in "[a-c]{1,3}",
1404 version in 1u64..1_000_000,
1405 ) {
1406 let event = Event::WindowClosed { label, version, crash_detected: false };
1407 let saga = match_trigger(&event)
1408 .expect("clean-close should spawn a saga");
1409 prop_assert_eq!(saga.name(), "window_cleanup_cascade");
1410 }
1411 }
1412
1413 /// **CPD-4 — concurrent same-kind sagas coexist without
1414 /// cross-talk.** Random sequences of trigger events drive the
1415 /// coordinator; the registry is allowed to hold multiple sagas of
1416 /// the same kind simultaneously (the pre-CPD-4 evict-and-replace
1417 /// gate is retired). Invariant under test: NO `SagaFailed` event
1418 /// fires with reason `"evicted"` — sagas only terminate via
1419 /// `Done` (their tagged echo lands) or via the saga timeout
1420 /// (~5-30s, far longer than this test's 50ms settling window).
1421 ///
1422 /// Replaces `f7_evict_and_replace_keeps_one_saga_per_kind`. The
1423 /// behavioral inversion is the core of CPD-4: pre-CPD-4 we
1424 /// asserted ≤1 in-flight per kind (and accepted premature
1425 /// `SagaFailed { reason: "evicted" }`); CPD-4 asserts no
1426 /// premature evictions.
1427 #[tokio::test]
1428 async fn cpd4_concurrent_sagas_no_eviction() {
1429 // Tiny LCG for deterministic per-seed sequences. Glibc's
1430 // constants — quality irrelevant; we just need reproducible
1431 // bit patterns across CI runs.
1432 fn lcg_next(state: &mut u64) -> u64 {
1433 *state = state
1434 .wrapping_mul(1_103_515_245)
1435 .wrapping_add(12_345);
1436 *state
1437 }
1438
1439 let seeds: &[u64] = &[1, 7, 42, 99, 1234, 5678, 0xDEADBEEF, 0xC0FFEE];
1440 for seed in seeds {
1441 let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(256);
1442 let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1443 let coord = Arc::new(SagaCoordinator::new(events_tx.clone(), Arc::clone(&state)));
1444 let mut witness = events_tx.subscribe();
1445 let coord_rx = events_tx.subscribe();
1446 let _handle = tokio::spawn(run_coordinator(Arc::clone(&coord), coord_rx));
1447 tokio::task::yield_now().await;
1448
1449 // Build a deterministic sequence from the seed.
1450 let mut rng = *seed;
1451 let labels = ["a", "b", "c"];
1452 for v in 1u64..=20 {
1453 let label_idx = (lcg_next(&mut rng) % labels.len() as u64) as usize;
1454 let label = labels[label_idx].to_string();
1455 let pick = lcg_next(&mut rng) % 3;
1456 let event = match pick {
1457 0 => Event::WindowClosed {
1458 label,
1459 version: v,
1460 crash_detected: false,
1461 },
1462 1 => Event::PoolWindowPromoted { label, version: v },
1463 _ => Event::Pong { nonce: v, version: v },
1464 };
1465 let _ = events_tx.send(event);
1466 // Yield so the coordinator processes one event before
1467 // we send the next. Tightens the concurrent-overlap
1468 // window.
1469 tokio::task::yield_now().await;
1470 }
1471 // Brief settling — give the coordinator time to process
1472 // the last few events + emit terminal lifecycles.
1473 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1474
1475 // Invariant: NO SagaFailed with "evicted" reason.
1476 // We don't assert the registry size — concurrent sagas
1477 // are allowed to pile up under per-saga correlation
1478 // because we never feed them their tagged terminal
1479 // events in this test. They'll reach their timeout
1480 // eventually, but not within 50ms.
1481 let mut evictions = 0u32;
1482 while let Ok(Ok(ev)) = tokio::time::timeout(
1483 std::time::Duration::from_millis(5),
1484 witness.recv(),
1485 )
1486 .await
1487 {
1488 if let Event::SagaFailed { reason, .. } = ev {
1489 if reason.contains("evicted") {
1490 evictions += 1;
1491 }
1492 }
1493 }
1494 assert_eq!(
1495 evictions, 0,
1496 "seed {} — CPD-4 invariant violated: observed {} SagaFailed events with reason='evicted'; \
1497 evict-and-replace should be retired so concurrent same-kind sagas coexist",
1498 seed, evictions,
1499 );
1500 }
1501 }
1502
1503 /// Saga forward-path emission contract: every saga that
1504 /// terminates emits exactly ONE terminal lifecycle event
1505 /// (`SagaCompleted` xor `SagaFailed`). Drives a small batch of
1506 /// triggers through the coordinator and counts emitted
1507 /// lifecycle events on the bus. Catches regressions where a
1508 /// saga's `Done` action somehow fires both `emit_completed`
1509 /// and `emit_failed`, or neither.
1510 #[tokio::test]
1511 async fn f7_each_saga_emits_exactly_one_terminal_event() {
1512 let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(256);
1513 let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1514 let coord = Arc::new(SagaCoordinator::new(events_tx.clone(), Arc::clone(&state)));
1515 let mut witness = events_tx.subscribe();
1516 let coord_rx = events_tx.subscribe();
1517 let _handle = tokio::spawn(run_coordinator(Arc::clone(&coord), coord_rx));
1518 tokio::task::yield_now().await;
1519
1520 // Drive ONE clean cleanup-cascade saga + ONE pool-respawn
1521 // saga to completion. CPD-4: route terminals by saga_id, so
1522 // we trigger each saga, capture its allocated id from
1523 // SagaStarted, then send the matching tagged terminal events.
1524 let _ = events_tx.send(Event::WindowClosed {
1525 label: "main".into(),
1526 version: 1,
1527 crash_detected: false,
1528 });
1529 let _ = events_tx.send(Event::PoolWindowPromoted {
1530 label: "pool-1".into(),
1531 version: 2,
1532 });
1533
1534 // Drain the bus with a deadline. Count one terminal event
1535 // (SagaCompleted xor SagaFailed) per saga_id. As each
1536 // SagaStarted lands, dispatch the matching tagged terminals.
1537 let mut started_ids = std::collections::HashSet::<u64>::new();
1538 let mut cleanup_saga_id: Option<u64> = None;
1539 let mut respawn_saga_id: Option<u64> = None;
1540 let mut version: u64 = 100;
1541 let mut terminal_for_id = std::collections::HashMap::<u64, &'static str>::new();
1542 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3);
1543 while std::time::Instant::now() < deadline {
1544 match tokio::time::timeout(std::time::Duration::from_millis(50), witness.recv()).await
1545 {
1546 Ok(Ok(Event::SagaStarted { saga_id, name, .. })) => {
1547 started_ids.insert(saga_id);
1548 if name == "window_cleanup_cascade" && cleanup_saga_id.is_none() {
1549 cleanup_saga_id = Some(saga_id);
1550 version += 1;
1551 let _ = events_tx.send(Event::PanesReaped {
1552 label: "main".into(),
1553 version,
1554 saga_id: Some(saga_id),
1555 });
1556 } else if name == "pool_respawn_on_promote" && respawn_saga_id.is_none() {
1557 respawn_saga_id = Some(saga_id);
1558 version += 1;
1559 let _ = events_tx.send(Event::PoolWindowAdded {
1560 label: "pool-2".into(),
1561 version,
1562 saga_id: Some(saga_id),
1563 });
1564 }
1565 }
1566 Ok(Ok(Event::PanesReaped { saga_id: Some(sid), .. })) => {
1567 if Some(sid) == cleanup_saga_id {
1568 version += 1;
1569 let _ = events_tx.send(Event::PoolDrained {
1570 label: "main".into(),
1571 version,
1572 saga_id: Some(sid),
1573 });
1574 }
1575 }
1576 Ok(Ok(Event::SagaCompleted { saga_id, .. })) => {
1577 let prev = terminal_for_id.insert(saga_id, "completed");
1578 assert!(
1579 prev.is_none(),
1580 "saga_id {} got two terminal events: prior={:?} now=completed",
1581 saga_id,
1582 prev,
1583 );
1584 }
1585 Ok(Ok(Event::SagaFailed { saga_id, .. })) => {
1586 let prev = terminal_for_id.insert(saga_id, "failed");
1587 assert!(
1588 prev.is_none(),
1589 "saga_id {} got two terminal events: prior={:?} now=failed",
1590 saga_id,
1591 prev,
1592 );
1593 }
1594 Ok(Ok(_)) => {}
1595 Ok(Err(_)) => break,
1596 Err(_) => {
1597 // No new events for 50ms — we've drained.
1598 if started_ids.len() == 2
1599 && started_ids.iter().all(|id| terminal_for_id.contains_key(id))
1600 {
1601 break;
1602 }
1603 }
1604 }
1605 }
1606 // Both sagas started.
1607 assert_eq!(
1608 started_ids.len(),
1609 2,
1610 "expected 2 SagaStarted events (one per saga), got {}: {:?}",
1611 started_ids.len(),
1612 started_ids,
1613 );
1614 // Each got exactly one terminal event.
1615 for id in &started_ids {
1616 assert!(
1617 terminal_for_id.contains_key(id),
1618 "saga_id {} never got a terminal lifecycle event",
1619 id,
1620 );
1621 }
1622 // No spurious terminal events for sagas we didn't observe
1623 // start.
1624 for id in terminal_for_id.keys() {
1625 assert!(
1626 started_ids.contains(id),
1627 "saga_id {} got a terminal event but no SagaStarted observed",
1628 id,
1629 );
1630 }
1631 }
1632
1633 // ---------- LSD-2 — coordinator <-> LauncherSagaLog wiring ------
1634 //
1635 // Three end-to-end tests that drive the coordinator with a real
1636 // in-memory `LauncherSagaLog` and assert lifecycle rows land in
1637 // the expected state. Mirror the test inventory pinned in
1638 // `docs/specs/SPEC_LAUNCHER_SAGA_DURABILITY_2026-05-01.md` §4 PR2.
1639 //
1640 // The tests use `pool_respawn::PoolRespawn` because its single-
1641 // step shape (one IssueCmd, one terminal echo) keeps assertions
1642 // tight; the `window_cleanup_cascade` two-step shape is exercised
1643 // implicitly by the existing F.6 coordinator tests + adds a
1644 // multi-step example in `saga_step_finish_records_event_payload`.
1645
1646 use crate::saga::log::LauncherSagaLog;
1647 use std::sync::Arc as StdArc;
1648
1649 /// Helper: spin up coordinator + in-memory log.
1650 fn spawn_coord_with_log()
1651 -> (StdArc<SagaCoordinator>, StdArc<LauncherSagaLog>, tokio::sync::broadcast::Sender<Event>)
1652 {
1653 let (events_tx, _) = tokio::sync::broadcast::channel::<Event>(256);
1654 let state = Arc::new(tokio::sync::Mutex::new(crate::state::State::default()));
1655 let log = StdArc::new(LauncherSagaLog::open_in_memory().expect("in-memory log"));
1656 let coord = StdArc::new(
1657 SagaCoordinator::new(events_tx.clone(), Arc::clone(&state))
1658 .with_log(StdArc::clone(&log))
1659 .expect("with_log on fresh in-memory log should succeed"),
1660 );
1661 (coord, log, events_tx)
1662 }
1663
1664 /// LSD-2 — drive a saga (PoolRespawn) through to completion and
1665 /// verify the durable log records:
1666 /// - `launcher_saga` row with state='completed', non-null
1667 /// ended_at, no failure_reason, input_json round-trips the
1668 /// `promoted_label`.
1669 /// - one step row in 'succeeded' state with output_json
1670 /// populated.
1671 #[tokio::test]
1672 async fn saga_completes_writes_lifecycle_to_log() {
1673 let (coord, log, events_tx) = spawn_coord_with_log();
1674 let mut witness = events_tx.subscribe();
1675 let coord_rx = events_tx.subscribe();
1676 let _handle = tokio::spawn(run_coordinator(StdArc::clone(&coord), coord_rx));
1677 tokio::task::yield_now().await;
1678
1679 // Trigger the saga.
1680 let _ = events_tx.send(Event::PoolWindowPromoted {
1681 label: "window-pool-abc".into(),
1682 version: 1,
1683 });
1684 // CPD-4: wait for SagaStarted to learn the saga_id, then send
1685 // the awaited terminal tagged with that id. Pre-CPD-4 we
1686 // could send `saga_id: None`; under per-saga correlation
1687 // the saga only consumes its own echo.
1688 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
1689 while std::time::Instant::now() < deadline {
1690 if let Ok(Ok(Event::SagaStarted { saga_id, .. })) =
1691 tokio::time::timeout(std::time::Duration::from_millis(50), witness.recv()).await
1692 {
1693 let _ = events_tx.send(Event::PoolWindowAdded {
1694 label: "window-pool-xyz".into(),
1695 version: 2,
1696 saga_id: Some(saga_id),
1697 });
1698 break;
1699 }
1700 }
1701
1702 // Settle: coordinator runs apply_action which awaits state
1703 // mutex + bus send. 200ms is generous for the in-process loop.
1704 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1705
1706 let snapshots = log.snapshot_recent(10).expect("snapshot_recent");
1707 assert_eq!(snapshots.len(), 1, "expected exactly one saga row");
1708 let s = &snapshots[0];
1709 assert_eq!(s.name, "pool_respawn_on_promote");
1710 assert_eq!(s.state, "completed");
1711 assert!(s.ended_at.is_some(), "ended_at must be set on completion");
1712 assert!(s.failure_reason.is_none());
1713 assert_eq!(s.step_count, 1, "expected exactly one succeeded step");
1714 // Input snapshot round-trips the saga's constructor arg.
1715 let parsed: serde_json::Value = serde_json::from_str(&s.input_json).unwrap();
1716 assert_eq!(parsed["promoted_label"], "window-pool-abc");
1717 }
1718
1719 /// **CPD-4 — two concurrent same-kind sagas BOTH complete
1720 /// cleanly.** Repurposed from `saga_fails_writes_failed_state`
1721 /// (which previously verified the evict-and-replace path).
1722 /// Per-saga event correlation lets two `pool_respawn_on_promote`
1723 /// sagas coexist; each consumes its own saga_id-tagged refill
1724 /// echo, so the durable log records two `completed` rows — no
1725 /// `failed` rows from the retired eviction policy.
1726 #[tokio::test]
1727 async fn saga_concurrent_same_kind_both_complete() {
1728 let (coord, log, events_tx) = spawn_coord_with_log();
1729 let mut witness = events_tx.subscribe();
1730 let coord_rx = events_tx.subscribe();
1731 let _handle = tokio::spawn(run_coordinator(StdArc::clone(&coord), coord_rx));
1732 tokio::task::yield_now().await;
1733
1734 // First promote — kicks off saga A.
1735 let _ = events_tx.send(Event::PoolWindowPromoted {
1736 label: "window-pool-a".into(),
1737 version: 1,
1738 });
1739 // Second promote BEFORE A's terminal arrives — under CPD-4
1740 // saga B coexists with A (no eviction).
1741 let _ = events_tx.send(Event::PoolWindowPromoted {
1742 label: "window-pool-b".into(),
1743 version: 2,
1744 });
1745
1746 // Capture both saga_ids from their `SagaStarted` brackets,
1747 // then fire each saga's tagged refill so both terminate
1748 // Completed. The order doesn't matter because per-saga
1749 // correlation routes events by id, not by FIFO.
1750 let mut ids = Vec::<u64>::new();
1751 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
1752 let mut version: u64 = 100;
1753 while std::time::Instant::now() < deadline {
1754 if let Ok(Ok(ev)) =
1755 tokio::time::timeout(std::time::Duration::from_millis(50), witness.recv()).await
1756 {
1757 if let Event::SagaStarted { saga_id, name, .. } = ev {
1758 if name == "pool_respawn_on_promote" {
1759 ids.push(saga_id);
1760 version += 1;
1761 let _ = events_tx.send(Event::PoolWindowAdded {
1762 label: format!("refill-for-{}", saga_id),
1763 version,
1764 saga_id: Some(saga_id),
1765 });
1766 }
1767 }
1768 if ids.len() == 2 {
1769 break;
1770 }
1771 }
1772 }
1773 assert_eq!(ids.len(), 2, "expected 2 SagaStarted events");
1774
1775 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1776
1777 let snapshots = log.snapshot_recent(10).expect("snapshot_recent");
1778 assert_eq!(
1779 snapshots.len(),
1780 2,
1781 "expected two saga rows (concurrent, no eviction)"
1782 );
1783 // BOTH must be 'completed' — eviction is retired.
1784 for s in &snapshots {
1785 assert_eq!(
1786 s.state, "completed",
1787 "CPD-4: both concurrent sagas must complete cleanly, got state={:?} reason={:?}",
1788 s.state, s.failure_reason,
1789 );
1790 assert!(s.failure_reason.is_none());
1791 }
1792 }
1793
1794 /// LSD-2 — when a saga's `on_event` consumes its awaited bus
1795 /// event, the coordinator calls `LauncherSagaLog::finish_step`
1796 /// with the event payload. To inspect `output_json` directly via
1797 /// the public LSD-1 API, we drive a saga only PART of the way —
1798 /// the cleanup cascade's Step 1 lands its echo (`PanesReaped`)
1799 /// but Step 2 is left in `pending` so the saga stays unresolved.
1800 /// `unresolved_sagas` then exposes the full step list including
1801 /// Step 0's `output_json`.
1802 #[tokio::test]
1803 async fn saga_step_finish_records_event_payload() {
1804 let (coord, log, events_tx) = spawn_coord_with_log();
1805 let mut witness = events_tx.subscribe();
1806 let coord_rx = events_tx.subscribe();
1807 let _handle = tokio::spawn(run_coordinator(StdArc::clone(&coord), coord_rx));
1808 tokio::task::yield_now().await;
1809
1810 // Trigger the cascade. Window-cleanup has two steps; we feed
1811 // only the first echo (`PanesReaped`) so Step 0 transitions
1812 // pending→succeeded but Step 1 (`DrainPoolIfLast`) stays
1813 // pending — the saga remains in_flight + queryable as
1814 // unresolved.
1815 //
1816 // CPD-4: tag PanesReaped with the saga's id so the saga
1817 // actually advances (per-saga correlation).
1818 let _ = events_tx.send(Event::WindowClosed {
1819 label: "main".into(),
1820 version: 1,
1821 crash_detected: false,
1822 });
1823 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
1824 while std::time::Instant::now() < deadline {
1825 if let Ok(Ok(Event::SagaStarted { saga_id, .. })) =
1826 tokio::time::timeout(std::time::Duration::from_millis(50), witness.recv()).await
1827 {
1828 let _ = events_tx.send(Event::PanesReaped {
1829 label: "main".into(),
1830 version: 2,
1831 saga_id: Some(saga_id),
1832 });
1833 break;
1834 }
1835 }
1836 // INTENTIONALLY no `PoolDrained` / `PoolNotLast`.
1837
1838 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1839
1840 let unresolved = log.unresolved_sagas().expect("unresolved_sagas");
1841 assert_eq!(unresolved.len(), 1, "expected one unresolved saga");
1842 let saga = &unresolved[0];
1843 assert_eq!(saga.name, "window_cleanup_cascade");
1844 assert_eq!(saga.state, "running");
1845 // Two step rows: Step 0 succeeded (PanesReaped echo), Step 1
1846 // pending (DrainPoolIfLast dispatched but no echo arrived).
1847 assert_eq!(saga.steps.len(), 2, "expected 2 step rows");
1848 let step0 = &saga.steps[0];
1849 assert_eq!(step0.step_index, 0);
1850 assert_eq!(step0.state, "succeeded");
1851 assert!(step0.ended_at.is_some(), "succeeded step needs ended_at");
1852 let output_json = step0
1853 .output_json
1854 .as_ref()
1855 .expect("output_json must be set after finish_step");
1856 // Round-trip — the JSON should deserialize to the awaited Event.
1857 let parsed: Event = serde_json::from_str(output_json).expect("event round-trips");
1858 match parsed {
1859 Event::PanesReaped { label, .. } => assert_eq!(label, "main"),
1860 other => panic!("expected PanesReaped, got {:?}", other),
1861 }
1862 let step1 = &saga.steps[1];
1863 assert_eq!(step1.step_index, 1);
1864 assert_eq!(step1.state, "pending");
1865 assert!(step1.output_json.is_none(), "pending step has no output");
1866 }
1867}