agentmux_srv/server/websocket.rs
1// Copyright 2025-2026, AgentMux Corp.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::sync::Arc;
5
6use axum::{
7 extract::{
8 ws::{Message, WebSocket},
9 State, WebSocketUpgrade,
10 },
11 response::Response,
12};
13use base64::Engine as _;
14use serde::Deserialize;
15use serde_json::json;
16
17use crate::backend::background_task_feed::publish_background_task_updated;
18use crate::backend::blockcontroller;
19use crate::backend::rpc::engine::WshRpcEngine;
20use crate::backend::rpc_types::{
21 AiRateLimitResult, AppInfoResult, NoArgsReq,
22 CommandBlockInputData, CommandControllerResyncData, CommandCreateSubBlockData,
23 CommandDeleteSubBlockData, CommandEventReadHistoryData,
24 CommandGetMetaData, CommandSetMetaData, CommandToolDecisionData,
25 RpcMessage, COMMAND_CONTROLLER_INPUT,
26 COMMAND_CONTROLLER_RESYNC, COMMAND_CREATE_SUB_BLOCK, COMMAND_DELETE_SUB_BLOCK,
27 COMMAND_EVENT_READ_HISTORY, COMMAND_EVENT_SUB, COMMAND_EVENT_UNSUB,
28 COMMAND_EVENT_UNSUB_ALL, COMMAND_GET_FULL_CONFIG, COMMAND_GET_META,
29 COMMAND_GET_AI_RATE_LIMIT, COMMAND_ROUTE_ANNOUNCE, COMMAND_ROUTE_UNANNOUNCE,
30 COMMAND_SET_META, COMMAND_SET_CONFIG, COMMAND_APP_INFO,
31 COMMAND_TOOL_DECISION, COMMAND_AGENT_ANSWER, COMMAND_AGENT_CANCEL,
32 COMMAND_AGENT_SHUTDOWN_KEEP, CommandAgentShutdownKeepData, AgentShutdownKeepResult,
33 CommandAgentAnswerData, CommandAgentCancelData,
34 COMMAND_AMBIENT_NARRATE, CommandAmbientNarrateData,
35 COMMAND_DOCK_NODE_STATUS, CommandDockNodeStatusData,
36 COMMAND_BACKGROUND_TASK_COMPLETION, CommandBackgroundTaskCompletionData,
37 COMMAND_BACKGROUND_TASK_PID, CommandBackgroundTaskPidData,
38 COMMAND_LIST_BACKGROUND_TASKS, CommandListBackgroundTasksData,
39};
40use crate::backend::base::normalize_working_dir;
41use crate::backend::obj::{Block, TermSize, MuxObjUpdate, mux_obj_to_value};
42use super::service::{update_object_meta, schedule_agent_zoom_mirror};
43
44use super::AppState;
45
46/// Incoming WebSocket message envelope.
47/// Supports both ping/pong messages and wscommand-based RPC.
48#[derive(Deserialize)]
49struct WSIncoming {
50 #[serde(rename = "type")]
51 msg_type: Option<String>,
52 #[allow(dead_code)]
53 stime: Option<i64>,
54 wscommand: Option<String>,
55 message: Option<RpcMessage>,
56 // Fields for setblocktermsize / blockinput
57 blockid: Option<String>,
58 inputdata64: Option<String>,
59 termsize: Option<serde_json::Value>,
60 // Fields for bus:* commands
61 agent_id: Option<String>,
62 from: Option<String>,
63 to: Option<String>,
64 target: Option<String>,
65 payload: Option<String>,
66 #[serde(rename = "bus_message")]
67 bus_message_text: Option<String>,
68 priority: Option<String>,
69}
70
71/// The `srvinfo` event sent once on every WebSocket connect, right after
72/// `config`: srv's version, which the UI compares with its own, and the
73/// machine srv runs on (`crate::srv_info`).
74fn srv_info_event(info: crate::srv_info::SrvInfo) -> serde_json::Value {
75 crate::backend::eventbus::EventBus::typed_frame::<crate::backend::mps_events::SrvInfo>(&info)
76}
77
78pub(super) async fn handle_ws(
79 State(state): State<AppState>,
80 ws: WebSocketUpgrade,
81) -> Response {
82 ws.on_upgrade(move |socket| handle_ws_connection(socket, state))
83}
84
85async fn handle_ws_connection(mut socket: WebSocket, state: AppState) {
86 let ws_start = std::time::Instant::now();
87 let conn_id = uuid::Uuid::new_v4().to_string();
88 let tab_id = String::new();
89
90 tracing::info!(conn_id = %conn_id, "WebSocket client connected");
91
92 // Two egress lanes per connection: interactive (terminal echo, RPC-routed
93 // wave events, obj updates) and background (droppable perf telemetry —
94 // sysinfo/blockstats). The select! below drains priority before background
95 // so typing never waits behind a perf tick. See
96 // docs/specs/SPEC_TERMINAL_INPUT_PRIORITY_OVER_SYSINFO_2026_06_16.md.
97 let crate::backend::eventbus::WsReceivers {
98 priority: mut priority_rx,
99 background: mut background_rx,
100 } = state.event_bus.register_ws(&conn_id, &tab_id);
101 tracing::info!("[ws-perf] register_ws: {:.2}ms", ws_start.elapsed().as_secs_f64() * 1000.0);
102
103 // Optional messagebus receiver — activated when pane sends bus:register
104 let mut bus_rx: Option<tokio::sync::mpsc::UnboundedReceiver<crate::backend::messagebus::BusMessage>> = None;
105 let mut bus_agent_id: Option<String> = None;
106
107 // Send initial "config" wave event via the RPC eventrecv path so the frontend
108 // populates fullConfigAtom (and shows the widget bar).
109 // Frontend only processes events via: {"eventtype":"rpc","data":{"command":"eventrecv","data":{"event":"config","data":{...}}}}
110 {
111 let t = std::time::Instant::now();
112 let config = state.config_watcher.get_full_config();
113 if let Ok(mut config_val) = serde_json::to_value(config.as_ref()) {
114 crate::backend::wconfig::redact_full_config_for_renderer(&mut config_val);
115 let config_event = crate::backend::eventbus::EventBus::typed_frame::<crate::backend::mps_events::Config>(&crate::backend::mps_events::ConfigEvent { fullconfig: config_val });
116 if let Ok(msg) = serde_json::to_string(&config_event) {
117 let _ = socket.send(Message::Text(msg.into())).await;
118 }
119 }
120 tracing::info!("[ws-perf] send_initial_config: {:.2}ms", t.elapsed().as_secs_f64() * 1000.0);
121 }
122
123 // Tell the client which srv it reached: the version, so a UI from another
124 // version (an update replaced one side) can ask the user to reload, and
125 // the machine, whose names and paths the UI shows and builds.
126 let info = crate::srv_info::srv_info(&state.version, &state.hostname);
127 if let Ok(msg) = serde_json::to_string(&srv_info_event(info)) {
128 let _ = socket.send(Message::Text(msg.into())).await;
129 }
130
131 // Create RPC engine for this connection
132 let t = std::time::Instant::now();
133 let (engine, mut rpc_output_rx) = WshRpcEngine::new();
134
135 // Register handlers
136 register_handlers(&engine, state.clone(), conn_id.clone());
137 tracing::info!("[ws-perf] create_engine+register_handlers: {:.2}ms", t.elapsed().as_secs_f64() * 1000.0);
138 tracing::info!("[ws-perf] TOTAL ws_setup: {:.2}ms", ws_start.elapsed().as_secs_f64() * 1000.0);
139
140 // Periodic ping interval (10 seconds)
141 let mut ping_interval = tokio::time::interval(std::time::Duration::from_secs(10));
142 ping_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
143
144 // Fairly-ordered priority-lane events awaiting their turn to be sent, one
145 // per `'ws` loop iteration (see the priority-lane branch below and the
146 // pending-priority-flush branch after it). Reagent P1 on this PR's first
147 // revision: forwarding an entire drained batch inside one `select!`
148 // branch body — even a fairly-ordered one — starves `socket.recv()`
149 // (this connection's OWN incoming keystrokes) for the whole batch, since
150 // `select!` only re-polls sibling branches after the chosen branch's body
151 // returns. Trickling the batch out one event per iteration keeps every
152 // send interleaved with a fresh `socket.recv()` poll, exactly like the
153 // pre-fix one-event-per-iteration cadence, while still applying the fair
154 // pane ordering to however the batch is drained.
155 let mut pending_priority: std::collections::VecDeque<serde_json::Value> =
156 std::collections::VecDeque::new();
157
158 'ws: loop {
159 tokio::select! {
160 // Biased: poll branches top-to-bottom so interactive terminal I/O
161 // always wins over droppable perf telemetry. Order = incoming
162 // keystrokes → RPC replies → agent bus → priority events (terminal
163 // echo) → background events (sysinfo/blockstats) → keepalive ping.
164 // See docs/specs/SPEC_TERMINAL_INPUT_PRIORITY_OVER_SYSINFO_2026_06_16.md.
165 biased;
166
167 // Incoming WebSocket messages → parse & dispatch (keystrokes first)
168 msg = socket.recv() => {
169 match msg {
170 Some(Ok(Message::Close(_))) | None => break,
171 Some(Ok(Message::Ping(data))) => {
172 let _ = socket.send(Message::Pong(data)).await;
173 }
174 Some(Ok(Message::Text(text))) => {
175 match handle_incoming_text(&text, &engine, &state, &mut socket).await {
176 Err(true) => break,
177 Ok(Some((new_rx, agent_id))) => {
178 // bus:register returned a new receiver
179 bus_rx = Some(new_rx);
180 bus_agent_id = Some(agent_id);
181 }
182 _ => {}
183 }
184 }
185 Some(Ok(_)) => {
186 // Binary or other message types — ignore
187 }
188 Some(Err(_)) => break,
189 }
190 }
191
192 // Forward RPC engine output → WebSocket (wrapped as eventtype:rpc)
193 Some(rpc_msg) = rpc_output_rx.recv() => {
194 let wrapped = json!({
195 "eventtype": "rpc",
196 "data": rpc_msg,
197 });
198 let msg = serde_json::to_string(&wrapped).unwrap_or_default();
199 if socket.send(Message::Text(msg.into())).await.is_err() {
200 break;
201 }
202 }
203
204 // Forward MessageBus messages → WebSocket (if registered as agent)
205 Some(bus_msg) = async {
206 match bus_rx.as_mut() {
207 Some(rx) => rx.recv().await,
208 None => std::future::pending().await,
209 }
210 } => {
211 let wrapped = json!({
212 "type": "bus:message",
213 "data": bus_msg,
214 });
215 let msg = serde_json::to_string(&wrapped).unwrap_or_default();
216 if socket.send(Message::Text(msg.into())).await.is_err() {
217 break;
218 }
219 }
220
221 // Priority event lane → WebSocket. Two sources feed it:
222 // 1. MPS Broker (via EventBusBridge) — already wrapped as
223 // { eventtype: "rpc", data: { command: "eventrecv", data: MuxEvent } }
224 // 2. Direct broadcasts (e.g., SetMeta's obj:update) — raw
225 // { eventtype: "waveobj:update", oref: "block:xxx", data: ... }
226 // This carries terminal echo output and all interactive events.
227 //
228 // Fairness: this lane is shared by EVERY pane on the connection, not
229 // just one. A single FIFO here means a noisy pane (heavy PTY output)
230 // can queue an unbounded number of frames ahead of a different pane's
231 // own keystroke echo, which arrives in the same channel. Draining all
232 // immediately-available events and round-robining them by pane (see
233 // `fair_drain_priority`) bounds that delay to one "round" per distinct
234 // active pane instead of the length of the noisy pane's backlog. See
235 // docs/analysis/ANALYSIS_CROSS_PANE_INPUT_DELAY_UNDER_OUTPUT_LOAD_2026_09_04.md.
236 //
237 // Only drains a fresh batch when `pending_priority` is empty — while
238 // it's still being flushed (one per iteration, below), this branch
239 // stays disabled so we don't pull more out of `priority_rx` than
240 // we've finished sending.
241 Some(event) = priority_rx.recv(), if pending_priority.is_empty() => {
242 let mut fair = fair_drain_priority(event, &mut priority_rx);
243 // `fair` always has >= 1 element (it contains at least `event`).
244 let first = fair.remove(0);
245 pending_priority.extend(fair);
246 if forward_event(&mut socket, first).await {
247 break 'ws;
248 }
249 }
250
251 // Flush one fairly-ordered priority event per iteration. A no-op
252 // branch guarded on `pending_priority` being non-empty — see the
253 // comment on `pending_priority`'s declaration above for why this
254 // is its own branch instead of a loop in the arm above.
255 _ = std::future::ready(()), if !pending_priority.is_empty() => {
256 let ev = pending_priority.pop_front().expect("guarded non-empty");
257 if forward_event(&mut socket, ev).await {
258 break 'ws;
259 }
260 }
261
262 // Background event lane → WebSocket. Droppable perf telemetry
263 // (sysinfo/blockstats) only; serviced when the priority lanes above
264 // are momentarily idle, so it can never delay a keystroke echo.
265 //
266 // Tradeoff of `biased;`: a SUSTAINED priority-lane flood (e.g.
267 // `! yes` pumping terminal output continuously) keeps the priority
268 // branches ready, so this lane is starved and events queue here
269 // until the flood pauses, then flush as a stale burst. As of B.2
270 // (SPEC_MEMORY_COMMIT_ATTRIBUTION_CORRECTION_2026_07_02) this is a
271 // bounded `mpsc::channel(BACKGROUND_LANE_CAPACITY)` with
272 // `try_send` drop-on-full semantics (see eventbus.rs), not an
273 // unbounded receiver — a starvation period long enough to fill
274 // all 256 slots drops each new event as it arrives (the queued
275 // events already in the channel are unaffected) rather than
276 // growing commit without limit. Acceptable for now because telemetry
277 // ingress is low-rate (sysinfo ~1/s) and a flood also saturates the
278 // socket writes — during which the priority lanes do drain, so this
279 // lane gets serviced — making true never-empty starvation the rare
280 // Phase 2: coalesce all immediately-available background events to the
281 // latest reading per (event-name:scope) key before forwarding. During
282 // sustained priority-lane activity, sysinfo/blockstats queue here; without
283 // coalescing they flush as a burst of stale frames that the browser
284 // processes back-to-back, delaying queued keypress events. Coalescing
285 // bounds the flush to O(distinct event×scope pairs) — at most 1 sysinfo +
286 // 1 blockstats per open block — regardless of how long the lane was starved.
287 // See SPEC_TERMINAL_INPUT_PRIORITY_OVER_SYSINFO_2026_06_16.md §4.2.
288 Some(first_bg) = background_rx.recv() => {
289 let coalesced = coalesce_background(first_bg, &mut background_rx);
290 for ev in coalesced {
291 if forward_event(&mut socket, ev).await {
292 break 'ws;
293 }
294 }
295 }
296
297 // Periodic ping
298 _ = ping_interval.tick() => {
299 let now = agentmux_common::time::now_ms();
300 let ping = json!({ "type": "ping", "stime": now });
301 let msg = serde_json::to_string(&ping).unwrap_or_default();
302 if socket.send(Message::Text(msg.into())).await.is_err() {
303 break;
304 }
305 }
306 }
307 }
308
309 tracing::info!(conn_id = %conn_id, "WebSocket client disconnected");
310 state.event_bus.unregister_ws(&conn_id);
311 state.broker.unsubscribe_all(&conn_id);
312 if let Some(r) = crate::backend::notify::router::get(&state.broker) {
313 r.disconnect(&conn_id);
314 }
315
316 // Unregister from messagebus if this connection was an agent
317 if let Some(ref agent_id) = bus_agent_id {
318 state.messagebus.unregister(agent_id);
319 }
320}
321
322/// Forward a queued event-bus value to the WebSocket. Returns `true` if the
323/// send failed (the caller should break the loop).
324///
325/// Two shapes arrive: already-RPC-wrapped values (from the MPS broker via
326/// EventBusBridge) are forwarded as-is; raw event-bus values (e.g. SetMeta's
327/// `waveobj:update`) are wrapped as an RPC `eventrecv` so the frontend
328/// WshRouter routes them to handleMuxEvent → updateMuxObject. Shared by both
329/// the priority and background egress lanes.
330async fn forward_event(socket: &mut WebSocket, event: serde_json::Value) -> bool {
331 let msg = if event["eventtype"] == "rpc" {
332 // Already an RPC message (from MPS broker via EventBusBridge)
333 serde_json::to_string(&event).unwrap_or_default()
334 } else {
335 // Raw event bus event — wrap as RPC eventrecv
336 // Re-wraps a raw event-bus event into the frame the renderer reads. It is typed where it
337 // was built (`EventBus::typed_ws_event`); check-untyped-rpc-handlers counts any raw event
338 // built by hand.
339 let mux_event = json!({
340 "event": event["eventtype"],
341 "scopes": [event["oref"]],
342 "data": event["data"],
343 });
344 let wrapped = json!({
345 "eventtype": "rpc",
346 "data": {
347 // untyped-relay: forwards an event built elsewhere, as it was built.
348 "command": "eventrecv",
349 "data": mux_event,
350 },
351 });
352 serde_json::to_string(&wrapped).unwrap_or_default()
353 };
354 socket.send(Message::Text(msg.into())).await.is_err()
355}
356
357/// Extract a coalescing key from a background-lane event.
358/// Background events are RPC-wrapped MPS events:
359/// { "eventtype": "rpc", "data": { "command": "eventrecv", "data": { "event": "sysinfo", "scopes": ["local"] } } }
360/// Key = "<event-name>:<first-scope>", e.g. "sysinfo:local" or "blockstats:block:abc123".
361/// Falls back to "_:_" for events that don't match the expected shape (safe: they coalesce
362/// to one slot, which is acceptable because non-sysinfo/blockstats never reach this lane).
363fn background_event_key(event: &serde_json::Value) -> String {
364 let inner = event.get("data").and_then(|d| d.get("data"));
365 let event_name = inner
366 .and_then(|d| d.get("event"))
367 .and_then(|e| e.as_str())
368 .unwrap_or("_");
369 let scope = inner
370 .and_then(|d| d.get("scopes"))
371 .and_then(|s| s.get(0))
372 .and_then(|s| s.as_str())
373 .unwrap_or("_");
374 format!("{event_name}:{scope}")
375}
376
377/// Drain all immediately-available events from `rx` (including `first`) and
378/// coalesce them to the latest reading per (event-name:scope) key. Iteration
379/// order in the returned Vec is first-seen (stable across coalescing rounds).
380fn coalesce_background(
381 first: serde_json::Value,
382 rx: &mut tokio::sync::mpsc::Receiver<serde_json::Value>,
383) -> Vec<serde_json::Value> {
384 let mut order: Vec<String> = Vec::new();
385 let mut map: std::collections::HashMap<String, serde_json::Value> = std::collections::HashMap::new();
386 let key = background_event_key(&first);
387 order.push(key.clone());
388 map.insert(key, first);
389 while let Ok(next) = rx.try_recv() {
390 let k = background_event_key(&next);
391 if !map.contains_key(&k) {
392 order.push(k.clone());
393 }
394 map.insert(k, next);
395 }
396 order.into_iter().filter_map(|k| map.remove(&k)).collect()
397}
398
399/// Sentinel key for every priority-lane event that is NOT a `blockfile`
400/// (terminal/PTY output) event. Not a valid `block:<id>` scope string, so it
401/// can never collide with a real pane key.
402const NON_PANE_EVENT_KEY: &str = "\u{0}non-pane-event";
403
404/// Extract a fairness key so a flood of PTY output from one pane can't queue
405/// an unbounded number of frames ahead of a different pane's own keystroke
406/// echo sitting in the same lane.
407///
408/// Reordering is scoped to `blockfile` events ONLY — that's the one event
409/// type a single pane can flood the shared lane with, and PTY bytes from
410/// different panes are causally independent (only same-pane ordering
411/// matters, which each key's own FIFO group preserves). Every OTHER event
412/// type on this lane (`waveobj:update`, `waveobj:batchedupdates`, the initial
413/// `config` push, RPC replies) returns the same `NON_PANE_EVENT_KEY` and so
414/// stays in ONE FIFO group, never reordered relative to each other.
415///
416/// This restriction is load-bearing, not a simplification: a `waveobj:update`
417/// for object X and a LATER `waveobj:batchedupdates` that also touches X must
418/// never be reordered relative to each other, but an unscoped batch has no
419/// single object id to key on — an earlier version of this function keyed
420/// updates by their individual `oref`/scope and left unscoped batches in
421/// their own group, which let round-robin interleave `[update, delete]` into
422/// `[update, delete, update]`-style reorderings; the frontend doesn't
423/// version-guard deletes, so a resurrected-then-redeleted object could render
424/// as resurrected until reload. See the review discussion on the PR that
425/// introduced this function.
426fn priority_pane_key(event: &serde_json::Value) -> Option<String> {
427 if event["eventtype"] != "rpc" {
428 // Raw (non-rpc) events on this lane are always object-model
429 // broadcasts (e.g. SetMeta's `waveobj:update`), never blockfile —
430 // see `forward_event`'s shape comment.
431 return Some(NON_PANE_EVENT_KEY.to_string());
432 }
433 let inner = event.get("data").and_then(|d| d.get("data"));
434 let is_blockfile = inner.and_then(|d| d.get("event")).and_then(|e| e.as_str())
435 == Some(crate::backend::mps::EVENT_BLOCK_FILE);
436 if !is_blockfile {
437 return Some(NON_PANE_EVENT_KEY.to_string());
438 }
439 inner
440 .and_then(|d| d.get("scopes"))
441 .and_then(|s| s.get(0))
442 .and_then(|s| s.as_str())
443 .map(|s| s.to_string())
444 .or_else(|| Some(NON_PANE_EVENT_KEY.to_string()))
445}
446
447fn enqueue_priority_event(
448 order: &mut Vec<String>,
449 groups: &mut std::collections::HashMap<String, std::collections::VecDeque<serde_json::Value>>,
450 event: serde_json::Value,
451) {
452 let key = priority_pane_key(&event).unwrap_or_default();
453 if !groups.contains_key(&key) {
454 order.push(key.clone());
455 }
456 groups.entry(key).or_default().push_back(event);
457}
458
459/// Drain all immediately-available events from `rx` (including `first`) and
460/// interleave them round-robin by pane (`priority_pane_key`), instead of
461/// forwarding in raw arrival order. Without this, a pane producing output
462/// faster than the socket can flush it fills this same lane with its own
463/// frames, and a different pane's keystroke-echo frame — queued in the same
464/// FIFO — waits behind the entire backlog. Round-robining bounds that wait to
465/// one slot per OTHER distinct pane with events currently queued, regardless
466/// of how deep any single pane's backlog is.
467///
468/// Ordering guarantee: events for the SAME pane are never reordered relative
469/// to each other (each pane's VecDeque is FIFO); only the INTERLEAVING across
470/// different panes changes.
471fn fair_drain_priority(
472 first: serde_json::Value,
473 rx: &mut tokio::sync::mpsc::Receiver<serde_json::Value>,
474) -> Vec<serde_json::Value> {
475 let mut order: Vec<String> = Vec::new();
476 let mut groups: std::collections::HashMap<String, std::collections::VecDeque<serde_json::Value>> =
477 std::collections::HashMap::new();
478
479 enqueue_priority_event(&mut order, &mut groups, first);
480 while let Ok(next) = rx.try_recv() {
481 enqueue_priority_event(&mut order, &mut groups, next);
482 }
483
484 // Common case — only one pane (or only one event) queued this round.
485 // Skip the round-robin bookkeeping and preserve arrival order directly.
486 if order.len() <= 1 {
487 return order
488 .into_iter()
489 .filter_map(|k| groups.remove(&k))
490 .flat_map(|q| q.into_iter())
491 .collect();
492 }
493
494 let total: usize = groups.values().map(|q| q.len()).sum();
495 let mut out = Vec::with_capacity(total);
496 loop {
497 let mut progressed = false;
498 for key in &order {
499 if let Some(q) = groups.get_mut(key) {
500 if let Some(ev) = q.pop_front() {
501 out.push(ev);
502 progressed = true;
503 }
504 }
505 }
506 if !progressed {
507 break;
508 }
509 }
510 out
511}
512
513/// Handle an incoming text message.
514/// Returns Err(true) if the socket send failed.
515/// Returns Ok(Some((rx, agent_id))) if a bus:register was processed.
516async fn handle_incoming_text(
517 text: &str,
518 engine: &Arc<WshRpcEngine>,
519 state: &AppState,
520 socket: &mut WebSocket,
521) -> Result<Option<(tokio::sync::mpsc::UnboundedReceiver<crate::backend::messagebus::BusMessage>, String)>, bool> {
522 let incoming: WSIncoming = match serde_json::from_str(text) {
523 Ok(v) => v,
524 Err(e) => {
525 tracing::warn!("ws: invalid JSON: {}", e);
526 return Ok(None);
527 }
528 };
529
530 // Handle ping/pong by type field
531 if let Some(ref msg_type) = incoming.msg_type {
532 match msg_type.as_str() {
533 "ping" => {
534 let now = agentmux_common::time::now_ms();
535 let pong = json!({ "type": "pong", "stime": now });
536 let msg = serde_json::to_string(&pong).unwrap_or_default();
537 if socket.send(Message::Text(msg.into())).await.is_err() {
538 return Err(true);
539 }
540 return Ok(None);
541 }
542 "pong" => {
543 return Ok(None);
544 }
545 "bus:register" => {
546 if let Some(ref agent_id) = incoming.agent_id {
547 let rx = state.messagebus.register(agent_id, "websocket");
548 // Stamp agent_id on the RPC context so MuxAPI S1 checks work.
549 engine.set_rpc_context(crate::backend::rpc_types::RpcContext {
550 agent_id: agent_id.clone(),
551 ..Default::default()
552 });
553 let ack = json!({ "type": "bus:registered", "agent_id": agent_id });
554 let msg = serde_json::to_string(&ack).unwrap_or_default();
555 if socket.send(Message::Text(msg.into())).await.is_err() {
556 return Err(true);
557 }
558 return Ok(Some((rx, agent_id.clone())));
559 }
560 return Ok(None);
561 }
562 "bus:send" => {
563 if let (Some(ref from), Some(ref to), Some(ref payload)) =
564 (&incoming.from, &incoming.to, &incoming.payload)
565 {
566 let priority = match incoming.priority.as_deref() {
567 Some("high") => crate::backend::messagebus::Priority::High,
568 Some("urgent") => crate::backend::messagebus::Priority::Urgent,
569 _ => crate::backend::messagebus::Priority::Normal,
570 };
571 let bus_msg = crate::backend::messagebus::BusMessage::new(
572 from, to, crate::backend::messagebus::MessageType::Send, payload, priority,
573 );
574 let msg_id = bus_msg.id.clone();
575 let _ = state.messagebus.send(bus_msg);
576 let ack = json!({ "type": "bus:sent", "message_id": msg_id });
577 let msg = serde_json::to_string(&ack).unwrap_or_default();
578 if socket.send(Message::Text(msg.into())).await.is_err() {
579 return Err(true);
580 }
581 }
582 return Ok(None);
583 }
584 "bus:inject" => {
585 let from = incoming.from.as_deref().unwrap_or("unknown");
586 if let (Some(ref target), Some(ref message)) =
587 (&incoming.target, &incoming.bus_message_text)
588 {
589 // Try direct PTY injection via ReactiveHandler first
590 let mut reactive_req = crate::backend::reactive::InjectionRequest {
591 target_agent: target.clone(),
592 message: message.clone(),
593 source_agent: Some(from.to_string()),
594 request_id: None,
595 priority: incoming.priority.clone(),
596 jekt_tier: None, // auto-detected from keywords
597 delivery_tier: Some("host".to_string()),
598 forward_hops: 0,
599 ..Default::default()
600 };
601 // reagentx P0 on PR #2565: same bypass as
602 // messagebus.rs::handle_inject — this WS message type
603 // built InjectionRequest from client-controlled
604 // `incoming.from` and called inject_message directly,
605 // never going through host-tier signature verification.
606 super::reactive::verify_jekt_signature(state, &mut reactive_req);
607 let resp = state.reactive_handler.inject_message(reactive_req);
608 if resp.success {
609 // Sender-side echo (SPEC_JEKT_SECURITY_AND_VISIBILITY §3.2)
610 super::reactive::echo_jekt_to_sender(
611 state,
612 Some(from),
613 target,
614 message,
615 &resp.request_id,
616 resp.effective_tier.as_deref(),
617 resp.requires_stop,
618 // Always host-tier here (this is the pane's own outbound
619 // send) — reagent/lan verification never applies.
620 super::reactive::EchoTrust {
621 delivery_tier: "host",
622 sig_verified: None,
623 reagent_verified: None,
624 lan_verified: None,
625 channel_verified: None,
626 },
627 incoming.priority.as_deref().unwrap_or("normal"),
628 );
629 let ack = json!({ "type": "bus:injected", "via": "pty", "block_id": resp.block_id });
630 let msg = serde_json::to_string(&ack).unwrap_or_default();
631 if socket.send(Message::Text(msg.into())).await.is_err() {
632 return Err(true);
633 }
634 return Ok(None);
635 }
636
637 // Non-"agent not found" error — report it
638 let is_not_found = resp.error.as_deref().map(|e| e.contains("not found")).unwrap_or(false);
639 if !is_not_found {
640 let err = json!({ "type": "bus:error", "error": resp.error });
641 let msg = serde_json::to_string(&err).unwrap_or_default();
642 if socket.send(Message::Text(msg.into())).await.is_err() {
643 return Err(true);
644 }
645 return Ok(None);
646 }
647
648 // Fall back to MessageBus WebSocket push
649 let priority = match incoming.priority.as_deref() {
650 Some("high") => crate::backend::messagebus::Priority::High,
651 Some("urgent") => crate::backend::messagebus::Priority::Urgent,
652 _ => crate::backend::messagebus::Priority::Normal,
653 };
654 match state.messagebus.inject(from, "", target, message, priority) {
655 Ok(msg_id) => {
656 // Sender-side echo for the messagebus fallback path.
657 // Tier is unknown here (no ReactiveHandler wrap) — None
658 // renders as the default "coord".
659 super::reactive::echo_jekt_to_sender(
660 state,
661 Some(from),
662 target,
663 message,
664 &msg_id,
665 None,
666 None,
667 super::reactive::EchoTrust {
668 delivery_tier: "host",
669 sig_verified: None,
670 reagent_verified: None,
671 lan_verified: None,
672 channel_verified: None,
673 },
674 incoming.priority.as_deref().unwrap_or("normal"),
675 );
676 let ack = json!({ "type": "bus:injected", "via": "messagebus", "message_id": msg_id });
677 let msg = serde_json::to_string(&ack).unwrap_or_default();
678 if socket.send(Message::Text(msg.into())).await.is_err() {
679 return Err(true);
680 }
681 }
682 Err(e) => {
683 let err = json!({ "type": "bus:error", "error": e });
684 let msg = serde_json::to_string(&err).unwrap_or_default();
685 if socket.send(Message::Text(msg.into())).await.is_err() {
686 return Err(true);
687 }
688 }
689 }
690 }
691 return Ok(None);
692 }
693 "bus:broadcast" => {
694 let from = incoming.from.as_deref().unwrap_or("unknown");
695 if let Some(ref payload) = incoming.payload {
696 let priority = match incoming.priority.as_deref() {
697 Some("high") => crate::backend::messagebus::Priority::High,
698 Some("urgent") => crate::backend::messagebus::Priority::Urgent,
699 _ => crate::backend::messagebus::Priority::Normal,
700 };
701 let _ = state.messagebus.broadcast(from, "", payload, priority);
702 }
703 return Ok(None);
704 }
705 _ => {}
706 }
707 }
708
709 // Handle wscommand-based messages
710 if let Some(ref wscommand) = incoming.wscommand {
711 match wscommand.as_str() {
712 "rpc" => {
713 if let Some(rpc_msg) = incoming.message {
714 engine.handle_message(rpc_msg);
715 } else {
716 tracing::warn!("ws: rpc wscommand missing message field");
717 }
718 }
719 "blockinput" => {
720 if let Some(ref block_id) = incoming.blockid {
721 if let Some(ref data64) = incoming.inputdata64 {
722 if !data64.is_empty() {
723 match base64::engine::general_purpose::STANDARD.decode(data64) {
724 Ok(data) => {
725 if let Err(e) = dispatch_blockinput(block_id, data) {
726 tracing::debug!("ws: blockinput error: {}", e);
727 }
728 }
729 Err(e) => {
730 tracing::warn!("ws: blockinput base64 decode error: {}", e);
731 }
732 }
733 }
734 }
735 }
736 }
737 "setblocktermsize" => {
738 if let Some(ref block_id) = incoming.blockid {
739 if let Some(ref ts_val) = incoming.termsize {
740 match serde_json::from_value::<TermSize>(ts_val.clone()) {
741 Ok(ts) => {
742 let input = blockcontroller::BlockInputUnion::resize(ts);
743 if let Err(e) = blockcontroller::send_input(block_id, input, None) {
744 tracing::debug!("ws: setblocktermsize error: {}", e);
745 }
746 }
747 Err(e) => {
748 tracing::warn!("ws: setblocktermsize parse error: {}", e);
749 }
750 }
751 }
752 }
753 }
754 other => {
755 tracing::warn!("ws: unknown wscommand: {}", other);
756 }
757 }
758 }
759
760 Ok(None)
761}
762
763/// How far back the fleet list (`COMMAND_LIST_BACKGROUND_TASKS` with an empty
764/// `blockid`) reaches for tasks that already ended. Comfortably longer than the
765/// longest finished-row retention the renderer applies (15 s for an error), so
766/// a finished row is still in the list for as long as it is shown.
767const FLEET_ENDED_WINDOW_MS: i64 = 60_000;
768
769pub(super) fn register_handlers(engine: &Arc<WshRpcEngine>, state: AppState, conn_id: String) {
770 // getfullconfig → return full config as JSON
771 let config_watcher = state.config_watcher.clone();
772 // The RESPONSE stays `serde_json::Value`, deliberately. Redaction runs on
773 // the serialized form (`redact_full_config_for_renderer` walks the JSON and
774 // strips secrets), so what goes on the wire is NOT `FullConfigType` -- it
775 // is `FullConfigType` minus fields that a type would still claim are
776 // there. Generating `FullConfigType` here would produce a binding that
777 // promises the renderer secrets it will never receive, which is worse
778 // than `unknown`.
779 engine.register_typed(
780 COMMAND_GET_FULL_CONFIG,
781 move |_req: Option<NoArgsReq>, _ctx| {
782 let cw = config_watcher.clone();
783 async move {
784 let config = cw.get_full_config();
785 let mut v = serde_json::to_value(config.as_ref())
786 .map_err(|e| format!("failed to serialize config: {}", e))?;
787 crate::backend::wconfig::redact_full_config_for_renderer(&mut v);
788 Ok(v)
789 }
790 },
791 );
792
793 // routeannounce → log + no-op (fire-and-forget, may have no reqid)
794 // `Value` request: fire-and-forget, may arrive with no reqid and any
795 // payload, and it only logs. A struct would start rejecting shapes this has
796 // always accepted.
797 engine.register_typed(
798 COMMAND_ROUTE_ANNOUNCE,
799 |data: serde_json::Value, _ctx| {
800 async move {
801 tracing::debug!("routeannounce: {:?}", data);
802 Ok(())
803 }
804 },
805 );
806
807 // routeunannounce → no-op
808 engine.register_typed(
809 COMMAND_ROUTE_UNANNOUNCE,
810 |_data: serde_json::Value, _ctx| async move { Ok(()) },
811 );
812
813 // eventsub → register subscription with the MPS broker
814 let broker_sub = state.broker.clone();
815 let conn_id_sub = conn_id.clone();
816 engine.register_typed(
817 COMMAND_EVENT_SUB,
818 move |sub: crate::backend::mps::SubscriptionRequest, _ctx| {
819 let broker = broker_sub.clone();
820 let conn_id = conn_id_sub.clone();
821 async move {
822 tracing::debug!("eventsub: event={} scopes={:?} allscopes={}", sub.event, sub.scopes, sub.allscopes);
823 broker.subscribe(&conn_id, sub);
824 Ok(())
825 }
826 },
827 );
828
829 // eventunsub → unsubscribe from the MPS broker
830 let broker_unsub = state.broker.clone();
831 let conn_id_unsub = conn_id.clone();
832 // The request is a BARE STRING, not an object -- the only command in the
833 // app shaped that way, which is exactly the sort of thing a hand-written
834 // stub gets wrong. Typing it as `String` makes the generated binding say
835 // so. `Option<String>` rather than `String` because the old handler
836 // answered a non-string payload with "" (no-op) rather than an error, and
837 // that tolerance is cheap to keep.
838 engine.register_typed(
839 COMMAND_EVENT_UNSUB,
840 move |data: serde_json::Value, _ctx| {
841 let broker = broker_unsub.clone();
842 let conn_id = conn_id_unsub.clone();
843 async move {
844 // `as_str().unwrap_or("")` -- byte-identical to the untyped
845 // handler. A number/object/array is a silent no-op, as before,
846 // NOT a deserialize error.
847 let event_name = data.as_str().unwrap_or("").to_string();
848 if !event_name.is_empty() {
849 broker.unsubscribe(&conn_id, &event_name);
850 }
851 Ok(())
852 }
853 },
854 );
855
856 // eventunsuball → unsubscribe all from the MPS broker
857 let broker_unsub_all = state.broker.clone();
858 let conn_id_unsub_all = conn_id.clone();
859 engine.register_typed(
860 COMMAND_EVENT_UNSUB_ALL,
861 move |_req: Option<NoArgsReq>, _ctx| {
862 let broker = broker_unsub_all.clone();
863 let conn_id = conn_id_unsub_all.clone();
864 async move {
865 broker.unsubscribe_all(&conn_id);
866 Ok(())
867 }
868 },
869 );
870
871 // setmeta → update object metadata in the DB, broadcast update event
872 let mstore_sm = state.mstore.clone();
873 let event_bus_sm = state.event_bus.clone();
874 // `Value` request, parsed into its struct below (which uses the hand-written ORef/MetaType); the stub keeps its
875 // hand-written request type (generated-manifest.json `dataType`).
876 engine.register_typed(COMMAND_SET_META, move |data: serde_json::Value, _ctx| {
877 let mstore = mstore_sm.clone();
878 let event_bus = event_bus_sm.clone();
879 async move {
880 let cmd: CommandSetMetaData =
881 serde_json::from_value(data).map_err(|e| format!("setmeta: {e}"))?;
882 let oref_str = cmd.oref.to_string();
883 let meta_keys: Vec<&String> = cmd.meta.keys().collect();
884 // debug, not info: fires on every metadata write (zoom, title,
885 // view-state edits); at info this became a meaningful slice of
886 // an unrotated launcher-log mirror (SPEC_WIN10_PAGEFILE_OOM_
887 // CRASH_2026_06_29 P1). Default production filter is info, so
888 // this is now suppressed unless RUST_LOG=debug is set.
889 tracing::debug!(oref = %oref_str, keys = ?meta_keys, "SetMeta");
890 crate::server::browser_owner::guard_client_meta_write(&oref_str, &cmd.meta)?;
891 update_object_meta(&mstore, &oref_str, &cmd.meta)?;
892 // Per-agent zoom persistence (SPEC_AGENT_ZOOM_PERSISTENCE): the
893 // frontend writes term:zoom via this WebSocket path, not the HTTP
894 // UpdateObjectMeta handler where the mirror was originally placed.
895 // Mirror term:zoom → ui:zoom here so the zoom survives pane close.
896 let oref_parsed = crate::backend::ORef::parse(&oref_str)
897 .map_err(|e| e.to_string())?;
898 if oref_parsed.otype == "block" && cmd.meta.contains_key("term:zoom") {
899 if let Ok(block) = mstore.must_get::<Block>(&oref_parsed.oid) {
900 let agent_id = block.meta.get("agentId")
901 .and_then(|v| v.as_str()).unwrap_or("").to_string();
902 if !agent_id.is_empty() {
903 let zoom = cmd.meta.get("term:zoom").and_then(|v| v.as_f64());
904 schedule_agent_zoom_mirror(mstore.clone(), agent_id, zoom);
905 }
906 }
907 }
908 // Read the updated object and broadcast a proper MuxObjUpdate
909 // so all WS clients refresh their atoms with the new data.
910 let oref = crate::backend::ORef::parse(&oref_str)
911 .map_err(|e| e.to_string())?;
912 // Only a block's update carries its object; any other went out with no data,
913 // which the frontend ignores, so it isn't sent.
914 if oref.otype == "block" {
915 if let Ok(block) = mstore.must_get::<Block>(&oref.oid) {
916 let update = MuxObjUpdate {
917 updatetype: "update".into(),
918 otype: oref.otype.clone(),
919 oid: oref.oid.clone(),
920 obj: Some(mux_obj_to_value(&block)),
921 };
922 event_bus.broadcast_event(&crate::backend::eventbus::EventBus::typed_ws_event::<crate::backend::mps_events::MuxObjUpdated>(&update, oref_str));
923 }
924 }
925 Ok(())
926 }
927 });
928
929 // getmeta → return metadata for a wave object
930 let mstore_gm = state.mstore.clone();
931 // `Value` request and response: the meta map is what the frontend models as
932 // MetaType (generated-manifest.json `dataType` / `returns`).
933 engine.register_typed(COMMAND_GET_META, move |data: serde_json::Value, _ctx| {
934 let mstore = mstore_gm.clone();
935 async move {
936 let cmd: CommandGetMetaData =
937 serde_json::from_value(data).map_err(|e| format!("getmeta: {e}"))?;
938 let obj: Option<serde_json::Value> = mstore
939 .get_raw(&cmd.oref.otype, &cmd.oref.oid)
940 .map_err(|e| format!("getmeta: {e}"))?;
941 match obj {
942 Some(val) => {
943 // Return the "meta" field if present, otherwise the full object
944 let meta = val.get("meta").cloned().unwrap_or(val);
945 Ok(meta)
946 }
947 None => Err(format!("getmeta: object {} not found", cmd.oref)),
948 }
949 }
950 });
951
952 // waveinfo → return version and build info
953 let version_info = state.version.clone();
954 engine.register_typed(
955 COMMAND_APP_INFO,
956 move |_req: Option<NoArgsReq>, _ctx| {
957 let version = version_info.clone();
958 async move { Ok(AppInfoResult { version }) }
959 },
960 );
961
962 // getwaveairatelimit → AgentMux has no rate limits; return unlimited/unknown
963 engine.register_typed(
964 COMMAND_GET_AI_RATE_LIMIT,
965 |_req: Option<NoArgsReq>, _ctx| async move {
966 Ok(AiRateLimitResult {
967 req: 9999,
968 reqlimit: 9999,
969 preq: 9999,
970 preqlimit: 9999,
971 resetepoch: 0,
972 unknown: true,
973 })
974 },
975 );
976
977 // controllerresync → load block from DB, create/restart controller with PTY
978 let mstore_resync = state.mstore.clone();
979 let broker_resync = state.broker.clone();
980 let event_bus_resync = state.event_bus.clone();
981 let filestore_resync = state.filestore.clone();
982 let id_store_resync = state.id_store.clone();
983 let identity_store_resync = state.identity_store.clone();
984 let boot_id_resync = state.boot_id.clone();
985 let auth_key_resync = state.auth_key.clone();
986 let config_watcher_resync = state.config_watcher.clone();
987 // `Value` request, parsed into its struct below (termsize and the rest); the stub keeps its
988 // hand-written request type (generated-manifest.json `dataType`).
989 engine.register_typed(COMMAND_CONTROLLER_RESYNC, move |data: serde_json::Value, _ctx| {
990 let mstore = mstore_resync.clone();
991 let broker = broker_resync.clone();
992 let event_bus = event_bus_resync.clone();
993 let filestore = filestore_resync.clone();
994 let id_store = id_store_resync.clone();
995 let identity_store = identity_store_resync.clone();
996 let boot_id = boot_id_resync.clone();
997 let auth_key = auth_key_resync.clone();
998 let config_watcher = config_watcher_resync.clone();
999 async move {
1000 let cmd: CommandControllerResyncData = serde_json::from_value(data)
1001 .map_err(|e| format!("controllerresync: {e}"))?;
1002 tracing::info!(
1003 block_id = %cmd.blockid,
1004 tab_id = %cmd.tabid,
1005 forcerestart = cmd.forcerestart,
1006 norespawn = cmd.norespawn,
1007 "ControllerResync"
1008 );
1009 let block: Block = mstore
1010 .get(&cmd.blockid)
1011 .map_err(|e| format!("controllerresync: load block: {e}"))?
1012 .ok_or_else(|| format!("controllerresync: block {} not found", cmd.blockid))?;
1013 let registry = mstore.shared_agent_registry();
1014 blockcontroller::resync_controller(
1015 &block,
1016 &cmd.tabid,
1017 cmd.rtopts,
1018 cmd.forcerestart,
1019 // respawn_if_done — see `norespawn`'s own doc comment.
1020 // Default (false → true here) preserves TermResyncHandler's
1021 // crash-recovery behaviour for every existing caller.
1022 !cmd.norespawn,
1023 Some(broker),
1024 Some(event_bus),
1025 Some(mstore),
1026 Some(filestore),
1027 Some(id_store),
1028 Some(identity_store),
1029 registry,
1030 boot_id,
1031 &auth_key,
1032 Some(config_watcher),
1033 )?;
1034 Ok(())
1035 }
1036 });
1037
1038 // controllerinput → route keyboard input / signals / resize to block controller
1039 //
1040 // Drops input for a block currently under an active PtyShell agent lock
1041 // (`term:agentlockuntil`, docs/specs/SPEC_AGENT_INTERACTIVE_PTY_SHELL_API_2026_09_10.md
1042 // §10.3) rather than forwarding it — Codex P1 on PR #3194: the
1043 // frontend's own gate (`AgentShellSubblock.tsx` dropping its
1044 // `sendDataHandler` calls while locked) is necessarily eventually-
1045 // consistent — it reacts to a WS-pushed meta value, not something
1046 // synchronous with the backend's own state — so a human keystroke
1047 // already in flight when the lock is set could otherwise still reach
1048 // `send_input` and interleave with the agent's own write, exactly the
1049 // collision the lease exists to prevent. This check, made at the
1050 // moment this handler actually runs against the backend's own current
1051 // state, is the real enforcement; the frontend gate is the UX layer
1052 // that keeps it from happening in the first place, not the
1053 // correctness boundary.
1054 //
1055 // Keep this check I/O-free. It reads `blockcontroller::agent_lock`, an
1056 // in-memory map, NOT the block's persisted meta: the original revision
1057 // called `mstore.get::<Block>()` here, putting a synchronous SQLite
1058 // query behind `Store`'s single process-wide `Mutex<Connection>` inside
1059 // an async handler, inline on a Tokio worker thread with no
1060 // `block_in_place` — the same failure class as the sysinfo incident
1061 // (#1782). This handler carries the Stop button's SIGINT and
1062 // `usePtyWidth`'s debounced resizes (which burst while a pane is being
1063 // dragged); it is NOT the keystroke path — those go through the
1064 // `blockinput` arm below, which has the same lock check via
1065 // `dispatch_blockinput` (closed 2026-09-16, previously `agent_lock`'s
1066 // documented "Known gap" — see its module doc history). See also
1067 // docs/analysis/ANALYSIS_CROSS_PANE_INPUT_DELAY_REGRESSION_2026_09_15.md.
1068 engine.register_typed(
1069 COMMAND_CONTROLLER_INPUT,
1070 move |cmd: CommandBlockInputData, _ctx| {
1071 async move {
1072 if blockcontroller::agent_lock::is_locked(&cmd.blockid) {
1073 tracing::debug!(block_id = %cmd.blockid, "controllerinput: dropped — agent lock active");
1074 return Ok(());
1075 }
1076 let input = parse_block_input(&cmd)?;
1077 blockcontroller::send_input(&cmd.blockid, input, cmd.seq)?;
1078 Ok(())
1079 }
1080 },
1081 );
1082
1083 // createsubblock → create a headless sub-block (no tab/layout entry)
1084 // parented to an existing block, e.g. a `term`-view PTY embedded in an
1085 // agent pane's details drawer.
1086 let mstore_csb = state.mstore.clone();
1087 // `Value` request, parsed into its struct below (a BlockDef); the stub keeps its
1088 // hand-written request type (generated-manifest.json `dataType`).
1089 engine.register_typed(COMMAND_CREATE_SUB_BLOCK, move |data: serde_json::Value, _ctx| {
1090 let mstore = mstore_csb.clone();
1091 async move {
1092 let cmd: CommandCreateSubBlockData = serde_json::from_value(data)
1093 .map_err(|e| format!("createsubblock: {e}"))?;
1094
1095 let mut meta = cmd.blockdef.meta.clone();
1096 let raw_cwd = meta
1097 .get(blockcontroller::META_KEY_CMD_CWD)
1098 .and_then(|v| v.as_str())
1099 .filter(|s| !s.is_empty())
1100 .map(str::to_string);
1101 if let Some(raw_cwd) = raw_cwd {
1102 // The PTY spawn path (blockcontroller/shell/lifecycle.rs)
1103 // reads cmd:cwd raw with no MSYS→Windows conversion, unlike
1104 // shellexec — normalize here so a Git-Bash-style path
1105 // doesn't hit os error 267 on Windows. Mirrors
1106 // server/mod.rs's shellexec cwd-fallback precedent: an
1107 // invalid/non-absolute value degrades to no cwd rather
1108 // than failing the whole call — this is a best-effort
1109 // convenience derived from the parent's meta, not a
1110 // caller-supplied value worth hard-rejecting.
1111 match normalize_working_dir(&raw_cwd).filter(|p| std::path::Path::new(p).is_absolute()) {
1112 Some(norm) => {
1113 meta.insert(
1114 blockcontroller::META_KEY_CMD_CWD.to_string(),
1115 serde_json::Value::String(norm),
1116 );
1117 }
1118 None => {
1119 tracing::warn!(
1120 raw_cwd = %raw_cwd,
1121 "createsubblock: invalid or non-absolute cmd:cwd, dropping (no cwd)"
1122 );
1123 meta.remove(blockcontroller::META_KEY_CMD_CWD);
1124 }
1125 }
1126 }
1127
1128 let child_id = uuid::Uuid::new_v4().to_string();
1129 let mut block = Block {
1130 oid: child_id.clone(),
1131 parentoref: format!("block:{}", cmd.parentblockid),
1132 meta,
1133 ..Default::default()
1134 };
1135 mstore
1136 .insert(&mut block)
1137 .map_err(|e| format!("createsubblock: insert: {e}"))?;
1138
1139 // Best-effort link into the parent's subblockids. Not a CAS
1140 // (Store::update is a blind version-bump) — an acceptable
1141 // race for a single lazily-created shell per agent pane.
1142 if let Ok(mut parent) = mstore.must_get::<Block>(&cmd.parentblockid) {
1143 parent
1144 .subblockids
1145 .get_or_insert_with(Vec::new)
1146 .push(child_id.clone());
1147 let _ = mstore.update(&mut parent);
1148 }
1149
1150 tracing::info!(
1151 child_id = %child_id,
1152 parent_id = %cmd.parentblockid,
1153 "CreateSubBlock"
1154 );
1155 // TS `CreateSubBlockCommand` resolves `Promise<ORef>`, and
1156 // `ORef` (srv-types.d.ts) is a plain string — return the
1157 // bare "block:<id>" string, not a wrapper object.
1158 Ok(format!("block:{child_id}"))
1159 }
1160 });
1161
1162 // deletesubblock → tear down a sub-block created via createsubblock.
1163 // Kill-first ordering matches sagas/delete_block.rs; unlike that saga,
1164 // this does NOT touch tab bookkeeping — sub-blocks are never
1165 // tab-referenced, so delete_block.rs's precondition would reject them.
1166 let mstore_dsb = state.mstore.clone();
1167 engine.register_typed(
1168 COMMAND_DELETE_SUB_BLOCK,
1169 move |cmd: CommandDeleteSubBlockData, _ctx| {
1170 let mstore = mstore_dsb.clone();
1171 async move {
1172
1173 // Kill process FIRST — a lingering PTY tree is worse than a
1174 // delayed row delete.
1175 crate::sagas::agent_teardown::discard(&cmd.blockid);
1176
1177 let parent_id = mstore
1178 .get::<Block>(&cmd.blockid)
1179 .ok()
1180 .flatten()
1181 .and_then(|b| b.parentoref.strip_prefix("block:").map(str::to_string));
1182
1183 mstore
1184 .delete::<Block>(&cmd.blockid)
1185 .map_err(|e| format!("deletesubblock: delete: {e}"))?;
1186
1187 // Best-effort unlink from the parent's subblockids — the
1188 // controller is already dead and the row is already gone
1189 // either way, so don't fail the call over this step.
1190 if let Some(parent_id) = parent_id {
1191 if let Ok(mut parent) = mstore.must_get::<Block>(&parent_id) {
1192 if let Some(ids) = parent.subblockids.as_mut() {
1193 ids.retain(|id| id != &cmd.blockid);
1194 let _ = mstore.update(&mut parent);
1195 }
1196 }
1197 }
1198
1199 tracing::info!(block_id = %cmd.blockid, "DeleteSubBlock");
1200 Ok(())
1201 }
1202 },
1203 );
1204
1205 // tooldecision → reply to a per-tool-call permission gate via the Agent SDK
1206 // **control protocol** — the `AgentDecisionPanel` Allow/Deny buttons, once
1207 // `should_route_to_decision_panel` (persistent/mod.rs) is flipped on for real
1208 // (that gate is currently hardcoded false, so `decide_tool_permission`
1209 // below will fail every call today with "no pending tool-permission
1210 // request" — expected until the Phase 2 policy decision is made; see
1211 // that function's own doc comment).
1212 //
1213 // The original PR-3a draft tried to write `y\n` / `n\n` to the
1214 // subprocess's stdin via `blockcontroller::send_input`. Codex P1
1215 // on PR #557 caught that this would fail: `SubprocessController::
1216 // send_input` (and `PersistentSubprocessController::send_input`)
1217 // both reject raw `input_data` with an `Err`. The deeper truth is
1218 // that AgentMux runs the agent CLI in non-interactive `--print`
1219 // mode — the CLI shows no y/n prompt and stdin carries only the
1220 // structured message stream, so a y/n write would be meaningless even if
1221 // the controller accepted it. Real delivery is the same
1222 // control_response mechanism `agent.answer` uses below for
1223 // AskUserQuestion. See SPEC_DECISION_PROMPT_2026_04_24.md §9.1
1224 // and SPEC_AGENT_CONTROL_PROTOCOL_2026_06_15.md.
1225 //
1226 // `cmd.request_id` is used here as the `tool_use_id` key
1227 // `pending_permissions`/`decide_tool_permission` are keyed on — NOT
1228 // as a separate correlator against the speculative
1229 // `PermissionRequestEvent` §4.1 describes, which was never built.
1230 // The eventual mechanism (still unbuilt — see the Phase 2 gate)
1231 // correlates a `pending_approval` ToolNode by `tool_use_id`, exactly
1232 // like AskUserQuestion's `awaiting_answer` already does; whatever
1233 // populates that field on the frontend side should set it to the
1234 // ToolNode's `tool_use_id`.
1235 //
1236 // `scope` is still validated but not yet acted on: §6's rules-
1237 // persistence layer (`permissions.json`) does not exist anywhere in
1238 // this tree. That is separate, still-open work — this PR only makes
1239 // the *transport* (control_response delivery) real, not repeat-
1240 // decision memory.
1241 engine.register_typed(
1242 COMMAND_TOOL_DECISION,
1243 move |cmd: CommandToolDecisionData, _ctx| async move {
1244 match cmd.outcome.as_str() {
1245 "allow" | "deny" => {}
1246 other => {
1247 return Err(format!(
1248 "tooldecision: invalid outcome '{}' (expected 'allow' or 'deny')",
1249 other
1250 ));
1251 }
1252 }
1253 // Validate scope so a future rules-persistence layer (§6)
1254 // can trust the value without re-checking. Reagent P1
1255 // round-3 on PR #557.
1256 match cmd.scope.as_str() {
1257 "once" | "session" | "project" | "global" => {}
1258 other => {
1259 return Err(format!(
1260 "tooldecision: invalid scope '{}' (expected 'once'/'session'/'project'/'global')",
1261 other
1262 ));
1263 }
1264 }
1265 tracing::info!(
1266 block_id = %cmd.blockid,
1267 request_id = %cmd.request_id,
1268 outcome = %cmd.outcome,
1269 scope = %cmd.scope,
1270 has_feedback = cmd.feedback.is_some(),
1271 "[tooldecision] delivering via control protocol"
1272 );
1273 let ctrl = blockcontroller::get_controller(&cmd.blockid)
1274 .ok_or_else(|| format!("tooldecision: no controller for block {}", cmd.blockid))?;
1275 if let Some(persistent_ctrl) = ctrl
1276 .as_any()
1277 .downcast_ref::<blockcontroller::persistent::PersistentSubprocessController>()
1278 {
1279 persistent_ctrl.decide_tool_permission(cmd.request_id, &cmd.outcome, cmd.feedback)
1280 } else {
1281 Err("tooldecision: UNSUPPORTED_CONTROLLER: per-tool permission decisions \
1282 require a persistent (host) agent; container/one-shot agents are not \
1283 yet supported (same limitation as agent.answer)".to_string())
1284 }
1285 },
1286 );
1287
1288 // docknodestatus → fire-and-forget push of a ToolNode's latest status,
1289 // cached in-memory per block for `muxspect dock` to read. See
1290 // docs/specs/SPEC_MUXSPECT_DOCK_DIAGNOSIS_AND_REMEDIATION_2026_08_06.md §3.1.
1291 //
1292 // Also mirrors a declared-background task's liveness into the durable
1293 // `db_background_tasks` registry (see
1294 // docs/status/STATUS_ATTACHED_TASK_AXIS_AND_DEV_LOOP_2026_08_15.md) —
1295 // `DockSnapshotCache` above is intentionally ephemeral (never persisted,
1296 // 1-hour eviction on read); the registry is the durable source of truth
1297 // consumers that need to survive a cache eviction or a session
1298 // reconnect (Swarm, #2492's teardown-survival) read from instead.
1299 //
1300 // `run_in_background == Some(true)` is ONLY ever sent once
1301 // `isAcceptedBackgroundLaunch()` holds client-side (`tool-adapter.ts`),
1302 // which by its own definition requires `status == "success"` — checking
1303 // `status == "running"` here (an earlier version of this handler did)
1304 // made the observe call unreachable, since the flag and that status
1305 // value are never sent together (reagentx P0 / codex P1 on PR #2590).
1306 // Best-effort: a registry write failure never blocks the dock push it
1307 // rides alongside — `dock_snapshots.push_delta` below already gives the
1308 // user the live UI signal regardless. Completion is NOT detected here —
1309 // see `COMMAND_BACKGROUND_TASK_COMPLETION` below for why this handler
1310 // structurally can't see it (the originating ToolNode's status never
1311 // changes again after acceptance).
1312 let dock_snapshots_dns = state.dock_snapshots.clone();
1313 // Ambient narration — generate a short user-facing line about something
1314 // AgentMux just did on its own, and broadcast it to the pane.
1315 //
1316 // Fire-and-forget in both directions: the renderer does not await a reply,
1317 // and the work runs in a detached task so a slow or capped model call never
1318 // holds the RPC open. Every failure path is silence — the narrated action
1319 // has already happened and the UI already reflects it, so a missing line is
1320 // a missing nicety, not a missing state change.
1321 let mstore_narrate = state.mstore.clone();
1322 let broker_narrate = state.broker.clone();
1323 let narrated = state.narrated_events.clone();
1324 engine.register_typed(
1325 COMMAND_AMBIENT_NARRATE,
1326 move |cmd: CommandAmbientNarrateData, _ctx| {
1327 let mstore = mstore_narrate.clone();
1328 let broker = broker_narrate.clone();
1329 let narrated = narrated.clone();
1330 async move {
1331
1332 // Dedupe per narrated event. A ToolNode can be re-observed, and
1333 // narrating one event twice is both noise and a wasted model
1334 // call. Checked before spawning so a repeat costs nothing.
1335 if !cmd.dedupe_key.is_empty() && !narrated.mark_new(&cmd.dedupe_key) {
1336 return Ok(());
1337 }
1338
1339 tokio::spawn(async move {
1340 let generation = agentmux_common::time::now_ms_u64();
1341 let Some(text) = crate::ambient::tasks::generate_ambient_narration(
1342 &mstore,
1343 &cmd.blockid,
1344 &cmd.dedupe_key,
1345 generation,
1346 &cmd.kind,
1347 &cmd.context,
1348 )
1349 .await
1350 else {
1351 return;
1352 };
1353 // A short task can finish while the model call is still
1354 // in flight, so by now the dock may already show it done.
1355 // Narrating it at that point is worse than silence: the line
1356 // describes a launch the user has already seen conclude.
1357 //
1358 // Belt and braces with the prompt, which is phrased to stay
1359 // true after completion — this drops the message entirely
1360 // rather than relying on wording alone. (codex P2 on #3169.)
1361 //
1362 // Only skips on a DEFINITE completion: an unknown id (no
1363 // registry row — the task was never observed, or this is a
1364 // narration kind that has none) falls through and publishes,
1365 // since absence of a row is not evidence of completion.
1366 if !cmd.dedupe_key.is_empty() {
1367 if let Ok(Some(task)) = mstore.background_task_get(&cmd.dedupe_key) {
1368 if task.status != crate::backend::storage::background_tasks::BackgroundTaskStatus::Running {
1369 return;
1370 }
1371 }
1372 }
1373 // Carries the text itself, unlike background-task-updated
1374 // next to it: there is no list query to re-read, so an
1375 // invalidation ping would have nothing to invalidate.
1376 let narration = crate::backend::mps_events::AmbientNarrationEvent { block_id: cmd.blockid.clone(), kind: cmd.kind.clone(), text: text.to_string() };
1377 broker.publish_event::<crate::backend::mps_events::AmbientNarration>(&narration, vec![format!("block:{}", cmd.blockid)], 0);
1378 });
1379 Ok(())
1380 }
1381 },
1382 );
1383
1384 let mstore_dns = state.mstore.clone();
1385 let pending_pids_dns = state.pending_background_pids.clone();
1386 let broker_dns = state.broker.clone();
1387 engine.register_typed(
1388 COMMAND_DOCK_NODE_STATUS,
1389 move |cmd: CommandDockNodeStatusData, _ctx| {
1390 let dock_snapshots = dock_snapshots_dns.clone();
1391 let mstore = mstore_dns.clone();
1392 let pending_pids = pending_pids_dns.clone();
1393 let broker = broker_dns.clone();
1394 async move {
1395 let observed_at = agentmux_common::time::now_ms();
1396
1397 if cmd.run_in_background == Some(true) {
1398 match mstore.background_task_observe(
1399 &cmd.node_id,
1400 &cmd.blockid,
1401 &cmd.tool_name,
1402 observed_at,
1403 observed_at,
1404 ) {
1405 Ok(()) => publish_background_task_updated(&broker, &cmd.blockid),
1406 Err(e) => tracing::warn!(
1407 target: "background_tasks",
1408 node_id = %cmd.node_id,
1409 error = %e,
1410 "failed to observe declared-background task in the durable registry",
1411 ),
1412 }
1413 // bashwrap's own pid publish (COMMAND_BACKGROUND_TASK_PID
1414 // below) routinely races ahead of the observe call above —
1415 // it fires at bashwrap's process start, before the frontend
1416 // even sees an accepted-background tool result. Check for
1417 // (and apply) a pid stashed ahead of this row's existence.
1418 // Runs unconditionally on observe failure too — a stashed
1419 // pid may still apply if the row already existed from an
1420 // earlier retry. Atomic with the pid handler's own
1421 // set_or_stash call for the same id — see
1422 // pending_background_pids.rs's module doc for why that
1423 // matters (Codex/reagentx findings on PR #2681).
1424 let node_id = cmd.node_id.clone();
1425 let mstore_apply = mstore.clone();
1426 let result: Result<(), crate::backend::storage::StoreError> = pending_pids
1427 .observe_and_apply(&cmd.node_id, observed_at, |pid| {
1428 mstore_apply.background_task_set_pid(&node_id, pid)
1429 });
1430 if let Err(e) = result {
1431 tracing::warn!(
1432 target: "background_tasks",
1433 node_id = %cmd.node_id,
1434 error = %e,
1435 "failed to apply a pid stashed ahead of this task's registry row",
1436 );
1437 }
1438 }
1439
1440 dock_snapshots.push_delta(
1441 &cmd.blockid,
1442 crate::backend::dock_snapshot::DockNodeSnapshot {
1443 node_id: cmd.node_id,
1444 tool_name: cmd.tool_name,
1445 status: cmd.status,
1446 timestamp: cmd.timestamp,
1447 observed_at,
1448 run_in_background: cmd.run_in_background,
1449 },
1450 );
1451 Ok(())
1452 }
1453 },
1454 );
1455
1456 // backgroundtaskcompletion → fire-and-forget push of a declared-
1457 // background task's REAL terminal outcome, parsed client-side from its
1458 // `<task-notification>` message (`tool-adapter.ts`'s
1459 // `parseTaskNotification`). This is a separate command from
1460 // `docknodestatus` above on purpose: the originating ToolNode's own
1461 // `status` field goes "success" at acceptance and never changes again
1462 // (that's the raw tool_result's outcome, not the background task's) —
1463 // there is no later `docknodestatus` push this handler could key off of
1464 // to learn a background task actually finished (reagentx P0 / codex P1
1465 // on PR #2590, corrected here). Routing this through `docknodestatus`
1466 // instead of a dedicated command would also corrupt `DockSnapshotCache`:
1467 // `push_delta` fully overwrites a node's snapshot, and this event has no
1468 // `tool_name`/original `run_in_background` value to carry forward — the
1469 // exact bug class #2520 already fixed once for a different call site.
1470 let mstore_btc = state.mstore.clone();
1471 let broker_btc = state.broker.clone();
1472 engine.register_typed(
1473 COMMAND_BACKGROUND_TASK_COMPLETION,
1474 move |cmd: CommandBackgroundTaskCompletionData, _ctx| {
1475 let mstore = mstore_btc.clone();
1476 let broker = broker_btc.clone();
1477 async move {
1478 let ended_at = cmd.timestamp.unwrap_or_else(|| {
1479 agentmux_common::time::now_ms()
1480 });
1481 let status = crate::backend::storage::background_tasks::BackgroundTaskStatus::from_str(&cmd.status);
1482 match mstore.background_task_complete(&cmd.node_id, status, ended_at) {
1483 Ok(_) => publish_background_task_updated(&broker, &cmd.blockid),
1484 Err(e) => tracing::warn!(
1485 target: "background_tasks",
1486 node_id = %cmd.node_id,
1487 error = %e,
1488 "failed to mark background task terminal in the durable registry",
1489 ),
1490 }
1491 Ok(())
1492 }
1493 },
1494 );
1495
1496 // backgroundtaskpid → fire-and-forget push of a declared-background
1497 // task's real OS pid, relayed from `agentmux-bashwrap`'s own MPS "pid"
1498 // chunk. Closes the gap where `db_background_tasks.pid` existed but
1499 // nothing in production ever wrote it (Phase A of
1500 // docs/specs/SPEC_BACKGROUND_TASK_PID_CAPTURE_2026_08_20.md). Best-effort,
1501 // same as the two handlers above: a write failure here doesn't block
1502 // anything else and is only ever a diagnostic/teardown-survival input,
1503 // never something the model-visible tool_result depends on.
1504 //
1505 // bashwrap publishes this essentially at process start — routinely
1506 // BEFORE `COMMAND_DOCK_NODE_STATUS` above has created this task's
1507 // `db_background_tasks` row (that requires a full round-trip through
1508 // the frontend recognizing an accepted background launch first).
1509 // `background_task_set_pid` silently no-ops on a missing row
1510 // (`Ok(false)`), which would lose the pid permanently with no way to
1511 // retry a write that will never succeed — stash it instead, and
1512 // `COMMAND_DOCK_NODE_STATUS`'s handler applies it the moment the row
1513 // exists. See Codex/reagentx findings on PR #2681.
1514 let mstore_btp = state.mstore.clone();
1515 let pending_pids_btp = state.pending_background_pids.clone();
1516 let broker_btp = state.broker.clone();
1517 engine.register_typed(
1518 COMMAND_BACKGROUND_TASK_PID,
1519 move |cmd: CommandBackgroundTaskPidData, _ctx| {
1520 let mstore = mstore_btp.clone();
1521 let pending_pids = pending_pids_btp.clone();
1522 let broker = broker_btp.clone();
1523 async move {
1524 let now_ms = agentmux_common::time::now_ms();
1525 let node_id = cmd.node_id.clone();
1526 let mstore_set = mstore.clone();
1527 let result = pending_pids.set_or_stash(&cmd.node_id, cmd.pid as i64, now_ms, |pid| {
1528 mstore_set.background_task_set_pid(&node_id, pid)
1529 });
1530 match result {
1531 Ok(()) => publish_background_task_updated(&broker, &cmd.blockid),
1532 Err(e) => tracing::warn!(
1533 target: "background_tasks",
1534 node_id = %cmd.node_id,
1535 error = %e,
1536 "failed to record background task pid in the durable registry",
1537 ),
1538 }
1539 Ok(())
1540 }
1541 },
1542 );
1543
1544 // listbackgroundtasks → request/response: this block's current
1545 // db_background_tasks rows, so the frontend can seed its attachedTask
1546 // axis from the durable registry on mount/reconnect instead of only
1547 // ever re-deriving it from this tab's own live transcript replay
1548 // (which has no way to know about a task that survived a session
1549 // restart under a controller generation with no transcript history of
1550 // ever launching it — Phase B of
1551 // docs/specs/SPEC_BACKGROUND_TASK_TEARDOWN_SURVIVAL_2026_08_20.md).
1552 // See docs/specs/SPEC_BACKGROUND_TASK_DASHBOARD_INTELLIGENCE_2026_08_20.md §3.1.
1553 let mstore_lbt = state.mstore.clone();
1554 engine.register_typed(
1555 COMMAND_LIST_BACKGROUND_TASKS,
1556 move |cmd: CommandListBackgroundTasksData, _ctx| {
1557 let mstore = mstore_lbt.clone();
1558 async move {
1559 // An empty blockid asks for the whole fleet (Swarm): every
1560 // running task plus those that ended recently enough to still
1561 // show a finished row. SPEC_BACKGROUND_TASK_STRUCTURED_FEED_
1562 // AND_SWARM_OWNERSHIP_2026_09_27.md §3.
1563 let tasks = if cmd.blockid.is_empty() {
1564 let now_ms = agentmux_common::time::now_ms();
1565 mstore.background_task_list_fleet(now_ms - FLEET_ENDED_WINDOW_MS)
1566 } else {
1567 mstore.background_task_list_for_block(&cmd.blockid)
1568 }
1569 .map_err(|e| format!("listbackgroundtasks: {e}"))?;
1570 let views: Vec<super::muxspect_handlers::BackgroundTaskView> =
1571 tasks.into_iter().map(Into::into).collect();
1572 Ok(views)
1573 }
1574 },
1575 );
1576
1577 // agent.answer → deliver an AskUserQuestion answer to the running agent CLI
1578 // via the Agent SDK **control protocol**: the persistent controller replies
1579 // to the CLI's parked `can_use_tool` control_request with a control_response
1580 // carrying `updatedInput.answers`. (Delivering a `tool_result` on stdin does
1581 // NOT work — the CLI auto-rejects AskUserQuestion within the turn; see spec
1582 // §2/§3.) Only the persistent controller speaks the control protocol;
1583 // container/one-shot subprocess agents return UNSUPPORTED_CONTROLLER (Phase 2).
1584 // Spec: docs/specs/SPEC_AGENT_CONTROL_PROTOCOL_2026_06_15.md.
1585 // `Value` request, parsed into its struct below (an answer to a question); the stub keeps its
1586 // hand-written request type (generated-manifest.json `dataType`).
1587 engine.register_typed(COMMAND_AGENT_ANSWER, move |data: serde_json::Value, _ctx| {
1588 async move {
1589 let cmd: CommandAgentAnswerData = serde_json::from_value(data)
1590 .map_err(|e| format!("agent.answer: {e}"))?;
1591 with_question_controller("agent.answer", "answering", &cmd.blockid, &cmd.tool_use_id, |ctrl| {
1592 ctrl.answer_question(cmd.tool_use_id.clone(), cmd.answers)
1593 })?;
1594 tracing::info!(
1595 block_id = %cmd.blockid,
1596 tool_use_id = %cmd.tool_use_id,
1597 "[agent.answer] control_response delivered to persistent stdin"
1598 );
1599 Ok(())
1600 }
1601 });
1602
1603 // agent.cancel → a REAL protocol-level decline of a pending AskUserQuestion
1604 // (Cancel button / Escape in AgentQuestionPanel.tsx), not a UI-only dismiss.
1605 // Sends `behavior: "deny"` over the same control protocol agent.answer uses
1606 // for `behavior: "allow"` — see `deny_question`'s doc comment for why this
1607 // is a general, documented Agent SDK mechanism rather than something
1608 // special-cased for AskUserQuestion. Its error strings come from
1609 // `with_question_controller`, shared with agent.answer (`no controller for
1610 // block`, `UNSUPPORTED_CONTROLLER`): the frontend's SAFE_TO_RETRY_VIA_FOLLOWUP
1611 // allowlist (useAgentQuestions.ts) matches on these substrings for both
1612 // commands. Spec: docs/specs/SPEC_AGENT_CONTROL_PROTOCOL_2026_06_15.md.
1613 engine.register_typed(
1614 COMMAND_AGENT_CANCEL,
1615 move |cmd: CommandAgentCancelData, _ctx| async move {
1616 with_question_controller("agent.cancel", "canceling", &cmd.blockid, &cmd.tool_use_id, |ctrl| {
1617 ctrl.deny_question(
1618 cmd.tool_use_id.clone(),
1619 blockcontroller::persistent::ASK_USER_QUESTION_DENY_MESSAGE.to_string(),
1620 )
1621 })?;
1622 tracing::info!(
1623 block_id = %cmd.blockid,
1624 tool_use_id = %cmd.tool_use_id,
1625 "[agent.cancel] deny control_response delivered to persistent stdin"
1626 );
1627 Ok(())
1628 },
1629 );
1630
1631 // The pending-shutdown banner's "Keep running"
1632 // (docs/specs/SPEC_AGENT_SELF_QUIT_2026_09_24.md §6.5).
1633 let state_keep = state.clone();
1634 engine.register_typed(
1635 COMMAND_AGENT_SHUTDOWN_KEEP,
1636 move |cmd: CommandAgentShutdownKeepData, _ctx| {
1637 let state = state_keep.clone();
1638 async move {
1639 let outcome = crate::sagas::pending_shutdown::keep(&state, &cmd.blockid, &cmd.request_id);
1640 Ok(AgentShutdownKeepResult { outcome: outcome.to_string() })
1641 }
1642 },
1643 );
1644
1645 // Agent input/stop + subprocess spawn handlers
1646 super::agent_handlers::register_agent_input_handlers(engine, &state);
1647
1648 // `/btw` slash command — one-shot, tool-less side question.
1649 super::agent_handlers::register_side_question_handlers(engine, &state);
1650
1651 // Shell exec/stop handlers
1652 super::shell_handlers::register_shell_handlers(engine, &state);
1653
1654 // Editor/file-ops + write_agent_config handlers
1655 super::editor_handlers::register_editor_handlers(engine, &state);
1656
1657 // Files pane: fs.list/places/watch/unwatch + mutations
1658 // (SPEC_FILE_BROWSER_PANE_2026_10_01.md §6.3)
1659 super::fs_handlers::register_fs_handlers(engine, &state);
1660
1661 // LSP handlers (lspstart, lspsend, lspstop)
1662 super::lsp_handlers::register_lsp_handlers(engine, &state);
1663
1664 // CLI handlers (resolvecli, checkcliauth, runclilogin), then widget packages (widgets.*)
1665 super::cli_handlers::register_cli_handlers(engine, &state);
1666 super::widget_handlers::register_widget_handlers(engine, &state);
1667
1668 // Tool store handlers (gettoolstatus, installtool)
1669 super::tool_handlers::register_tool_handlers(engine, &state);
1670
1671 // Provider model catalog (providers.models → authoritative /v1/models list)
1672 super::providers_handlers::register_providers_handlers(engine, &state);
1673
1674 // reactive.registrations → Stash "Registration" tab live status (#2696)
1675 super::reactive::register_reactive_ws_handlers(engine, &state);
1676
1677 // eventreadhistory → read persisted event history from the MPS broker
1678 let broker_history = state.broker.clone();
1679 engine.register_typed(
1680 COMMAND_EVENT_READ_HISTORY,
1681 move |cmd: CommandEventReadHistoryData, _ctx| {
1682 let broker = broker_history.clone();
1683 async move {
1684 let max_items = if cmd.maxitems == 0 { 1024 } else { cmd.maxitems };
1685 Ok(broker.read_event_history(&cmd.event, &cmd.scope, max_items))
1686 }
1687 },
1688 );
1689
1690 // setconfig → merge settings keys into settings.json AND update in-memory config immediately.
1691 // Writing to disk + broadcasting directly gives instant UI response without waiting for
1692 // the fs watcher (which has a ~300-800ms debounce + polling delay on Windows).
1693 // The fs watcher's subsequent reload is a no-op (settings already up to date).
1694 let config_watcher_setconfig = state.config_watcher.clone();
1695 let event_bus_setconfig = state.event_bus.clone();
1696
1697 let lan_listeners_setconfig = state.lan_listeners.clone();
1698 // `Value` request, parsed into its struct below (a partial settings object); the stub keeps its
1699 // hand-written request type (generated-manifest.json `dataType`).
1700 engine.register_typed(COMMAND_SET_CONFIG, move |data: serde_json::Value, _ctx| {
1701 let cw = config_watcher_setconfig.clone();
1702 let eb = event_bus_setconfig.clone();
1703
1704 let lan_listeners = lan_listeners_setconfig.clone();
1705 async move {
1706 let new_keys: serde_json::Map<String, serde_json::Value> =
1707 serde_json::from_value(data).map_err(|e| format!("setconfig: {e}"))?;
1708
1709 // 1. Write to disk (fs watcher will re-broadcast, harmlessly)
1710 crate::backend::config_watcher_fs::merge_settings_to_disk(new_keys.clone())
1711 .map_err(|e| format!("setconfig write: {e}"))?;
1712
1713 // 2. Update in-memory config immediately
1714 let merged_settings = crate::backend::config_watcher_fs::merge_settings_into_current(&cw, new_keys);
1715 let lan_enabled = merged_settings.network_lan_discovery;
1716 cw.update_settings(merged_settings);
1717
1718 // 3. Live-toggle LAN discovery if the key changed. `apply` is
1719 // idempotent so it's safe to call unconditionally — when the
1720 // daemon is already in the requested state, this is a no-op.
1721 // See docs/specs/lan-discovery-toggle.md.
1722 //
1723 // Only the LISTENER supervisor is driven here. It owns the
1724 // advertise/reachable pairing and re-gates `lan` itself on
1725 // every reconcile, so mDNS never advertises an address
1726 // nothing is listening on ("discovered but unreachable").
1727 // Calling `lan.apply` here as well would fight that gate —
1728 // it would advertise immediately, before any listener is
1729 // bound, which is the exact state being designed out.
1730 lan_listeners.apply(lan_enabled);
1731
1732 // 4. Broadcast updated config now — no waiting for fs watcher.
1733 // Shared with the settings-watcher reload path and the
1734 // browser-start-page watcher — see its own doc comment
1735 // for why this used to be duplicated inline here.
1736 crate::backend::config_watcher_fs::broadcast_full_config(&cw, &eb);
1737 Ok(())
1738 }
1739 });
1740
1741 // Agent handlers (definitions, content, skills, history, import, reseed)
1742 super::agent_handlers::register_agent_handlers(engine, &state);
1743
1744 // Drone handlers (issue #753 — Drone pane DAG executor)
1745 super::drone_handlers::register_drone_handlers(engine, &state);
1746
1747 // Pre-launch OAuth handlers (auth.start / poll / submitcallback /
1748 // cancel — see docs/specs/SPEC_PRE_LAUNCH_OAUTH_FLOW_2026_05_14.md)
1749 super::identity_handlers::register_identity_handlers(engine, &state);
1750
1751 // Install handlers (install.start / install.cancel — see
1752 // docs/specs/SPEC_AGENT_INSTALL_STAGE_2026_05_17.md)
1753 super::install_handlers::register_install_handlers(engine, &state);
1754
1755 // System-toolchain install handlers (toolchain.resolve_install_command /
1756 // toolchain.install_system_tool — see
1757 // docs/specs/SPEC_SYSTEM_TOOLCHAIN_INSTALLER_2026_08_24.md). Shares
1758 // state.install_sessions with the handlers just above (same
1759 // InstallSessionRegistry — install.cancel already works unchanged for
1760 // these sessions too, no new cancel command needed).
1761 super::system_install_handlers::register_system_install_handlers(engine, &state);
1762
1763 // MuxAPI handlers (agent.open, agent.send, agent.stop, agent.status, agent.list, agent.output)
1764 super::app_api::register_app_api_handlers(engine, &state);
1765
1766 // MuxBus cloud connectivity (muxbus.login / muxbus.login.cancel / muxbus.status / muxbus.disconnect / muxbus.cloudconfig)
1767 super::muxbus_handlers::register_muxbus_handlers(engine, &state);
1768
1769 // Native memory file browser (agent:memory:list / read_file / write_file)
1770 super::native_memory_handlers::register_native_memory_handlers(engine, &state);
1771
1772 // Memory delivery: the hidden reinjection's claim against the
1773 // SessionStart hook (memorydelivery:claim_fallback).
1774 super::memory_delivery_handlers::register_memory_delivery_handlers(engine, &state);
1775
1776 // OS notification Router (notify.emit / focus / ack / test / takeactivation
1777 // — docs/specs/SPEC_OS_NOTIFICATIONS_SYSTEM_2026_09_24.md §3.3).
1778 super::notify_handlers::register_notify_handlers(engine, &state, conn_id.clone());
1779}
1780
1781
1782/// Route a decoded `blockinput` payload — the actual human-keystroke path
1783/// (`termViewModel.ts`'s `blockinput` WS command, used by both the
1784/// standalone Terminal widget and an agent pane's composer-drawer shell) —
1785/// to its block's controller.
1786///
1787/// Mirrors `controllerinput`'s own lock check above: drops the input
1788/// outright while `block_id` is under an active PtyShell agent lease,
1789/// rather than forwarding it. This was `agent_lock`'s documented "Known
1790/// gap" — `blockinput`, not `controllerinput`, is where a human's real
1791/// keystrokes actually travel, and until now nothing server-side stopped
1792/// one from interleaving with an agent's own `PtyShellInput` write into the
1793/// same PTY. The frontend's `agentLocked()` gate is still the primary UX
1794/// layer (stops it before it's even sent), but it is eventually consistent;
1795/// this is the synchronous backstop against this handler's own state.
1796fn dispatch_blockinput(block_id: &str, data: Vec<u8>) -> Result<(), String> {
1797 if blockcontroller::agent_lock::is_locked(block_id) {
1798 tracing::debug!(block_id = %block_id, "blockinput: dropped — agent lock active");
1799 return Ok(());
1800 }
1801 let input = blockcontroller::BlockInputUnion::data(data);
1802 blockcontroller::send_input(block_id, input, None)
1803}
1804
1805/// Run `f` on `block_id`'s persistent controller for `agent.answer` /
1806/// `agent.cancel`. The error strings are matched by the frontend's
1807/// SAFE_TO_RETRY_VIA_FOLLOWUP allowlist (useAgentQuestions.ts).
1808fn with_question_controller<T>(
1809 command: &str,
1810 action: &str,
1811 block_id: &str,
1812 tool_use_id: &str,
1813 f: impl FnOnce(&blockcontroller::persistent::PersistentSubprocessController) -> Result<T, String>,
1814) -> Result<T, String> {
1815 if tool_use_id.is_empty() {
1816 return Err(format!("{command}: MISSING_ARG: tool_use_id"));
1817 }
1818 let ctrl = blockcontroller::get_controller(block_id)
1819 .ok_or_else(|| format!("{command}: no controller for block {block_id}"))?;
1820 match ctrl
1821 .as_any()
1822 .downcast_ref::<blockcontroller::persistent::PersistentSubprocessController>()
1823 {
1824 Some(persistent_ctrl) => f(persistent_ctrl),
1825 None => Err(format!(
1826 "{command}: UNSUPPORTED_CONTROLLER: {action} AskUserQuestion \
1827 requires a persistent (host) agent; container/one-shot agents are \
1828 not yet supported (Phase 2)"
1829 )),
1830 }
1831}
1832
1833/// Parse a CommandBlockInputData into a BlockInputUnion.
1834fn parse_block_input(
1835 cmd: &CommandBlockInputData,
1836) -> Result<blockcontroller::BlockInputUnion, String> {
1837 if !cmd.inputdata64.is_empty() {
1838 let data = base64::engine::general_purpose::STANDARD
1839 .decode(&cmd.inputdata64)
1840 .map_err(|e| format!("controllerinput: base64 decode: {e}"))?;
1841 return Ok(blockcontroller::BlockInputUnion::data(data));
1842 }
1843 if !cmd.signame.is_empty() {
1844 return Ok(blockcontroller::BlockInputUnion::signal(&cmd.signame));
1845 }
1846 if let Some(ref ts_val) = cmd.termsize {
1847 let ts: TermSize =
1848 serde_json::from_value(ts_val.clone()).map_err(|e| format!("controllerinput: {e}"))?;
1849 return Ok(blockcontroller::BlockInputUnion::resize(ts));
1850 }
1851 Err("controllerinput: no input data, signal, or termsize".to_string())
1852}
1853
1854// ====================================================================
1855// Tests
1856// ====================================================================
1857
1858#[cfg(test)]
1859mod tests {
1860 use super::*;
1861
1862 #[test]
1863 fn question_controller_errors_keep_the_strings_the_frontend_matches() {
1864 let missing = with_question_controller("agent.cancel", "canceling", "blk", "", |_| Ok(()));
1865 assert_eq!(missing.unwrap_err(), "agent.cancel: MISSING_ARG: tool_use_id");
1866 let absent = with_question_controller("agent.answer", "answering", "no-such-block-qc", "tu-1", |_| Ok(()));
1867 assert_eq!(absent.unwrap_err(), "agent.answer: no controller for block no-such-block-qc");
1868 }
1869
1870 /// Every no-argument stub on this connection sends `null`, not `{}`.
1871 ///
1872 /// `waveinfo`, `getfullconfig`, `eventunsuball`, `routeannounce` and
1873 /// `routeunannounce` are all `client.rpcCall("...", null, opts)`. That is
1874 /// the opposite of the assumption the earlier slices were built on -- those
1875 /// stubs send `{}` -- and it is why these register `Option<NoArgsReq>`
1876 /// rather than the bare struct: serde deserializes a struct only from an
1877 /// object, so a bare `NoArgsReq` would reject every one of these calls
1878 /// while compiling and passing every gate.
1879 #[test]
1880 fn the_no_argument_commands_accept_the_null_their_stubs_send() {
1881 for payload in [serde_json::Value::Null, serde_json::json!({})] {
1882 serde_json::from_value::<Option<crate::backend::rpc_types::NoArgsReq>>(payload.clone())
1883 .unwrap_or_else(|e| panic!("{payload} should deserialize, got {e}"));
1884 }
1885 // ...and the bare struct really does reject null, so the Option is
1886 // load-bearing rather than decoration.
1887 assert!(
1888 serde_json::from_value::<crate::backend::rpc_types::NoArgsReq>(serde_json::Value::Null)
1889 .is_err(),
1890 "if this ever passes, the Option wrappers here are redundant",
1891 );
1892 }
1893
1894 /// `eventunsub` takes a BARE STRING payload -- the only command in the app
1895 /// shaped that way. Its stub is `data: string`.
1896 ///
1897 /// Registered as `serde_json::Value`, NOT `Option<String>`. reagent caught
1898 /// that difference on #3348 and was right: the old handler read
1899 /// `data.as_str().unwrap_or("")`, so a number/object/array was a silent
1900 /// no-op, while `Option<String>` hard-errors on exactly those. Claiming it
1901 /// "keeps that tolerance" was false. `Value` keeps it for real.
1902 ///
1903 /// This test now covers "anything else" rather than only null, which is
1904 /// what its name always promised.
1905 #[test]
1906 fn eventunsub_takes_a_bare_string_and_tolerates_anything_else() {
1907 // Mirrors the handler exactly: deserialize as Value, then as_str().
1908 let name = |v: serde_json::Value| {
1909 serde_json::from_value::<serde_json::Value>(v)
1910 .expect("Value deserializes from anything")
1911 .as_str()
1912 .unwrap_or("")
1913 .to_string()
1914 };
1915 assert_eq!(name(serde_json::json!("blockfile:mux")), "blockfile:mux");
1916 // Each of these was a silent no-op before the migration and must stay one.
1917 for junk in [
1918 serde_json::Value::Null,
1919 serde_json::json!(42),
1920 serde_json::json!({ "event": "x" }),
1921 serde_json::json!(["x"]),
1922 serde_json::json!(true),
1923 ] {
1924 assert_eq!(name(junk.clone()), "", "{junk} must unsubscribe nothing, not error");
1925 }
1926 }
1927
1928 /// Codex P1 on PR #3194: the frontend's own lock gate
1929 /// (`AgentShellSubblock.tsx` dropping `sendDataHandler` while
1930 /// `term:agentlockuntil` is in the future) is necessarily eventually-
1931 /// consistent — it reacts to a WS-pushed meta value, not something
1932 /// synchronous with the backend's own state — so a human keystroke
1933 /// already in flight could otherwise still reach `send_input` and
1934 /// interleave with an agent's own write. `controllerinput` must reject
1935 /// it itself, checked against the backend's own current state at the
1936 /// moment it actually runs.
1937 ///
1938 /// Distinguishes "silently dropped by the lock gate" from "attempted
1939 /// and failed for lack of a live controller" by the response's `error`
1940 /// field: a target block here has no real controller registered at
1941 /// all, so an UNLOCKED call reaches `blockcontroller::send_input` and
1942 /// gets a real `"no controller for block"` error back — proving the
1943 /// call path is live. A LOCKED call must short-circuit before that
1944 /// point, returning cleanly with no error.
1945 #[tokio::test]
1946 async fn controllerinput_is_dropped_while_an_agent_lock_is_active() {
1947 use crate::backend::rpc::engine::WshRpcEngine;
1948 use crate::backend::rpc_types::RpcMessage;
1949 use crate::server::tests::test_state;
1950
1951 let state = test_state();
1952 let (engine, mut output_rx) = WshRpcEngine::new();
1953 register_handlers(&engine, state.clone(), "test-conn".to_string());
1954
1955 async fn send_input(
1956 engine: &std::sync::Arc<WshRpcEngine>,
1957 output_rx: &mut tokio::sync::mpsc::UnboundedReceiver<RpcMessage>,
1958 block_id: &str,
1959 ) -> RpcMessage {
1960 engine.handle_message(RpcMessage {
1961 command: COMMAND_CONTROLLER_INPUT.to_string(),
1962 reqid: uuid::Uuid::new_v4().to_string(),
1963 data: Some(serde_json::json!({ "blockid": block_id, "inputdata64": "eQ==" })), // "y"
1964 ..Default::default()
1965 });
1966 tokio::time::timeout(std::time::Duration::from_secs(2), output_rx.recv())
1967 .await
1968 .expect("handler should respond within 2s")
1969 .expect("response channel should not close")
1970 }
1971
1972 // Locked target: no live controller registered, but the check must
1973 // never reach that far. The lease is seeded through
1974 // `blockcontroller::agent_lock` — the in-memory registry this
1975 // handler actually enforces against — NOT through the block's
1976 // `term:agentlockuntil` meta, which is now the frontend-badge copy
1977 // only (see the handler's own comment and `agent_lock`'s module doc:
1978 // the meta read this replaced put a synchronous SQLite query on the
1979 // per-keystroke path).
1980 blockcontroller::agent_lock::lock_until(
1981 "ci-locked-shell",
1982 agentmux_common::time::now_ms() + 60_000,
1983 );
1984 let resp = send_input(&engine, &mut output_rx, "ci-locked-shell").await;
1985 assert!(
1986 resp.error.is_empty(),
1987 "a locked target must be silently dropped, not reach send_input at all: {}",
1988 resp.error
1989 );
1990
1991 // Unlocked target (no lease at all) — proves the call path above is
1992 // genuinely live: it must reach `send_input` and get a real "no
1993 // controller" error, not silently succeed.
1994 let resp = send_input(&engine, &mut output_rx, "ci-unlocked-shell").await;
1995 assert!(
1996 resp.error.contains("no controller"),
1997 "an unlocked target must reach send_input for real: {}",
1998 resp.error
1999 );
2000
2001 // A stale `term:agentlockuntil` in meta must NOT lock anything by
2002 // itself. This is the fail-open half of moving enforcement into
2003 // memory (`agent_lock`'s module doc): meta can outlive the process
2004 // that wrote it, memory cannot, and a lease that survives a restart
2005 // is a human who can't type `exit` in their own terminal.
2006 let mut stale_meta_block = crate::backend::obj::Block {
2007 oid: "ci-stale-meta-shell".to_string(),
2008 meta: {
2009 let mut m = crate::backend::obj::MetaMapType::new();
2010 m.insert(
2011 "term:agentlockuntil".to_string(),
2012 serde_json::json!(agentmux_common::time::now_ms() + 60_000),
2013 );
2014 m
2015 },
2016 ..Default::default()
2017 };
2018 state.mstore.insert(&mut stale_meta_block).unwrap();
2019 let resp = send_input(&engine, &mut output_rx, "ci-stale-meta-shell").await;
2020 assert!(
2021 resp.error.contains("no controller"),
2022 "a lease that exists only in stale meta must not lock the keyboard: {}",
2023 resp.error
2024 );
2025
2026 blockcontroller::agent_lock::release("ci-locked-shell");
2027 }
2028
2029 /// Regression test for the gap `agent_lock`'s own module doc calls out
2030 /// by name ("Known gap this does NOT close"): unlike `controllerinput`
2031 /// above (Stop/SIGINT/resize), `blockinput` is the ACTUAL human-keystroke
2032 /// path (`termViewModel.ts`, both the standalone Terminal widget and an
2033 /// agent pane's composer-drawer shell) and had no lease check at all —
2034 /// the frontend's own `agentLocked()` gate was the only thing standing
2035 /// between a human keystroke and an agent's own `PtyShellInput` write
2036 /// landing in the same PTY, and that gate is eventually consistent (it
2037 /// reacts to a WS-pushed meta value), not synchronous with this
2038 /// handler's own state. A keystroke slipping through that window
2039 /// interleaves at the byte level with whatever the agent is mid-write
2040 /// to — vim's cursor-addressed alternate-screen redraw does not degrade
2041 /// gracefully under that; it renders as a visibly garbled pane, not a
2042 /// hang. This is exactly what corrupted a live pane during a vim
2043 /// investigation on 2026-09-16.
2044 ///
2045 /// Same "no controller registered" trick as the `controllerinput` test
2046 /// above: a LOCKED call must short-circuit before ever reaching
2047 /// `blockcontroller::send_input`, returning `Ok(())` with no error even
2048 /// though no controller exists for this block id. An UNLOCKED call DOES
2049 /// reach `send_input` and gets back a real `"no controller for block"`
2050 /// error — proving the call path is live, not merely never exercised.
2051 #[test]
2052 fn blockinput_is_dropped_while_an_agent_lock_is_active() {
2053 let locked_id = "ws-blockinput-lock-test-locked";
2054 blockcontroller::agent_lock::lock_until(locked_id, agentmux_common::time::now_ms() + 60_000);
2055 let result = dispatch_blockinput(locked_id, b"y".to_vec());
2056 assert!(
2057 result.is_ok(),
2058 "a locked blockinput call must be dropped cleanly, not forwarded to send_input: {result:?}"
2059 );
2060 blockcontroller::agent_lock::release(locked_id);
2061
2062 let unlocked_id = "ws-blockinput-lock-test-unlocked";
2063 let result = dispatch_blockinput(unlocked_id, b"y".to_vec());
2064 assert_eq!(
2065 result,
2066 Err(format!("no controller for block {unlocked_id}")),
2067 "an unlocked blockinput call must still reach send_input for real"
2068 );
2069 }
2070
2071 /// The lease must stop dropping input the moment it lapses, with nobody
2072 /// calling `PtyShellStop` — an agent that crashed mid-write, or simply
2073 /// stopped writing, must not leave the human unable to type into their
2074 /// own shell (up to and including typing `exit` to close the pane,
2075 /// SPEC_TERM_EXIT_RESPAWN_LOOP_2026_09_15.md §10).
2076 #[tokio::test]
2077 async fn controllerinput_flows_again_once_the_agent_lease_lapses() {
2078 use crate::backend::rpc::engine::WshRpcEngine;
2079 use crate::backend::rpc_types::RpcMessage;
2080 use crate::server::tests::test_state;
2081
2082 let state = test_state();
2083 let (engine, mut output_rx) = WshRpcEngine::new();
2084 register_handlers(&engine, state, "test-conn-lapse".to_string());
2085
2086 let block_id = "ci-lapsing-shell";
2087 blockcontroller::agent_lock::lock_until(block_id, agentmux_common::time::now_ms() + 40);
2088
2089 let send = |engine: std::sync::Arc<WshRpcEngine>| {
2090 engine.handle_message(RpcMessage {
2091 command: COMMAND_CONTROLLER_INPUT.to_string(),
2092 reqid: uuid::Uuid::new_v4().to_string(),
2093 data: Some(serde_json::json!({ "blockid": block_id, "inputdata64": "eQ==" })),
2094 ..Default::default()
2095 });
2096 };
2097
2098 send(engine.clone());
2099 let resp = tokio::time::timeout(std::time::Duration::from_secs(2), output_rx.recv())
2100 .await
2101 .expect("handler should respond within 2s")
2102 .expect("response channel should not close");
2103 assert!(resp.error.is_empty(), "still leased: input must be dropped");
2104
2105 tokio::time::sleep(std::time::Duration::from_millis(80)).await;
2106
2107 send(engine.clone());
2108 let resp = tokio::time::timeout(std::time::Duration::from_secs(2), output_rx.recv())
2109 .await
2110 .expect("handler should respond within 2s")
2111 .expect("response channel should not close");
2112 assert!(
2113 resp.error.contains("no controller"),
2114 "a lapsed lease must let input through without any explicit release: {}",
2115 resp.error
2116 );
2117 }
2118
2119 fn rpc_event(block_id: &str) -> serde_json::Value {
2120 json!({
2121 "eventtype": "rpc",
2122 "data": {
2123 "command": "eventrecv",
2124 "data": {
2125 "event": "blockfile",
2126 "scopes": [format!("block:{block_id}")],
2127 "data": {"data64": "aGk="},
2128 },
2129 },
2130 })
2131 }
2132
2133 fn raw_waveobj_update(oref: &str) -> serde_json::Value {
2134 json!({
2135 "eventtype": "waveobj:update",
2136 "oref": oref,
2137 "data": {},
2138 })
2139 }
2140
2141 fn batched_updates_event() -> serde_json::Value {
2142 json!({
2143 "eventtype": "waveobj:batchedupdates",
2144 "data": [],
2145 })
2146 }
2147
2148 #[test]
2149 fn test_priority_pane_key_from_rpc_blockfile_scopes() {
2150 let event = rpc_event("abc123");
2151 assert_eq!(priority_pane_key(&event), Some("block:abc123".to_string()));
2152 }
2153
2154 /// Raw (non-rpc) object-model broadcasts are NEVER split by pane —
2155 /// splitting them would let round-robin reorder them relative to other
2156 /// object updates for the same object (see `priority_pane_key`'s doc
2157 /// comment for the concrete resurrected-object failure this prevents).
2158 #[test]
2159 fn test_priority_pane_key_raw_waveobj_update_is_non_pane() {
2160 let event = raw_waveobj_update("block:xyz789");
2161 assert_eq!(priority_pane_key(&event), Some(NON_PANE_EVENT_KEY.to_string()));
2162 }
2163
2164 #[test]
2165 fn test_priority_pane_key_non_pane_for_batched_updates() {
2166 let event = batched_updates_event();
2167 assert_eq!(priority_pane_key(&event), Some(NON_PANE_EVENT_KEY.to_string()));
2168 }
2169
2170 /// The core regression this exists to prevent: a noisy pane (A) producing
2171 /// many queued frames must not delay a quiet pane's (B) single frame by
2172 /// the full length of A's backlog. Round-robin interleaving means B's
2173 /// frame comes out within one "round" (bounded by the number of DISTINCT
2174 /// panes with queued output), not after every one of A's frames.
2175 #[tokio::test]
2176 async fn test_fair_drain_priority_interleaves_noisy_and_quiet_panes() {
2177 let (tx, mut rx) = tokio::sync::mpsc::channel(64);
2178
2179 // Pane A floods 10 frames; pane B contributes exactly 1, queued
2180 // in the middle of A's backlog (mirrors real arrival: B's keystroke
2181 // echo lands while A's flood is still being drained into the lane).
2182 for i in 0..5 {
2183 tx.try_send(rpc_event("A")).unwrap();
2184 let _ = i;
2185 }
2186 tx.try_send(rpc_event("B")).unwrap();
2187 for _ in 0..5 {
2188 tx.try_send(rpc_event("A")).unwrap();
2189 }
2190
2191 let first = rx.recv().await.unwrap();
2192 let drained = fair_drain_priority(first, &mut rx);
2193
2194 assert_eq!(drained.len(), 11, "all 11 queued events must be returned, none dropped");
2195
2196 // Find B's position in the fairly-drained output — it must appear
2197 // near the front (within the first "round" across 2 distinct panes),
2198 // not at position 6 where naive FIFO would place it.
2199 let b_pos = drained
2200 .iter()
2201 .position(|e| priority_pane_key(e).as_deref() == Some("block:B"))
2202 .expect("pane B's event must be present");
2203 assert!(
2204 b_pos <= 1,
2205 "pane B's frame should be at most 1 slot behind pane A in round-robin order, was at position {b_pos}"
2206 );
2207 }
2208
2209 #[tokio::test]
2210 async fn test_fair_drain_priority_single_pane_preserves_order() {
2211 let (tx, mut rx) = tokio::sync::mpsc::channel(64);
2212 for _ in 0..4 {
2213 tx.try_send(rpc_event("A")).unwrap();
2214 }
2215 let first = rx.recv().await.unwrap();
2216 let drained = fair_drain_priority(first, &mut rx);
2217 assert_eq!(drained.len(), 4);
2218 }
2219
2220 #[tokio::test]
2221 async fn test_fair_drain_priority_no_events_lost_across_many_panes() {
2222 let (tx, mut rx) = tokio::sync::mpsc::channel(256);
2223 let pane_ids = ["A", "B", "C", "D"];
2224 for (i, pane) in pane_ids.iter().cycle().take(40).enumerate() {
2225 let _ = i;
2226 tx.try_send(rpc_event(pane)).unwrap();
2227 }
2228 let first = rx.recv().await.unwrap();
2229 let drained = fair_drain_priority(first, &mut rx);
2230 assert_eq!(drained.len(), 40);
2231 }
2232
2233 /// Regression test for the object-ordering bug caught in review: a
2234 /// `waveobj:update` for object X followed by a LATER
2235 /// `waveobj:batchedupdates` that also touches X must come out in the
2236 /// same relative order they arrived in. An earlier version of
2237 /// `priority_pane_key` keyed the individual update by its oref and left
2238 /// the unscoped batch in its own group, so round-robin turned
2239 /// `[update1, update2, batch]` into `[update1, batch, update2]` —
2240 /// resurrecting a deleted object on the frontend until reload (the
2241 /// frontend doesn't version-guard deletes). Object-model events must
2242 /// never be split by pane key, only `blockfile` (terminal output) events
2243 /// may be.
2244 #[tokio::test]
2245 async fn test_fair_drain_priority_preserves_object_update_order() {
2246 let (tx, mut rx) = tokio::sync::mpsc::channel(64);
2247
2248 let update1 = raw_waveobj_update("tab:X");
2249 let update2 = raw_waveobj_update("tab:X");
2250 let batch_delete = batched_updates_event();
2251
2252 tx.try_send(update1.clone()).unwrap();
2253 tx.try_send(update2.clone()).unwrap();
2254 tx.try_send(batch_delete.clone()).unwrap();
2255
2256 let first = rx.recv().await.unwrap();
2257 let drained = fair_drain_priority(first, &mut rx);
2258
2259 assert_eq!(drained, vec![update1, update2, batch_delete], "object-model events must stay in pure arrival order, never reordered by round-robin");
2260 }
2261
2262 #[test]
2263 fn srv_info_event_carries_the_info_on_the_eventrecv_path() {
2264 let ev = super::srv_info_event(crate::srv_info::srv_info("1.2.3", "box"));
2265 assert_eq!(ev["eventtype"], "rpc");
2266 assert_eq!(ev["data"]["command"], "eventrecv");
2267 assert_eq!(ev["data"]["data"]["event"], "srvinfo");
2268 assert_eq!(ev["data"]["data"]["data"]["version"], "1.2.3");
2269 }
2270}