agentmux_srv/main.rs
1// Copyright 2025-2026, AgentMux Corp.
2// SPDX-License-Identifier: Apache-2.0
3
4mod agents;
5mod backend;
6mod config;
7mod event_log;
8mod identity;
9mod persist;
10mod persist_subscriber;
11mod reducer;
12mod registry;
13mod sagas;
14mod server;
15mod srv_ipc;
16mod state;
17mod drone;
18mod muxbus;
19#[cfg(windows)]
20mod crash_monitor;
21
22use std::future::IntoFuture;
23use std::sync::Arc;
24
25use clap::Parser;
26use config::CliArgs;
27use server::{AppState, build_router};
28use tokio::net::TcpListener;
29use tokio::signal;
30
31use backend::eventbus::EventBus;
32use backend::reactive::{self, Poller, PollerConfig};
33use backend::storage::filestore::FileStore;
34use backend::storage::migrations::OBJECT_SCHEMA_VERSION;
35use backend::storage::snapshot::maybe_snapshot_pre_migration;
36use backend::storage::store::Store;
37use backend::wps::Broker;
38use backend::wconfig;
39use backend::{docsite, sysinfo, base, wcore};
40
41/// Start a ppid polling watchdog on Linux/macOS.
42/// If the parent process dies, getppid() changes (reparented to init/launchd).
43/// This is safer than PR_SET_PDEATHSIG which tracks the parent *thread*, not process,
44/// and can fire spuriously with async runtimes like Tokio.
45#[cfg(any(target_os = "linux", target_os = "macos"))]
46fn start_ppid_watchdog() {
47 let original_ppid = unsafe { libc::getppid() };
48 std::thread::spawn(move || {
49 loop {
50 std::thread::sleep(std::time::Duration::from_secs(2));
51 let current_ppid = unsafe { libc::getppid() };
52 if current_ppid != original_ppid {
53 eprintln!(
54 "parent process died (ppid changed {} -> {}), shutting down",
55 original_ppid, current_ppid
56 );
57 std::process::exit(0);
58 }
59 }
60 });
61}
62
63/// Event-driven parent process watcher using kqueue (macOS) or pidfd (Linux).
64/// Monitors a specific PID and exits when that process terminates.
65/// Falls back to PPID polling on older Linux kernels without pidfd support.
66#[cfg(target_os = "macos")]
67fn start_parent_watcher(parent_pid: u32) {
68 std::thread::spawn(move || {
69 unsafe {
70 let kq = libc::kqueue();
71 if kq < 0 {
72 eprintln!(
73 "kqueue() failed (errno={}), falling back to ppid watchdog",
74 *libc::__error()
75 );
76 let _ = kq;
77 start_ppid_watchdog();
78 return;
79 }
80
81 // Register EVFILT_PROC + NOTE_EXIT on the parent PID.
82 let mut changelist: [libc::kevent; 1] = std::mem::zeroed();
83 changelist[0] = libc::kevent {
84 ident: parent_pid as usize,
85 filter: libc::EVFILT_PROC,
86 flags: libc::EV_ADD | libc::EV_ONESHOT,
87 fflags: libc::NOTE_EXIT,
88 data: 0,
89 udata: std::ptr::null_mut(),
90 };
91
92 let ret = libc::kevent(
93 kq,
94 changelist.as_ptr(),
95 1,
96 std::ptr::null_mut(),
97 0,
98 std::ptr::null(),
99 );
100
101 if ret < 0 {
102 let errno = *libc::__error();
103 libc::close(kq);
104 if errno == libc::ESRCH {
105 // Parent already dead
106 eprintln!(
107 "parent process {} already exited (ESRCH during kqueue registration), shutting down",
108 parent_pid
109 );
110 std::process::exit(0);
111 }
112 eprintln!(
113 "kevent() registration failed (errno={}), falling back to ppid watchdog",
114 errno
115 );
116 start_ppid_watchdog();
117 return;
118 }
119
120 eprintln!("kqueue EVFILT_PROC registered for parent pid {}", parent_pid);
121
122 // Race condition guard: check if the parent is still alive after registering.
123 // If it died between our registration and this check, we might miss the event.
124 if libc::kill(parent_pid as i32, 0) != 0 && *libc::__error() == libc::ESRCH {
125 libc::close(kq);
126 eprintln!(
127 "parent process {} already exited (post-registration check), shutting down",
128 parent_pid
129 );
130 std::process::exit(0);
131 }
132
133 // Block until the parent exits.
134 let mut eventlist: [libc::kevent; 1] = std::mem::zeroed();
135 let n = libc::kevent(
136 kq,
137 std::ptr::null(),
138 0,
139 eventlist.as_mut_ptr(),
140 1,
141 std::ptr::null(),
142 );
143 libc::close(kq);
144
145 if n > 0 {
146 eprintln!(
147 "parent process {} exited (kqueue EVFILT_PROC), shutting down",
148 parent_pid
149 );
150 } else {
151 eprintln!(
152 "kevent() wait returned {} (errno={}), shutting down",
153 n,
154 *libc::__error()
155 );
156 }
157 std::process::exit(0);
158 }
159 });
160}
161
162/// Event-driven parent process watcher using pidfd_open (Linux 5.3+).
163/// Falls back to PPID polling on older kernels without pidfd support.
164#[cfg(target_os = "linux")]
165fn start_parent_watcher(parent_pid: u32) {
166 std::thread::spawn(move || {
167 unsafe {
168 // Try pidfd_open (syscall 434 on x86_64, 434 on aarch64)
169 let pidfd = libc::syscall(libc::SYS_pidfd_open, parent_pid as libc::c_int, 0 as libc::c_int);
170
171 if pidfd < 0 {
172 let errno = *libc::__errno_location();
173 if errno == libc::ESRCH {
174 // Parent already dead
175 eprintln!(
176 "parent process {} already exited (ESRCH from pidfd_open), shutting down",
177 parent_pid
178 );
179 std::process::exit(0);
180 }
181 // ENOSYS means kernel doesn't support pidfd_open — fall back
182 eprintln!(
183 "pidfd_open() failed (errno={}), falling back to ppid watchdog",
184 errno
185 );
186 start_ppid_watchdog();
187 return;
188 }
189
190 let pidfd = pidfd as libc::c_int;
191
192 // Race condition guard: verify parent is still alive
193 if libc::kill(parent_pid as i32, 0) != 0 && *libc::__errno_location() == libc::ESRCH {
194 libc::close(pidfd);
195 eprintln!(
196 "parent process {} already exited (post-pidfd check), shutting down",
197 parent_pid
198 );
199 std::process::exit(0);
200 }
201
202 // poll() on the pidfd — blocks until the process exits
203 let mut pfd = libc::pollfd {
204 fd: pidfd,
205 events: libc::POLLIN,
206 revents: 0,
207 };
208
209 let ret = libc::poll(&mut pfd, 1, -1); // infinite timeout
210 libc::close(pidfd);
211
212 if ret > 0 {
213 eprintln!(
214 "parent process {} exited (pidfd poll), shutting down",
215 parent_pid
216 );
217 } else {
218 eprintln!(
219 "poll() on pidfd returned {} (errno={}), shutting down",
220 ret,
221 *libc::__errno_location()
222 );
223 }
224 std::process::exit(0);
225 }
226 });
227}
228
229#[tokio::main]
230async fn main() {
231 // -1. Crash monitor branch — must be checked before any other initialization.
232 // The monitor process runs a blocking minidumper::Server and exits when the
233 // main process disconnects. It does not run any backend logic.
234 #[cfg(windows)]
235 if std::env::args().any(|a| a == "--crash-monitor") {
236 crash_monitor::run_monitor();
237 return;
238 }
239
240 // 0. Start parent process watcher BEFORE tokio runtime does real work (Linux/macOS only).
241 // On Windows, the frontend uses a Job Object with KILL_ON_JOB_CLOSE instead.
242 // Uses getppid() to get the parent PID, then kqueue/pidfd to watch it (event-driven,
243 // zero CPU). Falls back to PPID polling if kqueue/pidfd setup fails or parent is init/launchd.
244 #[cfg(any(target_os = "linux", target_os = "macos"))]
245 {
246 let ppid = unsafe { libc::getppid() } as u32;
247 if ppid <= 1 {
248 // Parent is init/launchd — can't meaningfully watch it, use polling fallback
249 start_ppid_watchdog();
250 } else {
251 start_parent_watcher(ppid);
252 }
253 }
254
255 // 0b. Attach out-of-process crash dump handler (Windows only).
256 // Spawns self with --crash-monitor and installs a VEH handler.
257 // _crash_guard must stay alive — dropping it uninstalls the VEH handler.
258 // Non-fatal: if the monitor fails to start, the process continues normally
259 // and WER LocalDumps still captures __fastfail crashes independently.
260 #[cfg(windows)]
261 let _crash_guard = crash_monitor::spawn_and_attach();
262
263 // 1. Init tracing (stderr + rolling file)
264 let _log_guard = init_logging();
265
266 // 1b. Direct-launch PATH fallback. The host enriches the srv's PATH when it
267 // spawns it (sidecar.rs), so in normal operation this is a cheap no-op.
268 // It only does work when the srv is launched directly with a stripped
269 // launchd PATH (some dev paths), so installs/CLIs still resolve node/npm.
270 // See SPEC_TOOLCHAIN_MANAGER_2026-06-15 §3.1.
271 let path_source = agentmux_common::enrich_current_process_path();
272 if path_source != agentmux_common::PathSource::Inherited {
273 // Record the source for the Toolchain modal (the host sets this when it
274 // spawns the srv; on a direct launch we set it here after enriching).
275 std::env::set_var("AGENTMUX_PATH_SOURCE", path_source.as_str());
276 tracing::info!(
277 source = path_source.as_str(),
278 "Enriched srv PATH on direct launch (stripped PATH detected)"
279 );
280 }
281
282 // 2. Parse CLI args and build config
283 let args = CliArgs::parse();
284 let config = config::Config::from_env_and_args(&args).unwrap_or_else(|e| {
285 tracing::error!("Failed to load config: {}", e);
286 std::process::exit(1);
287 });
288
289 let version = config.version.to_string();
290 let build_time = config.build_time.to_string();
291
292 // Make the per-launch auth_key available to the cross-instance agent
293 // registry writer. Peers performing an HTTP forward of a missed inject
294 // use this to authenticate against the writing instance's sidecar.
295 // Must happen after Config::from_env_and_args (which removes
296 // AGENTMUX_AUTH_KEY from the process env) but before anything calls
297 // `agent_registry::write`.
298 crate::backend::reactive::registry::init_local_auth_key(&config.auth_key);
299
300 // 4. Initialize backend (matching Go cmd/server/main-server.go:374-590)
301 base::set_version(&version);
302 base::set_build_time(&build_time);
303
304 // Migrate ~/.waveterm → ~/.agentmux if needed (one-time, non-destructive)
305 base::migrate_legacy_data_dir();
306
307 // Set up data directory (uses AGENTMUX_DATA_HOME or default)
308 if !config.data_home.is_empty() {
309 std::env::set_var("AGENTMUX_DATA_HOME", &config.data_home);
310 }
311 if !config.config_home.is_empty() {
312 std::env::set_var("AGENTMUX_CONFIG_HOME", &config.config_home);
313 }
314 if !config.app_path.is_empty() {
315 std::env::set_var("AGENTMUX_APP_PATH", &config.app_path);
316 }
317
318 base::ensure_wave_data_dir().unwrap_or_else(|e| {
319 tracing::error!("Failed to ensure data dir: {}", e);
320 std::process::exit(1);
321 });
322 base::ensure_wave_db_dir().unwrap_or_else(|e| {
323 tracing::error!("Failed to ensure db dir: {}", e);
324 std::process::exit(1);
325 });
326
327 // Startup diagnostics
328 tracing::info!(
329 data_dir = %base::get_wave_data_dir().display(),
330 db_dir = %base::get_wave_db_dir().display(),
331 app_path = %config.app_path,
332 instance_id = %config.instance_id,
333 "backend directories initialized"
334 );
335
336 // Open databases
337 let db_dir = base::get_wave_db_dir();
338
339 // Pre-migration snapshot (Increment B.2 lean cut from
340 // SPEC_DATA_CHANNELS §3.4). Run BEFORE Store::open so the
341 // backup is taken before any DDL or table rename touches the DB.
342 // The safety lock inside Store::open is the upgrade-direction
343 // guard; this snapshot is the rollback aid for the much rarer case
344 // of a buggy forward migration.
345 //
346 // Failures are logged and ignored — refusing to boot when the
347 // snapshot can't be written would be worse than booting without a
348 // backup (the safety lock still prevents downgrade corruption).
349 let channel = std::env::var("AGENTMUX_CHANNEL").unwrap_or_else(|_| "stable".to_string());
350 let code_version = std::env::var("AGENTMUX_VERSION")
351 .unwrap_or_else(|_| env!("CARGO_PKG_VERSION").to_string());
352 // Snapshots live under the agentmux home root (sibling of `channels/`)
353 // so they survive channel switches and aren't counted against any one
354 // channel's data dir. Honor AGENTMUX_HOME_OVERRIDE for tests; else
355 // default to the OS-level `~/.agentmux/`. Matches `resolve_root` in
356 // agentmux-common — kept inline here to avoid threading the full
357 // DataPaths plumbing into main.rs for one path.
358 let snapshots_dir = std::env::var_os("AGENTMUX_HOME_OVERRIDE")
359 .filter(|s| !s.is_empty())
360 .map(std::path::PathBuf::from)
361 .unwrap_or_else(|| base::get_home_dir().join(".agentmux"))
362 .join("snapshots");
363 match maybe_snapshot_pre_migration(
364 &db_dir,
365 &snapshots_dir,
366 &channel,
367 &code_version,
368 OBJECT_SCHEMA_VERSION,
369 ) {
370 Ok(Some(path)) => tracing::info!(snapshot = %path.display(), "pre-migration snapshot written"),
371 Ok(None) => {}
372 Err(e) => tracing::warn!("pre-migration snapshot failed (continuing without backup): {}", e),
373 }
374
375 let wstore_raw = Store::open(&db_dir.join("objects.db")).unwrap_or_else(|e| {
376 tracing::error!("Failed to open object store: {}", e);
377 std::process::exit(1);
378 });
379 // Attach the cross-version named-agent registry. Falls back to a
380 // disabled registry when the shared home can't be resolved (CI,
381 // unusual envs); mutations still hit SQLite, just don't mirror.
382 // See docs/specs/SPEC_SHARED_AGENT_REGISTRY_2026_05_12.md.
383 if let Some(root) = registry::resolve_shared_registry_dir() {
384 match registry::Registry::open(root.clone()) {
385 Ok(reg) => {
386 // One-shot backfill from every channel/version + dev objects.db
387 // into the registry. Idempotent via the marker file in the
388 // registry root. Read-only on SQLite.
389 //
390 // Attach policy: the registry is attached whenever the migration
391 // RUNS (returns Ok), and intentionally serves the partial set of
392 // readable records — a corrupt/locked DB in an unrelated channel
393 // must not disable cross-channel My Agents (see the Ok arm below,
394 // codex P1 on #1389). On Err (e.g. the registry itself failed) or
395 // when the home can't be resolved, the registry stays detached and
396 // SQLite remains authoritative; the next launch retries.
397 // The generalized migration scans EVERY channel +dev tree under
398 // the true home, so derive ~/.agentmux from the now-global
399 // registry root: registry → agents → shared → <home>.
400 let home_dir = root.ancestors().nth(3).map(|p| p.to_path_buf());
401 let migration_ok = match home_dir {
402 Some(home) => match registry::migrate_from_sqlite_once(&home, ®) {
403 Ok(stats) => {
404 if stats.dbs_scanned > 0
405 || stats.records_written > 0
406 || stats.dbs_skipped > 0
407 {
408 tracing::info!(
409 dbs_scanned = stats.dbs_scanned,
410 dbs_skipped = stats.dbs_skipped,
411 rows_seen = stats.rows_seen,
412 records_written = stats.records_written,
413 records_skipped_existing = stats.records_skipped_existing,
414 records_skipped_unmappable = stats.records_skipped_unmappable,
415 complete = stats.complete,
416 "registry: cross-channel SQLite migration finished"
417 );
418 }
419 // Attach the registry whenever the migration ran —
420 // do NOT gate on `complete`. The scan now spans every
421 // channel + dev tree, so a single corrupt/locked
422 // objects.db in an unrelated channel must not disable
423 // cross-channel My Agents for everyone (codex P1 on
424 // #1389). The records that DID read are served now,
425 // and the live mirror backfills the current channel's
426 // named agents regardless of migration. On any skipped
427 // DB the migration leaves the marker deferred, so a
428 // future launch retries that source (idempotent via
429 // exists_anywhere).
430 //
431 // P0.4 backfill: records written by the P0.3b
432 // migration (or any pre-v3 mirror) lack
433 // source_agents_base, so a cross-channel read would
434 // re-join their working_dir under the wrong channel.
435 // Re-derive each one's source channel from SQLite and
436 // set just that field, preserving session_id etc. Own
437 // marker; runs once even though the migration marker
438 // is already present.
439 match registry::backfill_source_bases_once(&home, ®) {
440 Ok(bf)
441 if bf.records_updated > 0
442 || bf.records_unresolved > 0 =>
443 {
444 tracing::info!(
445 records_updated = bf.records_updated,
446 records_unresolved = bf.records_unresolved,
447 complete = bf.complete,
448 "registry: source-base backfill finished"
449 );
450 }
451 Ok(_) => {}
452 Err(e) => tracing::warn!(
453 error = %e,
454 "registry: source-base backfill errored (continuing; live mirror backfills on relaunch)"
455 ),
456 }
457 true
458 }
459 Err(e) => {
460 tracing::warn!(
461 error = %e,
462 "registry: SQLite migration errored — leaving registry detached; SQLite stays authoritative, next launch retries"
463 );
464 false
465 }
466 },
467 None => {
468 tracing::warn!(
469 root = %root.display(),
470 "registry: cannot resolve home (root has fewer than 3 ancestors) — leaving registry detached"
471 );
472 false
473 }
474 };
475 if migration_ok {
476 tracing::info!(root = %root.display(), "registry: shared agent registry attached");
477 wstore_raw.set_registry(Arc::new(reg));
478 // Now that the registry is GLOBAL for every mode, anchor the
479 // mirror/read working_directory base on the CURRENT channel's
480 // agents dir (AGENTMUX_AGENTS_DIR) — NOT the registry's parent
481 // (shared/agents), which no longer contains any instance. In
482 // dev this is ~/.agentmux/dev/<branch>/agents, in installed/
483 // portable it is channels/<ch>/agents; either way it is the
484 // correct per-channel anchor. This base is the fallback for
485 // legacy v1/v2 records only; v3 records carry their own
486 // source_agents_base (P0.4) and reconstruct against that,
487 // so cross-channel rows resolve to their real workspace.
488 if let Some(base) = std::env::var_os("AGENTMUX_AGENTS_DIR") {
489 if !base.is_empty() {
490 wstore_raw
491 .set_registry_agents_base(std::path::PathBuf::from(base));
492 }
493 }
494 }
495 }
496 Err(e) => tracing::warn!(
497 root = %root.display(),
498 error = %e,
499 "registry: failed to open shared agent registry — SQLite remains authoritative"
500 ),
501 }
502 } else {
503 tracing::warn!("registry: could not resolve shared registry dir — mirror disabled");
504 }
505 // Attach the GLOBAL (cross-channel) agent-definition store. Sibling of the
506 // instance registry above — since P0.3b both live under ~/.agentmux/shared/
507 // (definitions/ and registry/), so user agents created in one channel are
508 // visible in every channel. Best-effort: disabled when the shared dir can't
509 // be resolved. See SPEC_CROSS_CHANNEL_AGENT_PERSISTENCE_2026-06-13.md (P0.2/P0.3).
510 // Captured for the transcript backfill below (after the global transcript
511 // store opens): the user-agent definition ids to seed conversations for.
512 let mut backfill_def_ids: Vec<String> = Vec::new();
513 if let Some(def_dir) = registry::resolve_shared_definitions_dir() {
514 match registry::DefinitionStore::open(def_dir.clone()) {
515 Ok(def_store) => {
516 // P0.2d: one-shot backfill of EXISTING user agents from every
517 // channel's per-version objects.db into the global store, so
518 // agents created before this shipped become cross-channel
519 // without waiting for an edit. Idempotent; read-only on SQLite.
520 // home = def_dir/../../.. (definitions -> agents -> shared -> home).
521 if let Some(home) = def_dir.ancestors().nth(3) {
522 match registry::migrate_definitions_global_once(home, &def_store) {
523 Ok(stats) if stats.dbs_scanned > 0 => tracing::info!(
524 dbs_scanned = stats.dbs_scanned,
525 dbs_skipped = stats.dbs_skipped,
526 rows_seen = stats.rows_seen,
527 records_written = stats.records_written,
528 "def registry: global definition migration finished"
529 ),
530 Ok(_) => {}
531 Err(e) => tracing::warn!(error = %e, "def registry: global migration errored (continuing; live mirror backfills on edit)"),
532 }
533 }
534 // Capture user-agent ids for the transcript backfill (below).
535 backfill_def_ids = def_store
536 .list_active()
537 .map(|v| v.into_iter().map(|r| r.data.id).collect())
538 .unwrap_or_default();
539 wstore_raw.set_def_registry(Arc::new(def_store));
540 tracing::info!(dir = %def_dir.display(), "def registry: global definition store attached");
541 }
542 Err(e) => tracing::warn!(
543 dir = %def_dir.display(),
544 error = %e,
545 "def registry: failed to open global definition store — definitions stay channel-local"
546 ),
547 }
548 } else {
549 tracing::warn!("def registry: could not resolve shared definitions dir — global definitions disabled");
550 }
551 let wstore = Arc::new(wstore_raw);
552 let filestore = Arc::new(FileStore::open(&db_dir.join("filestore.db")).unwrap_or_else(|e| {
553 tracing::error!("Failed to open file store: {}", e);
554 std::process::exit(1);
555 }));
556 // GLOBAL transcript store — backs the `agent:<defId>:current` zone so a
557 // conversation loads when the agent is opened from any build/channel
558 // (finishes the cross-channel arc #1387–#1396). A second FileStore over an
559 // independent SQLite/WAL file is safe alongside the per-channel one. This is
560 // best-effort: if the shared root can't be resolved, or the store can't be
561 // opened, we log and fall back to the per-channel `filestore` (global
562 // transcripts disabled) — never fatal, unlike the per-channel store above.
563 // See `docs/analysis/ANALYSIS_CROSS_CHANNEL_CONVERSATION_HISTORY_2026_06_14.md`.
564 let global_transcript_store: Option<Arc<FileStore>> =
565 match registry::resolve_shared_transcripts_dir() {
566 Some(dir) => {
567 if let Err(e) = std::fs::create_dir_all(&dir) {
568 tracing::warn!(dir = %dir.display(), error = %e, "global transcripts: failed to create dir — disabled, falling back to per-channel store");
569 None
570 } else {
571 match FileStore::open(&dir.join("filestore.db")) {
572 Ok(fs) => {
573 tracing::info!(dir = %dir.display(), "global transcripts: store attached");
574 Some(Arc::new(fs))
575 }
576 Err(e) => {
577 tracing::warn!(dir = %dir.display(), error = %e, "global transcripts: failed to open store — disabled, falling back to per-channel store");
578 None
579 }
580 }
581 }
582 }
583 None => {
584 tracing::warn!("global transcripts: could not resolve shared transcripts dir — disabled, falling back to per-channel store");
585 None
586 }
587 };
588 // Install the process-global handle so the block-controller stdout-reader
589 // hot path can mirror agent `output` into the global zone without threading
590 // the store through `resync_controller` and every controller constructor.
591 if let Some(ref fs) = global_transcript_store {
592 crate::backend::agent_session::set_global_transcript_store(fs.clone());
593 // One-shot: seed pre-existing agents' conversations into the global zone
594 // so the 9 cross-channel agents (and any created before #1399) load
595 // their history when opened from a fresh channel. Runs before the
596 // frontend connects / controllers auto-start, so it seeds before any new
597 // turn writes. Marker-gated; best-effort. See transcript_backfill.rs.
598 if let Some(tdir) = registry::resolve_shared_transcripts_dir() {
599 if let Some(home) = tdir.ancestors().nth(3) {
600 let s = backend::transcript_backfill::backfill_transcripts_once(
601 home,
602 &tdir,
603 &backfill_def_ids,
604 fs,
605 );
606 if s.data_dirs_scanned > 0 || s.seeded > 0 {
607 tracing::info!(
608 agents = s.agents_seen,
609 data_dirs_scanned = s.data_dirs_scanned,
610 seeded = s.seeded,
611 skipped_no_source = s.skipped_no_source,
612 skipped_global_richer = s.skipped_global_richer,
613 "transcript backfill finished"
614 );
615 }
616 }
617 }
618 // Heal global snapshots poisoned before the normalize-on-mirror fix: a
619 // channel-local `sourceBlockId` mirrored into the global zone makes a
620 // cross-channel open render empty (the read fallback can't anchor a block
621 // that doesn't exist in the opening channel). Idempotent + cheap. See
622 // docs/retro/retro-legacy-agent-history-cross-channel-2026-06-16.md.
623 let healed =
624 backend::agent_session::heal_global_snapshot_source_block_ids(fs, &backfill_def_ids);
625 if healed > 0 {
626 tracing::info!(healed, "global transcripts: healed poisoned snapshot sourceBlockIds");
627 }
628 }
629
630 // Backfill the registry `session_id` from each agent's largest provider
631 // session, so a cross-channel / fresh-build open `--resume`s the ORIGINAL
632 // conversation instead of starting a new session that shadows it. The id is
633 // read on launch (picker → `--resume <sid>`) but was never written, so it was
634 // always null. Idempotent; once set, `--resume` keeps it stable across turns.
635 // See docs/retro/retro-cross-channel-conversation-continuity-regression-2026-06-16.md.
636 if let (Some(reg_root), Some(tdir)) = (
637 registry::resolve_shared_registry_dir(),
638 registry::resolve_shared_transcripts_dir(),
639 ) {
640 if let Some(shared) = tdir.ancestors().nth(2) {
641 if let Ok(reg) = registry::Registry::open(reg_root) {
642 // Pass the shared dir; the backfill resolves both the default
643 // `providers/claude/projects` and per-identity bundle roots.
644 let n = backend::session_backfill::backfill_session_ids(®, shared);
645 if n > 0 {
646 tracing::info!(
647 backfilled = n,
648 "registry: session_id backfill for cross-channel resume"
649 );
650 }
651 }
652 }
653 }
654
655 // Saga durability — see SPEC_SAGA_DURABILITY_2026-05-01.md.
656 // Backed by its own SQLite file (`sagas.db`) so saga writes
657 // commit independently of the wstore connection. Failure here
658 // is fatal: without the log, a srv crash mid-saga leaves
659 // unrecoverable state divergence.
660 let saga_log = Arc::new(
661 crate::sagas::log::SagaLog::open(&db_dir.join("sagas.db")).unwrap_or_else(|e| {
662 tracing::error!("Failed to open saga log: {}", e);
663 std::process::exit(1);
664 }),
665 );
666 // Seed `saga_id_alloc` from the highest persisted saga_id so
667 // restarts don't reuse IDs from prior runs (reagent P1 + codex
668 // P1 PR #631). With this seed + the plain INSERT (no OR REPLACE)
669 // in `start_saga`, ID collisions become impossible by
670 // construction.
671 let saga_id_seed = saga_log.max_saga_id().unwrap_or_else(|e| {
672 tracing::warn!(
673 "[saga] failed to read MAX(saga_id) for allocator seed: {} — defaulting to 0; ID collisions on restart possible until next successful query",
674 e
675 );
676 0
677 });
678 if saga_id_seed > 0 {
679 tracing::info!(
680 "[saga] seeded saga_id_alloc from durable log: next saga_id = {}",
681 saga_id_seed + 1
682 );
683 }
684
685 // Bootstrap data (creates Client/Window/Workspace/Tab on first launch)
686 let first_launch = wcore::ensure_initial_data(&wstore).unwrap_or_else(|e| {
687 tracing::error!("Failed to ensure initial data: {}", e);
688 std::process::exit(1);
689 });
690 if first_launch {
691 tracing::info!("First launch: created initial data");
692 }
693
694 // Seed ~/.agentmux/.gitignore so accidental git operations inside the
695 // data directory (e.g. an agent running `git init` or `git clone` in its
696 // cwd) don't stage anything by default. Idempotent — written once per
697 // install; we don't overwrite an existing user-customized file.
698 if let Some(home) = dirs::home_dir() {
699 let data_dir = home.join(".agentmux");
700 if data_dir.is_dir() {
701 let gitignore = data_dir.join(".gitignore");
702 if !gitignore.exists() {
703 let _ = std::fs::write(&gitignore, "*\n!.gitignore\n");
704 }
705 }
706 }
707
708 // Self-heal layouts: remove orphaned block nodes that cause blank panes.
709 // Runs on every startup to catch any corruption from prior sessions.
710 heal_all_layouts(&wstore);
711
712 // Option E (PR 1 of 2) — one-shot migration of per-block agent
713 // session zones into per-agent zones. Gated by a marker file under
714 // the data dir; a second startup is a no-op. Failures on
715 // individual blocks are logged but do not abort startup; the
716 // marker file is written even on partial failure so we don't
717 // retry indefinitely (operators can delete the marker to force a
718 // re-run). See
719 // docs/specs/SPEC_CONTINUATION_SESSION_PERSISTENCE_2026_05_23.md.
720 let _agent_zones_migration_stats = backend::agent_session::migrate_block_zones_v1(
721 &wstore,
722 &filestore,
723 &base::get_wave_data_dir(),
724 );
725
726 // Two-tier picker — Phase 1 (SPEC_AGENT_PICKER_TWO_TIER_2026_05_24.md).
727 // Mandatory companion to the picker UI split: any seeded template
728 // that currently carries a session zone (e.g. `agent:claude:current`
729 // with Maks's conversation) is promoted to a new user-owned
730 // definition with a sensible default name, and its zones +
731 // referencing instances are moved over. Without this step the
732 // freshly-introduced "Templates" section of the picker would
733 // silently reattach into pre-existing user sessions. Marker-file
734 // gated; second start is a no-op.
735 let _template_promote_stats = backend::agent_session::migrate_promote_template_sessions_v1(
736 &wstore,
737 &filestore,
738 &base::get_wave_data_dir(),
739 );
740
741 // Session recovery (Phase 4.2): scan for agent blocks that still have
742 // `session:active_pid` from a previous run — those sessions were killed
743 // by a crash/reboot. Transfer to `session:was_interrupted` so the
744 // frontend can show a reconnect banner.
745 let orphan_count = backend::blockcontroller::session_recovery::scan_orphans(&wstore);
746 if orphan_count > 0 {
747 tracing::info!(
748 orphan_count = orphan_count,
749 "session_recovery: flagged {} interrupted sessions for user reconnect",
750 orphan_count
751 );
752 }
753
754 // Auto-seed agent definitions on first launch (or empty DB)
755 backend::agent_seed::auto_seed_on_startup(&wstore);
756
757 // Phase 3a — `db_agents` consolidation backfill. Marker-file gated
758 // under the data dir; idempotent across restarts. WRITE-ONLY in
759 // Phase 3a: dual-write keeps `db_agents` fresh; reads still hit
760 // `db_agent_definitions` / `db_agent_instances`. Phase 3b will
761 // flip readers over. Failures here are logged + tolerated — the
762 // old tables remain authoritative; a future startup retries.
763 // See docs/specs/SPEC_AGENT_CONCEPT_CONSOLIDATION_2026_05_24.md.
764 match wstore.run_agents_consolidate(Some(&base::get_wave_data_dir())) {
765 Ok(stats) if stats.already_done => {
766 tracing::debug!("agents_consolidate: marker present; backfill already done");
767 }
768 Ok(stats) => {
769 tracing::info!(
770 templates_inserted = stats.templates_inserted,
771 user_defs_inserted = stats.user_defs_inserted,
772 instances_as_clone_inserted = stats.instances_as_clone_inserted,
773 instances_folded_into_def = stats.instances_folded_into_def,
774 instances_skipped_continuation = stats.instances_skipped_continuation,
775 instances_skipped_no_definition = stats.instances_skipped_no_definition,
776 instances_collision_warned = stats.instances_collision_warned,
777 "agents_consolidate: Phase 3a backfill done",
778 );
779 }
780 Err(e) => {
781 tracing::warn!(
782 error = %e,
783 "agents_consolidate: backfill failed; old tables remain authoritative",
784 );
785 }
786 }
787
788 // Gap-repair: backfill definitions written after the Phase 3a marker
789 // but before Phase 3b dual-write (they exist in db_agent_definitions
790 // but not in db_agents, making them invisible to Phase 3b readers).
791 match wstore.repair_agent_def_gaps() {
792 Ok(0) => {}
793 Ok(n) => {
794 tracing::info!(count = n, "agents_consolidate: gap-repair backfilled missing definitions");
795 }
796 Err(e) => {
797 tracing::warn!(error = %e, "agents_consolidate: gap-repair failed (non-fatal)");
798 }
799 }
800
801 // Event infrastructure
802 let event_bus = Arc::new(EventBus::new());
803 let broker = Arc::new(Broker::new());
804
805 // Bridge WPS events to WebSocket clients via EventBus
806 let bridge = backend::eventbus::EventBusBridge::new(event_bus.clone());
807 broker.set_client(Box::new(bridge));
808
809 // OAuth-bundles startup migration (PR E, spec §5):
810 // on first launch after an upgrade, detect ambient OAuth
811 // credentials in `<HOME>/.<auth_dir_name>/.credentials.json` for
812 // each oauth-class provider (claude / codex / openclaw) and seed a
813 // "Default" identity bundle whose binding points at the ambient
814 // dir via `SecretRef::OAuthConfigDir`. Idempotent across restarts —
815 // a second invocation sees the existing binding and exits early
816 // for each already-covered provider. Legacy empty / "blank"
817 // identity_id rows on `db_agent_instances` are back-filled to the
818 // Default bundle in the same pass. Pure no-op when no ambient
819 // creds exist (fresh install) or every oauth-class provider is
820 // already bound by a user-driven flow.
821 let _oauth_migration_stats = identity::migration::run_default_bundle_migration(
822 &wstore,
823 Some(&broker),
824 None,
825 );
826
827 // Config watcher (created before sysinfo loop so it can read telemetry:interval)
828 let config_watcher = Arc::new(wconfig::ConfigWatcher::with_config(wconfig::build_default_config()));
829
830 // Load user's settings.json from disk (merges with defaults)
831 backend::config_watcher_fs::load_settings_from_disk(&config_watcher);
832
833 // Watch settings.json for changes and broadcast to WebSocket clients
834 let _settings_watcher = backend::config_watcher_fs::spawn_settings_watcher(
835 config_watcher.clone(),
836 event_bus.clone(),
837 );
838
839 // Start sysinfo collection loop (interval configurable via telemetry:interval)
840 let sysinfo_broker = broker.clone();
841 let sysinfo_config = config_watcher.clone();
842 tokio::spawn(async move {
843 sysinfo::run_sysinfo_loop(sysinfo_broker, sysinfo_config, "local".to_string()).await;
844 });
845
846 // Start agent process watchdog (kills panes that exceed max-runtime or idle-output limits)
847 let watchdog_config = config_watcher.clone();
848 tokio::spawn(async move {
849 backend::blockcontroller::watchdog::run_watchdog_loop(watchdog_config).await;
850 });
851
852 // Reactive handler (global singleton) + poller
853 let reactive_handler = reactive::get_global_handler();
854 reactive_handler.set_input_sender(Arc::new(|block_id: &str, data: &[u8]| {
855 backend::blockcontroller::send_input(
856 block_id,
857 backend::blockcontroller::BlockInputUnion::data(data.to_vec()),
858 None,
859 )
860 }));
861 // Controller-aware delivery (SPEC_AGENT_CONTROL_PROTOCOL §6 / Phase 3): persistent
862 // stream-json and ACP agents have no PTY, so muxbus Tier-1 keystroke injection
863 // silently misses them. Route those through their structured channel (live stdin /
864 // session/prompt) — which also steers the agent mid-turn — and fall back to PTY
865 // keystrokes only for terminal-based agents.
866 reactive_handler.set_message_sender(Arc::new(|block_id: &str, message: &str| {
867 match backend::blockcontroller::deliver_agent_message(block_id, message) {
868 Ok(backend::blockcontroller::AgentDelivery::Structured) => Ok(true),
869 Ok(backend::blockcontroller::AgentDelivery::Pty) => Ok(false),
870 Err(e) => Err(e),
871 }
872 }));
873 let poller = Arc::new(Poller::new(
874 PollerConfig {
875 agentmux_url: None,
876 agentmux_token: None,
877 poll_interval_secs: reactive::DEFAULT_POLL_INTERVAL_SECS,
878 },
879 reactive_handler,
880 ));
881
882 // Cloud push subscriber — single WS connection per sidecar that the cloud
883 // uses to push reactive injections instead of polling.
884 // No-op until the user connects via muxbus.login.
885 crate::muxbus::cloud_subscriber::CloudSubscriber::init_global(wstore.clone());
886
887 // Set up docsite directory
888 if let Some(app_path) = base::get_wave_app_path() {
889 let docsite_dir = app_path.join("docsite");
890 docsite::set_docsite_dir(docsite_dir);
891 }
892
893 // Local MessageBus for inter-agent communication
894 let messagebus = Arc::new(backend::messagebus::MessageBus::new());
895
896 // Subagent watcher — monitors Claude Code session dirs for spawned subagents
897 let subagent_watcher = backend::subagent_watcher::SubagentWatcher::spawn(event_bus.clone());
898
899 // History service — discovers and indexes past CLI agent conversations
900 let history_service = Arc::new(backend::history::HistoryService::new());
901
902 // Session archiver — auto-archive sessions inactive for >7 days, cap at 2 GB.
903 // Skip if home directory can't be determined (would otherwise fall back to a
904 // relative path and create archives under the current working directory).
905 if let Some(archive_dir) = backend::session_archive::default_archive_dir() {
906 let archiver = Arc::new(backend::session_archive::SessionArchiver::new(
907 wstore.clone(),
908 filestore.clone(),
909 7, // inactive days
910 2 * 1024 * 1024 * 1024, // 2 GB max
911 archive_dir,
912 ));
913 tokio::spawn(async move {
914 loop {
915 tokio::time::sleep(std::time::Duration::from_secs(3600)).await;
916 match archiver.sweep().await {
917 Ok(stats) => tracing::info!(?stats, "session archiver sweep complete"),
918 Err(e) => tracing::warn!(error = %e, "session archiver sweep failed"),
919 }
920 }
921 });
922 } else {
923 tracing::warn!("session archiver: home dir unavailable, archiver disabled");
924 }
925
926 // 5. Bind 2 TCP listeners on 127.0.0.1:0 (web + ws — separate ports matching Go)
927 let web_listener = TcpListener::bind("127.0.0.1:0")
928 .await
929 .expect("failed to bind web listener");
930 let ws_listener = TcpListener::bind("127.0.0.1:0")
931 .await
932 .expect("failed to bind ws listener");
933
934 let web_addr = web_listener.local_addr().unwrap();
935 let ws_addr = ws_listener.local_addr().unwrap();
936 let local_web_url = format!("http://{}", web_addr);
937
938 // Make local backend URL available to child processes (PTY shells).
939 // agentbus-client reads AGENTMUX_LOCAL_URL and uses it for local PTY delivery
940 // instead of routing through the cloud agentbus.
941 std::env::set_var("AGENTMUX_LOCAL_URL", &local_web_url);
942
943 // LAN discovery via mDNS — opt-in to avoid Windows Firewall prompt.
944 // mDNS binds 0.0.0.0:5353 UDP which triggers the firewall dialog.
945 // The setting defaults to false; users opt in via the HostPopover toggle
946 // (or by editing settings.json). The controller supports live start/stop
947 // so flipping the setting does not require an app restart.
948 // See specs/lan-discovery-toggle.md.
949 let hostname = whoami::fallible::hostname().unwrap_or_else(|_| "unknown".to_string());
950 let lan_discovery = Arc::new(backend::lan_discovery::LanDiscoveryController::new(
951 config.instance_id.clone(),
952 hostname,
953 version.clone(),
954 web_addr.port(),
955 event_bus.clone(),
956 config.auth_key.clone(),
957 ));
958 // Honor the current setting at boot — starts the daemon if enabled.
959 lan_discovery.apply(config_watcher.get_settings().network_lan_discovery);
960
961 // LSP supervisor — owns LSP server child processes. Nothing spawned
962 // until the editor pane calls `lspstart`. Spec:
963 // specs/SPEC_EDITOR_LSP_AND_THEMES_2026-05-26.md
964 let lsp_supervisor = Arc::new(backend::lsp::LspSupervisor::new(event_bus.clone()));
965
966 // Clean up stale cross-instance agent registry entries (entries older than 4h).
967 backend::reactive::registry::cleanup_stale(
968 &base::get_wave_data_dir(),
969 4 * 60 * 60 * 1000,
970 );
971
972 // Tracks agent-spawned OS processes per block. Registered trackers
973 // live as long as their agent pane; the background poller emits
974 // delta events (`agent:process-added`/`-exited`) to the frontend.
975 let process_tracker = std::sync::Arc::new(
976 backend::process_tracker::registry::AgentProcessRegistry::new(Some(broker.clone())),
977 );
978 backend::process_tracker::registry::set_global(process_tracker.clone());
979 backend::process_tracker::registry::spawn_poller(process_tracker.clone());
980
981 // Phase E.2 / E.2c.2 — srv reducer plumbing, hoisted out of the
982 // (conditional) pipe-IPC bind block so HTTP/WS RPC handlers in
983 // dispatch_service can route through the reducer. State, event
984 // bus, event log, and persist subscriber all live unconditionally;
985 // the pipe IPC server is still conditional on
986 // `AGENTMUX_SRV_PIPE_PATH` being set (absent in `task dev` mode).
987 let wstore_for_persist = Arc::clone(&wstore);
988 let srv_state = std::sync::Arc::new(tokio::sync::Mutex::new(state::State::default()));
989 let (srv_events_tx, _) =
990 tokio::sync::broadcast::channel::<agentmux_common::ipc::Event>(1024);
991 let srv_event_log = std::sync::Arc::new(event_log::EventLog::new(Some(
992 base::get_wave_data_dir().join("srv-events.log"),
993 )));
994
995 // Bootstrap reducer state from SQLite. Always runs (even in
996 // `task dev` where there's no pipe IPC server) so RPC handlers
997 // dispatching through the reducer see populated state.
998 persist::bootstrap_state_from_wstore(&srv_state, &wstore_for_persist).await;
999
1000 // Spawn the disk writer (forensic log of every reducer event)
1001 // and the persist subscriber (idempotent SQLite write-back).
1002 let disk_writer_rx = srv_events_tx.subscribe();
1003 let log_for_writer = std::sync::Arc::clone(&srv_event_log);
1004 tokio::spawn(event_log::run_disk_writer(log_for_writer, disk_writer_rx));
1005 let subscriber_rx = srv_events_tx.subscribe();
1006 persist_subscriber::spawn_persist_subscriber(
1007 subscriber_rx,
1008 std::sync::Arc::clone(&wstore_for_persist),
1009 std::sync::Arc::clone(&srv_state),
1010 );
1011
1012 // Phase 1 of the WaveObjUpdate bridge: subscribe to srv_events_tx and
1013 // translate workspace mutations into `waveobj:update` WS broadcasts.
1014 // Fixes the workspace-rename reactivity gap where UpdateWorkspace
1015 // returned `success_empty()` and the response loop had nothing to
1016 // broadcast — see docs/specs/SPEC_OBJ_UPDATE_BRIDGE_2026-05-14.md.
1017 //
1018 // Watchdog: capture the JoinHandle and observe it from a sibling task
1019 // so a panic in the bridge's loop scaffolding (vs. an inner
1020 // dispatch_event panic, which is already caught per-event) is logged
1021 // loudly. Without this, a silent bridge death would manifest as
1022 // "renaming a workspace stopped propagating" with no log evidence.
1023 // (Per ReAgent P2 follow-up on PR #852.)
1024 let bridge_rx = srv_events_tx.subscribe();
1025 let bridge_handle = server::wave_obj_bridge::spawn_wave_obj_bridge(
1026 bridge_rx,
1027 std::sync::Arc::clone(&wstore_for_persist),
1028 std::sync::Arc::clone(&event_bus),
1029 );
1030 tokio::spawn(async move {
1031 match bridge_handle.await {
1032 Ok(()) => tracing::info!(
1033 target: "wave-obj-bridge",
1034 "bridge task exited normally (events channel closed at srv shutdown)"
1035 ),
1036 Err(e) if e.is_panic() => tracing::error!(
1037 target: "wave-obj-bridge",
1038 "bridge task PANICKED at top level — frontend WOS will stop receiving updates until srv restart. Panic: {}",
1039 e
1040 ),
1041 Err(e) => tracing::error!(
1042 target: "wave-obj-bridge",
1043 "bridge task terminated unexpectedly (non-panic JoinError): {}",
1044 e
1045 ),
1046 }
1047 });
1048
1049 let state = AppState {
1050 auth_key: config.auth_key.clone(),
1051 version: version.clone(),
1052 app_path: config.app_path.clone(),
1053 wstore,
1054 filestore,
1055 global_transcript_store,
1056 event_bus,
1057 broker,
1058 reactive_handler,
1059 poller,
1060 config_watcher,
1061 messagebus,
1062 subagent_watcher,
1063 history_service,
1064 lan_discovery,
1065 lsp_supervisor,
1066 local_web_url: local_web_url.clone(),
1067 http_client: reqwest::Client::new(),
1068 process_tracker,
1069 // Phase E.2c.2 — reducer state + event bus exposed to HTTP/WS
1070 // dispatch handlers. Workspace handlers route through the
1071 // reducer and publish events to `srv_events_tx`; the persist
1072 // subscriber writes back to SQLite asynchronously.
1073 srv_state: std::sync::Arc::clone(&srv_state),
1074 srv_events_tx: srv_events_tx.clone(),
1075 // Phase E.5.5 — saga-id allocator. Seeded from
1076 // `SagaLog::max_saga_id()` so restarts don't collide with
1077 // prior runs' IDs. First new saga after restart gets
1078 // `seed + 1`; on a fresh DB seed=0, first saga gets id 1.
1079 saga_id_alloc: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(saga_id_seed)),
1080 saga_log: Arc::clone(&saga_log),
1081 auth_session_manager: std::sync::Arc::new(
1082 crate::identity::auth_session::AuthSessionManager::new(),
1083 ),
1084 install_sessions: crate::server::install_handlers::InstallSessionRegistry::new(),
1085 container_manager: {
1086 match crate::backend::container::ContainerManager::connect() {
1087 Ok(mgr) => {
1088 // Ping is async — spawn a task; the manager is still exposed
1089 // so container agents can start even before the ping resolves.
1090 let mgr = std::sync::Arc::new(mgr);
1091 let mgr_check = mgr.clone();
1092 tokio::spawn(async move {
1093 match mgr_check.check_available().await {
1094 Ok(()) => tracing::info!("Docker daemon available — container agent panes enabled"),
1095 Err(e) => tracing::warn!(error = %e, "Docker daemon not reachable; container agent panes will fail to start"),
1096 }
1097 });
1098 Some(mgr)
1099 }
1100 Err(e) => {
1101 tracing::warn!(error = %e, "Docker not available; container agent panes disabled");
1102 None
1103 }
1104 }
1105 },
1106 shell_sessions: crate::backend::shell_node::ShellSessionRegistry::new(),
1107 };
1108
1109 // Saga durability PR 2 — resume-on-startup. Walk any sagas the
1110 // durable log says are unresolved (running / compensating /
1111 // failed) from a prior srv-process run, dispatch their inverse
1112 // commands, and mark them compensated. Runs AFTER reducer
1113 // bootstrap + persist subscriber spawn so the recovery's reducer
1114 // dispatches operate against fully-populated state, BUT BEFORE
1115 // the API server starts accepting requests so resumed
1116 // compensation can't interleave with new sagas.
1117 //
1118 // Failure here is non-fatal: the saga log read might be transient,
1119 // and starting up without recovery beats refusing to start.
1120 // Operator can still inspect via `--diag sagas` (PR 2 part 2).
1121 let resumed = sagas::recovery::compensate_unresolved(&state)
1122 .await
1123 .unwrap_or_else(|e| {
1124 tracing::error!(
1125 "[saga] resume-on-startup failed: {} — continuing; operator review needed",
1126 e
1127 );
1128 0
1129 });
1130 if resumed > 0 {
1131 tracing::info!(
1132 "[saga] resume-on-startup compensated {} unresolved saga(s) from prior run",
1133 resumed
1134 );
1135 }
1136
1137 // Phase E.1b — srv pipe IPC server. Bound when launcher passes
1138 // `AGENTMUX_SRV_PIPE_PATH`; absent in `task dev` mode (no
1139 // launcher in the loop).
1140 //
1141 // Phase E.2 — bootstrap reducer state from SQLite at startup
1142 // so the session-only projection starts populated. The persist
1143 // subscriber that mirrors pipe-event effects back to SQLite is
1144 // deferred to E.2c (alongside the RPC-through-reducer migration);
1145 // until then, HTTP/WS RPC continues writing directly via wcore
1146 // and pipe commands only mutate the reducer's session-only state.
1147 //
1148 // Bind happens BEFORE the AGENTMUXSRV-ESTART line so the
1149 // launcher knows the pipe is ready when host starts. Non-fatal
1150 // if the bind fails — srv keeps running with HTTP/WS only.
1151 #[cfg(target_os = "windows")]
1152 if let Ok(srv_pipe_path) = std::env::var("AGENTMUX_SRV_PIPE_PATH") {
1153 if !srv_pipe_path.is_empty() {
1154 match srv_ipc::server::bind_first_pipe_instance(&srv_pipe_path) {
1155 Ok(first_pipe) => {
1156 // Phase E.2c.2 — pipe IPC server reuses the
1157 // hoisted srv_state / events_tx / event_log so
1158 // pipe-originated commands and HTTP/WS-originated
1159 // commands mutate the same canonical state.
1160 let srv_ctx = srv_ipc::ServerCtx {
1161 srv_pid: std::process::id(),
1162 srv_version: version.clone(),
1163 state: std::sync::Arc::clone(&srv_state),
1164 events_tx: srv_events_tx.clone(),
1165 event_log: std::sync::Arc::clone(&srv_event_log),
1166 };
1167 let _srv_ipc_handle = srv_ipc::run_srv_ipc_server(
1168 srv_pipe_path.clone(),
1169 first_pipe,
1170 srv_ctx,
1171 );
1172 tracing::info!(
1173 target: "srv-ipc",
1174 "[srv-ipc] bound + spawned on {}",
1175 srv_pipe_path
1176 );
1177 }
1178 Err(e) => {
1179 tracing::error!(
1180 target: "srv-ipc",
1181 "[srv-ipc] bind failed on {}: {} — srv runs without pipe IPC",
1182 srv_pipe_path,
1183 e
1184 );
1185 }
1186 }
1187 }
1188 }
1189
1190 // 6. Emit AGENTMUXSRV-ESTART on stderr (exact format from cmd/server/main-server.go:617)
1191 eprintln!(
1192 "AGENTMUXSRV-ESTART ws:{} web:{} version:{} buildtime:{} instance:{}",
1193 ws_addr, web_addr, version, build_time, config.instance_id
1194 );
1195
1196 // 7. Build router and serve on both listeners
1197 // Keep a handle to the shell registry for shutdown cleanup — `state` is
1198 // moved into the router below. [reagent #1422 P2]
1199 let shell_sessions_shutdown = state.shell_sessions.clone();
1200 let router = build_router(state);
1201
1202 let web_server = axum::serve(web_listener, router.clone());
1203 let ws_server = axum::serve(ws_listener, router);
1204
1205 // 8. Spawn stdin watch thread (exit on EOF — matching Go's stdinReadWatch)
1206 let stdin_token = tokio_util::sync::CancellationToken::new();
1207 let stdin_shutdown = stdin_token.clone();
1208 std::thread::spawn(move || {
1209 use std::io::Read;
1210 let mut stdin = std::io::stdin().lock();
1211 let mut buf = [0u8; 1024];
1212 loop {
1213 match stdin.read(&mut buf) {
1214 Ok(0) => {
1215 eprintln!("stdin closed, shutting down");
1216 stdin_shutdown.cancel();
1217 break;
1218 }
1219 Ok(_) => {}
1220 Err(e) => {
1221 eprintln!("stdin read error: {}, shutting down", e);
1222 stdin_shutdown.cancel();
1223 break;
1224 }
1225 }
1226 }
1227 });
1228
1229 // 9. Spawn signal handler (SIGINT/SIGTERM → graceful shutdown)
1230 let signal_token = stdin_token.clone();
1231 tokio::spawn(async move {
1232 let ctrl_c = signal::ctrl_c();
1233 #[cfg(unix)]
1234 {
1235 let mut sigterm =
1236 signal::unix::signal(signal::unix::SignalKind::terminate()).unwrap();
1237 tokio::select! {
1238 _ = ctrl_c => {
1239 tracing::info!("received SIGINT, shutting down");
1240 }
1241 _ = sigterm.recv() => {
1242 tracing::info!("received SIGTERM, shutting down");
1243 }
1244 }
1245 }
1246 #[cfg(not(unix))]
1247 {
1248 ctrl_c.await.ok();
1249 tracing::info!("received Ctrl+C, shutting down");
1250 }
1251 signal_token.cancel();
1252 });
1253
1254 // Run both servers until shutdown
1255 tokio::select! {
1256 result = web_server.into_future() => {
1257 if let Err(e) = result {
1258 tracing::error!("web server error: {}", e);
1259 }
1260 }
1261 result = ws_server.into_future() => {
1262 if let Err(e) = result {
1263 tracing::error!("ws server error: {}", e);
1264 }
1265 }
1266 _ = stdin_token.cancelled() => {
1267 tracing::info!("shutdown signal received, exiting");
1268 }
1269 }
1270
1271 // Shutdown cleanup — tree-kill any persistent shells so long-running
1272 // children (`task dev` → task.exe/node) don't orphan on srv exit. stop_all()
1273 // fires each shell's cancel handle; the kill_tasks run taskkill/killpg
1274 // asynchronously, so give them a brief grace to complete before we exit.
1275 // (`kill_on_drop` only reaps the wrapper shell and doesn't fire on a clean
1276 // process exit, so this is the real orphan guard.) [reagent #1422 P2]
1277 let live = shell_sessions_shutdown.stop_all();
1278 if live > 0 {
1279 tracing::info!(count = live, "shutdown: stopping persistent shells");
1280 tokio::time::sleep(std::time::Duration::from_millis(800)).await;
1281 }
1282}
1283
1284/// Initialize tracing with dual output: JSON rolling file + human-readable stderr.
1285/// Returns a guard that must be held for the lifetime of the app to ensure log flushing.
1286fn init_logging() -> tracing_appender::non_blocking::WorkerGuard {
1287 use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter};
1288
1289 // Always log to ~/.agentmux/logs/ so all logs (host + sidecar) land in one
1290 // discoverable directory. AGENTMUX_DATA_HOME controls the data dir, not logs.
1291 // Version is embedded in the filename for side-by-side coexistence.
1292 let version = env!("CARGO_PKG_VERSION");
1293 let log_dir = dirs::home_dir()
1294 .unwrap_or_default()
1295 .join(".agentmux")
1296 .join("logs");
1297 let _ = std::fs::create_dir_all(&log_dir);
1298
1299 // Delete log files older than 7 days to prevent unbounded growth.
1300 cleanup_old_logs(&log_dir, 7);
1301
1302 // Rolling daily log file with JSON structured output
1303 let log_prefix = format!("agentmuxsrv-v{}.log", version);
1304 let file_appender = tracing_appender::rolling::daily(&log_dir, &log_prefix);
1305 let (non_blocking_file, guard) = tracing_appender::non_blocking(file_appender);
1306
1307 // Write pointer to current log file for zero-lookup agent discovery.
1308 // Version-qualified name so multi-instance doesn't clobber pointers.
1309 let today = chrono::Utc::now().format("%Y-%m-%d").to_string();
1310 let current_filename = format!("{}.{}", log_prefix, today);
1311 let pointer_name = format!("current-srv-v{}.path", version);
1312 let _ = std::fs::write(log_dir.join(&pointer_name), ¤t_filename);
1313
1314 // Spawn a background thread to refresh the pointer on UTC date rollover.
1315 // tracing_appender::rolling::daily creates a new file at midnight UTC.
1316 {
1317 let log_dir = log_dir.clone();
1318 let log_prefix = log_prefix.clone();
1319 let pointer_name = pointer_name.clone();
1320 std::thread::Builder::new()
1321 .name("srv-log-pointer".into())
1322 .spawn(move || {
1323 let mut last_date = chrono::Utc::now().format("%Y-%m-%d").to_string();
1324 loop {
1325 std::thread::sleep(std::time::Duration::from_secs(60));
1326 let today = chrono::Utc::now().format("%Y-%m-%d").to_string();
1327 if last_date != today {
1328 last_date = today.clone();
1329 let filename = format!("{}.{}", log_prefix, today);
1330 let _ = std::fs::write(log_dir.join(&pointer_name), &filename);
1331 }
1332 }
1333 })
1334 .ok();
1335 }
1336
1337 let subscriber = tracing_subscriber::registry()
1338 .with(
1339 EnvFilter::try_from_default_env()
1340 .unwrap_or_else(|_| EnvFilter::new("agentmuxsrv=info,info")),
1341 )
1342 .with(
1343 fmt::layer()
1344 .json()
1345 .with_writer(non_blocking_file)
1346 .with_target(true)
1347 .with_thread_ids(true),
1348 )
1349 .with(
1350 fmt::layer()
1351 .with_writer(std::io::stderr)
1352 .with_ansi(true),
1353 );
1354
1355 tracing::subscriber::set_global_default(subscriber).ok();
1356
1357 tracing::info!(
1358 version = env!("CARGO_PKG_VERSION"),
1359 os = std::env::consts::OS,
1360 arch = std::env::consts::ARCH,
1361 log_dir = %log_dir.display(),
1362 "agentmuxsrv starting"
1363 );
1364
1365 guard
1366}
1367
1368/// Delete log files (*.log.*) older than `days` to prevent unbounded growth.
1369/// Only touches files with `.log.` in the name — pointer files and other data are safe.
1370fn cleanup_old_logs(log_dir: &std::path::Path, days: u64) {
1371 let cutoff = std::time::SystemTime::now()
1372 - std::time::Duration::from_secs(days * 86400);
1373 let Ok(entries) = std::fs::read_dir(log_dir) else { return };
1374 for entry in entries.flatten() {
1375 let path = entry.path();
1376 if !path.to_string_lossy().contains(".log.") {
1377 continue;
1378 }
1379 if let Ok(meta) = entry.metadata() {
1380 if let Ok(modified) = meta.modified() {
1381 if modified < cutoff {
1382 let _ = std::fs::remove_file(&path);
1383 }
1384 }
1385 }
1386 }
1387}
1388
1389/// Walk all tabs and heal their layouts by removing orphaned block references.
1390fn heal_all_layouts(store: &Store) {
1391 use backend::obj::Tab;
1392
1393 let tabs: Vec<Tab> = match store.get_all::<Tab>() {
1394 Ok(tabs) => tabs,
1395 Err(e) => {
1396 tracing::warn!(error = %e, "heal_all_layouts: failed to list tabs");
1397 return;
1398 }
1399 };
1400
1401 let mut healed = 0;
1402 for tab in &tabs {
1403 match backend::wcore::heal_layout(store, &tab.oid) {
1404 Ok(true) => {
1405 tracing::info!(tab_id = %tab.oid, tab_name = %tab.name, "layout healed on startup");
1406 healed += 1;
1407 }
1408 Ok(false) => {}
1409 Err(e) => {
1410 tracing::warn!(tab_id = %tab.oid, error = %e, "heal_layout failed");
1411 }
1412 }
1413 }
1414 if healed > 0 {
1415 tracing::info!(tabs_healed = healed, "layout self-healing complete");
1416 } else {
1417 tracing::info!("layout self-healing: all layouts clean");
1418 }
1419}