agentmux_launcher/
stale_srv.rs

1// Copyright 2026, AgentMux Corp.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Reclaim the data dir from an srv that an earlier launcher left behind.
5//!
6//! srv takes an exclusive `flock` on `<data_dir>/srv.lock` and writes its PID
7//! into it (`crates/srv/src/backend/base.rs`, `acquire_data_dir_lock`). If an
8//! srv outlives its launcher, every later launch's srv fails on that lock and
9//! the app cannot be opened (2026-10-11: macOS "Quit & Reopen" left one
10//! behind). An srv still holding the lock whose launcher is gone (it was
11//! reparented to launchd/init) is stale: stop it, and start fresh.
12//!
13//! Only a process that is an `agentmux-srv` binary is stopped, and never one
14//! started headless (`--headless` / `AGENTMUX_HEADLESS=1`), which runs on its
15//! own by design, and never one whose parent is still an AgentMux launcher:
16//! winning the single-instance socket isn't proof enough on its own, since two
17//! local builds of one checkout can share a data dir with different sockets.
18
19use std::os::unix::io::AsRawFd;
20use std::path::Path;
21use std::time::{Duration, Instant};
22
23/// Name of srv's data-dir lock file. Mirrors `SRV_DATA_LOCK_FILE` in
24/// `crates/srv/src/backend/base.rs`.
25const SRV_LOCK_FILE: &str = "srv.lock";
26
27/// How long a stale srv gets to exit after SIGTERM (it closes its agents
28/// first) before its whole process group is killed.
29const TERM_GRACE: Duration = Duration::from_secs(10);
30
31/// If `<data_dir>/srv.lock` is held, stop the stale srv holding it. Returns
32/// once the lock is free, or the holder can't be stopped (the srv spawn that
33/// follows then fails and reports it as before).
34pub fn reclaim_stale_srv_lock(data_dir: &Path) {
35    let lock_path = data_dir.join(SRV_LOCK_FILE);
36    if lock_is_free(&lock_path) {
37        return;
38    }
39    // srv records its PID in the lock file; one from a release before that
40    // left it empty, so fall back to asking lsof who has the file open.
41    let recorded = std::fs::read_to_string(&lock_path)
42        .ok()
43        .and_then(|s| s.trim().parse::<i32>().ok())
44        .filter(|pid| *pid > 1);
45    let Some(pid) = recorded.or_else(|| lock_holder_via_lsof(&lock_path)) else {
46        crate::log(&format!(
47            "stale-srv: {} is held but its holder can't be identified; leaving it",
48            lock_path.display()
49        ));
50        return;
51    };
52    // `comm` is the executable as one field (a full path on macOS, which can
53    // hold spaces: `/Volumes/AgentMux 3/...`; the first 15 characters of the
54    // name on Linux). Only its file name ever goes into the log: another
55    // process' arguments and environment can hold secrets.
56    let Some(comm) = ps_field(pid, "comm=", false) else {
57        crate::log(&format!("stale-srv: lock names pid {pid}, which isn't running; leaving it"));
58        return;
59    };
60    let exe_name = Path::new(&comm)
61        .file_name()
62        .map(|n| n.to_string_lossy().into_owned())
63        .unwrap_or_default();
64    if !exe_name.starts_with("agentmux-srv") {
65        crate::log(&format!("stale-srv: lock names pid {pid} ({exe_name}), which isn't agentmux-srv; leaving it"));
66        return;
67    }
68    let headless_env = ps_field(pid, "command=", true)
69        .is_some_and(|with_env| with_env.split_whitespace().any(|w| w == "AGENTMUX_HEADLESS=1" || w == "AGENTMUX_HEADLESS=true"));
70    let headless_arg = ps_field(pid, "command=", false)
71        .is_some_and(|args| args.split_whitespace().any(|a| a == "--headless"));
72    if headless_arg || headless_env {
73        crate::log(&format!("stale-srv: pid {pid} is a headless srv using this data dir; leaving it"));
74        return;
75    }
76
77    // Only an srv whose launcher is really gone: winning this launcher's
78    // single-instance socket doesn't prove nobody else owns the data dir (two
79    // local builds of one checkout have different socket hashes but the same
80    // data dir). An orphan is reparented to launchd/init (PPID 1); one whose
81    // parent is still an AgentMux process belongs to a running app.
82    let ppid = ps_field(pid, "ppid=", false).and_then(|p| p.trim().parse::<i32>().ok());
83    if let Some(ppid) = ppid.filter(|p| *p > 1) {
84        let parent = ps_field(ppid, "comm=", false)
85            .and_then(|c| Path::new(&c).file_name().map(|n| n.to_string_lossy().into_owned()))
86            .unwrap_or_default();
87        if parent.to_ascii_lowercase().contains("agentmux") {
88            crate::log(&format!(
89                "stale-srv: srv pid {pid} still has its launcher (pid {ppid}, {parent}); another instance owns this data dir, leaving it"
90            ));
91            return;
92        }
93    }
94
95    crate::log(&format!(
96        "stale-srv: srv pid {pid} still holds {} with no launcher; stopping it so this launch can start",
97        lock_path.display()
98    ));
99    unsafe { libc::kill(pid, libc::SIGTERM) };
100    if wait_for_free(&lock_path, TERM_GRACE) {
101        crate::log(&format!("stale-srv: pid {pid} exited after SIGTERM; lock is free"));
102        return;
103    }
104    // srv leads its own process group (srv_spawner's process_group(0)), which
105    // holds its agent CLIs; take them down with it.
106    let target = if unsafe { libc::getpgid(pid) } == pid { -pid } else { pid };
107    unsafe { libc::kill(target, libc::SIGKILL) };
108    if wait_for_free(&lock_path, Duration::from_secs(3)) {
109        crate::log(&format!("stale-srv: pid {pid} killed; lock is free"));
110    } else {
111        crate::log(&format!("stale-srv: lock still held after killing pid {pid}"));
112    }
113}
114
115/// True when nobody holds the lock (or the file doesn't exist). Takes and
116/// immediately releases the lock; the srv spawned next takes it for real.
117fn lock_is_free(lock_path: &Path) -> bool {
118    let Ok(file) = std::fs::OpenOptions::new().read(true).open(lock_path) else {
119        return true;
120    };
121    let fd = file.as_raw_fd();
122    if unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) } == 0 {
123        unsafe { libc::flock(fd, libc::LOCK_UN) };
124        true
125    } else {
126        false
127    }
128}
129
130fn wait_for_free(lock_path: &Path, within: Duration) -> bool {
131    let deadline = Instant::now() + within;
132    while Instant::now() < deadline {
133        if lock_is_free(lock_path) {
134            return true;
135        }
136        std::thread::sleep(Duration::from_millis(100));
137    }
138    lock_is_free(lock_path)
139}
140
141/// The process that has `lock_path` open, for a lock file without a recorded
142/// PID: an agentmux-srv among the holders if there is one, else the first.
143/// The checks in [`reclaim_stale_srv_lock`] still apply to it.
144fn lock_holder_via_lsof(lock_path: &Path) -> Option<i32> {
145    let pids: Vec<i32> = processes_with_file_open(lock_path)
146        .into_iter()
147        .filter(|pid| *pid > 1 && *pid != std::process::id() as i32)
148        .collect();
149    pids.iter()
150        .copied()
151        .find(|pid| {
152            ps_field(*pid, "comm=", false)
153                .and_then(|c| Path::new(&c).file_name().map(|n| n.to_string_lossy().starts_with("agentmux-srv")))
154                .unwrap_or(false)
155        })
156        .or_else(|| pids.first().copied())
157}
158
159/// PIDs with `path` open. macOS: `lsof -t`, part of the base system.
160#[cfg(not(target_os = "linux"))]
161fn processes_with_file_open(path: &Path) -> Vec<i32> {
162    let Ok(out) = std::process::Command::new("/usr/sbin/lsof")
163        .arg("-t")
164        .arg("--")
165        .arg(path)
166        .output()
167    else {
168        return Vec::new();
169    };
170    String::from_utf8_lossy(&out.stdout)
171        .lines()
172        .filter_map(|l| l.trim().parse::<i32>().ok())
173        .collect()
174}
175
176/// PIDs with `path` open. Linux: read `/proc/<pid>/fd` directly, since
177/// `lsof` isn't installed everywhere (the AppImage doesn't bundle it and the
178/// packages declare no dependencies). Only this user's processes are
179/// readable, which is all an srv we could stop can be.
180#[cfg(target_os = "linux")]
181fn processes_with_file_open(path: &Path) -> Vec<i32> {
182    let Ok(target) = std::fs::canonicalize(path) else {
183        return Vec::new();
184    };
185    let Ok(procs) = std::fs::read_dir("/proc") else {
186        return Vec::new();
187    };
188    procs
189        .flatten()
190        .filter_map(|entry| entry.file_name().to_str()?.parse::<i32>().ok())
191        .filter(|pid| {
192            std::fs::read_dir(format!("/proc/{pid}/fd"))
193                .map(|fds| fds.flatten().any(|fd| std::fs::read_link(fd.path()).is_ok_and(|l| l == target)))
194                .unwrap_or(false)
195        })
196        .collect()
197}
198
199/// One `ps` field for `pid` (`ww`: untruncated; `with_env`: append the
200/// process' environment, `ps e`), or `None` if it isn't running. Never log
201/// the result: it can hold another process' secrets.
202fn ps_field(pid: i32, field: &str, with_env: bool) -> Option<String> {
203    let flags = if with_env { "eww" } else { "ww" };
204    let out = std::process::Command::new("/bin/ps")
205        .args([flags, "-o", field, "-p", &pid.to_string()])
206        .output()
207        .ok()?;
208    let s = String::from_utf8_lossy(&out.stdout).trim().to_string();
209    (out.status.success() && !s.is_empty()).then_some(s)
210}
211
212#[cfg(test)]
213mod tests {
214    use super::*;
215
216    #[test]
217    fn a_missing_or_unheld_lock_is_free() {
218        let dir = tempfile::tempdir().unwrap();
219        let lock = dir.path().join(SRV_LOCK_FILE);
220        assert!(lock_is_free(&lock), "missing file");
221        std::fs::write(&lock, "12345").unwrap();
222        assert!(lock_is_free(&lock), "file nobody holds");
223    }
224
225    #[test]
226    fn a_held_lock_is_not_free_and_is_left_alone_without_a_live_srv() {
227        let dir = tempfile::tempdir().unwrap();
228        let lock = dir.path().join(SRV_LOCK_FILE);
229        let holder = std::fs::OpenOptions::new()
230            .create(true)
231            .write(true)
232            .open(&lock)
233            .unwrap();
234        assert_eq!(unsafe { libc::flock(holder.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }, 0);
235        assert!(!lock_is_free(&lock));
236
237        // The lock names this test process, which isn't agentmux-srv: nothing
238        // may be signalled, and the lock stays held.
239        std::fs::write(&lock, std::process::id().to_string()).unwrap();
240        reclaim_stale_srv_lock(dir.path());
241        assert!(!lock_is_free(&lock));
242    }
243}
agentmux_launcher/stale_srv.rs

agentmux_launcher/
stale_srv.rs
