agentmux_srv/backend/
lan_mdns_health.rs

1// Copyright 2025-2026, AgentMux Corp.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Is our mDNS service really announced on an IPv4 interface?
5//!
6//! `mdns-sd` builds one socket per interface when its daemon is created. For
7//! IPv4 that is a bind to `0.0.0.0:5353`, a multicast-group join and a test
8//! packet; if any step fails the crate logs at `debug!` and **skips that
9//! interface** (mdns-sd's service daemon, "bind a socket to {}: {}. Skipped."). The
10//! daemon still starts and `LAN discovery started (mDNS)` is still logged, so a
11//! host can hear its peers (over IPv6) and never be heard. That is exactly what
12//! happened on Area54 on 2026-10-01 (SPEC_LAN_FIREWALL_SETUP_2026_10_01.md 8.1).
13//!
14//! The daemon already says which interfaces it announced on: its monitor
15//! channel emits `DaemonEvent::Announce(fullname, addresses)` only for
16//! interfaces that have a working socket. This module turns that into a health
17//! verdict. It opens **no socket of its own**: a second receiver on UDP 5353
18//! would compete with the daemon it is checking (Codex P1 on #4133), and
19//! `mdns-sd` always answers by multicast, so a legacy-unicast probe from an
20//! ephemeral port would never get an answer.
21
22use std::collections::{BTreeMap, BTreeSet};
23use std::net::{IpAddr, Ipv4Addr};
24use std::str::FromStr;
25
26/// How long after start the daemon gets to announce before "nothing announced"
27/// counts as a verdict. Probing plus the first announcement takes well under a
28/// second on a healthy host.
29pub const GRACE: std::time::Duration = std::time::Duration::from_secs(5);
30
31/// Consecutive undiscoverable verdicts, one per watchdog tick, required before
32/// the daemon is rebuilt. One miss can be a slow start, not a fault.
33pub const STRIKES_BEFORE_REBUILD: u32 = 2;
34
35/// Ticks to wait after rebuild attempt k (1-based) before attempt k+1; the last
36/// value repeats for ever. At the 15 s tick that is 30 s, 1 min, 2 min, 5 min, then
37/// every 10 min. There is NO cap on the number of attempts.
38///
39/// A cap of three, 30 s apart, was the first version and it failed on Area54
40/// (0.59.4, 2026-10-02): the holder of UDP 5353 was a long-lived program (Chrome had
41/// started 14 s before srv), all three attempts landed inside the same 90 s of
42/// contention, the watchdog gave up, and by 06:32 the port was free again with
43/// nothing left to retry. A rebuild costs milliseconds, so retrying slowly for ever
44/// is the right trade; each attempt names the cause (`lan_mdns_diag`).
45pub const RETRY_AFTER_TICKS: &[u32] = &[2, 4, 8, 20, 40];
46
47/// Ticks the watchdog will keep trying to bring the daemon back when LAN is
48/// wanted but no daemon is running (a rebuild whose new daemon failed to start).
49/// Without this a transient failure inside a rebuild would leave LAN dead until
50/// the user toggled it (ReAgent P2 on #4148).
51pub const MAX_DOWN_RETRIES: u32 = 6;
52
53/// Whether this verdict should be sent to the windows on this tick.
54///
55/// The frontend learns the verdict only from `laninstances:health` events, and
56/// a window opened (or a WebSocket reconnected) after the event has no way to
57/// ask. So a standing problem is **re-sent every tick**; sending it only on
58/// change left a late window showing `peers` for as long as the fault lasted
59/// (ReAgent P1 on #4148). `Healthy` is sent only on change: no news is the
60/// frontend's default for a window with nothing recorded. `Pending` is a gap
61/// between verdicts, never news.
62pub fn should_publish(health: &MdnsHealth, changed: bool) -> bool {
63    match health {
64        MdnsHealth::Pending => false,
65        MdnsHealth::Healthy => changed,
66        MdnsHealth::Degraded { .. } | MdnsHealth::Undiscoverable { .. } => true,
67    }
68}
69
70#[derive(Debug, Clone, PartialEq, Eq, Default)]
71pub enum MdnsHealth {
72    /// Too soon after start to say.
73    #[default]
74    Pending,
75    /// Announced on every IPv4 address we expect to be reachable on, or there
76    /// is no usable IPv4 address to announce on.
77    Healthy,
78    /// Announced on some expected IPv4 addresses but not all (usually a virtual
79    /// adapter). Worth a warning, not a flagged indicator.
80    Degraded { missing: Vec<Ipv4Addr> },
81    /// Announced on none of them: other machines cannot find this one.
82    Undiscoverable { missing: Vec<Ipv4Addr> },
83}
84
85impl MdnsHealth {
86    #[cfg(test)]
87    pub fn is_undiscoverable(&self) -> bool {
88        matches!(self, MdnsHealth::Undiscoverable { .. })
89    }
90
91    /// The value sent to the frontend as `state`.
92    pub fn wire_state(&self) -> &'static str {
93        match self {
94            MdnsHealth::Pending => "pending",
95            MdnsHealth::Healthy => "healthy",
96            MdnsHealth::Degraded { .. } => "degraded",
97            MdnsHealth::Undiscoverable { .. } => "undiscoverable",
98        }
99    }
100
101    pub fn missing(&self) -> &[Ipv4Addr] {
102        match self {
103            MdnsHealth::Degraded { missing } | MdnsHealth::Undiscoverable { missing } => missing,
104            _ => &[],
105        }
106    }
107}
108
109/// Every IPv4 address written in `payload`. `mdns-sd` 0.12 formats the address
110/// part of `DaemonEvent::Announce` two ways (a debug-formatted list at
111/// registration, `host:addr` on later announcements), so this scans for
112/// dotted quads and does not depend on either shape. IPv6 is ignored: only the
113/// IPv4 socket was ever missing.
114pub fn ipv4_in(payload: &str) -> Vec<Ipv4Addr> {
115    payload
116        .split(|c: char| !(c.is_ascii_digit() || c == '.'))
117        .filter(|t| !t.is_empty())
118        .filter_map(|t| Ipv4Addr::from_str(t).ok())
119        .collect()
120}
121
122/// The IPv4 addresses a LAN peer could reach us on, **grouped by interface**:
123/// not loopback, unspecified, link-local (169.254/16), multicast or broadcast.
124///
125/// Grouped because `mdns-sd` announces on ONE address per interface and IP
126/// version (`send_unsolicited_response`), so an interface with two IPv4
127/// addresses never announces its second. Counting each address would leave such
128/// a host `Degraded` for good. An interface is covered when any one of its
129/// addresses was announced.
130pub fn expected_by_interface(
131    local: impl IntoIterator<Item = (String, IpAddr)>,
132) -> Vec<BTreeSet<Ipv4Addr>> {
133    let mut by_name: BTreeMap<String, BTreeSet<Ipv4Addr>> = BTreeMap::new();
134    for (name, addr) in local {
135        let IpAddr::V4(v4) = addr else { continue };
136        if v4.is_loopback()
137            || v4.is_unspecified()
138            || v4.is_link_local()
139            || v4.is_multicast()
140            || v4.is_broadcast()
141        {
142            continue;
143        }
144        by_name.entry(name).or_default().insert(v4);
145    }
146    by_name.into_values().collect()
147}
148
149/// The verdict. Only addresses we expected count: an announcement on an address
150/// we do not listen on proves nothing about the ones we do. `missing` names one
151/// address (the lowest) per uncovered interface.
152pub fn evaluate(expected: &[BTreeSet<Ipv4Addr>], announced: &BTreeSet<Ipv4Addr>) -> MdnsHealth {
153    if expected.is_empty() {
154        return MdnsHealth::Healthy;
155    }
156    let missing: Vec<Ipv4Addr> = expected
157        .iter()
158        .filter(|iface| iface.is_disjoint(announced))
159        .filter_map(|iface| iface.iter().next().copied())
160        .collect();
161    if missing.is_empty() {
162        MdnsHealth::Healthy
163    } else if missing.len() == expected.len() {
164        MdnsHealth::Undiscoverable { missing }
165    } else {
166        MdnsHealth::Degraded { missing }
167    }
168}
169
170/// The verdict for a daemon of a given age.
171///
172/// `Pending` unless we are actually watching the daemon (`monitored`) and it has
173/// had its start-up grace. Without a monitor nothing ever fills `announced`, so
174/// evaluating would report a false `Undiscoverable` on any host with an IPv4
175/// address, rebuild three times for nothing and flag the indicator (ReAgent P1
176/// on #4148). "We cannot tell" is `Pending`, never a fault.
177pub fn verdict(
178    monitored: bool,
179    since_start: std::time::Duration,
180    expected: &[BTreeSet<Ipv4Addr>],
181    announced: &BTreeSet<Ipv4Addr>,
182) -> MdnsHealth {
183    if !monitored || since_start < GRACE {
184        return MdnsHealth::Pending;
185    }
186    evaluate(expected, announced)
187}
188
189#[derive(Debug, PartialEq, Eq)]
190pub enum Action {
191    Nothing,
192    /// Rebuild the daemon (turn LAN off and on).
193    Rebuild,
194}
195
196/// Decides, tick by tick, when an undiscoverable verdict justifies a rebuild.
197#[derive(Debug, Default)]
198pub struct Watchdog {
199    strikes: u32,
200    rebuilds: u32,
201    /// Ticks left before the next rebuild attempt is allowed.
202    cooldown: u32,
203    down_retries: u32,
204}
205
206impl Watchdog {
207    pub fn rebuilds(&self) -> u32 {
208        self.rebuilds
209    }
210
211    /// Ticks until the next attempt, right after a `Rebuild` (for the log line).
212    pub fn retry_in_ticks(&self) -> u32 {
213        self.cooldown
214    }
215
216    /// Forget everything, for when LAN is switched off.
217    pub fn reset(&mut self) {
218        *self = Watchdog::default();
219    }
220
221    /// LAN is wanted but no daemon is running. Returns whether to try starting
222    /// one again this tick. Deliberately does not touch the rebuild budget: a
223    /// failed rebuild must not wipe the count of rebuilds already spent.
224    pub fn observe_down(&mut self) -> bool {
225        if self.down_retries < MAX_DOWN_RETRIES {
226            self.down_retries += 1;
227            true
228        } else {
229            false
230        }
231    }
232
233    pub fn observe(&mut self, health: &MdnsHealth) -> Action {
234        // A daemon exists again, so the "down" retries start over.
235        self.down_retries = 0;
236        // Time passes on every tick, whatever the verdict.
237        self.cooldown = self.cooldown.saturating_sub(1);
238        match health {
239            // No verdict yet: neither a strike nor a recovery.
240            MdnsHealth::Pending => Action::Nothing,
241            MdnsHealth::Healthy => {
242                *self = Watchdog { down_retries: 0, ..Watchdog::default() };
243                Action::Nothing
244            }
245            MdnsHealth::Degraded { .. } => {
246                self.strikes = 0;
247                Action::Nothing
248            }
249            MdnsHealth::Undiscoverable { .. } => {
250                if self.rebuilds == 0 {
251                    // One miss can be a slow start; the first rebuild needs two in a row.
252                    self.strikes += 1;
253                    if self.strikes < STRIKES_BEFORE_REBUILD {
254                        return Action::Nothing;
255                    }
256                } else if self.cooldown > 0 {
257                    // Backing off between attempts.
258                    return Action::Nothing;
259                }
260                self.strikes = 0;
261                self.rebuilds += 1;
262                let step = (self.rebuilds as usize - 1).min(RETRY_AFTER_TICKS.len() - 1);
263                self.cooldown = RETRY_AFTER_TICKS[step];
264                Action::Rebuild
265            }
266        }
267    }
268}
269
270#[cfg(test)]
271mod tests {
272    use super::*;
273
274    fn ip(s: &str) -> Ipv4Addr {
275        s.parse().unwrap()
276    }
277    fn set(v: &[&str]) -> BTreeSet<Ipv4Addr> {
278        v.iter().map(|s| ip(s)).collect()
279    }
280    /// One entry per interface, one address each.
281    fn ifaces(v: &[&str]) -> Vec<BTreeSet<Ipv4Addr>> {
282        v.iter().map(|s| set(&[s])).collect()
283    }
284    fn undiscoverable() -> MdnsHealth {
285        MdnsHealth::Undiscoverable { missing: vec![ip("192.168.1.26")] }
286    }
287
288    #[test]
289    fn parses_the_registration_payload_shape() {
290        // `format!("{:?}", Vec<IpAddr>)`
291        let got = ipv4_in("[192.168.1.26, 172.17.16.1, fe80::1]");
292        assert_eq!(got, vec![ip("192.168.1.26"), ip("172.17.16.1")]);
293    }
294
295    #[test]
296    fn parses_the_host_colon_addr_payload_shape() {
297        assert_eq!(ipv4_in("Area54.local.:192.168.1.26"), vec![ip("192.168.1.26")]);
298        // A host label with digits must not be mistaken for an address.
299        assert!(ipv4_in("Area54.local.").is_empty());
300        assert!(ipv4_in("").is_empty());
301    }
302
303    #[test]
304    fn expected_drops_everything_a_peer_could_not_use() {
305        let local = [
306            ("lo", "127.0.0.1"),
307            ("any", "0.0.0.0"),
308            ("apipa", "169.254.21.81"),
309            ("mc", "224.0.0.251"),
310            ("bc", "255.255.255.255"),
311            ("Ethernet", "192.168.1.26"),
312            ("vEthernet", "172.17.16.1"),
313            ("v6only", "fe80::1"),
314        ]
315        .map(|(n, a)| (n.to_string(), a.parse::<IpAddr>().unwrap()));
316        assert_eq!(expected_by_interface(local), vec![set(&["192.168.1.26"]), set(&["172.17.16.1"])]);
317    }
318
319    #[test]
320    fn two_addresses_on_one_interface_are_one_group() {
321        let local = [("Ethernet", "192.168.1.26"), ("Ethernet", "192.168.1.27"), ("Wi-Fi", "10.0.0.5")]
322            .map(|(n, a)| (n.to_string(), a.parse::<IpAddr>().unwrap()));
323        assert_eq!(
324            expected_by_interface(local),
325            vec![set(&["192.168.1.26", "192.168.1.27"]), set(&["10.0.0.5"])]
326        );
327    }
328
329    #[test]
330    fn nothing_to_announce_on_is_not_a_fault() {
331        assert_eq!(evaluate(&[], &set(&[])), MdnsHealth::Healthy);
332    }
333
334    #[test]
335    fn announced_on_all_expected_is_healthy() {
336        let e = ifaces(&["192.168.1.26", "172.17.16.1"]);
337        assert_eq!(evaluate(&e, &set(&["192.168.1.26", "172.17.16.1"])), MdnsHealth::Healthy);
338    }
339
340    // ReAgent P2 on #4148: mdns-sd announces one address per interface, so the
341    // second address of a NIC is never announced and must not count as missing.
342    #[test]
343    fn an_interface_is_covered_by_any_one_of_its_addresses() {
344        let e = vec![set(&["192.168.1.26", "192.168.1.27"])];
345        assert_eq!(evaluate(&e, &set(&["192.168.1.26"])), MdnsHealth::Healthy);
346        assert_eq!(evaluate(&e, &set(&["192.168.1.27"])), MdnsHealth::Healthy);
347    }
348
349    #[test]
350    fn announcements_outside_the_expected_set_prove_nothing() {
351        let e = ifaces(&["192.168.1.26"]);
352        assert!(evaluate(&e, &set(&["10.9.9.9"])).is_undiscoverable());
353    }
354
355    #[test]
356    fn the_area54_case_is_undiscoverable() {
357        // Both IPv4 interfaces expected; the daemon announced on neither.
358        let e = ifaces(&["192.168.1.26", "172.17.16.1"]);
359        let h = evaluate(&e, &set(&[]));
360        assert_eq!(h.wire_state(), "undiscoverable");
361        assert_eq!(h.missing(), &[ip("192.168.1.26"), ip("172.17.16.1")]);
362    }
363
364    #[test]
365    fn a_missing_virtual_adapter_alone_is_only_degraded() {
366        let e = ifaces(&["192.168.1.26", "172.17.16.1"]);
367        let h = evaluate(&e, &set(&["192.168.1.26"]));
368        assert_eq!(h, MdnsHealth::Degraded { missing: vec![ip("172.17.16.1")] });
369        assert!(!h.is_undiscoverable());
370    }
371
372    #[test]
373    fn a_standing_problem_is_resent_every_tick_so_a_late_window_learns_of_it() {
374        // ReAgent P1 on #4148.
375        let bad = undiscoverable();
376        for _ in 0..3 {
377            assert!(should_publish(&bad, false), "unchanged but still broken: resend");
378        }
379        let degraded = MdnsHealth::Degraded { missing: vec![ip("172.17.16.1")] };
380        assert!(should_publish(&degraded, false));
381    }
382
383    #[test]
384    fn healthy_is_only_sent_on_change_and_pending_never() {
385        assert!(should_publish(&MdnsHealth::Healthy, true), "recovery must clear a warning");
386        assert!(!should_publish(&MdnsHealth::Healthy, false));
387        assert!(!should_publish(&MdnsHealth::Pending, true));
388        assert!(!should_publish(&MdnsHealth::Pending, false));
389    }
390
391    #[test]
392    fn a_down_daemon_is_retried_a_bounded_number_of_times() {
393        let mut w = Watchdog::default();
394        let tries = (0..20).filter(|_| w.observe_down()).count() as u32;
395        assert_eq!(tries, MAX_DOWN_RETRIES);
396    }
397
398    #[test]
399    fn a_failed_rebuild_does_not_wipe_the_rebuild_budget() {
400        // ReAgent P2 on #4148: rebuild, the new daemon fails to start, the watchdog
401        // sees "down" and retries. The spent rebuild must still be counted.
402        let mut w = Watchdog::default();
403        w.observe(&undiscoverable());
404        assert_eq!(w.observe(&undiscoverable()), Action::Rebuild);
405        assert_eq!(w.rebuilds(), 1);
406        assert!(w.observe_down());
407        assert!(w.observe_down());
408        assert_eq!(w.rebuilds(), 1, "still counted while the daemon is down");
409    }
410
411    #[test]
412    fn a_daemon_coming_back_restores_the_retries() {
413        let mut w = Watchdog::default();
414        while w.observe_down() {}
415        assert!(!w.observe_down(), "exhausted");
416        w.observe(&MdnsHealth::Pending); // a daemon exists again
417        assert!(w.observe_down(), "retries start over");
418    }
419
420    #[test]
421    fn no_monitor_means_no_verdict_however_long_it_has_been() {
422        // ReAgent P1 on #4148: nothing would ever fill `announced`.
423        let e = ifaces(&["192.168.1.26"]);
424        let long_ago = GRACE * 100;
425        assert_eq!(verdict(false, long_ago, &e, &set(&[])), MdnsHealth::Pending);
426        assert_eq!(verdict(false, long_ago, &e, &set(&["192.168.1.26"])), MdnsHealth::Pending);
427    }
428
429    #[test]
430    fn a_monitored_daemon_gets_its_start_up_grace_then_a_verdict() {
431        let e = ifaces(&["192.168.1.26"]);
432        assert_eq!(verdict(true, GRACE / 2, &e, &set(&[])), MdnsHealth::Pending);
433        assert!(verdict(true, GRACE, &e, &set(&[])).is_undiscoverable());
434        assert_eq!(verdict(true, GRACE, &e, &set(&["192.168.1.26"])), MdnsHealth::Healthy);
435    }
436
437    #[test]
438    fn one_miss_is_not_enough_to_rebuild() {
439        let mut w = Watchdog::default();
440        assert_eq!(w.observe(&undiscoverable()), Action::Nothing);
441        assert_eq!(w.rebuilds(), 0);
442    }
443
444    #[test]
445    fn two_consecutive_misses_rebuild_and_a_pending_verdict_does_not_count() {
446        let mut w = Watchdog::default();
447        assert_eq!(w.observe(&undiscoverable()), Action::Nothing);
448        // The rebuilt daemon reports Pending: not a strike, not a recovery.
449        assert_eq!(w.observe(&MdnsHealth::Pending), Action::Nothing);
450        assert_eq!(w.observe(&undiscoverable()), Action::Rebuild);
451        assert_eq!(w.rebuilds(), 1);
452    }
453
454    #[test]
455    fn a_healthy_tick_resets_the_strikes() {
456        let mut w = Watchdog::default();
457        w.observe(&undiscoverable());
458        w.observe(&MdnsHealth::Healthy);
459        assert_eq!(w.observe(&undiscoverable()), Action::Nothing, "strikes restarted");
460    }
461
462    /// Never gives up, and backs off: the gaps between attempts are 2, 4, 8, 20 then 40
463    /// ticks, for ever. (The first version stopped after three, 30 s apart, and failed on
464    /// Area54: the port was free again minutes later and nothing was left to retry.)
465    #[test]
466    fn rebuilds_back_off_and_never_stop() {
467        let mut w = Watchdog::default();
468        let mut at = Vec::new();
469        for tick in 1..=400u32 {
470            if w.observe(&undiscoverable()) == Action::Rebuild {
471                at.push(tick);
472            }
473        }
474        let gaps: Vec<u32> = at.windows(2).map(|p| p[1] - p[0]).collect();
475        assert_eq!(at[0], 2, "the first rebuild needs two misses in a row");
476        assert_eq!(&gaps[..5], &[2, 4, 8, 20, 40], "{at:?}");
477        assert!(gaps[5..].iter().all(|g| *g == 40), "settles at one attempt per 40 ticks: {gaps:?}");
478        assert!(400 - at.last().unwrap() < 40, "still retrying at the end: {at:?}");
479        assert!(at.len() > 10, "far more than the old cap of 3: {}", at.len());
480    }
481
482    /// The Area54 timeline: a long-lived program holds the port for minutes, then lets
483    /// go. Every attempt while it holds fails; the first one after it lets go succeeds.
484    /// Under the old cap this never recovered.
485    #[test]
486    fn a_holder_that_lets_go_minutes_later_is_still_recovered() {
487        const HOLDER_LEAVES_AT_TICK: u32 = 28; // 7 minutes at the 15 s tick
488        let mut w = Watchdog::default();
489        let mut recovered_ok = false;
490        let mut recovered_at = None;
491        for tick in 1..=200u32 {
492            let health = if recovered_ok { MdnsHealth::Healthy } else { undiscoverable() };
493            if w.observe(&health) == Action::Rebuild {
494                // A rebuilt daemon announces only if the port is free by then.
495                recovered_ok = tick >= HOLDER_LEAVES_AT_TICK;
496            }
497            if recovered_ok && recovered_at.is_none() {
498                recovered_at = Some(tick);
499            }
500        }
501        let at = recovered_at.expect("it must recover once the holder lets go");
502        assert!((HOLDER_LEAVES_AT_TICK..HOLDER_LEAVES_AT_TICK + 40).contains(&at), "recovered at tick {at}");
503        assert_eq!(w.rebuilds(), 0, "full health restores the budget");
504    }
505
506    #[test]
507    fn the_next_attempt_time_is_exposed_for_the_log_line() {
508        let mut w = Watchdog::default();
509        w.observe(&undiscoverable());
510        assert_eq!(w.observe(&undiscoverable()), Action::Rebuild);
511        assert_eq!(w.retry_in_ticks(), RETRY_AFTER_TICKS[0]);
512    }
513
514    #[test]
515    fn recovery_restores_the_rebuild_budget() {
516        let mut w = Watchdog::default();
517        for _ in 0..40 {
518            w.observe(&undiscoverable());
519        }
520        w.observe(&MdnsHealth::Healthy);
521        assert_eq!(w.rebuilds(), 0);
522        w.observe(&undiscoverable());
523        assert_eq!(w.observe(&undiscoverable()), Action::Rebuild);
524    }
525
526    #[test]
527    fn degraded_does_not_clear_the_rebuild_count_but_does_clear_strikes() {
528        let mut w = Watchdog::default();
529        w.observe(&undiscoverable());
530        w.observe(&undiscoverable()); // rebuild #1
531        assert_eq!(w.rebuilds(), 1);
532        w.observe(&MdnsHealth::Degraded { missing: vec![ip("172.17.16.1")] });
533        assert_eq!(w.rebuilds(), 1, "only full health restores the budget");
534    }
535}
agentmux_srv/backend/lan_mdns_health.rs

agentmux_srv/backend/
lan_mdns_health.rs
