agentmux_srv/backend/blockcontroller/
watchdog.rs

1// Copyright 2025, Command Line Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Agent process watchdog: kills agent panes that exceed max-runtime or idle-output limits.
5//!
6//! Runs every 60 seconds and inspects every running ShellController that has
7//! `is_agent_pane = true`. Two independent kill conditions:
8//!
9//!   A) `term:agentmaxruntimehours` — wall-clock runtime since spawn (0 = disabled)
10//!   B) `term:agentidletimeoutmins` — minutes since last PTY byte (0 = disabled)
11//!
12//! Both limits default to 0 (disabled) so the watchdog is opt-in.
13
14use std::sync::Arc;
15use std::time::Duration;
16use tokio::time::interval;
17
18use crate::backend::wconfig::ConfigState;
19use super::{get_all_controllers, STATUS_RUNNING};
20
21/// Check interval for the watchdog loop.
22const WATCHDOG_INTERVAL_SECS: u64 = 60;
23
24/// Run the agent watchdog loop. Never returns.
25pub async fn run_watchdog_loop(config: Arc<ConfigState>) {
26    let mut ticker = interval(Duration::from_secs(WATCHDOG_INTERVAL_SECS));
27    loop {
28        ticker.tick().await;
29        let settings = config.get_settings();
30        let max_runtime_hours = settings.term_agent_max_runtime_hours;
31        let idle_timeout_mins = settings.term_agent_idle_timeout_mins;
32
33        // Skip entire scan if both limits are disabled.
34        if max_runtime_hours <= 0.0 && idle_timeout_mins <= 0.0 {
35            continue;
36        }
37
38        let now_ms = agentmux_common::time::now_ms();
39
40        for (block_id, ctrl) in get_all_controllers() {
41            let status = ctrl.get_runtime_status();
42            if status.shellprocstatus != STATUS_RUNNING {
43                continue;
44            }
45            if !status.is_agent_pane {
46                continue;
47            }
48
49            // ── Condition A: max runtime ──────────────────────────────────
50            if max_runtime_hours > 0.0 {
51                if let Some(spawn_ms) = status.spawn_ts_ms {
52                    let elapsed_secs = ((now_ms - spawn_ms).max(0) as u64) / 1000;
53                    let limit_secs = (max_runtime_hours * 3600.0) as u64;
54                    if elapsed_secs >= limit_secs {
55                        tracing::warn!(
56                            block_id = %block_id,
57                            elapsed_hours = elapsed_secs / 3600,
58                            limit_hours = %max_runtime_hours,
59                            "watchdog: agent pane exceeded max-runtime, stopping"
60                        );
61                        let _ = crate::sagas::agent_teardown::stop_now(&block_id, true);
62                        continue;
63                    }
64                }
65            }
66
67            // ── Condition B: idle output timeout ─────────────────────────
68            if idle_timeout_mins > 0.0 {
69                if let Some(shell_ctrl) = ctrl.as_any().downcast_ref::<super::shell::ShellController>() {
70                    if let Some(idle_secs) = shell_ctrl.last_output_secs_ago() {
71                        let limit_secs = (idle_timeout_mins * 60.0) as u64;
72                        if idle_secs >= limit_secs {
73                            tracing::warn!(
74                                block_id = %block_id,
75                                idle_mins = idle_secs / 60,
76                                limit_mins = %idle_timeout_mins,
77                                "watchdog: agent pane exceeded idle-output timeout, stopping"
78                            );
79                            let _ = crate::sagas::agent_teardown::stop_now(&block_id, true);
80                        }
81                    }
82                }
83            }
84        }
85    }
86}
agentmux_srv/backend/blockcontroller/watchdog.rs

agentmux_srv/backend/blockcontroller/
watchdog.rs
