agentmux_srv/backend/attachments/
prompt.rs

1// Copyright 2026, AgentMux Corp.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Turning a message's attachments into what the agent receives
5//! (docs/specs/SPEC_AGENT_PANE_IMAGE_ATTACHMENTS_2026_09_26.md §6.6,
6//! SPEC_AGENT_PANE_FILE_ATTACHMENTS_2026_09_26.md §7):
7//!
8//! - every provider gets a numbered list appended to the text, in an
9//!   `<attached_files>` block: each image's send-copy, each other file as a
10//!   copy under its real name, plus notes (type, pages, text version), so
11//!   any agent that can open a local file can reach every attachment;
12//! - Claude (stream-json) additionally gets images inline as `image` blocks
13//!   and PDFs of up to 100 pages as `document` blocks, within a count and
14//!   size budget.
15//!
16//! The block is also how the pane rebuilds the tiles on reload: each line
17//! names the attachment and ends with a path that carries the attachment id
18//! (a `<sha256>.` file name or a `named/<sha256>/` folder).
19
20use std::path::PathBuf;
21
22use base64::Engine;
23
24use super::Service;
25use crate::backend::rpc_types::AttachmentRef;
26
27/// Inline at most this many base64 bytes in one Claude message. The API
28/// caps a request at 32 MB, and the whole conversation is resent each turn.
29pub const INLINE_MAX_B64_BYTES: usize = 20 * 1024 * 1024;
30pub const DEFAULT_INLINE_MAX_COUNT: usize = 20;
31/// Inline at most this much base64 across one Claude session. Claude Code
32/// keeps every inline image in its own session transcript.
33pub const DEFAULT_SESSION_INLINE_MB: u64 = 50;
34/// Claude's PDF support takes at most 100 pages per request on models with
35/// less than a 1M-token context; longer PDFs go by path and text version.
36pub const INLINE_PDF_MAX_PAGES: u32 = 100;
37
38pub const BLOCK_OPEN: &str = "<attached_files>";
39pub const BLOCK_CLOSE: &str = "</attached_files>";
40
41/// One attachment resolved to the file the agent gets.
42#[derive(Debug, Clone, PartialEq, Eq)]
43pub struct Resolved {
44    /// 1-based, matching the composer's tile badges.
45    pub number: usize,
46    pub name: String,
47    pub path: PathBuf,
48    pub mime: String,
49    pub page_count: Option<u32>,
50    /// Shown in square brackets after the name.
51    pub note: Option<String>,
52}
53
54impl Resolved {
55    fn is_image(&self) -> bool {
56        self.mime.starts_with("image/") && self.mime != "image/svg+xml"
57    }
58
59    /// Whether Claude can take this inline: a decoded image, or a PDF short
60    /// enough for its document support.
61    fn inlineable(&self) -> bool {
62        self.is_image()
63            || (self.mime == "application/pdf"
64                && self.page_count.is_some_and(|n| n <= INLINE_PDF_MAX_PAGES))
65    }
66}
67
68/// Everything a turn needs from its attachments.
69#[derive(Debug, Clone, Default)]
70pub struct Prepared {
71    /// Appended to the message text.
72    pub list: String,
73    /// Anthropic `image` / `document` content blocks, in order. Empty unless
74    /// asked for.
75    pub inline: Vec<serde_json::Value>,
76}
77
78/// Resolve refs to what the agent gets. Refs whose files are gone (swept, or
79/// never finished processing) come back as names in the second list.
80/// Blocking.
81pub fn resolve(svc: &Service, refs: &[AttachmentRef]) -> (Vec<Resolved>, Vec<String>) {
82    let mut found = Vec::new();
83    let mut missing = Vec::new();
84    for (i, r) in refs.iter().enumerate() {
85        let name = clean_name(&r.name);
86        match svc.send_target(&r.id, &name) {
87            Some(t) => found.push(Resolved {
88                number: i + 1,
89                name,
90                path: t.path,
91                mime: t.mime,
92                page_count: t.page_count,
93                note: t.note,
94            }),
95            None => missing.push(name),
96        }
97    }
98    (found, missing)
99}
100
101/// Build the list and (when `inline_max_count > 0`) the inline blocks, using
102/// at most `inline_budget` base64 bytes (further capped at
103/// [`INLINE_MAX_B64_BYTES`] per message). Reads the inlined files from disk.
104/// Blocking.
105pub fn prepare(
106    items: &[Resolved],
107    missing: &[String],
108    inline_max_count: usize,
109    inline_budget: u64,
110) -> Prepared {
111    let mut inline = Vec::new();
112    let mut inlined: Vec<usize> = Vec::new();
113    let mut budget = (INLINE_MAX_B64_BYTES as u64).min(inline_budget) as usize;
114    for item in items.iter().filter(|i| i.inlineable()) {
115        if inline.len() >= inline_max_count {
116            break;
117        }
118        let Ok(bytes) = std::fs::read(&item.path) else {
119            continue;
120        };
121        let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes);
122        if encoded.len() > budget {
123            // A smaller one further down may still fit.
124            continue;
125        }
126        budget -= encoded.len();
127        inline.push(if item.is_image() {
128            serde_json::json!({
129                "type": "image",
130                "source": { "type": "base64", "media_type": item.mime, "data": encoded },
131            })
132        } else {
133            serde_json::json!({
134                "type": "document",
135                "source": { "type": "base64", "media_type": "application/pdf", "data": encoded },
136                "title": item.name,
137            })
138        });
139        inlined.push(item.number);
140    }
141    Prepared {
142        list: list_block(items, missing, &inlined),
143        inline,
144    }
145}
146
147/// Base64 bytes carried by inline blocks, for the session budget.
148pub fn inline_bytes(blocks: &[serde_json::Value]) -> u64 {
149    blocks
150        .iter()
151        .filter_map(|b| b.pointer("/source/data").and_then(|d| d.as_str()))
152        .map(|d| d.len() as u64)
153        .sum()
154}
155
156/// The `<attached_files>` block. `inlined` holds the numbers of the items
157/// Claude was also given inline.
158pub fn list_block(items: &[Resolved], missing: &[String], inlined: &[usize]) -> String {
159    let total = items.len() + missing.len();
160    let all_images = missing.is_empty() && items.iter().all(|i| i.is_image());
161    let (one, many) = if all_images {
162        ("image", "images")
163    } else {
164        ("file", "files")
165    };
166    let mut out = String::new();
167    out.push_str(BLOCK_OPEN);
168    out.push('\n');
169    let noun = if total == 1 { one } else { many };
170    out.push_str(&format!(
171        "The user attached {total} {noun}. The numbers match how the user refers to them."
172    ));
173    if !inlined.is_empty() && inlined.len() == items.len() {
174        out.push_str(" They are shown above; the files are listed here too.");
175    } else if !inlined.is_empty() {
176        // Name the inlined ones by their real numbers: a missing or
177        // non-inlined attachment shifts them off 1..=n.
178        let (one, many) = if all_images {
179            ("Image", "Images")
180        } else {
181            ("File", "Files")
182        };
183        let noun = if inlined.len() == 1 { one } else { many };
184        let verb = if inlined.len() == 1 { "is" } else { "are" };
185        out.push_str(&format!(
186            " {noun} {} {verb} shown above. Open the others from these paths with your file or image viewing tool when you need them.",
187            number_list(inlined)
188        ));
189    } else if !items.is_empty() {
190        out.push_str(
191            " Open them from these paths with your file or image viewing tool when you need them.",
192        );
193    }
194    out.push('\n');
195    for item in items {
196        let note = item
197            .note
198            .as_deref()
199            .map(|n| format!(" [{n}]"))
200            .unwrap_or_default();
201        out.push_str(&format!(
202            "{}. {}{note} — {}\n",
203            item.number,
204            item.name,
205            item.path.display()
206        ));
207    }
208    for name in missing {
209        out.push_str(&format!("- {name} — (no longer available)\n"));
210    }
211    out.push_str(BLOCK_CLOSE);
212    out
213}
214
215/// "1–3", "2, 4 and 5", "1 and 2": runs of three or more consecutive
216/// numbers collapse to a range.
217fn number_list(nums: &[usize]) -> String {
218    let mut parts: Vec<String> = Vec::new();
219    let mut i = 0;
220    while i < nums.len() {
221        let mut j = i;
222        while j + 1 < nums.len() && nums[j + 1] == nums[j] + 1 {
223            j += 1;
224        }
225        if j - i >= 2 {
226            parts.push(format!("{}–{}", nums[i], nums[j]));
227        } else {
228            parts.extend(nums[i..=j].iter().map(|n| n.to_string()));
229        }
230        i = j + 1;
231    }
232    match parts.len() {
233        0 => String::new(),
234        1 => parts.remove(0),
235        n => format!("{} and {}", parts[..n - 1].join(", "), parts[n - 1]),
236    }
237}
238
239/// The user's text with the attachment list after it.
240pub fn append(message: &str, list: &str) -> String {
241    if list.is_empty() {
242        return message.to_string();
243    }
244    let trimmed = message.trim_end();
245    if trimmed.is_empty() {
246        list.to_string()
247    } else {
248        format!("{trimmed}\n\n{list}")
249    }
250}
251
252/// Names go into a one-line list entry; keep them on one line and short.
253fn clean_name(name: &str) -> String {
254    let one_line: String = name
255        .chars()
256        .map(|c| if c.is_control() { ' ' } else { c })
257        .collect();
258    let trimmed = one_line.trim();
259    let short: String = trimmed.chars().take(120).collect();
260    if short.is_empty() {
261        "image".to_string()
262    } else {
263        short
264    }
265}
266
267/// A Claude stream-json user line. With no images, `content` is the plain
268/// string it always was; with images, the image blocks come first and the
269/// text last, as the Messages API recommends.
270pub fn claude_user_line(message: String, images: Vec<serde_json::Value>) -> String {
271    let content = if images.is_empty() {
272        serde_json::Value::String(message)
273    } else {
274        let mut blocks = images;
275        blocks.push(serde_json::json!({ "type": "text", "text": message }));
276        serde_json::Value::Array(blocks)
277    };
278    serde_json::json!({
279        "type": "user",
280        "message": { "role": "user", "content": content }
281    })
282    .to_string()
283}
284
285/// The form of a Claude stdin line that AgentMux stores for replay: image
286/// blocks dropped and the content collapsed back to its text, which still
287/// holds the `<attached_images>` list. Every reader of stored user lines
288/// expects `content` to be a string, and a stored line should never carry
289/// megabytes of base64. Lines without an array `content` pass through.
290pub fn persisted_line(json_str: &str) -> std::borrow::Cow<'_, str> {
291    let Ok(mut v) = serde_json::from_str::<serde_json::Value>(json_str) else {
292        return json_str.into();
293    };
294    let Some(content) = v.pointer_mut("/message/content") else {
295        return json_str.into();
296    };
297    let Some(blocks) = content.as_array() else {
298        return json_str.into();
299    };
300    let text: Vec<&str> = blocks
301        .iter()
302        .filter(|b| b.get("type").and_then(|t| t.as_str()) == Some("text"))
303        .filter_map(|b| b.get("text").and_then(|t| t.as_str()))
304        .collect();
305    *content = serde_json::Value::String(text.join("\n\n"));
306    v.to_string().into()
307}
308
309#[cfg(test)]
310mod tests {
311    use super::*;
312
313    fn item(n: usize, name: &str, path: &str) -> Resolved {
314        Resolved {
315            number: n,
316            name: name.into(),
317            path: path.into(),
318            mime: "image/png".into(),
319            page_count: None,
320            note: None,
321        }
322    }
323
324    fn file(
325        n: usize,
326        name: &str,
327        path: &str,
328        mime: &str,
329        pages: Option<u32>,
330        note: Option<&str>,
331    ) -> Resolved {
332        Resolved {
333            number: n,
334            name: name.into(),
335            path: path.into(),
336            mime: mime.into(),
337            page_count: pages,
338            note: note.map(str::to_string),
339        }
340    }
341
342    #[test]
343    fn list_numbers_items_and_names_missing_ones() {
344        let items = vec![
345            item(1, "a.png", "/s/aa.v1-e2000.send.png"),
346            item(3, "c.png", "/s/cc.v1-e2000.send.png"),
347        ];
348        let list = list_block(&items, &["b.png".into()], &[]);
349        assert!(list.starts_with(BLOCK_OPEN));
350        assert!(list.ends_with(BLOCK_CLOSE));
351        assert!(
352            list.contains("The user attached 3 files."),
353            "a missing one isn't known to be an image"
354        );
355        assert!(list.contains("\n1. a.png — /s/aa.v1-e2000.send.png\n"));
356        assert!(list.contains("\n3. c.png — /s/cc.v1-e2000.send.png\n"));
357        assert!(list.contains("- b.png — (no longer available)"));
358    }
359
360    #[test]
361    fn list_says_which_images_are_inline() {
362        let items = vec![item(1, "a", "/a"), item(2, "b", "/b")];
363        assert!(list_block(&items, &[], &[1, 2]).contains("They are shown above"));
364        assert!(list_block(&items, &[], &[1]).contains("Image 1 is shown above"));
365        assert!(list_block(&items, &[], &[]).contains("Open them from these paths"));
366        assert!(list_block(&items, &[], &[]).contains("The user attached 2 images."));
367    }
368
369    #[test]
370    fn inline_text_uses_real_numbers_when_an_earlier_image_is_missing() {
371        // refs = [missing, #2, #3]; one inlined.
372        let items = vec![item(2, "b", "/b"), item(3, "c", "/c")];
373        let list = list_block(&items, &["a".into()], &[2]);
374        assert!(list.contains("File 2 is shown above"), "{list}");
375        assert!(!list.contains("File 1 "));
376    }
377
378    #[test]
379    fn files_carry_notes_and_mixed_lists_say_files() {
380        let items = vec![
381            item(1, "shot.png", "/s/aa.v1-e2000.send.png"),
382            file(
383                2,
384                "spec.pdf",
385                "/s/named/cc/spec.pdf",
386                "application/pdf",
387                Some(12),
388                Some("PDF, 12 pages; text version: /t.txt"),
389            ),
390            file(
391                3,
392                "notes.md",
393                "/s/named/dd/notes.md",
394                "text/plain; charset=utf-8",
395                None,
396                None,
397            ),
398        ];
399        let list = list_block(&items, &[], &[1, 2]);
400        assert!(list.contains("The user attached 3 files."), "{list}");
401        assert!(list.contains("Files 1 and 2 are shown above"), "{list}");
402        assert!(
403            list.contains(
404                "\n2. spec.pdf [PDF, 12 pages; text version: /t.txt] — /s/named/cc/spec.pdf\n"
405            ),
406            "{list}"
407        );
408        assert!(
409            list.contains("\n3. notes.md — /s/named/dd/notes.md\n"),
410            "{list}"
411        );
412    }
413
414    #[test]
415    fn short_pdfs_go_inline_as_documents_long_ones_and_text_do_not() {
416        let d = tempfile::tempdir().unwrap();
417        let write = |name: &str| {
418            let p = d.path().join(name);
419            std::fs::write(&p, b"%PDF-1.4 tiny").unwrap();
420            p
421        };
422        let items = vec![
423            file(
424                1,
425                "short.pdf",
426                write("s.pdf").to_str().unwrap(),
427                "application/pdf",
428                Some(3),
429                None,
430            ),
431            file(
432                2,
433                "long.pdf",
434                write("l.pdf").to_str().unwrap(),
435                "application/pdf",
436                Some(250),
437                None,
438            ),
439            file(
440                3,
441                "unknown.pdf",
442                write("u.pdf").to_str().unwrap(),
443                "application/pdf",
444                None,
445                None,
446            ),
447            file(
448                4,
449                "notes.md",
450                write("n.md").to_str().unwrap(),
451                "text/plain; charset=utf-8",
452                None,
453                None,
454            ),
455        ];
456        let p = prepare(&items, &[], 20, u64::MAX);
457        assert_eq!(p.inline.len(), 1);
458        assert_eq!(p.inline[0]["type"], "document");
459        assert_eq!(p.inline[0]["source"]["media_type"], "application/pdf");
460        assert_eq!(p.inline[0]["title"], "short.pdf");
461        assert!(p.list.contains("File 1 is shown above"), "{}", p.list);
462        // The stored line drops document blocks like image blocks.
463        let line = claude_user_line("see".into(), p.inline.clone());
464        assert_eq!(
465            serde_json::from_str::<serde_json::Value>(&persisted_line(&line)).unwrap()["message"]
466                ["content"],
467            "see"
468        );
469    }
470
471    #[test]
472    fn number_lists_read_naturally() {
473        assert_eq!(number_list(&[1]), "1");
474        assert_eq!(number_list(&[1, 2]), "1 and 2");
475        assert_eq!(number_list(&[1, 2, 3]), "1–3");
476        assert_eq!(number_list(&[2, 4, 5]), "2, 4 and 5");
477        assert_eq!(number_list(&[1, 2, 3, 7]), "1–3 and 7");
478    }
479
480    #[test]
481    fn append_puts_the_list_after_the_text() {
482        assert_eq!(append("look\n", "<L>"), "look\n\n<L>");
483        assert_eq!(append("   ", "<L>"), "<L>");
484        assert_eq!(append("hi", ""), "hi");
485    }
486
487    #[test]
488    fn names_stay_on_one_line() {
489        assert_eq!(clean_name("a\nb\tc.png"), "a b c.png");
490        assert_eq!(clean_name("  "), "image");
491        assert_eq!(clean_name(&"x".repeat(500)).chars().count(), 120);
492    }
493
494    #[test]
495    fn prepare_inlines_within_count_and_reads_files() {
496        let d = tempfile::tempdir().unwrap();
497        let paths: Vec<_> = (0..3)
498            .map(|i| {
499                let p = d.path().join(format!("{i}.png"));
500                std::fs::write(&p, [0x89, b'P', b'N', b'G', i]).unwrap();
501                p
502            })
503            .collect();
504        let items: Vec<_> = paths
505            .iter()
506            .enumerate()
507            .map(|(i, p)| Resolved {
508                number: i + 1,
509                name: format!("{i}.png"),
510                path: p.clone(),
511                mime: "image/png".into(),
512                page_count: None,
513                note: None,
514            })
515            .collect();
516        let p = prepare(&items, &[], 2, u64::MAX);
517        assert_eq!(p.inline.len(), 2);
518        assert_eq!(p.inline[0]["source"]["media_type"], "image/png");
519        assert!(p.list.contains("Images 1 and 2 are shown above"));
520        assert_eq!(
521            inline_bytes(&p.inline),
522            16,
523            "two 5-byte files are 8 base64 chars each"
524        );
525        assert!(prepare(&items, &[], 0, u64::MAX).inline.is_empty());
526        // A spent session budget means paths only.
527        let spent = prepare(&items, &[], 20, 10);
528        assert_eq!(spent.inline.len(), 1);
529        assert!(prepare(&items, &[], 20, 0).inline.is_empty());
530    }
531
532    #[test]
533    fn claude_line_without_images_is_unchanged_and_with_images_puts_text_last() {
534        // Exactly the expression `send_message_from` used before images existed.
535        let before = serde_json::json!({
536            "type": "user",
537            "message": { "role": "user", "content": "hi" }
538        })
539        .to_string();
540        assert_eq!(claude_user_line("hi".into(), vec![]), before);
541        let img = serde_json::json!({ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "AA" } });
542        let line = claude_user_line("look".into(), vec![img.clone()]);
543        let v: serde_json::Value = serde_json::from_str(&line).unwrap();
544        assert_eq!(v["message"]["content"][0], img);
545        assert_eq!(
546            v["message"]["content"][1],
547            serde_json::json!({ "type": "text", "text": "look" })
548        );
549        // What gets stored is the text alone.
550        let stored: serde_json::Value = serde_json::from_str(&persisted_line(&line)).unwrap();
551        assert_eq!(stored["message"]["content"], "look");
552    }
553
554    #[test]
555    fn persisted_line_drops_images_and_keeps_text_as_a_string() {
556        let line = serde_json::json!({
557            "type": "user",
558            "message": { "role": "user", "content": [
559                { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "AAAA" } },
560                { "type": "text", "text": "hello\n\n<attached_images>\n1. a — /p\n</attached_images>" }
561            ]}
562        })
563        .to_string();
564        let stored = persisted_line(&line);
565        let v: serde_json::Value = serde_json::from_str(&stored).unwrap();
566        assert_eq!(
567            v["message"]["content"],
568            "hello\n\n<attached_images>\n1. a — /p\n</attached_images>"
569        );
570        assert!(!stored.contains("AAAA"));
571        // Plain string content is untouched, byte for byte.
572        let plain = r#"{"type":"user","message":{"role":"user","content":"hi"}}"#;
573        assert_eq!(persisted_line(plain), plain);
574    }
575}
agentmux_srv/backend/attachments/prompt.rs

agentmux_srv/backend/attachments/
prompt.rs
