修复DirectProject历史图片注入超限

压缩MCP大图回传并限制恢复图片预算保留canonical历史并更新恢复文档
2026-09-16 02:30:10 +08:00
parent 1a8c30c865
commit e90f1473c6
3 changed files with 167 additions and 10 deletions
@@ -7,17 +7,89 @@ use super::direct_project_history_injection_oversize_error;
 use serde_json::Value;
 use std::path::Path;

+const DIRECT_PROJECT_HISTORY_IMAGE_TOTAL_MAX_BYTES: usize = 8 * 1024 * 1024;
+const DIRECT_PROJECT_HISTORY_IMAGE_OMITTED_TEXT: &str =
+    "[历史图片预览已省略：本次恢复图片预算已用尽]";
+
+fn omit_image_block(object: &mut serde_json::Map<String, Value>, text_type: &str) {
+    object.clear();
+    object.insert("type".to_string(), Value::String(text_type.to_string()));
+    object.insert(
+        "text".to_string(),
+        Value::String(DIRECT_PROJECT_HISTORY_IMAGE_OMITTED_TEXT.to_string()),
+    );
+}
+
+fn compact_history_images(value: &mut Value, remaining_bytes: &mut usize) {
+    match value {
+        Value::Array(values) => values
+            .iter_mut()
+            .for_each(|value| compact_history_images(value, remaining_bytes)),
+        Value::Object(object) => {
+            let is_image_block = object.get("type").and_then(Value::as_str) == Some("image");
+            if is_image_block {
+                if let Some(data) = object.get("data").and_then(Value::as_str) {
+                    if let Some((preview, mime_type)) = crate::agent::compact_mcp_image_data(data) {
+                        if preview.len() > *remaining_bytes {
+                            omit_image_block(object, "text");
+                        } else {
+                            *remaining_bytes -= preview.len();
+                            object.insert("data".to_string(), Value::String(preview));
+                            object.insert(
+                                "mimeType".to_string(),
+                                Value::String(mime_type.to_string()),
+                            );
+                        }
+                    }
+                }
+            }
+            if object.get("type").and_then(Value::as_str) == Some("input_image") {
+                if let Some(url) = object
+                    .get("image_url")
+                    .and_then(Value::as_str)
+                    .map(str::to_string)
+                {
+                    if let Some((header, data)) = url.split_once(",") {
+                        if header.ends_with(";base64") {
+                            if let Some((preview, mime_type)) =
+                                crate::agent::compact_mcp_image_data(data)
+                            {
+                                if preview.len() > *remaining_bytes {
+                                    omit_image_block(object, "input_text");
+                                } else {
+                                    *remaining_bytes -= preview.len();
+                                    object.insert(
+                                        "image_url".to_string(),
+                                        Value::String(format!("data:{mime_type};base64,{preview}")),
+                                    );
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            object
+                .values_mut()
+                .for_each(|value| compact_history_images(value, remaining_bytes));
+        }
+        Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => {}
+    }
+}
+
 pub(super) fn build_direct_project_history_injection_params(
    history_root: &Path,
    thread_id: &str,
 ) -> Result<Value, platform_llm::LlmError> {
    let canonical_items = read_direct_project_history_items_at(history_root)
        .map_err(platform_llm::LlmError::InvalidRequest)?;
+    let mut remaining_image_bytes = DIRECT_PROJECT_HISTORY_IMAGE_TOTAL_MAX_BYTES;
    let items = canonical_items
        .iter()
        .map(|item| {
-            direct_codex_user_item_to_response_item(history_root, item)
-                .map_err(platform_llm::LlmError::InvalidRequest)
+            let mut projected = direct_codex_user_item_to_response_item(history_root, item)
+                .map_err(platform_llm::LlmError::InvalidRequest)?;
+            compact_history_images(&mut projected, &mut remaining_image_bytes);
+            Ok(projected)
        })
        .collect::<Result<Vec<_>, _>>()?;
    let params = serde_json::json!({"threadId": thread_id, "items": items});
@@ -30,3 +102,27 @@ pub(super) fn build_direct_project_history_injection_params(
    }
    Ok(params)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{compact_history_images, DIRECT_PROJECT_HISTORY_IMAGE_OMITTED_TEXT};
+    use serde_json::json;
+
+    #[test]
+    fn history_image_budget_omits_only_wire_preview_when_exhausted() {
+        let mut item = json!({
+            "type": "function_call_output",
+            "output": {"content": [{
+                "type": "image",
+                "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
+                "mimeType": "image/png"
+            }]}
+        });
+        let mut remaining = 1;
+        compact_history_images(&mut item, &mut remaining);
+        let block = &item["output"]["content"][0];
+        assert_eq!(block["type"], "text");
+        assert_eq!(block["text"], DIRECT_PROJECT_HISTORY_IMAGE_OMITTED_TEXT);
+        assert_eq!(remaining, 1);
+    }
+}
@@ -19,6 +19,8 @@ const DIRECT_TOOL_BRIDGE_MAX_WRITE_CONTENT_BYTES: usize = 1_500_000;
 const DIRECT_TOOL_BRIDGE_MAX_ART_BRIEF_CHARS: usize = 4_000;
 const DIRECT_TOOL_BRIDGE_MAX_IMAGE_PROMPT_CHARS: usize = 32_000;
 const DIRECT_TOOL_BRIDGE_MAX_IMAGE_BYTES: u64 = 6 * 1024 * 1024;
+const DIRECT_TOOL_BRIDGE_IMAGE_PREVIEW_MAX_BYTES: usize = 256 * 1024;
+const DIRECT_TOOL_BRIDGE_IMAGE_PREVIEW_MAX_DIMENSION: u32 = 1024;
 const DIRECT_TOOL_BRIDGE_MAX_SEARCH_QUERY_CHARS: usize = 400;
 const DIRECT_TOOL_BRIDGE_MAX_SEARCH_RESULTS: usize = 5;
 const DIRECT_TOOL_BRIDGE_SEARCH_URL: &str = "https://www.bing.com/search?format=rss";
@@ -694,14 +696,44 @@ fn direct_tool_bridge_state_with_search(
    })
 }

+/// 将 MCP 图片 block 限制为可安全回显和持久化的预览。
+///
+/// 工具结果会被 Codex 原样写入 DirectProject 历史；这里保留小图的原始
+/// PNG，大图则缩放并转成 JPEG。项目文件中的原图不受影响，历史恢复仍有
+/// 可见证据，但不会把多张几 MiB 的截图永久复制进上下文。
+pub(crate) fn compact_mcp_image_data(data: &str) -> Option<(String, &'static str)> {
+    let bytes = BASE64_STANDARD.decode(data).ok()?;
+    if bytes.is_empty() {
+        return None;
+    }
+    if bytes.len() <= DIRECT_TOOL_BRIDGE_IMAGE_PREVIEW_MAX_BYTES {
+        return Some((data.to_string(), "image/png"));
+    }
+
+    let image = image::load_from_memory(&bytes).ok()?;
+    let mut preview = image.thumbnail(
+        DIRECT_TOOL_BRIDGE_IMAGE_PREVIEW_MAX_DIMENSION,
+        DIRECT_TOOL_BRIDGE_IMAGE_PREVIEW_MAX_DIMENSION,
+    );
+    for (dimension, quality) in [(1024, 78), (768, 70), (512, 60), (384, 50)] {
+        if preview.width() > dimension || preview.height() > dimension {
+            preview = image.thumbnail(dimension, dimension);
+        }
+        let mut encoded = Vec::new();
+        let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut encoded, quality);
+        preview.write_with_encoder(encoder).ok()?;
+        if encoded.len() <= DIRECT_TOOL_BRIDGE_IMAGE_PREVIEW_MAX_BYTES {
+            return Some((BASE64_STANDARD.encode(encoded), "image/jpeg"));
+        }
+    }
+    None
+}
+
 fn bridge_tool_result(text: String, images: Vec<String>, is_error: bool) -> Value {
    let mut content = vec![json!({ "type": "text", "text": text })];
-    content.extend(images.into_iter().map(|data| {
-        json!({
-            "type": "image",
-            "data": data,
-            "mimeType": "image/png"
-        })
+    content.extend(images.into_iter().filter_map(|data| {
+        let (data, mime_type) = compact_mcp_image_data(&data).unwrap_or((data, "image/png"));
+        Some(json!({ "type": "image", "data": data, "mimeType": mime_type }))
    }));
    json!({ "content": content, "isError": is_error })
 }
@@ -2672,7 +2704,7 @@ pub(crate) async fn start_direct_tool_bridge(
 #[cfg(test)]
 mod tests {
    use super::*;
-    use std::io::{Read, Write};
+    use std::io::{Cursor, Read, Write};

    #[tokio::test]
    async fn controlled_search_client_omits_agc_marker() {
@@ -2766,6 +2798,35 @@ mod tests {
        assert!(bridge_search_max_results(&json!({ "maxResults": 6 })).is_err());
    }

+    #[test]
+    fn large_mcp_images_are_reduced_to_bounded_jpeg_previews() {
+        let image = image::RgbaImage::from_fn(1600, 1200, |x, y| {
+            image::Rgba([
+                (x % 251) as u8,
+                (y % 251) as u8,
+                ((x.wrapping_mul(31) + y.wrapping_mul(17)) % 251) as u8,
+                u8::MAX,
+            ])
+        });
+        let mut png = Cursor::new(Vec::new());
+        image::DynamicImage::ImageRgba8(image)
+            .write_to(&mut png, image::ImageFormat::Png)
+            .expect("encode image fixture");
+        assert!(png.get_ref().len() > DIRECT_TOOL_BRIDGE_IMAGE_PREVIEW_MAX_BYTES);
+
+        let (preview, mime_type) =
+            compact_mcp_image_data(&BASE64_STANDARD.encode(png.into_inner()))
+                .expect("large valid image should produce preview");
+        assert_eq!(mime_type, "image/jpeg");
+        assert!(
+            BASE64_STANDARD
+                .decode(preview)
+                .expect("preview base64")
+                .len()
+                <= DIRECT_TOOL_BRIDGE_IMAGE_PREVIEW_MAX_BYTES
+        );
+    }
+
    #[test]
    fn search_parser_accepts_only_bounded_public_https_results() {
        let body = r#"<rss><channel><item><title>Tauri &amp; Rust</title><link>https://tauri.app/</link><description>&lt;b&gt;Cross-platform apps&lt;/b&gt;</description></item><item><title>Private</title><link>http://127.0.0.1:8082/private</link><description>private</description></item><item><title>Credentials</title><link>https://user:pass@example.test/path</link><description>private</description></item><item><title>Loopback host</title><link>https://localhost/private</link><description>private</description></item><item><title>Local host</title><link>https://service.internal/private</link><description>private</description></item></channel></rss>"#;
@@ -49,7 +49,7 @@ Codex 启动时注入的 `host_skills.instructions`、`permissions.instructions`

 ## 恢复

-创建新的 ephemeral thread 后，读取 `project.jsonl` 中所有 `response_item.payload`，按文件行顺序一次调用 `thread/inject_items`，再执行新的 `turn/start`。Codex 负责上下文窗口管理；注入失败直接失败，AGC 不截断、摘要或改写历史。新 thread 已进入连接池但历史读取或注入失败时，必须先从池中淘汰并取消订阅该 thread，重试只能创建新 thread 并重新注入。
+创建新的 ephemeral thread 后，读取 `project.jsonl` 中所有 `response_item.payload`，按文件行顺序一次调用 `thread/inject_items`，再执行新的 `turn/start`。Codex 负责上下文窗口管理。磁盘上的 canonical 历史永不改写；恢复 wire 载荷会把 MCP `image` block（以及 `input_image` data URL）转换为每张最多 `256 KiB` 的 JPEG 预览，整次恢复图片预算为 `8 MiB`，保留图片证据并避免旧项目把完整 PNG Base64 重复注入。预算耗尽的图片只在 wire 载荷中替换为省略标记。工具新回传图片也在进入 Codex 前执行同一预览上限。除图片二进制预览外，不截断、摘要或改写历史；若其它内容仍超过单行上限，继续失败关闭并指出 `itemId`。新 thread 已进入连接池但历史读取或注入失败时，必须先从池中淘汰并取消订阅该 thread，重试只能创建新 thread 并重新注入。

 `clientUserMessageId` 仅作为 Codex 用户消息的稳定标识随 `turn/start` 发送，不等价于 turn 级 exactly-once 幂等。断线后的重试仍须由项目侧持久化 turn ledger 或服务端去重合同决定，不能仅凭该字段再次执行。