From 9778ac1660dba9a13b0d3dbd85f6c05d863f18de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Thu, 20 Aug 2026 15:54:54 +0800 Subject: [PATCH] =?UTF-8?q?=E9=99=90=E5=88=B6UI=E7=BB=84=E4=BB=B6=E7=BB=91?= =?UTF-8?q?=E5=AE=9A=E8=B5=84=E6=BA=90=E4=B8=8E=E5=93=8D=E5=BA=94=E8=A7=84?= =?UTF-8?q?=E6=A8=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 组件绑定图片改用五MiB有界阻塞读取 工具参数增加一MiB及组件栈数量预检 补充超限回归测试并同步技术契约 --- .../src/ui_editor/commands/binding.rs | 114 ++++++++++++++---- .../src-tauri/src/ui_editor/persistence.rs | 4 +- ...案】AI游戏创作智能体App实施计划-2026-06-24.md | 4 +- 3 files changed, 92 insertions(+), 30 deletions(-) diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/binding.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/binding.rs index 2752078d8..92ecb2157 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/binding.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/binding.rs @@ -1,10 +1,14 @@ use crate::config::build_game_creator_llm_client_from_config; -use crate::ui_editor::commands::utils::strict_json_schema; +use crate::ui_editor::commands::utils::{ + parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, strict_json_schema, +}; use crate::ui_editor::component::Component; use crate::ui_editor::layout::node::{Node, StageStatus}; +use crate::ui_editor::persistence::{ + UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE, UI_DESIGN_STATE_MAX_NODES, +}; use crate::ui_editor::state::State; use crate::ui_editor::utils::{NodeId, SpriteAssetId}; -use base64::Engine as _; use platform_llm::{ LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, }; @@ -77,19 +81,6 @@ fn binding_json_schema() -> Result { strict_json_schema::() } -fn image_data_url(path: &Path, bytes: &[u8]) -> Result { - let mime = match path.extension().and_then(|value| value.to_str()) { - Some("png") => "image/png", - Some("jpg") | Some("jpeg") => "image/jpeg", - Some("webp") => "image/webp", - _ => return Err(format!("不支持的图片格式：{}", path.display())), - }; - Ok(format!( - "data:{mime};base64,{}", - base64::engine::general_purpose::STANDARD.encode(bytes) - )) -} - fn collect_editable_nodes<'a>(node: &'a Node, output: &mut Vec>) { if node.metadata.allow_llm_edit_component { output.push(EditableNodeContext { @@ -104,6 +95,41 @@ fn collect_editable_nodes<'a>(node: &'a Node, output: &mut Vec Result<(), String> { + let changes = value + .get("changes") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| "组件绑定工具参数缺少 changes 数组".to_string())?; + let max_changes = editable_node_count.min(UI_DESIGN_STATE_MAX_NODES); + if changes.len() > max_changes { + return Err(format!("组件绑定 changes 不能超过 {max_changes} 条")); + } + for change in changes { + let components = change + .get("components") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| "组件绑定 change 缺少 components 数组".to_string())?; + if components.len() > UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE { + return Err(format!( + "单个组件绑定栈不能超过 {UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE} 个组件" + )); + } + } + Ok(()) +} + +fn parse_binding_response( + arguments: &str, + editable_node_count: usize, +) -> Result { + let value = parse_limited_llm_tool_arguments(arguments)?; + validate_binding_response_shape(&value, editable_node_count)?; + serde_json::from_value(value).map_err(|error| format!("组件绑定工具参数无效：{error}")) +} + fn validate_and_materialize( changes: Vec, editable_ids: &HashSet, @@ -200,7 +226,9 @@ pub(crate) async fn bind_components_impl( }); for (id, image) in &state.ui_design_images { let absolute = crate::project::resolve_local_project_path(root, &image.path)?; - let bytes = std::fs::read(&absolute).map_err(|error| format!("读取界面图失败：{error}"))?; + let image_url = read_ui_reference_image_data_url(absolute) + .await + .map_err(|error| format!("读取界面图失败：{error}"))?; parts.push(LlmMessageContentPart::InputText { text: format!( "UI_REFERENCE id={} pixel_size={:?}", @@ -208,9 +236,7 @@ pub(crate) async fn bind_components_impl( image.pixel_size ), }); - parts.push(LlmMessageContentPart::InputImage { - image_url: image_data_url(&absolute, &bytes)?, - }); + parts.push(LlmMessageContentPart::InputImage { image_url }); } for id in &sprite_ids { let sprite = state @@ -218,8 +244,9 @@ pub(crate) async fn bind_components_impl( .get(id) .ok_or_else(|| format!("独立素材不存在：{}", id.as_str()))?; let absolute = crate::project::resolve_local_project_path(root, &sprite.path)?; - let bytes = - std::fs::read(&absolute).map_err(|error| format!("读取独立素材失败：{error}"))?; + let image_url = read_ui_reference_image_data_url(absolute) + .await + .map_err(|error| format!("读取独立素材失败：{error}"))?; parts.push(LlmMessageContentPart::InputText { text: format!( "SPRITE id={} name={} asset_type={} pixel_size={:?}", @@ -229,9 +256,7 @@ pub(crate) async fn bind_components_impl( sprite.pixel_size ), }); - parts.push(LlmMessageContentPart::InputImage { - image_url: image_data_url(&absolute, &bytes)?, - }); + parts.push(LlmMessageContentPart::InputImage { image_url }); } let client = build_game_creator_llm_client_from_config()?; let tool = LlmFunctionTool::new( @@ -256,8 +281,7 @@ pub(crate) async fn bind_components_impl( .iter() .find(|call| call.name == "bind_ui_components") .ok_or_else(|| "LLM 未返回 bind_ui_components 工具调用".to_string())?; - let parsed = serde_json::from_str::(&call.arguments) - .map_err(|error| format!("组件绑定工具参数无效：{error}"))?; + let parsed = parse_binding_response(&call.arguments, editable_nodes.len())?; let known_sprite_ids = state.sprite_assets.keys().cloned().collect::>(); let result = validate_and_materialize(parsed.changes, &editable_ids, &known_sprite_ids)?; eprintln!( @@ -345,6 +369,44 @@ mod tests { assert_eq!(result.changes[0].components_status, StageStatus::NoProblem); } + #[test] + fn binding_response_rejects_oversized_tool_arguments_before_dto_conversion() { + let oversized = + "x".repeat(crate::ui_editor::commands::utils::LLM_TOOL_ARGUMENT_MAX_BYTES + 1); + assert!(parse_binding_response(&oversized, 1) + .expect_err("oversized tool arguments must fail") + .contains("字节上限")); + } + + #[test] + fn binding_response_bounds_changes_and_each_component_stack() { + let too_many_changes = serde_json::json!({ + "changes": [{"components": []}, {"components": []}] + }); + assert!(validate_binding_response_shape(&too_many_changes, 1).is_err()); + + let too_many_components = serde_json::json!({ + "changes": [{ + "components": (0..=UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE) + .map(|_| serde_json::Value::Null) + .collect::>() + }] + }); + assert!(validate_binding_response_shape(&too_many_components, 1).is_err()); + } + + #[tokio::test] + async fn binding_image_input_rejects_oversized_files_via_bounded_reader() { + let directory = tempfile::tempdir().expect("binding image fixture"); + let path = directory.path().join("oversized.png"); + let file = std::fs::File::create(&path).expect("create sparse binding image"); + file.set_len((crate::ui_editor::commands::utils::UI_REFERENCE_IMAGE_MAX_BYTES + 1) as u64) + .expect("size sparse binding image"); + drop(file); + + assert!(read_ui_reference_image_data_url(path).await.is_err()); + } + #[test] fn exports_ui_editor_types() { let config = Config::from_env(); diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs index a3dcb6adc..fbdf0970c 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs @@ -14,9 +14,9 @@ const UI_DESIGN_STATE_SCHEMA_VERSION: &str = "game-creator-ui-design-state.v1"; const UI_DESIGN_STATE_MAX_BYTES: usize = 2 * 1024 * 1024; const UI_DESIGN_STATE_MAX_IMAGES: usize = 4; const UI_DESIGN_STATE_MAX_SPRITES: usize = 1_024; -const UI_DESIGN_STATE_MAX_NODES: usize = 10_000; +pub(crate) const UI_DESIGN_STATE_MAX_NODES: usize = 10_000; const UI_DESIGN_STATE_MAX_DEPTH: usize = 128; -const UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE: usize = 64; +pub(crate) const UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE: usize = 64; const UI_DESIGN_STATE_MAX_SAFE_REVISION: u64 = 9_007_199_254_740_991; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] diff --git a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md index 26a608321..8a52592d8 100644 --- a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md +++ b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md @@ -40,7 +40,7 @@ UI Editor Inspector 的全局只读状态唯一来源是 `controller.editor.isLo UI Editor 当前把“识别界面结构”定义为结构草稿阶段，而不是完整视觉还原阶段。识别 DTO 只负责输出节点层级、几何、名称、描述和置信度；节点组件暂为空，由后续“绑定视觉素材”阶段补齐 Image / Text 组件。`applyRecognitionResult` 可以整体替换当前 `ui_trees`，但该替换只代表结构结果，不能宣称已经保留截图中的视觉内容；组件状态使用 `NoProblem`，前置检查仍会根据空组件和素材绑定情况阻止跳过绑定阶段。 -结构识别、界面语义建议和多图合并只接受不超过 `1 MiB` 的 LLM 工具调用 arguments，并在递归业务类型反序列化前先解析为通用 JSON、迭代检查结构预算。结构识别按每棵返回树独立限制为最多 `512` 个 LLM 节点和 `32` 层，不汇总多棵树的节点数，也不计 Rust 自动补建的页面根；界面语义建议最多 `4` 个节点和 `4` 层；合并计划最多 `512` 个计划节点和 `32` 层，`Simple.children` 与 `Merged.merged_from` 使用同一计数和深度口径。任何超限结果均整次拒绝，不截断、不返回部分结果，也不把工具 arguments 正文写入日志。 +结构识别、界面语义建议、多图合并和组件绑定只接受不超过 `1 MiB` 的 LLM 工具调用 arguments，并在递归业务类型反序列化前先解析为通用 JSON、迭代检查结构预算。结构识别按每棵返回树独立限制为最多 `512` 个 LLM 节点和 `32` 层，不汇总多棵树的节点数，也不计 Rust 自动补建的页面根；界面语义建议最多 `4` 个节点和 `4` 层；合并计划最多 `512` 个计划节点和 `32` 层，`Simple.children` 与 `Merged.merged_from` 使用同一计数和深度口径；组件绑定 `changes` 不得超过当前可编辑节点数且绝对上限为 `10,000`，每个 change 的完整组件栈最多 `64` 个组件。任何超限结果均整次拒绝，不截断、不返回部分结果，也不把工具 arguments 正文写入日志。 多图合并使用 LLM 返回的语义投影树。`Simple` 和 `Merged` 只列出希望进入结果树的原始节点；未被计划引用的源节点表示本次合并判定为冗余或不属于目标公共结构，允许被丢弃，不要求 `used_original_ids` 覆盖全部输入节点。重复 ID 和未知 ID 仍然是错误；省略不是隐式复制或随机删除，而是合并计划的正式语义。 @@ -48,7 +48,7 @@ UI Editor 当前把“识别界面结构”定义为结构草稿阶段，而不 UI Editor 的图片、字体和远程素材导入采用增量提交合同：输入先完成各入口已有的预检，随后按顺序逐项写入和登记；前面已成功的项目在后续写入或登记失败时保持有效，调用方必须重新读取 manifest 取得已提交集合。远程素材目标文件名必须同时包含清洗后的可读 ID 与原始稳定 ID 的确定性摘要，不能让不同 ID 因清洗结果相同而覆盖已有文件；同一批次计算出的重复目标路径必须在写入前拒绝。该入口不提供整批事务回滚，也不承诺失败后“零文件、零 manifest 变化”；后续如需原子批量导入，必须新增明确的 transaction / reconciliation 合同，不能把当前入口静默改成另一种语义。 -作为 LLM 参考图使用的 UI 设计图单张文件上限为 `5 MiB`，不设置多图合计大小上限。前端设计图 AssetImporter 对本地和远端入口使用相同单文件限制；结构识别与界面语义建议在 Rust 中先检查文件元数据、再有限读取，并把读取与 base64 编码放在 blocking worker，避免阻塞 Tokio async worker。本阶段不新增图片像素数限制或命令超时。 +作为 LLM 视觉输入使用的 UI 设计图和独立素材单张文件上限为 `5 MiB`，不设置多图合计大小上限。前端设计图 AssetImporter 对本地和远端入口使用相同单文件限制；结构识别、界面语义建议与组件绑定在 Rust 中先检查文件元数据、再有限读取，并把读取与 base64 编码放在 blocking worker，避免阻塞 Tokio async worker。本阶段不新增图片像素数限制或命令超时。 ## 2026-08-18 UI Editor 左侧节点树跨界面移动