限制 UI 编辑器 LLM 树与参考图资源
LLM 工具参数限制为 1 MiB,识别与合并按每棵树 512 节点、32 层拒绝超限。 参考图只限制单张 5 MiB,并将有限读取与 base64 编码移入 blocking worker。 补充定向测试和权威文档,移除已完成 TODO。
This commit is contained in:
@@ -1,13 +1,16 @@
|
||||
use crate::config::build_game_creator_llm_client_from_config;
|
||||
use crate::ui_editor::commands::utils::strict_json_schema;
|
||||
use crate::ui_editor::commands::utils::{parse_limited_llm_tool_arguments, strict_json_schema};
|
||||
use crate::ui_editor::state::{State, UITree};
|
||||
use platform_llm::{LlmFunctionTool, LlmMessage, LlmRunRequest, LlmToolChoice};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use ts_rs::TS;
|
||||
|
||||
// TODO 13. `merge.rs`:合并命令接受任意 State,未在序列化和 LLM 请求前执行节点数、深度、序列化大小及超时/取消限制。需要确定与持久化限制一致的上限。
|
||||
|
||||
const MERGE_TOOL_NAME: &str = "merge_ui_trees";
|
||||
const MAX_MERGE_INPUT_NODES: usize = 512;
|
||||
const MAX_MERGE_INPUT_DEPTH: usize = 32;
|
||||
const MAX_MERGE_INPUT_BYTES: usize = 2 * 1024 * 1024;
|
||||
const MAX_MERGE_PLAN_NODES: usize = 512;
|
||||
const MAX_MERGE_PLAN_DEPTH: usize = 32;
|
||||
|
||||
const SYSTEM_PROMPT: &str = r#"
|
||||
角色:
|
||||
@@ -324,6 +327,66 @@ mod materialize {
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_merge_input_state(state: &State) -> Result<(), String> {
|
||||
for tree in &state.ui_trees {
|
||||
let mut stack = vec![(&tree.root, 1usize)];
|
||||
let mut node_count = 0usize;
|
||||
while let Some((node, depth)) = stack.pop() {
|
||||
if depth > MAX_MERGE_INPUT_DEPTH {
|
||||
return Err(format!(
|
||||
"单棵待合并 UI 树最大深度不能超过 {MAX_MERGE_INPUT_DEPTH}"
|
||||
));
|
||||
}
|
||||
node_count += 1;
|
||||
if node_count > MAX_MERGE_INPUT_NODES {
|
||||
return Err(format!(
|
||||
"单棵待合并 UI 树最多包含 {MAX_MERGE_INPUT_NODES} 个节点"
|
||||
));
|
||||
}
|
||||
stack.extend(node.children.iter().map(|child| (child, depth + 1)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_merge_plan_shape(value: &serde_json::Value) -> Result<(), String> {
|
||||
let root = value
|
||||
.get("root")
|
||||
.ok_or_else(|| "UI 合并工具参数缺少 root".to_string())?;
|
||||
let mut stack = vec![(root, 1usize)];
|
||||
let mut node_count = 0usize;
|
||||
while let Some((node, depth)) = stack.pop() {
|
||||
if depth > MAX_MERGE_PLAN_DEPTH {
|
||||
return Err(format!(
|
||||
"UI 合并计划最大深度不能超过 {MAX_MERGE_PLAN_DEPTH}"
|
||||
));
|
||||
}
|
||||
node_count += 1;
|
||||
if node_count > MAX_MERGE_PLAN_NODES {
|
||||
return Err(format!("UI 合并计划最多包含 {MAX_MERGE_PLAN_NODES} 个节点"));
|
||||
}
|
||||
let object = node
|
||||
.as_object()
|
||||
.filter(|object| object.len() == 1)
|
||||
.ok_or_else(|| "UI 合并计划节点结构无效".to_string())?;
|
||||
let children = if let Some(simple) = object.get("Simple") {
|
||||
simple
|
||||
.get("children")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.ok_or_else(|| "Simple 合并计划节点缺少 children 数组".to_string())?
|
||||
} else if let Some(merged) = object.get("Merged") {
|
||||
merged
|
||||
.get("merged_from")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.ok_or_else(|| "Merged 合并计划节点缺少 merged_from 数组".to_string())?
|
||||
} else {
|
||||
return Err("UI 合并计划节点类型无效".to_string());
|
||||
};
|
||||
stack.extend(children.iter().map(|child| (child, depth + 1)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct MergeDTO {
|
||||
@@ -335,6 +398,10 @@ pub(crate) async fn merge_ui_impl(state: State) -> Result<MergeDTO, String> {
|
||||
eprintln!("ui_merge.error stage=validate reason=no_trees");
|
||||
return Err("请先完成 UI 结构识别".to_string());
|
||||
}
|
||||
validate_merge_input_state(&state).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=validate_input error={error}");
|
||||
error
|
||||
})?;
|
||||
let priorities = priority::for_state(&state).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=build_priority error={error}");
|
||||
error
|
||||
@@ -347,6 +414,13 @@ pub(crate) async fn merge_ui_impl(state: State) -> Result<MergeDTO, String> {
|
||||
eprintln!("ui_merge.error stage=serialize_input error={error}");
|
||||
format!("序列化 UI 合并输入失败:{error}")
|
||||
})?;
|
||||
if records_json.len() > MAX_MERGE_INPUT_BYTES {
|
||||
eprintln!(
|
||||
"ui_merge.error stage=serialize_input reason=too_large bytes={} limit={MAX_MERGE_INPUT_BYTES}",
|
||||
records_json.len()
|
||||
);
|
||||
return Err(format!("UI 合并输入超过 {MAX_MERGE_INPUT_BYTES} 字节上限"));
|
||||
}
|
||||
let client = build_game_creator_llm_client_from_config().map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=build_client error={error}");
|
||||
error
|
||||
@@ -383,8 +457,16 @@ pub(crate) async fn merge_ui_impl(state: State) -> Result<MergeDTO, String> {
|
||||
eprintln!("ui_merge.error stage=parse_tool_call reason=missing_tool_call");
|
||||
format!("LLM 未返回 {MERGE_TOOL_NAME} 工具调用")
|
||||
})?;
|
||||
let arguments = parse_limited_llm_tool_arguments(&call.arguments).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=parse_arguments error={error}");
|
||||
format!("UI 合并工具参数无效:{error}")
|
||||
})?;
|
||||
validate_merge_plan_shape(&arguments).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=validate_arguments error={error}");
|
||||
format!("UI 合并工具参数无效:{error}")
|
||||
})?;
|
||||
let parsed =
|
||||
serde_json::from_str::<llm_contract::MergeResponse>(&call.arguments).map_err(|error| {
|
||||
serde_json::from_value::<llm_contract::MergeResponse>(arguments).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=parse_arguments error={error}");
|
||||
format!("UI 合并工具参数无效:{error}")
|
||||
})?;
|
||||
@@ -398,7 +480,10 @@ pub(crate) async fn merge_ui_impl(state: State) -> Result<MergeDTO, String> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::llm_contract::{MergedNode, Node as PlanNode, SimpleNode};
|
||||
use super::materialize;
|
||||
use super::{
|
||||
materialize, validate_merge_input_state, validate_merge_plan_shape, MAX_MERGE_INPUT_DEPTH,
|
||||
MAX_MERGE_INPUT_NODES, MAX_MERGE_PLAN_DEPTH, MAX_MERGE_PLAN_NODES,
|
||||
};
|
||||
use crate::ui_editor::component::Component;
|
||||
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
|
||||
use crate::ui_editor::layout::control_layout::ControlLayout;
|
||||
@@ -459,7 +544,87 @@ mod tests {
|
||||
result.root.children_display_mode,
|
||||
ChildrenDisplayMode::Exclusive
|
||||
);
|
||||
assert_eq!(result.root.metadata.components_status, StageStatus::NoProblem);
|
||||
assert_eq!(
|
||||
result.root.metadata.components_status,
|
||||
StageStatus::NoProblem
|
||||
);
|
||||
assert_eq!(result.root.children.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_plan_rejects_node_and_depth_overflow() {
|
||||
let leaf = || serde_json::json!({"Simple": {"id": "leaf", "children": []}});
|
||||
let oversized = (0..MAX_MERGE_PLAN_NODES)
|
||||
.map(|_| leaf())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(validate_merge_plan_shape(&serde_json::json!({
|
||||
"root": {"Simple": {"id": "root", "children": oversized}}
|
||||
}))
|
||||
.is_err());
|
||||
|
||||
let mut nested = leaf();
|
||||
for _ in 0..MAX_MERGE_PLAN_DEPTH {
|
||||
nested = serde_json::json!({
|
||||
"Merged": {"name": "层", "description": "", "merged_from": [nested]}
|
||||
});
|
||||
}
|
||||
assert!(validate_merge_plan_shape(&serde_json::json!({"root": nested})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_input_limits_apply_per_tree_without_summing_trees() {
|
||||
let tree = |page: &str, prefix: &str| UITree {
|
||||
src_ui_design: UIDesignImageId::new(page).expect("valid page id"),
|
||||
root: node(
|
||||
&format!("{prefix}-root"),
|
||||
(1..MAX_MERGE_INPUT_NODES)
|
||||
.map(|index| node(&format!("{prefix}-{index}"), vec![]))
|
||||
.collect(),
|
||||
),
|
||||
};
|
||||
let state = State {
|
||||
ui_trees: vec![tree("page-a", "a"), tree("page-b", "b")],
|
||||
ui_design_images: HashMap::new(),
|
||||
sprite_assets: HashMap::new(),
|
||||
font_assets: HashMap::new(),
|
||||
};
|
||||
validate_merge_input_state(&state)
|
||||
.expect("each source tree independently fits the node limit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_input_rejects_node_and_depth_overflow() {
|
||||
let page_id = UIDesignImageId::new("page").expect("valid page id");
|
||||
let wide_root = node(
|
||||
"root",
|
||||
(0..MAX_MERGE_INPUT_NODES)
|
||||
.map(|index| node(&format!("child-{index}"), vec![]))
|
||||
.collect(),
|
||||
);
|
||||
let wide_state = State {
|
||||
ui_trees: vec![UITree {
|
||||
src_ui_design: page_id.clone(),
|
||||
root: wide_root,
|
||||
}],
|
||||
ui_design_images: HashMap::new(),
|
||||
sprite_assets: HashMap::new(),
|
||||
font_assets: HashMap::new(),
|
||||
};
|
||||
assert!(validate_merge_input_state(&wide_state).is_err());
|
||||
|
||||
let mut deep_root = node("leaf", vec![]);
|
||||
for depth in 0..MAX_MERGE_INPUT_DEPTH {
|
||||
deep_root = node(&format!("depth-{depth}"), vec![deep_root]);
|
||||
}
|
||||
let deep_state = State {
|
||||
ui_trees: vec![UITree {
|
||||
src_ui_design: page_id,
|
||||
root: deep_root,
|
||||
}],
|
||||
ui_design_images: HashMap::new(),
|
||||
sprite_assets: HashMap::new(),
|
||||
font_assets: HashMap::new(),
|
||||
};
|
||||
assert!(validate_merge_input_state(&deep_state).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use crate::config::build_game_creator_llm_client_from_config;
|
||||
use crate::ui_editor::commands::utils::strict_json_schema;
|
||||
use crate::ui_editor::commands::utils::{
|
||||
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, strict_json_schema,
|
||||
};
|
||||
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
|
||||
use crate::ui_editor::layout::control_layout::ControlLayout;
|
||||
use crate::ui_editor::layout::dimension::UIRect;
|
||||
@@ -8,7 +10,6 @@ use crate::ui_editor::layout::transform::Transform;
|
||||
use crate::ui_editor::resource::ui_design_image::UIDesignImage;
|
||||
use crate::ui_editor::state::{State, UITree};
|
||||
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
|
||||
use base64::Engine as _;
|
||||
use nalgebra::{Point2, Vector2};
|
||||
use platform_llm::{
|
||||
LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice,
|
||||
@@ -19,8 +20,9 @@ use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use ts_rs::TS;
|
||||
|
||||
// TODO 14. `recognition.rs`:LLM 返回的 children 树在递归反序列化和转换前没有深度、节点数、响应体上限,存在资源耗尽/栈溢出风险。需要确定契约上限及拒绝方式。
|
||||
const MAX_REFERENCES: usize = 4;
|
||||
const MAX_RECOGNITION_TREE_NODES: usize = 512;
|
||||
const MAX_RECOGNITION_TREE_DEPTH: usize = 32;
|
||||
|
||||
const SYSTEM_PROMPT: &str = r#"
|
||||
角色:
|
||||
@@ -130,17 +132,44 @@ fn recognition_json_schema() -> Result<serde_json::Value, String> {
|
||||
strict_json_schema::<RecognitionResponse>()
|
||||
}
|
||||
|
||||
fn image_data_url(path: &Path, bytes: &[u8]) -> Result<String, String> {
|
||||
let mime = match path.extension().and_then(|value| value.to_str()) {
|
||||
Some("png") => "image/png",
|
||||
Some("jpg") | Some("jpeg") => "image/jpeg",
|
||||
Some("webp") => "image/webp",
|
||||
_ => return Err(format!("不支持的界面图格式:{}", path.display())),
|
||||
};
|
||||
Ok(format!(
|
||||
"data:{mime};base64,{}",
|
||||
base64::engine::general_purpose::STANDARD.encode(bytes)
|
||||
))
|
||||
fn validate_recognition_response_shape(value: &serde_json::Value) -> Result<(), String> {
|
||||
let trees = value
|
||||
.get("trees")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.ok_or_else(|| "识别工具参数缺少 trees 数组".to_string())?;
|
||||
if trees.len() > MAX_REFERENCES {
|
||||
return Err(format!("识别结果最多包含 {MAX_REFERENCES} 棵界面树"));
|
||||
}
|
||||
for tree in trees {
|
||||
let children = tree
|
||||
.get("children")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.ok_or_else(|| "识别树缺少 children 数组".to_string())?;
|
||||
let mut stack = children
|
||||
.iter()
|
||||
.map(|node| (node, 1usize))
|
||||
.collect::<Vec<_>>();
|
||||
let mut node_count = 0usize;
|
||||
while let Some((node, depth)) = stack.pop() {
|
||||
if depth > MAX_RECOGNITION_TREE_DEPTH {
|
||||
return Err(format!(
|
||||
"单棵识别树最大深度不能超过 {MAX_RECOGNITION_TREE_DEPTH}"
|
||||
));
|
||||
}
|
||||
node_count += 1;
|
||||
if node_count > MAX_RECOGNITION_TREE_NODES {
|
||||
return Err(format!(
|
||||
"单棵识别树最多包含 {MAX_RECOGNITION_TREE_NODES} 个节点"
|
||||
));
|
||||
}
|
||||
let children = node
|
||||
.get("children")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.ok_or_else(|| "识别节点缺少 children 数组".to_string())?;
|
||||
stack.extend(children.iter().map(|child| (child, depth + 1)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn anchor_ranges(anchor: &Anchor) -> Result<(Vector2<f32>, Vector2<f32>), String> {
|
||||
@@ -387,6 +416,41 @@ mod tests {
|
||||
assert_no_one_of(&schema);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognition_limits_apply_per_tree_without_summing_trees() {
|
||||
let leaves = (0..MAX_RECOGNITION_TREE_NODES)
|
||||
.map(|_| serde_json::json!({"children": []}))
|
||||
.collect::<Vec<_>>();
|
||||
let response = serde_json::json!({
|
||||
"trees": [
|
||||
{"children": leaves.clone()},
|
||||
{"children": leaves}
|
||||
]
|
||||
});
|
||||
validate_recognition_response_shape(&response)
|
||||
.expect("each tree independently fits the node limit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognition_rejects_tree_node_and_depth_overflow() {
|
||||
let oversized = (0..=MAX_RECOGNITION_TREE_NODES)
|
||||
.map(|_| serde_json::json!({"children": []}))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(validate_recognition_response_shape(&serde_json::json!({
|
||||
"trees": [{"children": oversized}]
|
||||
}))
|
||||
.is_err());
|
||||
|
||||
let mut nested = serde_json::json!({"children": []});
|
||||
for _ in 0..MAX_RECOGNITION_TREE_DEPTH {
|
||||
nested = serde_json::json!({"children": [nested]});
|
||||
}
|
||||
assert!(validate_recognition_response_shape(&serde_json::json!({
|
||||
"trees": [{"children": [nested]}]
|
||||
}))
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preset_anchor_ranges_use_top_left_image_coordinates() {
|
||||
let (min, max) = anchor_ranges(&Anchor::Preset(PresetAnchor {
|
||||
@@ -573,14 +637,16 @@ pub(crate) async fn recognize_ui_impl(
|
||||
);
|
||||
error
|
||||
})?;
|
||||
let bytes = std::fs::read(&absolute).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=read_image root={} image={} error={error}",
|
||||
root_id.as_str(),
|
||||
context_id.as_str()
|
||||
);
|
||||
format!("读取界面图失败:{error}")
|
||||
})?;
|
||||
let image_url = read_ui_reference_image_data_url(absolute)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=read_image root={} image={} error={error}",
|
||||
root_id.as_str(),
|
||||
context_id.as_str()
|
||||
);
|
||||
error
|
||||
})?;
|
||||
parts.push(LlmMessageContentPart::InputText {
|
||||
text: format!(
|
||||
"{} id={} pixel_size={:?}",
|
||||
@@ -589,9 +655,7 @@ pub(crate) async fn recognize_ui_impl(
|
||||
image.pixel_size
|
||||
),
|
||||
});
|
||||
parts.push(LlmMessageContentPart::InputImage {
|
||||
image_url: image_data_url(&absolute, &bytes)?,
|
||||
});
|
||||
parts.push(LlmMessageContentPart::InputImage { image_url });
|
||||
}
|
||||
let tool = LlmFunctionTool::new(
|
||||
"recognize_ui_structure",
|
||||
@@ -617,10 +681,10 @@ pub(crate) async fn recognize_ui_impl(
|
||||
format!("UI 结构识别失败(根界面图 {}):{error}", root_id.as_str())
|
||||
})?;
|
||||
eprintln!(
|
||||
"ui_recognition.llm_output root={} text={:?} tool_calls={:?}",
|
||||
"ui_recognition.llm_output root={} text_present={} tool_call_count={}",
|
||||
root_id.as_str(),
|
||||
response.text,
|
||||
response.tool_calls
|
||||
!response.text.trim().is_empty(),
|
||||
response.tool_calls.len()
|
||||
);
|
||||
let call = response
|
||||
.tool_calls
|
||||
@@ -636,14 +700,27 @@ pub(crate) async fn recognize_ui_impl(
|
||||
root_id.as_str()
|
||||
)
|
||||
})?;
|
||||
let parsed =
|
||||
serde_json::from_str::<RecognitionResponse>(&call.arguments).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=parse_arguments root={} error={error}",
|
||||
root_id.as_str()
|
||||
);
|
||||
format!("根界面图 {} 的识别工具参数无效:{error}", root_id.as_str())
|
||||
})?;
|
||||
let arguments = parse_limited_llm_tool_arguments(&call.arguments).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=parse_arguments root={} error={error}",
|
||||
root_id.as_str()
|
||||
);
|
||||
format!("根界面图 {} 的识别工具参数无效:{error}", root_id.as_str())
|
||||
})?;
|
||||
validate_recognition_response_shape(&arguments).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=validate_arguments root={} error={error}",
|
||||
root_id.as_str()
|
||||
);
|
||||
format!("根界面图 {} 的识别工具参数无效:{error}", root_id.as_str())
|
||||
})?;
|
||||
let parsed = serde_json::from_value::<RecognitionResponse>(arguments).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=parse_arguments root={} error={error}",
|
||||
root_id.as_str()
|
||||
);
|
||||
format!("根界面图 {} 的识别工具参数无效:{error}", root_id.as_str())
|
||||
})?;
|
||||
validate_tree_image_ids(&parsed.trees, &context_ids).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=validate_trees root={} error={error}",
|
||||
|
||||
+67
-30
@@ -1,9 +1,10 @@
|
||||
use crate::config::build_game_creator_llm_client_from_config;
|
||||
use crate::ui_editor::commands::utils::strict_json_schema;
|
||||
use crate::ui_editor::commands::utils::{
|
||||
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, strict_json_schema,
|
||||
};
|
||||
use crate::ui_editor::resource::ui_design_image::UIDesignImageRole;
|
||||
use crate::ui_editor::state::State;
|
||||
use crate::ui_editor::utils::UIDesignImageId;
|
||||
use base64::Engine as _;
|
||||
use platform_llm::{
|
||||
LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice,
|
||||
};
|
||||
@@ -13,8 +14,6 @@ use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use ts_rs::TS;
|
||||
|
||||
// TODO `ui_design_suggestion.rs`:异步命令同步读取图片并进行 base64 编码,未设置输入大小上限。需要确定参考图大小策略,并决定 `tokio::fs` 还是 blocking worker 的执行边界。
|
||||
|
||||
const SYSTEM_PROMPT: &str = r#"
|
||||
请识别这些 UI 参考图的界面语义,并调用 suggest_ui_design_semantics 工具返回结果。
|
||||
不要在工具调用外输出 JSON、Markdown、代码围栏、注释、额外字段或解释文字。
|
||||
@@ -61,22 +60,32 @@ fn ui_design_suggestion_json_schema() -> Result<serde_json::Value, String> {
|
||||
}
|
||||
|
||||
const MAX_REFERENCES: usize = 4;
|
||||
const MAX_SUGGESTION_TREE_DEPTH: usize = 4;
|
||||
|
||||
fn image_data_url(path: &Path, bytes: &[u8]) -> Result<String, String> {
|
||||
let extension = path
|
||||
.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(str::to_ascii_lowercase);
|
||||
let mime = match extension.as_deref() {
|
||||
Some("png") => "image/png",
|
||||
Some("jpg") | Some("jpeg") => "image/jpeg",
|
||||
Some("webp") => "image/webp",
|
||||
_ => return Err(format!("不支持的界面图格式:{}", path.display())),
|
||||
};
|
||||
Ok(format!(
|
||||
"data:{mime};base64,{}",
|
||||
base64::engine::general_purpose::STANDARD.encode(bytes)
|
||||
))
|
||||
fn validate_suggestion_response_shape(value: &serde_json::Value) -> Result<(), String> {
|
||||
let roots = value
|
||||
.get("ui_designs")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.ok_or_else(|| "UI 语义建议工具参数缺少 ui_designs 数组".to_string())?;
|
||||
let mut stack = roots.iter().map(|node| (node, 1usize)).collect::<Vec<_>>();
|
||||
let mut node_count = 0usize;
|
||||
while let Some((node, depth)) = stack.pop() {
|
||||
if depth > MAX_SUGGESTION_TREE_DEPTH {
|
||||
return Err(format!(
|
||||
"UI 语义建议树最大深度不能超过 {MAX_SUGGESTION_TREE_DEPTH}"
|
||||
));
|
||||
}
|
||||
node_count += 1;
|
||||
if node_count > MAX_REFERENCES {
|
||||
return Err(format!("UI 语义建议最多包含 {MAX_REFERENCES} 个节点"));
|
||||
}
|
||||
let children = node
|
||||
.get("children")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.ok_or_else(|| "UI 语义建议节点缺少 children 数组".to_string())?;
|
||||
stack.extend(children.iter().map(|child| (child, depth + 1)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_suggestions(
|
||||
@@ -149,19 +158,19 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
|
||||
);
|
||||
error
|
||||
})?;
|
||||
let bytes = std::fs::read(&absolute).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_design_suggestion.error stage=read_image id={} error={error}",
|
||||
id.as_str()
|
||||
);
|
||||
format!("读取界面图失败:{error}")
|
||||
})?;
|
||||
let image_url = read_ui_reference_image_data_url(absolute)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_design_suggestion.error stage=read_image id={} error={error}",
|
||||
id.as_str()
|
||||
);
|
||||
error
|
||||
})?;
|
||||
parts.push(LlmMessageContentPart::InputText {
|
||||
text: format!("REFERENCE id:{} metadata:{}", id.as_str(), image.metadata,),
|
||||
});
|
||||
parts.push(LlmMessageContentPart::InputImage {
|
||||
image_url: image_data_url(&absolute, &bytes)?,
|
||||
});
|
||||
parts.push(LlmMessageContentPart::InputImage { image_url });
|
||||
}
|
||||
let client = build_game_creator_llm_client_from_config().map_err(|error| {
|
||||
eprintln!("ui_design_suggestion.error stage=build_client error={error}");
|
||||
@@ -204,7 +213,15 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
|
||||
eprintln!("ui_design_suggestion.error stage=parse_tool_call reason=missing_tool_call");
|
||||
"LLM 响应无效(详情:未返回 suggest_ui_design_semantics 工具调用)".to_string()
|
||||
})?;
|
||||
let suggestions = serde_json::from_str::<UIDesignSuggestion>(&call.arguments)
|
||||
let arguments = parse_limited_llm_tool_arguments(&call.arguments).map_err(|error| {
|
||||
eprintln!("ui_design_suggestion.error stage=parse_arguments error={error}");
|
||||
format!("LLM 响应无效(详情:UI 语义建议工具参数无效:{error})")
|
||||
})?;
|
||||
validate_suggestion_response_shape(&arguments).map_err(|error| {
|
||||
eprintln!("ui_design_suggestion.error stage=validate_arguments error={error}");
|
||||
format!("LLM 响应无效(详情:{error})")
|
||||
})?;
|
||||
let suggestions = serde_json::from_value::<UIDesignSuggestion>(arguments)
|
||||
.map_err(|error| {
|
||||
eprintln!("ui_design_suggestion.error stage=parse_arguments error={error}");
|
||||
format!("LLM 响应无效(详情:UI 语义建议工具参数无效:{error})")
|
||||
@@ -253,6 +270,26 @@ mod tests {
|
||||
assert!(validate_suggestions(&suggestions, &image_ids(&["page", "section"])).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggestion_shape_rejects_more_than_four_nodes_or_four_levels() {
|
||||
let oversized = (0..=MAX_REFERENCES)
|
||||
.map(|_| serde_json::json!({"children": []}))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(validate_suggestion_response_shape(&serde_json::json!({
|
||||
"ui_designs": oversized
|
||||
}))
|
||||
.is_err());
|
||||
|
||||
let mut nested = serde_json::json!({"children": []});
|
||||
for _ in 0..MAX_SUGGESTION_TREE_DEPTH {
|
||||
nested = serde_json::json!({"children": [nested]});
|
||||
}
|
||||
assert!(validate_suggestion_response_shape(&serde_json::json!({
|
||||
"ui_designs": [nested]
|
||||
}))
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_duplicate_and_unknown_ids() {
|
||||
let duplicate = vec![
|
||||
|
||||
@@ -1,4 +1,70 @@
|
||||
use base64::Engine as _;
|
||||
use schemars::JsonSchema;
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub(crate) const LLM_TOOL_ARGUMENT_MAX_BYTES: usize = 1024 * 1024;
|
||||
pub(crate) const UI_REFERENCE_IMAGE_MAX_BYTES: usize = 5 * 1024 * 1024;
|
||||
|
||||
pub(crate) fn parse_limited_llm_tool_arguments(
|
||||
arguments: &str,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
if arguments.len() > LLM_TOOL_ARGUMENT_MAX_BYTES {
|
||||
return Err(format!(
|
||||
"LLM 工具参数超过 {LLM_TOOL_ARGUMENT_MAX_BYTES} 字节上限"
|
||||
));
|
||||
}
|
||||
serde_json::from_str(arguments).map_err(|error| format!("LLM 工具参数不是有效 JSON:{error}"))
|
||||
}
|
||||
|
||||
pub(crate) async fn read_ui_reference_image_data_url(path: PathBuf) -> Result<String, String> {
|
||||
tokio::task::spawn_blocking(move || read_ui_reference_image_data_url_blocking(&path))
|
||||
.await
|
||||
.map_err(|error| format!("读取界面图任务失败:{error}"))?
|
||||
}
|
||||
|
||||
fn read_ui_reference_image_data_url_blocking(path: &Path) -> Result<String, String> {
|
||||
let extension = path
|
||||
.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(str::to_ascii_lowercase);
|
||||
let mime = match extension.as_deref() {
|
||||
Some("png") => "image/png",
|
||||
Some("jpg") | Some("jpeg") => "image/jpeg",
|
||||
Some("webp") => "image/webp",
|
||||
_ => return Err(format!("不支持的界面图格式:{}", path.display())),
|
||||
};
|
||||
let mut file =
|
||||
File::open(path).map_err(|error| format!("读取界面图失败:{}: {error}", path.display()))?;
|
||||
let metadata = file
|
||||
.metadata()
|
||||
.map_err(|error| format!("读取界面图信息失败:{}: {error}", path.display()))?;
|
||||
if !metadata.is_file() {
|
||||
return Err(format!("界面图不是普通文件:{}", path.display()));
|
||||
}
|
||||
if metadata.len() > UI_REFERENCE_IMAGE_MAX_BYTES as u64 {
|
||||
return Err(format!(
|
||||
"单张界面图不能超过 {} MiB",
|
||||
UI_REFERENCE_IMAGE_MAX_BYTES / 1024 / 1024
|
||||
));
|
||||
}
|
||||
let mut bytes = Vec::with_capacity(metadata.len() as usize);
|
||||
Read::by_ref(&mut file)
|
||||
.take((UI_REFERENCE_IMAGE_MAX_BYTES + 1) as u64)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|error| format!("读取界面图失败:{}: {error}", path.display()))?;
|
||||
if bytes.len() > UI_REFERENCE_IMAGE_MAX_BYTES {
|
||||
return Err(format!(
|
||||
"单张界面图不能超过 {} MiB",
|
||||
UI_REFERENCE_IMAGE_MAX_BYTES / 1024 / 1024
|
||||
));
|
||||
}
|
||||
Ok(format!(
|
||||
"data:{mime};base64,{}",
|
||||
base64::engine::general_purpose::STANDARD.encode(bytes)
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn strict_json_schema<T: JsonSchema>() -> Result<serde_json::Value, String> {
|
||||
let mut settings = schemars::generate::SchemaSettings::default();
|
||||
@@ -45,3 +111,30 @@ fn normalize_strict_schema(schema: &mut serde_json::Value) {
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn llm_tool_arguments_limit_counts_utf8_bytes() {
|
||||
let oversized = "界".repeat(LLM_TOOL_ARGUMENT_MAX_BYTES / 3 + 1);
|
||||
assert!(parse_limited_llm_tool_arguments(&oversized).is_err());
|
||||
assert_eq!(
|
||||
parse_limited_llm_tool_arguments(r#"{"ok":true}"#).expect("valid arguments"),
|
||||
serde_json::json!({"ok": true})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_image_rejects_file_over_five_mib_before_reading() {
|
||||
let directory = tempfile::tempdir().expect("reference image fixture");
|
||||
let path = directory.path().join("oversized.png");
|
||||
let file = File::create(&path).expect("create sparse reference image");
|
||||
file.set_len((UI_REFERENCE_IMAGE_MAX_BYTES + 1) as u64)
|
||||
.expect("size sparse reference image");
|
||||
drop(file);
|
||||
|
||||
assert!(read_ui_reference_image_data_url_blocking(&path).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,7 +92,10 @@ function imageImporterSettings(
|
||||
}
|
||||
|
||||
export const DESIGN_IMAGE_IMPORTER_SETTINGS: AssetImporterSettings =
|
||||
imageImporterSettings({ ...imageRequirements, maxItems: 4 });
|
||||
imageImporterSettings({
|
||||
maxItems: 4,
|
||||
maxFileSizeBytes: 5 * 1024 * 1024,
|
||||
});
|
||||
|
||||
export const SPRITE_IMPORTER_SETTINGS: AssetImporterSettings =
|
||||
imageImporterSettings(imageRequirements);
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { DESIGN_IMAGE_IMPORTER_SETTINGS } from '../src/components/AssetImporter/settings';
|
||||
|
||||
describe('UI design image importer limits', () => {
|
||||
it('limits design references to four files and five MiB each', () => {
|
||||
const expected = {
|
||||
maxItems: 4,
|
||||
maxFileSizeBytes: 5 * 1024 * 1024,
|
||||
};
|
||||
|
||||
expect(DESIGN_IMAGE_IMPORTER_SETTINGS.local.requirements).toEqual(expected);
|
||||
expect(DESIGN_IMAGE_IMPORTER_SETTINGS.remote?.requirements).toEqual(
|
||||
expected,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,12 @@
|
||||
# 决策记录
|
||||
|
||||
## 2026-08-20 UI Editor LLM 递归输出与参考图单文件限制
|
||||
|
||||
- 背景:结构识别、界面语义建议和多图合并直接把 LLM 工具 arguments 反序列化为递归树;结构识别与语义建议还在 async command 中同步读取并 base64 编码参考图。模型异常输出或过大图片可能造成不受控内存、栈和 async worker 占用。
|
||||
- 决策:三个工具调用的 arguments 统一限制为 `1 MiB`,先解析通用 JSON 并迭代检查,再进入递归业务类型。结构识别按每棵树独立限制 `512` 个 LLM 节点 / `32` 层,不跨树求和且不计 Rust 页面根;语义建议限制 `4` 节点 / `4` 层;合并计划限制 `512` 节点 / `32` 层。超限整次拒绝,不截断或交付部分结果,日志不记录 arguments 正文。
|
||||
- 输入边界:`merge_ui` 继续直接接收 `State`,不修改 Tauri/frontend IPC 参数;进入 Rust 后、发起 LLM 前按每棵源树独立限制 `512` 节点 / `32` 层,不跨树求和,并限制 `2 MiB` 序列化投影。UI 设计参考图只设单张 `5 MiB` 上限,不设批次合计或像素数上限;元数据检查、有限读取和 base64 编码进入 blocking worker,不新增命令超时。
|
||||
- 关联文档:`docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md`。
|
||||
|
||||
## 2026-08-15 AGC game-chat 主代码 Run 接管直属美术 delivery
|
||||
|
||||
- 背景:真实 `gpt-5.6-sol / max` 验收中,`art-director` 失败后已形成 `ready + needs-repair` delivery,但认领、合同读取、claim observation 和完成 blocker 均硬编码为 Supervisor-only;实际直属父 Run `code-prototype` 无法消费回执,随后又发起 29 次 Provider 请求。
|
||||
|
||||
@@ -40,10 +40,16 @@ UI Editor Inspector 的全局只读状态唯一来源是 `controller.editor.isLo
|
||||
|
||||
UI Editor 当前把“识别界面结构”定义为结构草稿阶段,而不是完整视觉还原阶段。识别 DTO 只负责输出节点层级、几何、名称、描述和置信度;节点组件暂为空,由后续“绑定视觉素材”阶段补齐 Image / Text 组件。`applyRecognitionResult` 可以整体替换当前 `ui_trees`,但该替换只代表结构结果,不能宣称已经保留截图中的视觉内容;组件状态使用 `NoProblem`,前置检查仍会根据空组件和素材绑定情况阻止跳过绑定阶段。
|
||||
|
||||
结构识别、界面语义建议和多图合并只接受不超过 `1 MiB` 的 LLM 工具调用 arguments,并在递归业务类型反序列化前先解析为通用 JSON、迭代检查结构预算。结构识别按每棵返回树独立限制为最多 `512` 个 LLM 节点和 `32` 层,不汇总多棵树的节点数,也不计 Rust 自动补建的页面根;界面语义建议最多 `4` 个节点和 `4` 层;合并计划最多 `512` 个计划节点和 `32` 层,`Simple.children` 与 `Merged.merged_from` 使用同一计数和深度口径。任何超限结果均整次拒绝,不截断、不返回部分结果,也不把工具 arguments 正文写入日志。
|
||||
|
||||
多图合并使用 LLM 返回的语义投影树。`Simple` 和 `Merged` 只列出希望进入结果树的原始节点;未被计划引用的源节点表示本次合并判定为冗余或不属于目标公共结构,允许被丢弃,不要求 `used_original_ids` 覆盖全部输入节点。重复 ID 和未知 ID 仍然是错误;省略不是隐式复制或随机删除,而是合并计划的正式语义。
|
||||
|
||||
`merge_ui` 的 Tauri 命令参数继续保持 `State`,不新增前端字符串 payload 或 IPC 参数上限。命令进入 Rust 后、构建 LLM 请求前,按每棵源树独立限制 `512` 个节点和 `32` 层,不跨树求和;序列化投影超过 `2 MiB` 时同样拒绝。本边界不宣称覆盖 Tauri 自身的入参反序列化阶段。
|
||||
|
||||
UI Editor 的图片、字体和远程素材导入采用增量提交合同:输入先完成各入口已有的预检,随后按顺序逐项写入和登记;前面已成功的项目在后续写入或登记失败时保持有效,调用方必须重新读取 manifest 取得已提交集合。该入口不提供整批事务回滚,也不承诺失败后“零文件、零 manifest 变化”;后续如需原子批量导入,必须新增明确的 transaction / reconciliation 合同,不能把当前入口静默改成另一种语义。
|
||||
|
||||
作为 LLM 参考图使用的 UI 设计图单张文件上限为 `5 MiB`,不设置多图合计大小上限。前端设计图 AssetImporter 对本地和远端入口使用相同单文件限制;结构识别与界面语义建议在 Rust 中先检查文件元数据、再有限读取,并把读取与 base64 编码放在 blocking worker,避免阻塞 Tokio async worker。本阶段不新增图片像素数限制或命令超时。
|
||||
|
||||
## 2026-08-18 UI Editor 左侧节点树跨界面移动
|
||||
|
||||
左侧 UI 节点树使用 UI-only 的虚拟超级节点统一承载现有 `ui_trees` 的页面根节点,仅改变树视图,不写入 State 或持久化契约。页面根节点和超级节点不可拖动;普通节点可在不同页面根节点之间拖动,整个子树随节点移动并保留原 `transform`。这里的“保留”只表示复制节点原本的局部布局参数,不承诺跨界面后的页面像素位置稳定:不同界面图拥有独立的画布尺寸、像素密度和父节点坐标空间,跨树坐标换算没有可靠的默认语义,因此本次不做换算,也不把跨树移动后的视觉位置描述为“保持不变”。
|
||||
|
||||
Reference in New Issue
Block a user