diff --git a/apps/ai-game-creator-shell/src-tauri/Cargo.lock b/apps/ai-game-creator-shell/src-tauri/Cargo.lock index 98aa5134d..2d5b61287 100644 --- a/apps/ai-game-creator-shell/src-tauri/Cargo.lock +++ b/apps/ai-game-creator-shell/src-tauri/Cargo.lock @@ -1666,6 +1666,7 @@ dependencies = [ "portable-pty", "reqwest 0.12.28", "rmcp", + "schemars 1.2.1", "serde", "serde_json", "serde_yaml", @@ -4453,7 +4454,7 @@ checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ "dyn-clone", "indexmap 1.9.3", - "schemars_derive", + "schemars_derive 0.8.22", "serde", "serde_json", "url", @@ -4480,6 +4481,7 @@ checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", + "schemars_derive 1.2.1", "serde", "serde_json", ] @@ -4496,6 +4498,18 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "schemars_derive" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.118", +] + [[package]] name = "scopeguard" version = "1.2.0" diff --git a/apps/ai-game-creator-shell/src-tauri/Cargo.toml b/apps/ai-game-creator-shell/src-tauri/Cargo.toml index 55ec23f55..45fe12272 100644 --- a/apps/ai-game-creator-shell/src-tauri/Cargo.toml +++ b/apps/ai-game-creator-shell/src-tauri/Cargo.toml @@ -36,6 +36,7 @@ rmcp = { version = "2.2.0", default-features = false, features = ["client", "req serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" +schemars = "1.2.1" sha2 = "0.10" similar = "2.7" platform-llm = { path = "../../../server-rs/crates/platform-llm" } diff --git a/apps/ai-game-creator-shell/src-tauri/src/main.rs b/apps/ai-game-creator-shell/src-tauri/src/main.rs index 5e3245728..28ae96d72 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/main.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/main.rs @@ -118,6 +118,14 @@ async fn suggest_ui_design_semantic( ui_editor::commands::suggest_ui_design_semantic_impl(project_path, state).await } +#[tauri::command] +async fn recognize_ui( + project_path: String, + state: ui_editor::state::State, +) -> Result { + ui_editor::commands::recognize_ui_impl(project_path, state).await +} + #[derive(Debug, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] struct InitLocalProjectResult { @@ -2252,6 +2260,7 @@ fn main() { import_ui_editor_local_files, import_ui_editor_remote_assets, suggest_ui_design_semantic, + recognize_ui, generate_platform_art_asset, open_canvas_project, get_game_creation_agent_capabilities, diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs index a13a9c852..94c273f9a 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs @@ -1,5 +1,8 @@ -pub mod ui_design_suggestion; pub mod recognition; +pub mod ui_design_suggestion; +pub mod utils; +pub(crate) use recognition::recognize_ui_impl; +pub use recognition::RecognitionDTO; pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl; pub use ui_design_suggestion::UIDesignSuggestion; diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs new file mode 100644 index 000000000..9d1cd43f8 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs @@ -0,0 +1,539 @@ +use crate::config::build_game_creator_llm_client_from_config; +use crate::ui_editor::commands::utils::strict_json_schema; +use crate::ui_editor::layout::dimension::UIRect; +use crate::ui_editor::layout::node::{Node as LayoutNode, NodeMetadata, NodeSource, NodeStatus}; +use crate::ui_editor::layout::transform::Transform; +use crate::ui_editor::resource::ui_design_image::UIDesignImage; +use crate::ui_editor::state::{State, UITree}; +use crate::ui_editor::utils::{NodeId, UIDesignImageId}; +use base64::Engine as _; +use nalgebra::{Point2, Vector2}; +use platform_llm::{ + LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::Path; +use ts_rs::TS; + +const MAX_REFERENCES: usize = 4; + +const SYSTEM_PROMPT: &str = r#" +你是游戏 UI 结构识别器。当前请求只处理一个 Page 参考图，以及直接归属于该 Page 的 slave 参考图。 +第一张标记为 PAGE 的图片是唯一输出树的画布和归属页面；其它标记为 SLAVE 的图片只提供视觉上下文。 +本次调用只能调用 recognize_ui_structure 工具返回一个节点列表，不得返回树数组、树 ID 或其它页面的节点。 +每个节点必须使用 src_ui_design_image_id 指定坐标来源，并在该图片的原始像素坐标系中输出 +global_pos_x_px、global_pos_y_px、width_px、height_px；坐标相对于来源图片左上角 +"#; + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +enum HorizonalAnchor { + Left, + Center, + Right, + Stretch, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +enum VerticalAnchor { + Top, + Center, + Bottom, + Stretch, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +struct PresetAnchor { + horizontal: HorizonalAnchor, + vertical: VerticalAnchor, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +struct CustomMinMaxAnchor { + min_x: f32, + min_y: f32, + max_x: f32, + max_y: f32, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +enum Anchor { + Preset(PresetAnchor), + CustomMinMax(CustomMinMaxAnchor), +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub enum Confidence { + Confident, + UnSure(String), +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +struct RecognitionNode { + global_pos_x_px: u32, + global_pos_y_px: u32, + width_px: u32, + height_px: u32, + /// 视觉证据来源；只允许当前 PAGE 或其直接 slave，不能用来改变树归属或坐标系。 + #[schemars(with = "String")] + src_ui_design_image_id: UIDesignImageId, + local_anchor: Anchor, + name: String, + description: String, + children: Vec, + confidence: Confidence, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +struct RecognitionResponse { + children: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct RecognitionDTO { + pub ui_trees: Vec, +} + +fn recognition_json_schema() -> Result { + strict_json_schema::() +} + +fn image_data_url(path: &Path, bytes: &[u8]) -> Result { + let mime = match path.extension().and_then(|value| value.to_str()) { + Some("png") => "image/png", + Some("jpg") | Some("jpeg") => "image/jpeg", + Some("webp") => "image/webp", + _ => return Err(format!("不支持的界面图格式：{}", path.display())), + }; + Ok(format!( + "data:{mime};base64,{}", + base64::engine::general_purpose::STANDARD.encode(bytes) + )) +} + +fn anchor_ranges(anchor: &Anchor) -> Result<(Vector2, Vector2), String> { + fn horizontal(anchor: &HorizonalAnchor) -> (f32, f32) { + match anchor { + HorizonalAnchor::Left => (0.0, 0.0), + HorizonalAnchor::Center => (0.5, 0.5), + HorizonalAnchor::Right => (1.0, 1.0), + HorizonalAnchor::Stretch => (0.0, 1.0), + } + } + fn vertical(anchor: &VerticalAnchor) -> (f32, f32) { + match anchor { + VerticalAnchor::Top => (0.0, 0.0), + VerticalAnchor::Center => (0.5, 0.5), + VerticalAnchor::Bottom => (1.0, 1.0), + VerticalAnchor::Stretch => (0.0, 1.0), + } + } + let (min, max) = match anchor { + Anchor::Preset(value) => { + let (min_x, max_x) = horizontal(&value.horizontal); + let (min_y, max_y) = vertical(&value.vertical); + (Vector2::new(min_x, min_y), Vector2::new(max_x, max_y)) + } + Anchor::CustomMinMax(value) => ( + Vector2::new(value.min_x, value.min_y), + Vector2::new(value.max_x, value.max_y), + ), + }; + if !min.iter().chain(max.iter()).all(|value| value.is_finite()) + || min.x < 0.0 + || min.y < 0.0 + || max.x > 1.0 + || max.y > 1.0 + || min.x > max.x + || min.y > max.y + { + return Err("锚点范围无效".to_string()); + } + Ok((min, max)) +} + +fn random_node_id() -> Result { + NodeId::new(uuid::Uuid::new_v4().simple().to_string()) + .map_err(|error| format!("生成节点 ID 失败：{error}")) +} + +fn image_layout_size(image: &UIDesignImage) -> Result, String> { + let pixels_per_unit = image.pixels_per_unit.get(); + if !pixels_per_unit.is_finite() || pixels_per_unit <= 0.0 { + return Err("界面图 pixels_per_unit 无效".to_string()); + } + let size = image.pixel_size / pixels_per_unit; + if !size.iter().all(|value| value.is_finite() && *value > 0.0) { + return Err("界面图尺寸无效".to_string()); + } + Ok(size) +} + +fn convert_node( + source: &RecognitionNode, + page_image: &UIDesignImage, + source_image: &UIDesignImage, + images: &HashMap, + parent_rect: UIRect, +) -> Result { + // LLM 输出的是 source_image 原始像素。Rust 先按来源图尺寸归一化， + // 再缩放到 Page 图像素尺寸，最后使用 Page 的 pixels_per_unit 转成布局单位。 + let source_size_px = source_image.pixel_size; + let page_size_px = page_image.pixel_size; + let source_size_valid = source_size_px + .iter() + .all(|value| value.is_finite() && *value > 0.0); + if !source_size_valid + || !page_size_px + .iter() + .all(|value| value.is_finite() && *value > 0.0) + { + return Err("界面图尺寸无效".to_string()); + } + let source_min_px = Vector2::new(source.global_pos_x_px as f32, source.global_pos_y_px as f32); + let source_size_node_px = Vector2::new(source.width_px as f32, source.height_px as f32); + let source_max_px = source_min_px + source_size_node_px; + let mut blocked = source.width_px == 0 || source.height_px == 0; + if source_min_px.x < 0.0 + || source_min_px.y < 0.0 + || source_max_px.x > source_size_px.x + || source_max_px.y > source_size_px.y + { + blocked = true; + } + let scale = page_size_px.component_div(&source_size_px); + let page_min_px = source_min_px.component_mul(&scale); + let page_size_node_px = source_size_node_px.component_mul(&scale); + let ppu = page_image.pixels_per_unit.get(); + let target_min = Point2::new(page_min_px.x / ppu, page_min_px.y / ppu); + let target_rect = UIRect::new( + target_min, + Vector2::new(page_size_node_px.x / ppu, page_size_node_px.y / ppu), + ); + let (anchor_min, anchor_max) = anchor_ranges(&source.local_anchor) + .unwrap_or((Vector2::new(0.5, 0.5), Vector2::new(0.5, 0.5))); + if target_rect.min.x < parent_rect.min.x + || target_rect.min.y < parent_rect.min.y + || target_rect.max().x > parent_rect.max().x + || target_rect.max().y > parent_rect.max().y + { + blocked = true; + } + if anchor_ranges(&source.local_anchor).is_err() { + blocked = true; + } + if source.name.trim().is_empty() { + blocked = true; + } + let mut transform = Transform::new(anchor_min, anchor_max, Vector2::zeros(), Vector2::zeros()); + transform.set_resolved_rect(&parent_rect, target_rect); + let status = if blocked { + NodeStatus::Blocked + } else { + match &source.confidence { + Confidence::Confident => NodeStatus::Passed, + Confidence::UnSure(reason) => NodeStatus::NeedReview(reason.clone()), + } + }; + let children = source + .children + .iter() + .map(|child| { + let child_source_image = + images.get(&child.src_ui_design_image_id).ok_or_else(|| { + format!( + "节点引用了缺失界面图 {}", + child.src_ui_design_image_id.as_str() + ) + })?; + convert_node(child, page_image, child_source_image, images, target_rect) + }) + .collect::, _>>()?; + Ok(LayoutNode { + id: random_node_id()?, + transform, + metadata: NodeMetadata { + name: source.name.clone(), + description: source.description.clone(), + status, + source: NodeSource::Llm, + }, + // TODO: 等识别 DTO 增加 type 字段后，再映射 Image/Text Component。 + components: Vec::new(), + children, + }) +} + +fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> { + for node in nodes { + if let Confidence::UnSure(reason) = &node.confidence { + if reason.trim().is_empty() { + return Err("UnSure 必须包含审阅原因".to_string()); + } + } + validate_confidence(&node.children)?; + } + Ok(()) +} + +fn validate_source_image_ids( + nodes: &[RecognitionNode], + allowed_ids: &[UIDesignImageId], +) -> Result<(), String> { + for node in nodes { + if !allowed_ids + .iter() + .any(|allowed| allowed == &node.src_ui_design_image_id) + { + return Err("LLM 返回了不属于当前 Page 上下文的节点来源界面图 ID".to_string()); + } + validate_source_image_ids(&node.children, allowed_ids)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn preset_anchor_ranges_use_top_left_image_coordinates() { + let (min, max) = anchor_ranges(&Anchor::Preset(PresetAnchor { + horizontal: HorizonalAnchor::Center, + vertical: VerticalAnchor::Top, + })) + .expect("preset anchor is valid"); + assert_eq!(min, Vector2::new(0.5, 0.0)); + assert_eq!(max, Vector2::new(0.5, 0.0)); + } + + #[test] + fn invalid_custom_anchor_is_rejected() { + let result = anchor_ranges(&Anchor::CustomMinMax(CustomMinMaxAnchor { + min_x: 0.8, + min_y: 0.0, + max_x: 0.2, + max_y: 1.0, + })); + assert!(result.is_err()); + } + + #[test] + fn unsure_without_reason_is_rejected() { + let node = RecognitionNode { + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + src_ui_design_image_id: UIDesignImageId::new("page").expect("image id"), + local_anchor: Anchor::Preset(PresetAnchor { + horizontal: HorizonalAnchor::Left, + vertical: VerticalAnchor::Top, + }), + name: "node".to_string(), + description: String::new(), + children: Vec::new(), + confidence: Confidence::UnSure(String::new()), + }; + assert!(validate_confidence(&[node]).is_err()); + } +} + +fn page_image_ids(state: &State) -> Vec { + state + .ui_design_images + .iter() + .filter_map(|(id, image)| { + (image.metadata.role + == Some(crate::ui_editor::resource::ui_design_image::UIDesignImageRole::Page)) + .then(|| id.clone()) + }) + .collect() +} + +fn slave_image_ids(state: &State, page_id: &UIDesignImageId) -> Vec { + state + .ui_design_images + .iter() + .filter_map(|(id, image)| { + (image.metadata.slave_to.as_ref() == Some(page_id)).then(|| id.clone()) + }) + .collect() +} + +pub(crate) async fn recognize_ui_impl( + project_path: String, + state: State, +) -> Result { + if state.ui_design_images.is_empty() { + eprintln!("ui_recognition.error stage=validate reason=no_images"); + return Err("请先导入界面图".to_string()); + } + if state.ui_design_images.len() > MAX_REFERENCES { + eprintln!( + "ui_recognition.error stage=validate reason=too_many_images count={}", + state.ui_design_images.len() + ); + return Err("界面图最多 4 张".to_string()); + } + let page_ids = page_image_ids(&state); + if page_ids.is_empty() { + eprintln!("ui_recognition.error stage=validate reason=no_page"); + return Err("至少需要一张 role 为 Page 的界面图".to_string()); + } + let client = build_game_creator_llm_client_from_config().map_err(|error| { + eprintln!("ui_recognition.error stage=build_client error={error}"); + error + })?; + let schema = recognition_json_schema().map_err(|error| { + eprintln!("ui_recognition.error stage=build_schema error={error}"); + error + })?; + let root = Path::new(project_path.trim()); + let mut ui_trees = Vec::with_capacity(page_ids.len()); + for page_id in page_ids { + let page = state + .ui_design_images + .get(&page_id) + .ok_or_else(|| "缺少 Page 界面图资源".to_string())?; + let mut context_ids = vec![page_id.clone()]; + context_ids.extend(slave_image_ids(&state, &page_id)); + // The model may cite only this Page or its direct slave images as evidence. + let allowed_source_ids = context_ids.clone(); + let mut parts = Vec::with_capacity(context_ids.len() * 2); + for (index, context_id) in context_ids.iter().enumerate() { + let image = state + .ui_design_images + .get(context_id) + .ok_or_else(|| "缺少识别上下文界面图资源".to_string())?; + let absolute = + crate::project::resolve_local_project_path(root, &image.path).map_err(|error| { + eprintln!( + "ui_recognition.error stage=resolve_image page={} image={} error={error}", + page_id.as_str(), + context_id.as_str() + ); + error + })?; + let bytes = std::fs::read(&absolute).map_err(|error| { + eprintln!( + "ui_recognition.error stage=read_image page={} image={} error={error}", + page_id.as_str(), + context_id.as_str() + ); + format!("读取界面图失败：{error}") + })?; + parts.push(LlmMessageContentPart::InputText { + text: format!( + "{} id={} pixel_size={:?}", + if index == 0 { "PAGE" } else { "SLAVE" }, + context_id.as_str(), + image.pixel_size + ), + }); + parts.push(LlmMessageContentPart::InputImage { + image_url: image_data_url(&absolute, &bytes)?, + }); + } + let tool = LlmFunctionTool::new( + "recognize_ui_structure", + "识别当前 Page 及其 slave 参考图中的 UI 节点，并只返回这一棵树的节点列表", + schema.clone(), + ) + .with_strict(true); + let response = client + .run( + LlmRunRequest::new(vec![ + LlmMessage::system(SYSTEM_PROMPT), + LlmMessage::user_multimodal(parts), + ]) + .with_function_tools(vec![tool]) + .with_tool_choice(LlmToolChoice::Required), + ) + .await + .map_err(|error| { + eprintln!( + "ui_recognition.error stage=llm_request page={} error={error}", + page_id.as_str() + ); + format!("UI 结构识别失败（Page {}）：{error}", page_id.as_str()) + })?; + eprintln!( + "ui_recognition.llm_output page={} text={:?} tool_calls={:?}", + page_id.as_str(), + response.text, + response.tool_calls + ); + let call = response + .tool_calls + .iter() + .find(|call| call.name == "recognize_ui_structure") + .ok_or_else(|| { + eprintln!( + "ui_recognition.error stage=parse_tool_call page={} reason=missing_tool_call", + page_id.as_str() + ); + format!( + "Page {} 的 LLM 未返回 recognize_ui_structure 工具调用", + page_id.as_str() + ) + })?; + let parsed = + serde_json::from_str::(&call.arguments).map_err(|error| { + eprintln!( + "ui_recognition.error stage=parse_arguments page={} error={error}", + page_id.as_str() + ); + format!("Page {} 的识别工具参数无效：{error}", page_id.as_str()) + })?; + validate_confidence(&parsed.children).map_err(|error| { + eprintln!( + "ui_recognition.error stage=validate_confidence page={} error={error}", + page_id.as_str() + ); + error + })?; + validate_source_image_ids(&parsed.children, &allowed_source_ids).map_err(|error| { + eprintln!( + "ui_recognition.error stage=validate_source_ids page={} error={error}", + page_id.as_str() + ); + error + })?; + let size = image_layout_size(page)?; + let root_rect = UIRect::new(Point2::origin(), size); + let children = parsed + .children + .iter() + .map(|node| { + let source_image = state + .ui_design_images + .get(&node.src_ui_design_image_id) + .ok_or_else(|| { + format!( + "Page {} 的节点引用了缺失界面图 {}", + page_id.as_str(), + node.src_ui_design_image_id.as_str() + ) + })?; + convert_node(node, page, source_image, &state.ui_design_images, root_rect) + }) + .collect::, _>>()?; + // Tree identity is assigned by Rust from the Page image, never chosen by the model. + ui_trees.push(UITree { + src_ui_design: page_id, + children, + }); + } + Ok(RecognitionDTO { ui_trees }) +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/ui_design_suggestion.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/ui_design_suggestion.rs index 040a395de..9f7515c8e 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/ui_design_suggestion.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/ui_design_suggestion.rs @@ -1,67 +1,117 @@ use crate::config::build_game_creator_llm_client_from_config; +use crate::ui_editor::commands::utils::strict_json_schema; use crate::ui_editor::resource::ui_design_image::UIDesignImageRole; use crate::ui_editor::state::State; use crate::ui_editor::utils::UIDesignImageId; use base64::Engine as _; -use platform_llm::{LlmMessage, LlmMessageContentPart, LlmRunRequest}; +use platform_llm::{ + LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, +}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::path::Path; use ts_rs::TS; const SYSTEM_PROMPT: &str = r#" -请识别这些 UI 参考图的界面语义。你的响应必须是严格 JSON 数组；不要输出 Markdown、代码围栏、注释、额外字段或解释文字。 - -TypeScript 等价类型如下（仅用于说明契约，实际响应仍必须是 JSON）： -type UIDesignSuggestion = { - ui_design_image_id: string; - name: string | null; - description: string | null; - role: "Page" | "Section" | "Modal" | "Drawer" | "Popover" | "State" | "Scrolled" | "Detail" | null; - slave_to: string | null; -}; +请识别这些 UI 参考图的界面语义，并调用 suggest_ui_design_semantics 工具返回结果。 +不要在工具调用外输出 JSON、Markdown、代码围栏、注释、额外字段或解释文字。 字段含义和 null 规则： - ui_design_image_id：必填，必须逐字匹配当前输入参考图的 id。它标识“这条建议属于哪张图”，不是新 ID，不能为 null。 - name：要写入该图片 metadata 的简短、可读名称； - description：要写入该图片 metadata 的简短语义描述，例如“带底部导航的主游戏页面”。 -- role：要写入该图片 metadata 的界面角色。只能使用 Page、Section、Modal、Drawer、Popover、State、Scrolled、Detail 之一；。Page 表示完整主页面，Section 表示同一主页面中的子界面或页签，Modal/Drawer/Popover 表示浮层或局部覆盖界面，State 表示同一界面的状态变体，Scrolled 表示滚动或分页后的内容，Detail 表示局部详情或补充证据。 -- slave_to：要写入该图片 metadata 的归属页面 id。只有当当前图片明显是另一张输入图的局部/子界面时才填写那个宿主图的 id；如果不应修改现有 slave_to，必须填 null。Page 不得设置宿主。不能填写自身 id，也不能填写不在输入参考图中的 id。 +- role：要写入该图片 metadata 的界面角色。 + Page 表示完整主页面， + Section 表示同一主页面中的子界面或页签， + Modal/Drawer/Popover 表示浮层或局部覆盖界面， + State 表示同一界面的状态变体， + Scrolled 表示滚动或分页后的内容， + Detail 表示局部详情或补充证据。 +- slave_to：要写入该图片 metadata 的归属页面 id。只有当当前图片明显是另一张输入图的局部/子界面时才填写那个宿主图的 id； + Page 不得设置宿主。 所有字段都必须出现；nullable 字段使用 JSON null 表示“不要修改该字段”，不要省略字段。 - -每张参考图最多返回一条建议，且不得重复 ui_design_image_id。示例： -[ - { - "ui_design_image_id": "image-main", - "name": "主游戏页面", - "description": "带顶部资源栏和底部导航的完整游戏主页面", - "role": "Page", - "slave_to": null - }, - { - "ui_design_image_id": "image-inventory", - "description": "从主页面进入的物品列表子界面", - "role": "Section", - "slave_to": "image-main" - }, -] -示例中的 image-main、image-inventory、image-detail 只是占位符；必须替换为本次输入中真实存在的 id。 - +每张参考图最多返回一条建议，且不得重复 ui_design_image_id。 以下UI设计图中的metadata部分信息已确定, 请补全未确定/为空的信息 "#; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS, JsonSchema)] +#[schemars(deny_unknown_fields)] +#[serde(deny_unknown_fields)] #[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] pub struct UIDesignSuggestion { pub ui_design_image_id: UIDesignImageId, + #[schemars(required)] pub name: Option, + #[schemars(required)] pub description: Option, + #[schemars(required)] pub role: Option, + #[schemars(required)] pub slave_to: Option, } +#[derive(Clone, Debug, Deserialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +#[serde(deny_unknown_fields)] +struct UIDesignSuggestionResponse { + suggestions: Vec, +} + +fn ui_design_suggestion_json_schema() -> Result { + strict_json_schema::() +} + const MAX_REFERENCES: usize = 4; +#[cfg(test)] +mod tests { + use super::*; + + fn assert_no_refs(value: &serde_json::Value) { + match value { + serde_json::Value::Object(object) => { + assert!(!object.contains_key("$ref")); + assert!(!object.contains_key("$defs")); + for child in object.values() { + assert_no_refs(child); + } + } + serde_json::Value::Array(values) => { + for child in values { + assert_no_refs(child); + } + } + _ => {} + } + } + + #[test] + fn strict_schema_is_inline_and_requires_all_properties() { + let schema = ui_design_suggestion_json_schema().expect("schema"); + assert_no_refs(&schema); + assert_eq!(schema["required"], serde_json::json!(["suggestions"])); + let required = schema["properties"]["suggestions"]["items"]["required"] + .as_array() + .expect("item required array") + .iter() + .filter_map(serde_json::Value::as_str) + .collect::>(); + assert_eq!( + required, + [ + "ui_design_image_id", + "name", + "description", + "role", + "slave_to" + ] + .into_iter() + .collect() + ); + } +} + fn image_data_url(path: &Path, bytes: &[u8]) -> Result { let mime = match path.extension().and_then(|value| value.to_str()) { Some("png") => "image/png", @@ -105,9 +155,14 @@ pub(crate) async fn suggest_ui_design_semantic_impl( state: State, ) -> Result, String> { if state.ui_design_images.is_empty() { + eprintln!("ui_design_suggestion.error stage=validate reason=no_images"); return Err("请先导入界面图".to_string()); } if state.ui_design_images.len() > MAX_REFERENCES { + eprintln!( + "ui_design_suggestion.error stage=validate reason=too_many_images count={}", + state.ui_design_images.len() + ); return Err("界面图最多 4 张".to_string()); } let root = Path::new(project_path.trim()); @@ -116,8 +171,21 @@ pub(crate) async fn suggest_ui_design_semantic_impl( let mut ids = HashSet::new(); for (id, image) in &state.ui_design_images { ids.insert(id.clone()); - let absolute = crate::project::resolve_local_project_path(root, &image.path)?; - let bytes = std::fs::read(&absolute).map_err(|error| format!("读取界面图失败：{error}"))?; + let absolute = + crate::project::resolve_local_project_path(root, &image.path).map_err(|error| { + eprintln!( + "ui_design_suggestion.error stage=resolve_image id={} error={error}", + id.as_str() + ); + error + })?; + let bytes = std::fs::read(&absolute).map_err(|error| { + eprintln!( + "ui_design_suggestion.error stage=read_image id={} error={error}", + id.as_str() + ); + format!("读取界面图失败：{error}") + })?; parts.push(LlmMessageContentPart::InputText { text: format!( "REFERENCE id={} metadata={} pixel_size={:?}", @@ -130,17 +198,56 @@ pub(crate) async fn suggest_ui_design_semantic_impl( image_url: image_data_url(&absolute, &bytes)?, }); } - let client = build_game_creator_llm_client_from_config()?; + let client = build_game_creator_llm_client_from_config().map_err(|error| { + eprintln!("ui_design_suggestion.error stage=build_client error={error}"); + error + })?; + let schema = ui_design_suggestion_json_schema().map_err(|error| { + eprintln!("ui_design_suggestion.error stage=build_schema error={error}"); + error + })?; + let tool = LlmFunctionTool::new( + "suggest_ui_design_semantics", + "为所有 UI 参考图补全界面语义 metadata", + schema, + ) + .with_strict(true); let response = client - .run(LlmRunRequest::new(vec![ - LlmMessage::system(SYSTEM_PROMPT), - LlmMessage::user_multimodal(parts), - ])) + .run( + LlmRunRequest::new(vec![ + LlmMessage::system(SYSTEM_PROMPT), + LlmMessage::user_multimodal(parts), + ]) + .with_function_tools(vec![tool]) + .with_tool_choice(LlmToolChoice::Required), + ) .await - .map_err(|error| format!("UI 参考图语义识别失败：{error}"))?; - let suggestions = serde_json::from_str::>(response.text.trim()) - .map_err(|error| format!("LLM 返回的 UI 语义 JSON 无效：{error}"))?; - validate_suggestions(&suggestions, &ids)?; + .map_err(|error| { + eprintln!("ui_design_suggestion.error stage=llm_request error={error}"); + format!("UI 参考图语义识别失败：{error}") + })?; + eprintln!( + "ui_design_suggestion.llm_output text={:?} tool_calls={:?}", + response.text, response.tool_calls + ); + let call = response + .tool_calls + .iter() + .find(|call| call.name == "suggest_ui_design_semantics") + .ok_or_else(|| { + eprintln!("ui_design_suggestion.error stage=parse_tool_call reason=missing_tool_call"); + "LLM 未返回 suggest_ui_design_semantics 工具调用".to_string() + })?; + let suggestions = serde_json::from_str::(&call.arguments) + .map_err(|error| { + eprintln!("ui_design_suggestion.error stage=parse_arguments error={error}"); + format!("UI 语义建议工具参数无效：{error}") + })? + .suggestions; + validate_suggestions(&suggestions, &ids).map_err(|error| { + eprintln!("ui_design_suggestion.error stage=validate_result error={error}"); + error + })?; // TODO: future single-page policy may reject multiple Page suggestions. Ok(suggestions) } diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs new file mode 100644 index 000000000..832621028 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs @@ -0,0 +1,43 @@ +use schemars::JsonSchema; + +pub(crate) fn strict_json_schema() -> Result { + let mut settings = schemars::generate::SchemaSettings::default(); + settings.inline_subschemas = true; + let generator = schemars::SchemaGenerator::new(settings); + let mut schema = serde_json::to_value(generator.into_root_schema_for::()) + .map_err(|error| format!("生成 JSON Schema 失败：{error}"))?; + normalize_strict_schema(&mut schema); + Ok(schema) +} + +fn normalize_strict_schema(schema: &mut serde_json::Value) { + match schema { + serde_json::Value::Object(object) => { + if let Some(properties) = object.get("properties").and_then(|value| value.as_object()) { + object.insert( + "required".to_string(), + serde_json::Value::Array( + properties + .keys() + .cloned() + .map(serde_json::Value::String) + .collect(), + ), + ); + object.insert( + "additionalProperties".to_string(), + serde_json::Value::Bool(false), + ); + } + for value in object.values_mut() { + normalize_strict_schema(value); + } + } + serde_json::Value::Array(values) => { + for value in values { + normalize_strict_schema(value); + } + } + _ => {} + } +} diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/recognition.ts b/apps/ai-game-creator-shell/src/features/ui-editor/recognition.ts new file mode 100644 index 000000000..024d910be --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/recognition.ts @@ -0,0 +1,13 @@ +import type { RecognitionDTO } from './types/RecognitionDTO'; +import type { State } from './types/State'; + +/** 识别结果是整棵树的替换结果，不与旧树逐节点合并。 */ +export function applyRecognitionResult( + state: State, + result: RecognitionDTO, +): State { + return { + ...structuredClone(state), + ui_trees: structuredClone(result.ui_trees), + }; +}