新增 UI 结构识别功能

- 实现 Rust 中的 `recognize_ui_impl` 函数，支持基于多图的 UI 树识别。
- 增加 TypeScript 定义的 `RecognitionDTO` 数据结构，集成到前端。
- 支持通过 Tauri 命令调用 `recognize_ui`。
- 引入 `utils` 模块，提供严格 JSON Schema 支持。
- 更新依赖添加 `schemars` 和其衍生模块，支持 JSON Schema 生成和验证。
- 扩展 Cargo 配置和锁文件适配新依赖。

apps/ai-game-creator-shell/src-tauri/Cargo.lock

@@ -1666,6 +1666,7 @@ dependencies = [
  "portable-pty",
  "reqwest 0.12.28",
  "rmcp",
+ "schemars 1.2.1",
  "serde",
  "serde_json",
  "serde_yaml",
@@ -4453,7 +4454,7 @@ checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615"
 dependencies = [
  "dyn-clone",
  "indexmap 1.9.3",
- "schemars_derive",
+ "schemars_derive 0.8.22",
  "serde",
  "serde_json",
  "url",
@@ -4480,6 +4481,7 @@ checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc"
 dependencies = [
  "dyn-clone",
  "ref-cast",
+ "schemars_derive 1.2.1",
  "serde",
  "serde_json",
 ]
@@ -4496,6 +4498,18 @@ dependencies = [
  "syn 2.0.118",
 ]
 
+[[package]]
+name = "schemars_derive"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde_derive_internals",
+ "syn 2.0.118",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"

apps/ai-game-creator-shell/src-tauri/Cargo.toml

@@ -36,6 +36,7 @@ rmcp = { version = "2.2.0", default-features = false, features = ["client", "req
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yaml = "0.9"
+schemars = "1.2.1"
 sha2 = "0.10"
 similar = "2.7"
 platform-llm = { path = "../../../server-rs/crates/platform-llm" }

apps/ai-game-creator-shell/src-tauri/src/main.rs

@@ -118,6 +118,14 @@ async fn suggest_ui_design_semantic(
     ui_editor::commands::suggest_ui_design_semantic_impl(project_path, state).await
 }
 
+#[tauri::command]
+async fn recognize_ui(
+    project_path: String,
+    state: ui_editor::state::State,
+) -> Result<ui_editor::commands::RecognitionDTO, String> {
+    ui_editor::commands::recognize_ui_impl(project_path, state).await
+}
+
 #[derive(Debug, Eq, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct InitLocalProjectResult {
@@ -2252,6 +2260,7 @@ fn main() {
             import_ui_editor_local_files,
             import_ui_editor_remote_assets,
             suggest_ui_design_semantic,
+            recognize_ui,
             generate_platform_art_asset,
             open_canvas_project,
             get_game_creation_agent_capabilities,

apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs

@@ -1,5 +1,8 @@
-pub mod ui_design_suggestion;
 pub mod recognition;
+pub mod ui_design_suggestion;
+pub mod utils;
 
+pub(crate) use recognition::recognize_ui_impl;
+pub use recognition::RecognitionDTO;
 pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl;
 pub use ui_design_suggestion::UIDesignSuggestion;

apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/ui_design_suggestion.rs

@@ -1,67 +1,117 @@
 use crate::config::build_game_creator_llm_client_from_config;
+use crate::ui_editor::commands::utils::strict_json_schema;
 use crate::ui_editor::resource::ui_design_image::UIDesignImageRole;
 use crate::ui_editor::state::State;
 use crate::ui_editor::utils::UIDesignImageId;
 use base64::Engine as _;
-use platform_llm::{LlmMessage, LlmMessageContentPart, LlmRunRequest};
+use platform_llm::{
+    LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice,
+};
+use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 use std::path::Path;
 use ts_rs::TS;
 
 const SYSTEM_PROMPT: &str = r#"
-请识别这些 UI 参考图的界面语义。你的响应必须是严格 JSON 数组；不要输出 Markdown、代码围栏、注释、额外字段或解释文字。
-
-TypeScript 等价类型如下（仅用于说明契约，实际响应仍必须是 JSON）：
-type UIDesignSuggestion = {
-  ui_design_image_id: string;
-  name: string | null;
-  description: string | null;
-  role: "Page" | "Section" | "Modal" | "Drawer" | "Popover" | "State" | "Scrolled" | "Detail" | null;
-  slave_to: string | null;
-};
+请识别这些 UI 参考图的界面语义，并调用 suggest_ui_design_semantics 工具返回结果。
+不要在工具调用外输出 JSON、Markdown、代码围栏、注释、额外字段或解释文字。
 
 字段含义和 null 规则：
 - ui_design_image_id：必填，必须逐字匹配当前输入参考图的 id。它标识“这条建议属于哪张图”，不是新 ID，不能为 null。
 - name：要写入该图片 metadata 的简短、可读名称；
 - description：要写入该图片 metadata 的简短语义描述，例如“带底部导航的主游戏页面”。
-- role：要写入该图片 metadata 的界面角色。只能使用 Page、Section、Modal、Drawer、Popover、State、Scrolled、Detail 之一；。Page 表示完整主页面，Section 表示同一主页面中的子界面或页签，Modal/Drawer/Popover 表示浮层或局部覆盖界面，State 表示同一界面的状态变体，Scrolled 表示滚动或分页后的内容，Detail 表示局部详情或补充证据。
-- slave_to：要写入该图片 metadata 的归属页面 id。只有当当前图片明显是另一张输入图的局部/子界面时才填写那个宿主图的 id；如果不应修改现有 slave_to，必须填 null。Page 不得设置宿主。不能填写自身 id，也不能填写不在输入参考图中的 id。
+- role：要写入该图片 metadata 的界面角色。
+  Page 表示完整主页面，
+  Section 表示同一主页面中的子界面或页签，
+  Modal/Drawer/Popover 表示浮层或局部覆盖界面，
+  State 表示同一界面的状态变体，
+  Scrolled 表示滚动或分页后的内容，
+  Detail 表示局部详情或补充证据。
+- slave_to：要写入该图片 metadata 的归属页面 id。只有当当前图片明显是另一张输入图的局部/子界面时才填写那个宿主图的 id；
+  Page 不得设置宿主。
 
 所有字段都必须出现；nullable 字段使用 JSON null 表示“不要修改该字段”，不要省略字段。
-
-每张参考图最多返回一条建议，且不得重复 ui_design_image_id。示例：
-[
-  {
-    "ui_design_image_id": "image-main",
-    "name": "主游戏页面",
-    "description": "带顶部资源栏和底部导航的完整游戏主页面",
-    "role": "Page",
-    "slave_to": null
-  },
-  {
-    "ui_design_image_id": "image-inventory",
-    "description": "从主页面进入的物品列表子界面",
-    "role": "Section",
-    "slave_to": "image-main"
-  },
-]
-示例中的 image-main、image-inventory、image-detail 只是占位符；必须替换为本次输入中真实存在的 id。
-
+每张参考图最多返回一条建议，且不得重复 ui_design_image_id。
 以下UI设计图中的metadata部分信息已确定, 请补全未确定/为空的信息
 "#;
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS, JsonSchema)]
+#[schemars(deny_unknown_fields)]
+#[serde(deny_unknown_fields)]
 #[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
 pub struct UIDesignSuggestion {
     pub ui_design_image_id: UIDesignImageId,
+    #[schemars(required)]
     pub name: Option<String>,
+    #[schemars(required)]
     pub description: Option<String>,
+    #[schemars(required)]
     pub role: Option<UIDesignImageRole>,
+    #[schemars(required)]
     pub slave_to: Option<UIDesignImageId>,
 }
 
+#[derive(Clone, Debug, Deserialize, JsonSchema)]
+#[schemars(deny_unknown_fields)]
+#[serde(deny_unknown_fields)]
+struct UIDesignSuggestionResponse {
+    suggestions: Vec<UIDesignSuggestion>,
+}
+
+fn ui_design_suggestion_json_schema() -> Result<serde_json::Value, String> {
+    strict_json_schema::<UIDesignSuggestionResponse>()
+}
+
 const MAX_REFERENCES: usize = 4;
 
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_no_refs(value: &serde_json::Value) {
+        match value {
+            serde_json::Value::Object(object) => {
+                assert!(!object.contains_key("$ref"));
+                assert!(!object.contains_key("$defs"));
+                for child in object.values() {
+                    assert_no_refs(child);
+                }
+            }
+            serde_json::Value::Array(values) => {
+                for child in values {
+                    assert_no_refs(child);
+                }
+            }
+            _ => {}
+        }
+    }
+
+    #[test]
+    fn strict_schema_is_inline_and_requires_all_properties() {
+        let schema = ui_design_suggestion_json_schema().expect("schema");
+        assert_no_refs(&schema);
+        assert_eq!(schema["required"], serde_json::json!(["suggestions"]));
+        let required = schema["properties"]["suggestions"]["items"]["required"]
+            .as_array()
+            .expect("item required array")
+            .iter()
+            .filter_map(serde_json::Value::as_str)
+            .collect::<std::collections::BTreeSet<_>>();
+        assert_eq!(
+            required,
+            [
+                "ui_design_image_id",
+                "name",
+                "description",
+                "role",
+                "slave_to"
+            ]
+            .into_iter()
+            .collect()
+        );
+    }
+}
+
 fn image_data_url(path: &Path, bytes: &[u8]) -> Result<String, String> {
     let mime = match path.extension().and_then(|value| value.to_str()) {
         Some("png") => "image/png",
@@ -105,9 +155,14 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
     state: State,
 ) -> Result<Vec<UIDesignSuggestion>, String> {
     if state.ui_design_images.is_empty() {
+        eprintln!("ui_design_suggestion.error stage=validate reason=no_images");
         return Err("请先导入界面图".to_string());
     }
     if state.ui_design_images.len() > MAX_REFERENCES {
+        eprintln!(
+            "ui_design_suggestion.error stage=validate reason=too_many_images count={}",
+            state.ui_design_images.len()
+        );
         return Err("界面图最多 4 张".to_string());
     }
     let root = Path::new(project_path.trim());
@@ -116,8 +171,21 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
     let mut ids = HashSet::new();
     for (id, image) in &state.ui_design_images {
         ids.insert(id.clone());
-        let absolute = crate::project::resolve_local_project_path(root, &image.path)?;
-        let bytes = std::fs::read(&absolute).map_err(|error| format!("读取界面图失败：{error}"))?;
+        let absolute =
+            crate::project::resolve_local_project_path(root, &image.path).map_err(|error| {
+                eprintln!(
+                    "ui_design_suggestion.error stage=resolve_image id={} error={error}",
+                    id.as_str()
+                );
+                error
+            })?;
+        let bytes = std::fs::read(&absolute).map_err(|error| {
+            eprintln!(
+                "ui_design_suggestion.error stage=read_image id={} error={error}",
+                id.as_str()
+            );
+            format!("读取界面图失败：{error}")
+        })?;
         parts.push(LlmMessageContentPart::InputText {
             text: format!(
                 "REFERENCE id={} metadata={} pixel_size={:?}",
@@ -130,17 +198,56 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
             image_url: image_data_url(&absolute, &bytes)?,
         });
     }
-    let client = build_game_creator_llm_client_from_config()?;
+    let client = build_game_creator_llm_client_from_config().map_err(|error| {
+        eprintln!("ui_design_suggestion.error stage=build_client error={error}");
+        error
+    })?;
+    let schema = ui_design_suggestion_json_schema().map_err(|error| {
+        eprintln!("ui_design_suggestion.error stage=build_schema error={error}");
+        error
+    })?;
+    let tool = LlmFunctionTool::new(
+        "suggest_ui_design_semantics",
+        "为所有 UI 参考图补全界面语义 metadata",
+        schema,
+    )
+    .with_strict(true);
     let response = client
-        .run(LlmRunRequest::new(vec![
-            LlmMessage::system(SYSTEM_PROMPT),
-            LlmMessage::user_multimodal(parts),
-        ]))
+        .run(
+            LlmRunRequest::new(vec![
+                LlmMessage::system(SYSTEM_PROMPT),
+                LlmMessage::user_multimodal(parts),
+            ])
+            .with_function_tools(vec![tool])
+            .with_tool_choice(LlmToolChoice::Required),
+        )
         .await
-        .map_err(|error| format!("UI 参考图语义识别失败：{error}"))?;
-    let suggestions = serde_json::from_str::<Vec<UIDesignSuggestion>>(response.text.trim())
-        .map_err(|error| format!("LLM 返回的 UI 语义 JSON 无效：{error}"))?;
-    validate_suggestions(&suggestions, &ids)?;
+        .map_err(|error| {
+            eprintln!("ui_design_suggestion.error stage=llm_request error={error}");
+            format!("UI 参考图语义识别失败：{error}")
+        })?;
+    eprintln!(
+        "ui_design_suggestion.llm_output text={:?} tool_calls={:?}",
+        response.text, response.tool_calls
+    );
+    let call = response
+        .tool_calls
+        .iter()
+        .find(|call| call.name == "suggest_ui_design_semantics")
+        .ok_or_else(|| {
+            eprintln!("ui_design_suggestion.error stage=parse_tool_call reason=missing_tool_call");
+            "LLM 未返回 suggest_ui_design_semantics 工具调用".to_string()
+        })?;
+    let suggestions = serde_json::from_str::<UIDesignSuggestionResponse>(&call.arguments)
+        .map_err(|error| {
+            eprintln!("ui_design_suggestion.error stage=parse_arguments error={error}");
+            format!("UI 语义建议工具参数无效：{error}")
+        })?
+        .suggestions;
+    validate_suggestions(&suggestions, &ids).map_err(|error| {
+        eprintln!("ui_design_suggestion.error stage=validate_result error={error}");
+        error
+    })?;
     // TODO: future single-page policy may reject multiple Page suggestions.
     Ok(suggestions)
 }

apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs

@@ -0,0 +1,43 @@
+use schemars::JsonSchema;
+
+pub(crate) fn strict_json_schema<T: JsonSchema>() -> Result<serde_json::Value, String> {
+    let mut settings = schemars::generate::SchemaSettings::default();
+    settings.inline_subschemas = true;
+    let generator = schemars::SchemaGenerator::new(settings);
+    let mut schema = serde_json::to_value(generator.into_root_schema_for::<T>())
+        .map_err(|error| format!("生成 JSON Schema 失败：{error}"))?;
+    normalize_strict_schema(&mut schema);
+    Ok(schema)
+}
+
+fn normalize_strict_schema(schema: &mut serde_json::Value) {
+    match schema {
+        serde_json::Value::Object(object) => {
+            if let Some(properties) = object.get("properties").and_then(|value| value.as_object()) {
+                object.insert(
+                    "required".to_string(),
+                    serde_json::Value::Array(
+                        properties
+                            .keys()
+                            .cloned()
+                            .map(serde_json::Value::String)
+                            .collect(),
+                    ),
+                );
+                object.insert(
+                    "additionalProperties".to_string(),
+                    serde_json::Value::Bool(false),
+                );
+            }
+            for value in object.values_mut() {
+                normalize_strict_schema(value);
+            }
+        }
+        serde_json::Value::Array(values) => {
+            for value in values {
+                normalize_strict_schema(value);
+            }
+        }
+        _ => {}
+    }
+}

apps/ai-game-creator-shell/src/features/ui-editor/recognition.ts

@@ -0,0 +1,13 @@
+import type { RecognitionDTO } from './types/RecognitionDTO';
+import type { State } from './types/State';
+
+/** 识别结果是整棵树的替换结果，不与旧树逐节点合并。 */
+export function applyRecognitionResult(
+  state: State,
+  result: RecognitionDTO,
+): State {
+  return {
+    ...structuredClone(state),
+    ui_trees: structuredClone(result.ui_trees),
+  };
+}