diff --git a/apps/ai-game-creator-shell/scripts/check-config.mjs b/apps/ai-game-creator-shell/scripts/check-config.mjs index 0149d953a..f2ef7b4b8 100644 --- a/apps/ai-game-creator-shell/scripts/check-config.mjs +++ b/apps/ai-game-creator-shell/scripts/check-config.mjs @@ -108,6 +108,8 @@ const rustSharedContractSource = fs.readFileSync( ); const allowedUncalledTauriCommands = [ 'append_direct_project_conversation_message', + // TODO: Remove the retired binding command after the legacy runtime path is removed. + 'bind_components', 'chat_with_game_creator_agent', 'check_ui_editor_font_glyph_coverage', 'create_ui_design_resource', diff --git a/apps/ai-game-creator-shell/src-tauri/src/agent/prompt.rs b/apps/ai-game-creator-shell/src-tauri/src/agent/prompt.rs index 460d8dfad..94abf882b 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/agent/prompt.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/agent/prompt.rs @@ -1832,7 +1832,7 @@ mod tests { assert!(with_canvas.contains("由 Runtime 在收束门内同时核对固定 owner 文档")); assert!(!with_canvas.contains("成功动作本身就是当前 revision 的验证")); assert!(with_canvas.contains("调用 ui.workflow.run")); - assert!(with_canvas.contains("visual-binding 最终编辑器路由")); + assert!(with_canvas.contains("asset-separation 最终编辑器路由")); assert!(with_canvas.contains("每个功能页面各写一行 @genarrative-ui-page")); assert!(with_canvas.contains("ui.workflow.run 的 discover")); assert!(with_canvas.contains("assets/ui-pages/{pageId}.png")); diff --git a/apps/ai-game-creator-shell/src-tauri/src/agent/runtime_actions/action_audit.rs b/apps/ai-game-creator-shell/src-tauri/src/agent/runtime_actions/action_audit.rs index 235177a31..52af400ac 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/agent/runtime_actions/action_audit.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/agent/runtime_actions/action_audit.rs @@ -801,7 +801,7 @@ fn agent_runtime_action_receipt_safe_detail_with_owner( let initial_step = route.get("initialStep")?.as_str()?; let render_mode = route.get("renderMode")?.as_str()?; if resource_id.is_empty() - || initial_step != "visual-binding" + || initial_step != "asset-separation" || render_mode != "final-preview" { return None; diff --git a/apps/ai-game-creator-shell/src-tauri/src/main.rs b/apps/ai-game-creator-shell/src-tauri/src/main.rs index e163cad21..1f51a98ae 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/main.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/main.rs @@ -328,6 +328,39 @@ async fn recognize_ui( ui_editor::commands::recognize_ui_impl(project_path, state).await } +#[tauri::command] +async fn separate_ui( + project_path: String, + asset_id: String, + state: ui_editor::state::State, +) -> Result { + ui_editor::commands::separate_ui_impl(project_path, asset_id, state).await +} + +#[tauri::command] +fn inspect_separation_recovery( + project_path: String, + asset_id: String, +) -> Result { + let root = Path::new(project_path.trim()); + enforce_project_permission_policy(root, "asset.list")?; + ui_editor::commands::separation::inspect_separation_recovery(root, &asset_id) +} + +#[tauri::command] +fn finalize_separation(project_path: String, asset_id: String) -> Result<(), String> { + let root = Path::new(project_path.trim()); + enforce_project_permission_policy(root, "asset.register")?; + ui_editor::commands::separation::finalize_separation(root, &asset_id) +} + +#[tauri::command] +fn discard_separation_recovery(project_path: String, asset_id: String) -> Result<(), String> { + let root = Path::new(project_path.trim()); + enforce_project_permission_policy(root, "asset.register")?; + ui_editor::commands::separation::finalize_separation(root, &asset_id) +} + #[tauri::command] async fn merge_ui(state: ui_editor::state::State) -> Result { ui_editor::commands::merge_ui_impl(state).await @@ -2559,6 +2592,10 @@ fn main() { check_ui_editor_font_glyph_coverage, suggest_ui_design_semantic, recognize_ui, + separate_ui, + inspect_separation_recovery, + finalize_separation, + discard_separation_recovery, merge_ui, bind_components, load_ui_design_state, diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/binding.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/binding.rs index 4585eee85..17979cbca 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/binding.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/binding.rs @@ -5,11 +5,9 @@ use crate::ui_editor::commands::utils::{ strict_json_schema, }; use crate::ui_editor::component::text::FontSource; -use crate::ui_editor::component::Component; +use crate::ui_editor::component::{Component, NodeComponent}; use crate::ui_editor::layout::node::{Node, StageStatus}; -use crate::ui_editor::persistence::{ - UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE, UI_DESIGN_STATE_MAX_NODES, -}; +use crate::ui_editor::persistence::UI_DESIGN_STATE_MAX_NODES; use crate::ui_editor::state::State; use crate::ui_editor::utils::{FontAssetId, NodeId, SpriteAssetId}; use platform_llm::{ @@ -31,10 +29,10 @@ const SYSTEM_PROMPT: &str = r#" 你是游戏 UI 组件绑定器。你会看到全部 UI 参考图、可编辑节点说明，以及本批独立素材的真实像素。 * 只对视觉上确实需要改变组件的节点返回 changes； -* 每个 change 的 components 是该节点完整的新渲染栈，空数组表示明确清空。数组顺序从底到顶渲染。 -* 对每个 Component，直接完整返回其全部参数. +* 每个 change 的 component 是该节点完整的新组件；纯结构节点返回 "PureNode"，有组件返回 {"WithComponent": <完整 Component>}。 +* 对 Component，直接完整返回其全部参数. * 有任何困难或者不确定把状态设为 NeedReview，说明中文原因。 -* 纯结构节点可以返回空数组并标为 NoProblem。 +* 纯结构节点可以返回 "PureNode" 并标为 NoProblem。 * 容器背景等推荐使用Simple + preserve_aspect: false 实现与node大小一致 * 面向用户的 reason 使用中文。 @@ -53,8 +51,8 @@ enum DraftStatus { #[schemars(deny_unknown_fields)] struct BindingChangeDraft { node_id: NodeId, - components: Vec, - components_status: DraftStatus, + component: NodeComponent, + component_status: DraftStatus, } #[derive(Clone, Debug, Deserialize, JsonSchema)] @@ -68,8 +66,8 @@ struct BindingResponse { #[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] pub struct BindingChange { pub node_id: NodeId, - pub components: Vec, - pub components_status: StageStatus, + pub component: NodeComponent, + pub component_status: StageStatus, } #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] @@ -83,7 +81,7 @@ struct EditableNodeContext<'a> { node_id: &'a NodeId, name: &'a str, description: &'a str, - components: &'a [Component], + component: Option<&'a Component>, } #[derive(Debug, Serialize)] @@ -106,7 +104,7 @@ fn collect_editable_nodes<'a>(node: &'a Node, output: &mut Vec UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE { - return Err(format!( - "单个组件绑定栈不能超过 {UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE} 个组件" - )); + if !change + .as_object() + .is_some_and(|object| object.contains_key("component")) + { + return Err("组件绑定 change 缺少 component 字段".to_string()); } } Ok(()) @@ -212,7 +207,7 @@ fn validate_and_materialize( if !changed_ids.insert(change.node_id.clone()) { return Err(format!("组件绑定重复返回节点：{}", change.node_id.as_str())); } - for component in &change.components { + if let NodeComponent::WithComponent(component) = &change.component { match component { Component::Image(image) => { if image @@ -232,18 +227,22 @@ fn validate_and_materialize( } } } - let components = change.components; - let components_status = match change.components_status { + let component_status = match change.component_status { DraftStatus::NoProblem => StageStatus::NoProblem, DraftStatus::NeedReview(reason) if reason.trim().is_empty() => { return Err("组件待审状态必须包含原因".to_string()) } - DraftStatus::NeedReview(reason) => StageStatus::NeedReview(reason), + DraftStatus::NeedReview(reason) => { + if matches!(&change.component, NodeComponent::PureNode) { + return Err("纯结构节点不能标记为组件待审".to_string()); + } + StageStatus::NeedReview(reason) + } }; materialized.push(BindingChange { node_id: change.node_id, - components, - components_status, + component: change.component, + component_status, }); } Ok(BindingDTO { @@ -440,8 +439,8 @@ mod tests { ]); let unapproved = BindingChangeDraft { node_id: id("other"), - components: Vec::new(), - components_status: DraftStatus::NoProblem, + component: NodeComponent::PureNode, + component_status: DraftStatus::NoProblem, }; assert!( validate_and_materialize(vec![unapproved], &editable, &known, &HashSet::new()).is_err() @@ -450,7 +449,7 @@ mod tests { // References to sprites from another batch are allowed once they exist in the project. let other_batch = BindingChangeDraft { node_id: id("editable"), - components: vec![Component::Image( + component: NodeComponent::WithComponent(Component::Image( crate::ui_editor::component::image::ImageComponent { target_graphic: Some( SpriteAssetId::new("other-batch-sprite").expect("valid sprite"), @@ -459,8 +458,8 @@ mod tests { preserve_aspect: false, }, }, - )], - components_status: DraftStatus::NoProblem, + )), + component_status: DraftStatus::NoProblem, }; assert!( validate_and_materialize(vec![other_batch], &editable, &known, &HashSet::new()).is_ok() @@ -469,15 +468,15 @@ mod tests { // References to sprites that do not exist in the project at all are still rejected. let unknown = BindingChangeDraft { node_id: id("editable"), - components: vec![Component::Image( + component: NodeComponent::WithComponent(Component::Image( crate::ui_editor::component::image::ImageComponent { target_graphic: Some(SpriteAssetId::new("unknown").expect("valid sprite")), image_type: crate::ui_editor::component::image::ImageType::Simple { preserve_aspect: false, }, }, - )], - components_status: DraftStatus::NoProblem, + )), + component_status: DraftStatus::NoProblem, }; assert!( validate_and_materialize(vec![unknown], &editable, &known, &HashSet::new()).is_err() @@ -491,8 +490,8 @@ mod tests { text.font = FontSource::Bound(FontAssetId::new("unknown-font").expect("valid font")); let change = BindingChangeDraft { node_id: id("editable"), - components: vec![Component::Text(text)], - components_status: DraftStatus::NoProblem, + component: NodeComponent::WithComponent(Component::Text(text)), + component_status: DraftStatus::NoProblem, }; let error = validate_and_materialize( @@ -506,13 +505,13 @@ mod tests { } #[test] - fn materialization_preserves_changed_only_empty_component_lists() { + fn materialization_preserves_pure_node_change() { let editable = HashSet::from([id("editable")]); let result = validate_and_materialize( vec![BindingChangeDraft { node_id: id("editable"), - components: Vec::new(), - components_status: DraftStatus::NoProblem, + component: NodeComponent::PureNode, + component_status: DraftStatus::NoProblem, }], &editable, &HashSet::new(), @@ -520,8 +519,28 @@ mod tests { ) .expect("valid changed-only clear"); assert_eq!(result.changes.len(), 1); - assert!(result.changes[0].components.is_empty()); - assert_eq!(result.changes[0].components_status, StageStatus::NoProblem); + assert!(matches!( + result.changes[0].component, + NodeComponent::PureNode + )); + assert_eq!(result.changes[0].component_status, StageStatus::NoProblem); + } + + #[test] + fn materialization_rejects_problematic_pure_node() { + let editable = HashSet::from([id("editable")]); + let error = validate_and_materialize( + vec![BindingChangeDraft { + node_id: id("editable"), + component: NodeComponent::PureNode, + component_status: DraftStatus::NeedReview("缺少可确认的组件".to_string()), + }], + &editable, + &HashSet::new(), + &HashSet::new(), + ) + .expect_err("pure node cannot carry a component review status"); + assert!(error.contains("纯结构节点")); } #[test] @@ -575,20 +594,26 @@ mod tests { } #[test] - fn binding_response_bounds_changes_and_each_component_stack() { + fn binding_response_bounds_changes_and_uses_single_component_shape() { let too_many_changes = serde_json::json!({ - "changes": [{"components": []}, {"components": []}] + "changes": [{"component": "PureNode"}, {"component": "PureNode"}] }); assert!(validate_binding_response_shape(&too_many_changes, 1).is_err()); - let too_many_components = serde_json::json!({ + let one_component = serde_json::json!({ "changes": [{ - "components": (0..=UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE) - .map(|_| serde_json::Value::Null) - .collect::>() + "node_id": "editable", + "component": "PureNode", + "component_status": "NoProblem" }] }); - assert!(validate_binding_response_shape(&too_many_components, 1).is_err()); + assert!(validate_binding_response_shape(&one_component, 1).is_ok()); + let parsed = parse_binding_response(&one_component.to_string(), 1) + .expect("explicit PureNode payload should parse"); + assert!(matches!( + parsed.changes[0].component, + NodeComponent::PureNode + )); } #[tokio::test] diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/merge.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/merge.rs index e8a789948..9fe0ad71f 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/merge.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/merge.rs @@ -295,12 +295,12 @@ mod materialize { name: container_name, description: container_description, layout_status: StageStatus::NoProblem, - components_status: StageStatus::NoProblem, + component_status: StageStatus::NoProblem, allow_llm_edit_layout: true, allow_llm_edit_component: true, source: NodeSource::Llm, }, - components: Vec::new(), + component: None, children_display_mode: ChildrenDisplayMode::Exclusive, children: members.into_iter().map(|member| member.node).collect(), }, @@ -542,12 +542,12 @@ mod tests { name: id.to_string(), description: String::new(), layout_status: StageStatus::NoProblem, - components_status: StageStatus::NoProblem, + component_status: StageStatus::NoProblem, allow_llm_edit_layout: true, allow_llm_edit_component: true, source: NodeSource::Human, }, - components: Vec::::new(), + component: None, children_display_mode: ChildrenDisplayMode::Stack, children, } @@ -586,7 +586,7 @@ mod tests { ChildrenDisplayMode::Exclusive ); assert_eq!( - result.root.metadata.components_status, + result.root.metadata.component_status, StageStatus::NoProblem ); assert_eq!(result.root.children.len(), 2); diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs index e2e1a1bc5..d85432e07 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs @@ -1,6 +1,7 @@ pub mod binding; pub mod merge; pub mod recognition; +pub mod separation; pub mod ui_design_suggestion; pub mod utils; @@ -10,5 +11,7 @@ pub use merge::MergeDTO; pub(crate) use merge::{merge_ui_impl, merge_ui_impl_with_provider}; pub use recognition::RecognitionDTO; pub(crate) use recognition::{recognize_ui_impl, recognize_ui_impl_with_provider}; +pub(crate) use separation::separate_ui_impl; +pub use separation::{SeparationDTO, SeparationRecoveryDTO}; pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl; pub use ui_design_suggestion::UIDesignSuggestionTreeNode; diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs index 081db0bb1..15e71f2aa 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs @@ -4,6 +4,7 @@ use crate::ui_editor::commands::utils::{ parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm, strict_json_schema, }; +use crate::ui_editor::component::{Component, NodeComponent}; use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode; use crate::ui_editor::layout::control_layout::ControlLayout; use crate::ui_editor::layout::dimension::UIRect; @@ -27,16 +28,13 @@ const MAX_RECOGNITION_TREE_NODES: usize = 512; const MAX_RECOGNITION_TREE_DEPTH: usize = 32; const SYSTEM_PROMPT: &str = r#" -角色： -你是游戏 UI 多图结构识别器。 - 任务： 同时分析同一 UI 系统的全部参考图，建立UI树 用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构 识别规则： -* 只识别 UI，不识别场景人物、地形、建筑、光影和背景装饰。 +* 只识别 UI元素. 要区分动态内容, 不要白费力气识别应该由程序生成/绘制的内容.(此类内容应该用一个整体节点+自然语言描述) 除此之外必须完整包含所有元素,结构. * 无法确定类型、层级、关系时，在 UnSure 中写明原因。 * 返回的 trees 必须与输入图片一一对应，每张输入图片只能有一棵树，不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id。 * 每棵树必须使用自己的输入图片原始像素坐标系(0,0 as left top)输出 @@ -45,8 +43,15 @@ const SYSTEM_PROMPT: &str = r#" * 面向用户的字段如名称描述等请用中文 * 由于每个截图未必是完整的, 可能是局部的, 每棵树描述清楚每个截图上UI的层次结构即可 * 不同树的共用框架/层次/...请使用使用相同的名称描述. 不同状态/变体名称使用相同的前缀, 用后缀区别 -* 粒度要求: 尽可能细致, 最小单元举例: 进度条的底槽、填充和外框; slider的底槽, dragger等 - +* 粒度要求: 尽可能细致, 以可交互,方便程序化控制的最小单位为准. 包括不限于: icon, 进度条的底槽、填充和外框; slider的底槽, dragger等. +* 为每个节点直接返回 component. + 无背景的逻辑容器返回 "PureNode"，不要返回 null. + 有背景的容器推荐使用Simple+不锁定宽高比的Image component. + 目前我们只做识别, 不要求图片字体参数. + 每个节点最多返回一个 component；需要多个视觉层时拆成多个节点。 + 文字组件要求: 艺术字等作为图片组件, 其余正常文字要作为单独的节点识别. +* 不鼓励兄弟节点相互重叠. +* 对于面板等容器的背景等, 必须作为父节点的组件, 禁止新增冗余的所谓"背景节点". "#; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] @@ -109,6 +114,7 @@ struct RecognitionNode { description: String, children: Vec, confidence: Confidence, + component: NodeComponent, } #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] @@ -307,15 +313,12 @@ fn convert_node( name: source.name.clone(), description: source.description.clone(), layout_status: status, - components_status: StageStatus::NoProblem, + component_status: StageStatus::NoProblem, allow_llm_edit_layout: true, allow_llm_edit_component: true, source: NodeSource::Llm, }, - // V1 有意把识别结果限定为“结构草稿”：组件绑定属于后续独立阶段。 - // 因此空组件不是丢失数据，而是等待 visual-binding 阶段补齐 Image/Text。 - // 约定见 docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md。 - components: Vec::new(), + component: source.component.clone().into_option(), children_display_mode: ChildrenDisplayMode::Stack, children, }) @@ -323,6 +326,26 @@ fn convert_node( fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> { for node in nodes { + if let NodeComponent::WithComponent(component) = &node.component { + if matches!( + component, + Component::Image(crate::ui_editor::component::image::ImageComponent { + target_graphic: Some(_), + .. + }) + ) { + return Err("识别阶段不能返回已绑定的 SpriteAssetId".to_string()); + } + if matches!( + component, + Component::Text(crate::ui_editor::component::text::TextComponent { + font: crate::ui_editor::component::text::FontSource::Bound(_), + .. + }) + ) { + return Err("识别阶段不能返回已绑定的字体素材".to_string()); + } + } if let Confidence::UnSure(reason) = &node.confidence { if reason.trim().is_empty() { return Err("UnSure 必须包含审阅原因".to_string()); @@ -387,6 +410,7 @@ mod tests { description: String::new(), children: Vec::new(), confidence: Confidence::Confident, + component: NodeComponent::PureNode, } } @@ -490,6 +514,31 @@ mod tests { description: String::new(), children: Vec::new(), confidence: Confidence::UnSure(String::new()), + component: NodeComponent::PureNode, + }; + assert!(validate_confidence(&[node]).is_err()); + } + + #[test] + fn recognition_rejects_bound_font_references() { + let mut text = crate::ui_editor::component::text::TextComponent::default(); + text.font = crate::ui_editor::component::text::FontSource::Bound( + crate::ui_editor::utils::FontAssetId::new("font").expect("valid font id"), + ); + let node = RecognitionNode { + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + local_anchor: Anchor::Preset(PresetAnchor { + horizontal: HorizontalAnchor::Left, + vertical: VerticalAnchor::Top, + }), + name: "文本".to_string(), + description: String::new(), + children: Vec::new(), + confidence: Confidence::Confident, + component: NodeComponent::WithComponent(Component::Text(text)), }; assert!(validate_confidence(&[node]).is_err()); } @@ -506,7 +555,7 @@ mod tests { converted.layout.transform.resolve(&root_rect), UIRect::new(Point2::new(50.0, 25.0), Vector2::new(100.0, 50.0)), ); - assert_eq!(converted.metadata.components_status, StageStatus::NoProblem); + assert_eq!(converted.metadata.component_status, StageStatus::NoProblem); } #[test] @@ -787,12 +836,12 @@ pub(crate) async fn recognize_ui_impl_with_provider( name: "页面根节点".to_string(), description: String::new(), layout_status: StageStatus::NoProblem, - components_status: StageStatus::NoProblem, + component_status: StageStatus::NoProblem, allow_llm_edit_layout: true, allow_llm_edit_component: true, source: NodeSource::System, }, - components: Vec::new(), + component: None, children_display_mode: ChildrenDisplayMode::Stack, children, }; diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/area.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/area.rs new file mode 100644 index 000000000..739492b6c --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/area.rs @@ -0,0 +1,433 @@ +use super::model::BindingArea; +use image::RgbaImage; +use std::time::Instant; + +/// Each edge may move by at most this percentage of the corresponding area +/// dimension returned by the visual model. Keep this policy explicit so +/// changing it is an intentional workflow decision rather than a scattered +/// numeric literal. +pub(crate) const MAX_BINDING_AREA_EDGE_ADJUSTMENT_PERCENT: u32 = 100; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct NormalizedBindingArea { + pub(crate) area: BindingArea, + pub(crate) changed: bool, + pub(crate) clamped: bool, + pub(crate) transparent: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum EdgeDirection { + Inward, + Outward, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Rect { + left: u32, + top: u32, + right: u32, + bottom: u32, +} + +impl Rect { + fn from_area(area: BindingArea) -> Self { + Self { + left: area.global_pos_x_px, + top: area.global_pos_y_px, + right: area.global_pos_x_px + area.width_px, + bottom: area.global_pos_y_px + area.height_px, + } + } + + fn into_area(self) -> BindingArea { + BindingArea { + global_pos_x_px: self.left, + global_pos_y_px: self.top, + width_px: self.right - self.left, + height_px: self.bottom - self.top, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Edge { + Left, + Right, + Top, + Bottom, +} + +impl Edge { + const ALL: [Self; 4] = [Self::Left, Self::Right, Self::Top, Self::Bottom]; +} + +fn edge_has_visible_pixel(image: &RgbaImage, rect: Rect, edge: Edge) -> bool { + match edge { + Edge::Left | Edge::Right => { + let x = if edge == Edge::Left { + rect.left + } else { + rect.right - 1 + }; + (rect.top..rect.bottom).any(|y| image.get_pixel(x, y).0[3] > 0) + } + Edge::Top | Edge::Bottom => { + let y = if edge == Edge::Top { + rect.top + } else { + rect.bottom - 1 + }; + (rect.left..rect.right).any(|x| image.get_pixel(x, y).0[3] > 0) + } + } +} + +fn rect_has_visible_pixel(image: &RgbaImage, rect: Rect) -> bool { + (rect.top..rect.bottom).any(|y| (rect.left..rect.right).any(|x| image.get_pixel(x, y).0[3] > 0)) +} + +fn max_edge_adjustment(dimension: u32) -> u32 { + ((u64::from(dimension) * u64::from(MAX_BINDING_AREA_EDGE_ADJUSTMENT_PERCENT)) / 100) + .min(u64::from(u32::MAX)) as u32 +} + +fn edge_direction(image: &RgbaImage, rect: Rect, edge: Edge) -> EdgeDirection { + if edge_has_visible_pixel(image, rect, edge) { + EdgeDirection::Outward + } else { + EdgeDirection::Inward + } +} + +fn move_edge(rect: &mut Rect, edge: Edge, direction: EdgeDirection) { + match (edge, direction) { + (Edge::Left, EdgeDirection::Inward) => rect.left += 1, + (Edge::Left, EdgeDirection::Outward) => rect.left -= 1, + (Edge::Right, EdgeDirection::Inward) => rect.right -= 1, + (Edge::Right, EdgeDirection::Outward) => rect.right += 1, + (Edge::Top, EdgeDirection::Inward) => rect.top += 1, + (Edge::Top, EdgeDirection::Outward) => rect.top -= 1, + (Edge::Bottom, EdgeDirection::Inward) => rect.bottom -= 1, + (Edge::Bottom, EdgeDirection::Outward) => rect.bottom += 1, + } +} + +fn edge_coordinate(rect: Rect, edge: Edge) -> u32 { + match edge { + Edge::Left => rect.left, + Edge::Right => rect.right, + Edge::Top => rect.top, + Edge::Bottom => rect.bottom, + } +} + +fn edge_displacement(original: Rect, current: Rect, edge: Edge) -> u32 { + edge_coordinate(original, edge).abs_diff(edge_coordinate(current, edge)) +} + +fn edge_adjustment_limit(original: Rect, edge: Edge) -> u32 { + max_edge_adjustment(match edge { + Edge::Left | Edge::Right => original.right - original.left, + Edge::Top | Edge::Bottom => original.bottom - original.top, + }) +} + +fn reached_adjustment_limit(original: Rect, current: Rect, edge: Edge) -> bool { + edge_displacement(original, current, edge) >= edge_adjustment_limit(original, edge) +} + +fn can_move_geometrically( + image: &RgbaImage, + current: Rect, + edge: Edge, + direction: EdgeDirection, +) -> bool { + match (edge, direction) { + (Edge::Left, EdgeDirection::Inward) => current.left + 1 < current.right, + (Edge::Left, EdgeDirection::Outward) => current.left > 0, + (Edge::Right, EdgeDirection::Inward) => current.right > current.left + 1, + (Edge::Right, EdgeDirection::Outward) => current.right < image.width(), + (Edge::Top, EdgeDirection::Inward) => current.top + 1 < current.bottom, + (Edge::Top, EdgeDirection::Outward) => current.top > 0, + (Edge::Bottom, EdgeDirection::Inward) => current.bottom > current.top + 1, + (Edge::Bottom, EdgeDirection::Outward) => current.bottom < image.height(), + } +} + +fn next_edge_rect(rect: Rect, edge: Edge, direction: EdgeDirection) -> Option { + let mut next = rect; + match (edge, direction) { + (Edge::Left, EdgeDirection::Inward) if rect.left + 1 < rect.right => next.left += 1, + (Edge::Left, EdgeDirection::Outward) if rect.left > 0 => next.left -= 1, + (Edge::Right, EdgeDirection::Inward) if rect.right > rect.left + 1 => next.right -= 1, + (Edge::Right, EdgeDirection::Outward) => next.right = next.right.checked_add(1)?, + (Edge::Top, EdgeDirection::Inward) if rect.top + 1 < rect.bottom => next.top += 1, + (Edge::Top, EdgeDirection::Outward) if rect.top > 0 => next.top -= 1, + (Edge::Bottom, EdgeDirection::Inward) if rect.bottom > rect.top + 1 => next.bottom -= 1, + (Edge::Bottom, EdgeDirection::Outward) => next.bottom = next.bottom.checked_add(1)?, + _ => return None, + } + Some(next) +} + +fn edge_requires_move(image: &RgbaImage, rect: Rect, edge: Edge, direction: EdgeDirection) -> bool { + match direction { + EdgeDirection::Inward => !edge_has_visible_pixel(image, rect, edge), + EdgeDirection::Outward => { + if !edge_has_visible_pixel(image, rect, edge) { + return false; + } + if !can_move_geometrically(image, rect, edge, direction) { + return true; + } + next_edge_rect(rect, edge, direction) + .is_some_and(|next| edge_has_visible_pixel(image, next, edge)) + } + } +} + +fn apply_edge_step( + image: &RgbaImage, + original: Rect, + current: Rect, + edge: Edge, + direction: EdgeDirection, +) -> (Rect, bool, bool) { + if !edge_requires_move(image, current, edge, direction) { + return (current, false, false); + } + if reached_adjustment_limit(original, current, edge) + || !can_move_geometrically(image, current, edge, direction) + { + return (current, false, true); + } + let mut next = current; + move_edge(&mut next, edge, direction); + (next, true, false) +} + +/// Normalizes a model-provided area using visible pixels on the processed +/// transparent image. Each edge chooses inward/outward direction once from +/// its initial scan and then moves monotonically, so sparse pixels cannot make +/// the boundary oscillate. The four edge steps are calculated from the same +/// rectangle on each round. +pub(crate) fn normalize_binding_area( + image: &RgbaImage, + original_area: BindingArea, +) -> Result { + let started = Instant::now(); + if let Err(error) = original_area.validate_in(image.width(), image.height()) { + app_log!( + "ui_separation.area.timing outcome=error elapsed_us={} rounds=0 image_width={} image_height={} area=({}, {}, {}, {})", + started.elapsed().as_micros(), + image.width(), + image.height(), + original_area.global_pos_x_px, + original_area.global_pos_y_px, + original_area.width_px, + original_area.height_px + ); + return Err(error); + } + let original = Rect::from_area(original_area); + let directions = Edge::ALL.map(|edge| edge_direction(image, original, edge)); + let mut current = original; + let mut clamped = false; + let mut active = [true; 4]; + let mut rounds = 0u32; + + // TODO: Replace the deliberately simple pixel-by-pixel scan if real UI + // design sizes show this path to be a measurable bottleneck. + while active.iter().any(|value| *value) { + rounds = rounds.saturating_add(1); + let before = current; + let mut next = current; + let mut moved = [false; 4]; + for (index, edge) in Edge::ALL.into_iter().enumerate() { + if !active[index] { + continue; + } + let (candidate, did_move, reached_limit) = + apply_edge_step(image, original, current, edge, directions[index]); + if reached_limit { + clamped = true; + active[index] = false; + } else if !did_move { + active[index] = false; + } + moved[index] = did_move; + match edge { + Edge::Left => next.left = candidate.left, + Edge::Right => next.right = candidate.right, + Edge::Top => next.top = candidate.top, + Edge::Bottom => next.bottom = candidate.bottom, + } + } + if next.left >= next.right { + clamped = true; + if moved[0] { + active[0] = false; + } + if moved[1] { + active[1] = false; + } + next.left = current.left; + next.right = current.right; + } + if next.top >= next.bottom { + clamped = true; + if moved[2] { + active[2] = false; + } + if moved[3] { + active[3] = false; + } + next.top = current.top; + next.bottom = current.bottom; + } + current = next; + if current == before { + break; + } + } + + let area = current.into_area(); + let result = Ok(NormalizedBindingArea { + changed: area != original_area, + area, + clamped, + transparent: !rect_has_visible_pixel(image, current), + }); + app_log!( + "ui_separation.area.timing outcome=ok elapsed_us={} rounds={} image_width={} image_height={} area=({}, {}, {}, {}) changed={} clamped={} transparent={}", + started.elapsed().as_micros(), + rounds, + image.width(), + image.height(), + original_area.global_pos_x_px, + original_area.global_pos_y_px, + original_area.width_px, + original_area.height_px, + result.as_ref().expect("normalization result exists").changed, + result.as_ref().expect("normalization result exists").clamped, + result.as_ref().expect("normalization result exists").transparent + ); + result +} + +#[cfg(test)] +mod tests { + use super::*; + use image::{Rgba, RgbaImage}; + + fn image_with_rect( + width: u32, + height: u32, + left: u32, + top: u32, + right: u32, + bottom: u32, + ) -> RgbaImage { + let mut image = RgbaImage::from_pixel(width, height, Rgba([0, 0, 0, 0])); + for y in top..bottom { + for x in left..right { + image.put_pixel(x, y, Rgba([255, 255, 255, 255])); + } + } + image + } + + fn area(x: u32, y: u32, width: u32, height: u32) -> BindingArea { + BindingArea { + global_pos_x_px: x, + global_pos_y_px: y, + width_px: width, + height_px: height, + } + } + + #[test] + fn shrinks_empty_edges_to_visible_bounds() { + let image = image_with_rect(32, 32, 10, 11, 16, 18); + let result = normalize_binding_area(&image, area(6, 7, 14, 16)).unwrap(); + assert_eq!(result.area, area(10, 11, 6, 7)); + assert!(result.changed); + assert!(!result.clamped); + assert!(!result.transparent); + } + + #[test] + fn expands_visible_edges_to_cover_the_element() { + let image = image_with_rect(32, 32, 10, 11, 16, 18); + let result = normalize_binding_area(&image, area(11, 12, 4, 5)).unwrap(); + assert_eq!(result.area, area(10, 11, 6, 7)); + assert!(result.changed); + assert!(!result.clamped); + assert!(!result.transparent); + } + + #[test] + fn adjusts_each_edge_independently() { + let image = image_with_rect(32, 32, 10, 11, 16, 18); + let result = normalize_binding_area(&image, area(10, 12, 10, 3)).unwrap(); + assert_eq!(result.area, area(10, 11, 6, 7)); + } + + #[test] + fn keeps_nonzero_alpha_antialias_pixels() { + let mut image = RgbaImage::from_pixel(16, 16, Rgba([0, 0, 0, 0])); + image.put_pixel(5, 6, Rgba([255, 255, 255, 1])); + image.put_pixel(7, 8, Rgba([255, 255, 255, 255])); + let result = normalize_binding_area(&image, area(4, 5, 5, 5)).unwrap(); + assert_eq!(result.area, area(5, 6, 3, 3)); + } + + #[test] + fn fully_transparent_image_uses_the_same_path() { + let image = RgbaImage::from_pixel(32, 32, Rgba([0, 0, 0, 0])); + let result = normalize_binding_area(&image, area(10, 10, 10, 10)).unwrap(); + assert_eq!(result.area, area(14, 14, 2, 2)); + assert!(result.changed); + assert!(result.transparent); + } + + #[test] + fn caps_each_edge_at_original_dimension() { + let image = image_with_rect(64, 64, 0, 0, 64, 64); + let result = normalize_binding_area(&image, area(16, 16, 8, 8)).unwrap(); + assert_eq!(result.area, area(8, 8, 24, 24)); + assert!(result.clamped); + } + + #[test] + fn clamps_expansion_to_image_edges() { + let image = image_with_rect(16, 16, 0, 0, 4, 4); + let result = normalize_binding_area(&image, area(1, 1, 2, 2)).unwrap(); + assert_eq!(result.area, area(0, 0, 4, 4)); + assert!(result.clamped); + } + + #[test] + fn exact_split_at_adjustment_limit_is_not_clamped() { + let image = image_with_rect(16, 16, 4, 4, 8, 8); + let result = normalize_binding_area(&image, area(5, 5, 2, 2)).unwrap(); + assert_eq!(result.area, area(4, 4, 4, 4)); + assert!(!result.clamped); + } + + #[test] + fn one_pixel_area_expands_with_configured_adjustment_limit() { + let image = image_with_rect(8, 8, 2, 2, 5, 5); + let result = normalize_binding_area(&image, area(3, 3, 1, 1)).unwrap(); + assert_eq!(result.area, area(2, 2, 3, 3)); + assert!(!result.clamped); + } + + #[test] + fn rejects_zero_sized_or_out_of_bounds_model_areas() { + let image = RgbaImage::from_pixel(16, 16, Rgba([0, 0, 0, 0])); + assert!(normalize_binding_area(&image, area(0, 0, 0, 1)).is_err()); + assert!(normalize_binding_area(&image, area(15, 15, 2, 2)).is_err()); + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/marker.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/marker.rs new file mode 100644 index 000000000..117c266b7 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/marker.rs @@ -0,0 +1,364 @@ +use super::model::SeparationNode; +use base64::Engine as _; +use std::path::Path; +use std::path::PathBuf; +use std::time::Instant; + +const MARKER_LINE_WIDTH: u32 = 2; +const PURPLE_FILL: image::Rgba = image::Rgba([180, 0, 180, 120]); + +pub async fn build_marked_image( + source_url: String, + nodes: Vec, + target: PathBuf, +) -> Result { + let node_count = nodes.len(); + let started = Instant::now(); + let result = match tokio::task::spawn_blocking(move || { + let blocking_started = Instant::now(); + let result = build_marked_image_blocking(&source_url, &nodes, &target); + app_log!( + "ui_separation.marker.blocking_timing outcome={} elapsed_ms={} nodes={}", + if result.is_ok() { "ok" } else { "error" }, + blocking_started.elapsed().as_millis(), + node_count + ); + result + }) + .await + { + Ok(result) => result, + Err(error) => Err(format!("构建标记图任务失败：{error}")), + }; + app_log!( + "ui_separation.marker.timing outcome={} elapsed_ms={} nodes={}", + if result.is_ok() { "ok" } else { "error" }, + started.elapsed().as_millis(), + node_count + ); + result +} + +fn build_marked_image_blocking( + source_url: &str, + nodes: &[SeparationNode], + target: &Path, +) -> Result { + let encoded = source_url + .split_once(',') + .map(|(_, d)| d) + .ok_or_else(|| "源图 data URL 无效".to_string())?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|e| format!("解码源图失败：{e}"))?; + let mut image = image::load_from_memory(&bytes) + .map_err(|e| format!("读取源图失败：{e}"))? + .to_rgba8(); + let width = image.width(); + let height = image.height(); + // 先填充紫色重建区域，随后绘制绿色框，确保绿色框位于最上层。 + for node in nodes { + for child in &node.children { + fill_rect( + &mut image, + child.global_pos_x_px, + child.global_pos_y_px, + child.width_px, + child.height_px, + PURPLE_FILL, + width, + height, + ); + } + for mask in &node.text_mask_areas { + fill_rect( + &mut image, + mask.global_pos_x_px, + mask.global_pos_y_px, + mask.width_px, + mask.height_px, + PURPLE_FILL, + width, + height, + ); + } + } + for node in nodes { + draw_frame( + &mut image, + node.global_pos_x_px, + node.global_pos_y_px, + node.width_px, + node.height_px, + width, + height, + ); + } + let image = image::DynamicImage::ImageRgba8(image); + image + .save_with_format(target, image::ImageFormat::Png) + .map_err(|e| format!("写入标记图失败：{e}"))?; + let mut png = Vec::new(); + image + .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) + .map_err(|e| format!("编码标记图失败：{e}"))?; + Ok(format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(png) + )) +} + +fn clipped_rect( + x: u32, + y: u32, + w: u32, + h: u32, + width: u32, + height: u32, +) -> Option<(u32, u32, u32, u32)> { + if width == 0 || height == 0 || w == 0 || h == 0 { + return None; + } + if x >= width || y >= height { + return None; + } + let x0 = x; + let y0 = y; + let x1 = x.saturating_add(w).min(width).saturating_sub(1); + let y1 = y.saturating_add(h).min(height).saturating_sub(1); + (x0 <= x1 && y0 <= y1).then_some((x0, y0, x1, y1)) +} + +fn fill_rect( + image: &mut image::RgbaImage, + x: u32, + y: u32, + w: u32, + h: u32, + color: image::Rgba, + width: u32, + height: u32, +) { + let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else { + return; + }; + for yy in y0..=y1 { + for xx in x0..=x1 { + image.put_pixel(xx, yy, color); + } + } +} + +fn draw_frame( + image: &mut image::RgbaImage, + x: u32, + y: u32, + w: u32, + h: u32, + width: u32, + height: u32, +) { + let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else { + return; + }; + let green = image::Rgba([0, 255, 0, 255]); + draw_line( + image, + (x0, y0), + (x1, y0), + green, + MARKER_LINE_WIDTH, + (x0, y0, x1, y1), + ); + draw_line( + image, + (x0, y1), + (x1, y1), + green, + MARKER_LINE_WIDTH, + (x0, y0, x1, y1), + ); + draw_line( + image, + (x0, y0), + (x0, y1), + green, + MARKER_LINE_WIDTH, + (x0, y0, x1, y1), + ); + draw_line( + image, + (x1, y0), + (x1, y1), + green, + MARKER_LINE_WIDTH, + (x0, y0, x1, y1), + ); + draw_line( + image, + (x0, y0), + (x1, y1), + green, + MARKER_LINE_WIDTH, + (x0, y0, x1, y1), + ); + draw_line( + image, + (x1, y0), + (x0, y1), + green, + MARKER_LINE_WIDTH, + (x0, y0, x1, y1), + ); +} + +fn draw_line( + image: &mut image::RgbaImage, + start: (u32, u32), + end: (u32, u32), + color: image::Rgba, + line_width: u32, + bounds: (u32, u32, u32, u32), +) { + let mut x = start.0 as i64; + let mut y = start.1 as i64; + let target_x = end.0 as i64; + let target_y = end.1 as i64; + let dx = (target_x - x).abs(); + let sx = if x < target_x { 1 } else { -1 }; + let dy = -(target_y - y).abs(); + let sy = if y < target_y { 1 } else { -1 }; + let mut error = dx + dy; + + loop { + draw_brush(image, x, y, color, line_width, bounds); + if x == target_x && y == target_y { + break; + } + let twice_error = error * 2; + if twice_error >= dy { + error += dy; + x += sx; + } + if twice_error <= dx { + error += dx; + y += sy; + } + } +} + +fn draw_brush( + image: &mut image::RgbaImage, + x: i64, + y: i64, + color: image::Rgba, + line_width: u32, + bounds: (u32, u32, u32, u32), +) { + let (x0, y0, x1, y1) = bounds; + let line_width = line_width.max(1) as i64; + let before = (line_width - 1) / 2; + let after = line_width / 2; + let min_x = (x - before).max(x0 as i64); + let max_x = (x + after).min(x1 as i64); + let min_y = (y - before).max(y0 as i64); + let max_y = (y + after).min(y1 as i64); + for yy in min_y..=max_y { + for xx in min_x..=max_x { + image.put_pixel(xx as u32, yy as u32, color); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ui_editor::commands::separation::{SeparationNote, TextMaskArea}; + + #[test] + fn draw_frame_adds_green_cross_corner_lines() { + let mut image = image::RgbaImage::from_pixel(8, 6, image::Rgba([1, 2, 3, 255])); + draw_frame(&mut image, 1, 1, 5, 3, 8, 6); + let green = image::Rgba([0, 255, 0, 255]); + + for &(x, y) in &[(1, 1), (5, 1), (1, 3), (5, 3), (3, 2)] { + assert_eq!(*image.get_pixel(x, y), green, "pixel ({x}, {y})"); + } + assert_eq!(*image.get_pixel(3, 1), green); + assert_eq!(*image.get_pixel(3, 3), green); + assert_eq!(*image.get_pixel(2, 2), green); + assert_eq!(*image.get_pixel(4, 2), green); + assert_eq!(*image.get_pixel(0, 0), image::Rgba([1, 2, 3, 255])); + } + + #[test] + fn draw_frame_keeps_cross_inside_clipped_rect() { + let mut image = image::RgbaImage::from_pixel(4, 4, image::Rgba([1, 2, 3, 255])); + draw_frame(&mut image, 2, 2, 4, 4, 4, 4); + let green = image::Rgba([0, 255, 0, 255]); + for y in 2..4 { + for x in 2..4 { + assert_eq!(*image.get_pixel(x, y), green); + } + } + assert_eq!(*image.get_pixel(1, 1), image::Rgba([1, 2, 3, 255])); + } + + #[test] + fn clipped_rect_ignores_rectangles_starting_outside_image() { + assert_eq!(clipped_rect(4, 0, 1, 1, 4, 4), None); + assert_eq!(clipped_rect(0, 4, 1, 1, 4, 4), None); + } + + #[test] + fn text_mask_is_purple_before_green_frame() { + let mut image = image::RgbaImage::from_pixel(8, 8, image::Rgba([1, 2, 3, 255])); + let node = SeparationNode { + id: crate::ui_editor::utils::NodeId::new("image").unwrap(), + global_pos_x_px: 1, + global_pos_y_px: 1, + width_px: 6, + height_px: 6, + note: SeparationNote::default(), + text_mask_areas: vec![ + TextMaskArea { + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + }, + TextMaskArea { + global_pos_x_px: 1, + global_pos_y_px: 1, + width_px: 1, + height_px: 1, + }, + ], + children: vec![], + rework_count: 0, + }; + for mask in &node.text_mask_areas { + fill_rect( + &mut image, + mask.global_pos_x_px, + mask.global_pos_y_px, + mask.width_px, + mask.height_px, + PURPLE_FILL, + 8, + 8, + ); + } + draw_frame( + &mut image, + node.global_pos_x_px, + node.global_pos_y_px, + node.width_px, + node.height_px, + 8, + 8, + ); + assert_eq!(*image.get_pixel(0, 0), PURPLE_FILL); + assert_eq!(*image.get_pixel(1, 1), image::Rgba([0, 255, 0, 255])); + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs new file mode 100644 index 000000000..fa3cc23d2 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs @@ -0,0 +1,389 @@ +mod area; +mod marker; +mod model; +mod persistence; +mod prompt; +mod tree; +mod workflow; + +pub(crate) use marker::build_marked_image; +pub use model::*; +pub use persistence::*; +pub use tree::*; +pub use workflow::apply_batch_patch; +pub(crate) use workflow::separate_ui_impl; +#[cfg(test)] +mod tests { + use super::*; + use crate::ui_editor::component::image::{ImageComponent, ImageType}; + use crate::ui_editor::component::text::TextComponent; + use crate::ui_editor::component::Component; + use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode; + use crate::ui_editor::layout::control_layout::ControlLayout; + use crate::ui_editor::layout::node::Node; + use crate::ui_editor::layout::node::{NodeMetadata, NodeSource, StageStatus}; + use crate::ui_editor::resource::ui_design_image::UIDesignImage; + use crate::ui_editor::state::{State, UITree}; + use crate::ui_editor::utils::{NodeId, UIDesignImageId}; + use nalgebra::Vector2; + use std::collections::HashMap; + use std::path::Path; + use typed_floats::tf32::StrictlyPositiveFinite; + + fn node(id: &str, component: Option, children: Vec) -> Node { + Node { + id: NodeId::new(id).unwrap(), + layout: ControlLayout::default(), + metadata: NodeMetadata { + name: id.to_string(), + description: String::new(), + layout_status: StageStatus::NoProblem, + component_status: StageStatus::NoProblem, + allow_llm_edit_layout: true, + allow_llm_edit_component: true, + source: NodeSource::Llm, + }, + component, + children_display_mode: ChildrenDisplayMode::Stack, + children, + } + } + fn state(root: Node) -> State { + let image_id = UIDesignImageId::new("page").unwrap(); + State { + ui_trees: vec![UITree { + src_ui_design: image_id.clone(), + root, + }], + ui_design_images: HashMap::from([( + image_id, + UIDesignImage { + metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata { + name: "page".to_string(), + description: String::new(), + role: None, + slave_to: None, + }, + path: "page.png".to_string(), + pixel_size: Vector2::new(100.0, 100.0), + pixels_per_unit: StrictlyPositiveFinite::new(1.0).unwrap(), + }, + )]), + sprite_assets: HashMap::new(), + font_assets: HashMap::new(), + } + } + #[test] + fn construction_filters_pure_nodes_and_passes_children_through() { + let image = Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + }); + let root = node( + "root", + None, + vec![node( + "container", + None, + vec![node("image", Some(image), vec![])], + )], + ); + let result = construct_separation_state(&state(root)); + assert_eq!(result.trees[0].root.children[0].id.as_str(), "image"); + } + + #[test] + fn construction_keeps_real_root_for_root_image() { + let image = Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + }); + let root = node("root-image", Some(image), vec![]); + let result = construct_separation_state(&state(root)); + let tree = &result.trees[0]; + assert_eq!(tree.root.id.as_str(), "root-image"); + assert!(tree.root_extractable); + } + + #[test] + fn construction_attaches_text_mask_to_nearest_unbound_image() { + let image = Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + }); + let text = Component::Text(TextComponent::new("按钮")); + let root = node( + "root", + None, + vec![node( + "outer-image", + Some(image.clone()), + vec![node("text", Some(text.clone()), vec![])], + )], + ); + let result = construct_separation_state(&state(root)); + let outer = &result.trees[0].root.children[0]; + assert_eq!(outer.id.as_str(), "outer-image"); + assert_eq!(outer.text_mask_areas.len(), 1); + assert!(outer.children.is_empty()); + + let nested_root = node( + "root", + None, + vec![node( + "outer-image", + Some(image.clone()), + vec![node( + "inner-image", + Some(image), + vec![node("text", Some(text), vec![])], + )], + )], + ); + let nested = construct_separation_state(&state(nested_root)); + let inner = &nested.trees[0].root.children[0].children[0]; + assert_eq!(inner.text_mask_areas.len(), 1); + assert!(nested.trees[0].root.children[0].text_mask_areas.is_empty()); + } + + #[test] + fn root_image_receives_text_mask() { + let root = node( + "root-image", + Some(Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + })), + vec![node( + "text", + Some(Component::Text(TextComponent::new("标题"))), + vec![], + )], + ); + let result = construct_separation_state(&state(root)); + assert_eq!(result.trees[0].root.text_mask_areas.len(), 1); + } + #[test] + fn binding_validation_requires_exact_batch_coverage() { + let node = SeparationNode { + id: NodeId::new("image").unwrap(), + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + note: SeparationNote { + description: "image".to_string(), + rework_notes: Vec::new(), + }, + text_mask_areas: vec![], + children: vec![], + rework_count: 0, + }; + assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err()); + } + + #[test] + fn separation_note_prompt_keeps_rework_notes_in_order() { + let without_notes = SeparationNote { + description: "按钮".to_string(), + rework_notes: Vec::new(), + }; + assert_eq!(without_notes.as_prompt(), "desc: 按钮"); + + let with_notes = SeparationNote { + description: "按钮".to_string(), + rework_notes: vec!["保留圆角".to_string(), "去掉阴影".to_string()], + }; + assert_eq!( + with_notes.as_prompt(), + "desc: 按钮\nprevious rework notes:\n- 保留圆角\n- 去掉阴影" + ); + } + + #[test] + fn need_rework_appends_note_and_final_attempt_becomes_problematic() { + let image = Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + }); + let mut separation = construct_separation_state(&state(node( + "root", + None, + vec![node("image", Some(image), vec![])], + ))); + let id = NodeId::new("image").unwrap(); + let paths = HashMap::new(); + + for note in ["第一次意见", "第二次意见", "最后一次意见"] { + apply_batch_patch( + &mut separation, + 0, + &[BindingDecision::NeedRework { + to_node: id.clone(), + advice: note.to_string(), + }], + &paths, + ) + .unwrap(); + } + + let node = &separation.trees[0].root.children[0]; + assert_eq!( + node.note.rework_notes, + ["第一次意见", "第二次意见", "最后一次意见"] + ); + assert_eq!(separation.problematic_nodes.len(), 1); + assert_eq!( + separation.problematic_nodes[0].rework_count, + MAX_REWORK_COUNT + ); + assert_eq!(node.rework_count, MAX_REWORK_COUNT); + assert!(next_leaf_batch(&separation, &separation.trees[0]).is_empty()); + } + + #[test] + fn next_extract_prompt_contains_previous_rework_notes() { + let note = SeparationNote { + description: "图标".to_string(), + rework_notes: vec!["不要带父背景".to_string()], + }; + let prompt = super::prompt::gen_extract_prompt(vec![note]); + assert!(prompt.contains("previous rework notes:\n- 不要带父背景")); + } + + #[test] + fn binding_validation_rejects_overlong_rework_note() { + let node = SeparationNode { + id: NodeId::new("image").unwrap(), + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + note: SeparationNote::default(), + text_mask_areas: vec![], + children: vec![], + rework_count: 0, + }; + let decision = BindingDecision::NeedRework { + to_node: node.id.clone(), + advice: "x".repeat(MAX_REWORK_NOTE_CHARS + 1), + }; + assert!(validate_binding_response( + &BindingResp { + decisions: vec![decision] + }, + &[&node] + ) + .is_err()); + } + #[test] + fn sidecar_name_uses_asset_id_digest() { + let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap(); + assert!(dir.to_string_lossy().contains("ui_1-")); + assert!(dir.to_string_lossy().ends_with("-separation")); + } + #[test] + fn patch_collects_bound_and_keeps_tree_topology() { + let image = Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + }); + let mut state = construct_separation_state(&state(node( + "root", + None, + vec![node("image", Some(image), vec![])], + ))); + let id = NodeId::new("image").unwrap(); + let decisions = vec![BindingDecision::Ok { + to_node: id.clone(), + extracted_area: BindingArea { + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + }, + }]; + let paths = HashMap::from([(id.clone(), "ui/.sidecar/cut.png".to_string())]); + apply_batch_patch(&mut state, 0, &decisions, &paths).unwrap(); + assert_eq!(state.bound[0].node_id, id); + assert_eq!(state.trees[0].root.children.len(), 1); + } + + #[test] + fn batch_selection_greedily_skips_overlapping_leaves() { + let a = SeparationNode { + id: NodeId::new("a").unwrap(), + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 10, + height_px: 10, + note: SeparationNote::default(), + text_mask_areas: vec![], + children: vec![], + rework_count: 0, + }; + let b = SeparationNode { + id: NodeId::new("b").unwrap(), + global_pos_x_px: 5, + global_pos_y_px: 5, + width_px: 10, + height_px: 10, + note: SeparationNote::default(), + text_mask_areas: vec![], + children: vec![], + rework_count: 0, + }; + let c = SeparationNode { + id: NodeId::new("c").unwrap(), + global_pos_x_px: 20, + global_pos_y_px: 0, + width_px: 5, + height_px: 5, + note: SeparationNote::default(), + text_mask_areas: vec![], + children: vec![], + rework_count: 0, + }; + let tree = SeparationTree { + src_ui_design: UIDesignImageId::new("page").unwrap(), + root: SeparationNode { + id: NodeId::new("root").unwrap(), + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 100, + height_px: 100, + note: SeparationNote::default(), + text_mask_areas: vec![], + children: vec![a, b, c], + rework_count: 0, + }, + root_extractable: false, + }; + let state = SeparationState { + schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(), + trees: vec![tree.clone()], + bound: vec![], + problematic_nodes: vec![], + }; + let batch = next_leaf_batch(&state, &tree); + assert_eq!( + batch + .iter() + .map(|node| node.id.as_str()) + .collect::>(), + vec!["a", "c"] + ); + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/binding.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/binding.rs new file mode 100644 index 000000000..d24f249eb --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/binding.rs @@ -0,0 +1,50 @@ +use crate::ui_editor::utils::NodeId; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct BindingArea { + pub global_pos_x_px: u32, + pub global_pos_y_px: u32, + pub width_px: u32, + pub height_px: u32, +} + +impl BindingArea { + pub fn validate_in(&self, w: u32, h: u32) -> Result<(), String> { + if self.width_px == 0 || self.height_px == 0 { + return Err("BindingArea 宽度和高度必须大于 0".into()); + } + if self + .global_pos_x_px + .checked_add(self.width_px) + .is_none_or(|v| v > w) + || self + .global_pos_y_px + .checked_add(self.height_px) + .is_none_or(|v| v > h) + { + return Err("BindingArea 超出处理图边界".into()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub enum BindingDecision { + Ok { + extracted_area: BindingArea, + to_node: NodeId, + }, + NeedRework { + advice: String, + to_node: NodeId, + }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +pub struct BindingResp { + pub decisions: Vec, +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/mod.rs new file mode 100644 index 000000000..7e1032bac --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/mod.rs @@ -0,0 +1,13 @@ +mod binding; +mod node; +mod note; +mod result; + +pub use binding::*; +pub use node::*; +pub use note::*; +pub use result::*; + +pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1"; +pub const MAX_REWORK_COUNT: u32 = 3; +pub const MAX_REWORK_NOTE_CHARS: usize = 512; diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/node.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/node.rs new file mode 100644 index 000000000..e0ab3c8fb --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/node.rs @@ -0,0 +1,44 @@ +use crate::ui_editor::utils::{NodeId, UIDesignImageId}; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct TextMaskArea { + pub global_pos_x_px: u32, + pub global_pos_y_px: u32, + pub width_px: u32, + pub height_px: u32, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationNode { + pub id: NodeId, + pub global_pos_x_px: u32, + pub global_pos_y_px: u32, + pub width_px: u32, + pub height_px: u32, + pub note: super::SeparationNote, + pub text_mask_areas: Vec, + pub children: Vec, + pub rework_count: u32, +} + +impl SeparationNode { + pub fn as_prompt(&self) -> String { + format!( + "node_id={} note: {}", + self.id.as_str(), + self.note.as_prompt() + ) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationTree { + pub src_ui_design: UIDesignImageId, + pub root: SeparationNode, + pub root_extractable: bool, +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/note.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/note.rs new file mode 100644 index 000000000..15c500b59 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/note.rs @@ -0,0 +1,23 @@ +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationNote { + pub description: String, + pub rework_notes: Vec, +} + +impl SeparationNote { + pub fn as_prompt(&self) -> String { + let mut prompt = format!("desc: {}", self.description); + if !self.rework_notes.is_empty() { + prompt.push_str("\nprevious rework notes:"); + for note in &self.rework_notes { + prompt.push_str("\n- "); + prompt.push_str(note); + } + } + prompt + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/result.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/result.rs new file mode 100644 index 000000000..82b34bdcd --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model/result.rs @@ -0,0 +1,43 @@ +use crate::ui_editor::utils::NodeId; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct BoundNode { + pub node_id: NodeId, + pub cut_image_path: String, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct ProblematicNode { + pub node_id: NodeId, + pub problem_description: String, + pub rework_count: u32, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationState { + pub schema_version: String, + pub trees: Vec, + pub bound: Vec, + pub problematic_nodes: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationDTO { + pub bound_nodes: Vec, + pub problematic_nodes: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationRecoveryDTO { + pub exists: bool, + pub bound_node_count: usize, + pub problematic_node_count: usize, + pub has_pending_tree: bool, +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/persistence.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/persistence.rs new file mode 100644 index 000000000..80fd9f0a2 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/persistence.rs @@ -0,0 +1,167 @@ +use super::model::*; +use crate::ui_editor::commands::separation::*; +use std::fs; +use std::path::{Path, PathBuf}; +pub fn separation_sidecar_dir(root: &Path, asset_id: &str) -> Result { + if asset_id.trim().is_empty() || asset_id.trim() != asset_id { + app_log!("ui_separation.error stage=sidecar_dir reason=invalid_asset_id"); + return Err("UI 资源 ID 无效".to_string()); + } + let dir = root.join("ui").join(format!( + ".{}-separation", + crate::ui_editor::persistence::generated_file_stem(asset_id) + )); + if !dir.starts_with(root) { + app_log!("ui_separation.error stage=sidecar_dir reason=path_escape"); + return Err("separation sidecar 路径越界".to_string()); + } + app_log!( + "ui_separation.sidecar_resolved asset_id={} directory={}", + asset_id, + dir.file_name() + .and_then(|name| name.to_str()) + .unwrap_or("") + ); + Ok(dir) +} + +pub fn separation_state_path(root: &Path, asset_id: &str) -> Result { + Ok(separation_sidecar_dir(root, asset_id)?.join("state.json")) +} + +pub fn project_relative_path(root: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix(root) + .map_err(|_| "separation 产物必须位于项目目录内".to_string())?; + let value = relative.to_string_lossy().replace('\\', "/"); + if value.is_empty() || value.starts_with('/') || value.split('/').any(|part| part == "..") { + return Err("separation 产物相对路径无效".to_string()); + } + Ok(value) +} + +pub fn write_separation_state(path: &Path, state: &SeparationState) -> Result<(), String> { + if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION { + app_log!("ui_separation.error stage=state_write reason=schema_mismatch"); + return Err("不支持的 separation state schema".to_string()); + } + app_log!( + "ui_separation.state_write.start file={} trees={} bound={} problematic={}", + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + state.trees.len(), + state.bound.len(), + state.problematic_nodes.len() + ); + let bytes = serde_json::to_vec_pretty(state).map_err(|error| { + app_log!("ui_separation.error stage=state_write reason=serialize error={error}"); + format!("序列化 separation state 失败：{error}") + })?; + let parent = path.parent().ok_or_else(|| { + app_log!("ui_separation.error stage=state_write reason=missing_parent"); + "separation state 路径缺少父目录".to_string() + })?; + fs::create_dir_all(parent).map_err(|error| { + app_log!("ui_separation.error stage=state_write reason=create_parent error={error}"); + format!("创建 separation sidecar 失败：{error}") + })?; + let temporary = path.with_extension("json.tmp"); + fs::write(&temporary, bytes).map_err(|error| { + app_log!("ui_separation.error stage=state_write reason=write_temp error={error}"); + format!("写入 separation state 失败：{error}") + })?; + fs::rename(&temporary, path).map_err(|error| { + app_log!("ui_separation.error stage=state_write reason=install error={error}"); + format!("安装 separation state 失败：{error}") + })?; + app_log!( + "ui_separation.state_write.completed file={} bytes={} trees={} bound={} problematic={}", + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + fs::metadata(path) + .map(|metadata| metadata.len()) + .unwrap_or(0), + state.trees.len(), + state.bound.len(), + state.problematic_nodes.len() + ); + Ok(()) +} + +pub fn read_separation_state(path: &Path) -> Result { + app_log!( + "ui_separation.state_read.start file={}", + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or("") + ); + let bytes = fs::read(path).map_err(|error| { + app_log!("ui_separation.error stage=state_read reason=read error={error}"); + format!("读取 separation state 失败：{error}") + })?; + let state: SeparationState = serde_json::from_slice(&bytes).map_err(|error| { + app_log!("ui_separation.error stage=state_read reason=parse error={error}"); + format!("解析 separation state 失败：{error}") + })?; + if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION { + app_log!("ui_separation.error stage=state_read reason=schema_mismatch"); + return Err("不支持的 separation state schema".to_string()); + } + app_log!( + "ui_separation.state_read.completed bytes={} trees={} bound={} problematic={}", + bytes.len(), + state.trees.len(), + state.bound.len(), + state.problematic_nodes.len() + ); + Ok(state) +} + +pub fn separation_dto(state: &SeparationState) -> SeparationDTO { + app_log!( + "ui_separation.dto bound_nodes={} problematic_nodes={} remaining_trees={}", + state.bound.len(), + state.problematic_nodes.len(), + state.trees.len() + ); + SeparationDTO { + bound_nodes: state.bound.clone(), + problematic_nodes: state.problematic_nodes.clone(), + } +} + +pub fn inspect_separation_recovery( + root: &Path, + asset_id: &str, +) -> Result { + let state_path = separation_state_path(root, asset_id)?; + if !state_path.exists() { + return Ok(SeparationRecoveryDTO { + exists: false, + bound_node_count: 0, + problematic_node_count: 0, + has_pending_tree: false, + }); + } + let state = read_separation_state(&state_path)?; + Ok(SeparationRecoveryDTO { + exists: true, + bound_node_count: state.bound.len(), + problematic_node_count: state.problematic_nodes.len(), + has_pending_tree: state + .trees + .iter() + .any(|tree| !tree.root.children.is_empty() || tree.root_extractable), + }) +} + +pub fn finalize_separation(root: &Path, asset_id: &str) -> Result<(), String> { + let state_path = separation_state_path(root, asset_id)?; + match fs::remove_file(&state_path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(format!("删除 separation state 失败：{error}")), + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs new file mode 100644 index 000000000..a91a94889 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs @@ -0,0 +1,70 @@ +use crate::ui_editor::commands::separation::{SeparationNode, SeparationNote}; + +const SHARED_SEPARATION_REQ: &str = r#" + + MUST hard edges; preserve no glow/blur beyond the exact visible shape. + NEVER keep its parent's background with it. + NEVER include any text unless requested. + + UI elements that needs to extract has been marked with GREEN line frames box with crossline inside. (only for mark purpose, NEVER wrap a frame in your extraction). + On some UI elements, there is some PURPLE filled area, they were removed UI elements, reconstruct the background under where they were. + + MUST extract exactly these marked UI elements area. +"#; +pub(super) fn gen_extract_prompt(separation_notes: Vec) -> String { + let extract_system_prompt = format!( + r#" + This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction. + Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element. + + MUST keep each element at its original position on a transparent canvas. + {SHARED_SEPARATION_REQ} + here are UI elements to extract: + + "# + ); + let mut result = extract_system_prompt; + result.reserve(512); + for elem in separation_notes { + result.push_str(&elem.as_prompt()); + result.push('\n'); + } + result +} +pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String { + let binding_system_prompt = format!( + r#" + You will be given a src UI design image and a processed image, where some ui elements are separated. + You need to recognize and review the separation using the given tool. + field notes: + * extracted_area MUST be the recognized area from the processed image, INSTEAD OF from the src image. + The processed image is the only authoritative image for extracted_area. + Return the pixel bounding box of the extracted element as it appears in the processed image. + Do not copy, infer, or reuse the source node rectangle. + The src image is only for identifying which semantic UI element belongs to to_node. + + Here were the separation requirements: + ``` + {SHARED_SEPARATION_REQ} + ``` + And you should also review if the extracted's successfully meet the src image: + * shape + * color + * style + * edge process + ... + + if not, use `NeedRework` data structure in the tool to indicate the (it should be from which) node id and advice. + your advice (less than 20 words) will be used to improve the separation in the next time. + + these node need handle: + "# + ); + let mut result = binding_system_prompt; + result.reserve(512); + for elem in nodes { + result.push_str(&elem.as_prompt()); + result.push('\n'); + } + result +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs new file mode 100644 index 000000000..6e942e260 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs @@ -0,0 +1,273 @@ +use super::model::*; +use crate::ui_editor::component::{image::ImageComponent, Component}; +use crate::ui_editor::layout::node::Node; +use crate::ui_editor::state::State; +use crate::ui_editor::utils::NodeId; +use std::collections::HashMap; +use std::collections::HashSet; + +fn is_unbound_image(node: &Node) -> bool { + matches!( + node.component.as_ref(), + Some(Component::Image(ImageComponent { + target_graphic: None, + .. + })) + ) +} + +fn has_image_component(node: &Node) -> bool { + matches!(node.component.as_ref(), Some(Component::Image(_))) +} + +fn has_text_component(node: &Node) -> bool { + matches!(node.component.as_ref(), Some(Component::Text(_))) +} + +fn node_pixel_rect( + node: &Node, + parent: &crate::ui_editor::layout::dimension::UIRect, + ppu: f32, +) -> (u32, u32, u32, u32) { + let rect = node.layout.transform.resolve(parent); + ( + (rect.min.x * ppu).max(0.0).round() as u32, + (rect.min.y * ppu).max(0.0).round() as u32, + (rect.size.x * ppu).max(0.0).round() as u32, + (rect.size.y * ppu).max(0.0).round() as u32, + ) +} + +fn node_description(node: &Node) -> String { + let name = node.metadata.name.trim(); + let description = node.metadata.description.trim(); + match (name.is_empty(), description.is_empty()) { + (true, true) => "未命名 UI 图片元素".to_string(), + (false, true) => name.to_string(), + (true, false) => description.to_string(), + (false, false) => format!("{name}：{description}"), + } +} + +fn collect_todo_nodes( + node: &Node, + parent: &crate::ui_editor::layout::dimension::UIRect, + ppu: f32, + output: &mut Vec, + text_masks: &mut HashMap>, +) { + let rect = node.layout.transform.resolve(parent); + let mut children = Vec::new(); + for child in &node.children { + collect_todo_nodes(child, &rect, ppu, &mut children, text_masks); + } + if is_unbound_image(node) { + let (x, y, w, h) = node_pixel_rect(node, parent, ppu); + output.push(SeparationNode { + id: node.id.clone(), + global_pos_x_px: x, + global_pos_y_px: y, + width_px: w, + height_px: h, + note: SeparationNote { + description: node_description(node), + rework_notes: Vec::new(), + }, + text_mask_areas: text_masks.remove(&node.id).unwrap_or_default(), + children, + rework_count: 0, + }); + } else { + output.extend(children); + } +} + +fn collect_text_masks( + node: &Node, + parent: &crate::ui_editor::layout::dimension::UIRect, + ppu: f32, + nearest_image: Option, + output: &mut HashMap>, +) { + let node_is_image = is_unbound_image(node); + let nearest_image = if node_is_image { + Some(node.id.clone()) + } else { + nearest_image + }; + if has_text_component(node) && !has_image_component(node) { + if let Some(image_id) = nearest_image.clone() { + let (x, y, w, h) = node_pixel_rect(node, parent, ppu); + output.entry(image_id).or_default().push(TextMaskArea { + global_pos_x_px: x, + global_pos_y_px: y, + width_px: w, + height_px: h, + }); + } + } + let rect = node.layout.transform.resolve(parent); + for child in &node.children { + collect_text_masks(child, &rect, ppu, nearest_image.clone(), output); + } +} + +pub fn construct_separation_state(state: &State) -> SeparationState { + let trees = state + .ui_trees + .iter() + .filter_map(|tree| { + let image = state.ui_design_images.get(&tree.src_ui_design)?; + let ppu = image.pixels_per_unit.get(); + let size = image.pixel_size / ppu; + let root_rect = + crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size); + let mut text_masks = HashMap::new(); + collect_text_masks(&tree.root, &root_rect, ppu, None, &mut text_masks); + let mut children = Vec::new(); + for child in &tree.root.children { + collect_todo_nodes(child, &root_rect, ppu, &mut children, &mut text_masks); + } + let root_extractable = is_unbound_image(&tree.root); + if !root_extractable && children.is_empty() { + return None; + } + let (x, y, w, h) = node_pixel_rect(&tree.root, &root_rect, ppu); + Some(SeparationTree { + src_ui_design: tree.src_ui_design.clone(), + root: SeparationNode { + id: tree.root.id.clone(), + global_pos_x_px: x, + global_pos_y_px: y, + width_px: w, + height_px: h, + note: SeparationNote { + description: node_description(&tree.root), + rework_notes: Vec::new(), + }, + text_mask_areas: text_masks.remove(&tree.root.id).unwrap_or_default(), + children, + rework_count: 0, + }, + root_extractable, + }) + }) + .collect(); + SeparationState { + schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(), + trees, + bound: Vec::new(), + problematic_nodes: Vec::new(), + } +} + +fn terminal_ids(state: &SeparationState) -> HashSet { + state + .bound + .iter() + .map(|n| n.node_id.clone()) + .chain(state.problematic_nodes.iter().map(|n| n.node_id.clone())) + .collect() +} + +fn logical_leaves<'a>( + node: &'a SeparationNode, + extractable: bool, + terminal: &HashSet, + output: &mut Vec<&'a SeparationNode>, +) { + let is_terminal = terminal.contains(&node.id); + let children_terminal = node + .children + .iter() + .all(|child| terminal.contains(&child.id)); + if extractable && !is_terminal && children_terminal { + output.push(node); + return; + } + for child in &node.children { + logical_leaves(child, true, terminal, output); + } +} + +fn overlaps(a: &SeparationNode, b: &SeparationNode) -> bool { + let ax1 = a.global_pos_x_px as u64 + a.width_px as u64; + let ay1 = a.global_pos_y_px as u64 + a.height_px as u64; + let bx1 = b.global_pos_x_px as u64 + b.width_px as u64; + let by1 = b.global_pos_y_px as u64 + b.height_px as u64; + let width = ax1 + .min(bx1) + .saturating_sub(a.global_pos_x_px.max(b.global_pos_x_px) as u64); + let height = ay1 + .min(by1) + .saturating_sub(a.global_pos_y_px.max(b.global_pos_y_px) as u64); + width > 0 && height > 0 +} + +pub fn next_leaf_batch<'a>( + state: &SeparationState, + tree: &'a SeparationTree, +) -> Vec<&'a SeparationNode> { + let terminal = terminal_ids(state); + let mut candidates = Vec::new(); + logical_leaves( + &tree.root, + tree.root_extractable, + &terminal, + &mut candidates, + ); + let mut selected: Vec<&'a SeparationNode> = Vec::new(); + for candidate in candidates { + if selected.iter().all(|other| !overlaps(candidate, other)) { + selected.push(candidate); + } + } + app_log!( + "ui_separation.batch_selected image_id={} leaf_nodes={}", + tree.src_ui_design.as_str(), + selected.len() + ); + selected +} + +pub fn validate_binding_response( + response: &BindingResp, + batch: &[&SeparationNode], +) -> Result<(), String> { + let expected = batch + .iter() + .map(|node| node.id.clone()) + .collect::>(); + let mut seen = HashSet::new(); + for decision in &response.decisions { + let node_id = match decision { + BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => { + to_node + } + }; + if !expected.contains(node_id) { + return Err(format!("视觉绑定返回了未知节点：{}", node_id.as_str())); + } + if !seen.insert(node_id.clone()) { + return Err(format!("视觉绑定重复返回节点：{}", node_id.as_str())); + } + if let BindingDecision::NeedRework { + advice, + .. + } = decision + { + if advice.trim().is_empty() { + return Err("NeedRework 必须包含问题描述".to_string()); + } + if advice.chars().count() > MAX_REWORK_NOTE_CHARS { + return Err(format!( + "NeedRework 问题描述不能超过 {MAX_REWORK_NOTE_CHARS} 个字符" + )); + } + } + } + if seen.len() != expected.len() { + return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string()); + } + Ok(()) +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs new file mode 100644 index 000000000..da0c9e6b0 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs @@ -0,0 +1,730 @@ +use super::area::normalize_binding_area; +use super::model::*; +use super::prompt::{gen_binding_prompt, gen_extract_prompt}; +use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config}; +use crate::platform_session::current_platform_session; +use crate::ui_editor::commands::separation::*; +use crate::ui_editor::commands::utils::{ + parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm, + run_with_repair_history, strict_json_schema, +}; +use crate::ui_editor::state::State; +use crate::ui_editor::utils::NodeId; +use base64::Engine as _; +use image::ImageFormat; +use platform_llm::{ + LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, +}; +use serde::Deserialize; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Instant; +pub fn apply_batch_patch( + state: &mut SeparationState, + tree_index: usize, + decisions: &[BindingDecision], + cut_paths: &std::collections::HashMap, +) -> Result<(), String> { + app_log!( + "ui_separation.batch_patch.start tree_index={} decisions={} cut_paths={}", + tree_index, + decisions.len(), + cut_paths.len() + ); + let batch_nodes = { + let tree = state + .trees + .get(tree_index) + .ok_or_else(|| "separation tree 索引无效".to_string())?; + next_leaf_batch(state, tree) + .into_iter() + .cloned() + .collect::>() + }; + let batch = batch_nodes.iter().collect::>(); + validate_binding_response( + &BindingResp { + decisions: decisions.to_vec(), + }, + &batch, + )?; + let rework_counts = batch_nodes + .iter() + .map(|node| (node.id.clone(), node.rework_count)) + .collect::>(); + let tree = state.trees.get_mut(tree_index).expect("tree index checked"); + for decision in decisions { + match decision { + BindingDecision::Ok { to_node, .. } => { + let path = cut_paths + .get(to_node) + .ok_or_else(|| format!("缺少节点 {} 的 cut 图片", to_node.as_str()))?; + state.bound.push(BoundNode { + node_id: to_node.clone(), + cut_image_path: path.clone(), + }); + } + BindingDecision::NeedRework { + to_node, + advice: problem_description, + } => { + append_rework_note(&mut tree.root, to_node, problem_description); + let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1; + increment_rework_count(&mut tree.root, to_node, count); + if count >= MAX_REWORK_COUNT { + state.problematic_nodes.push(ProblematicNode { + node_id: to_node.clone(), + problem_description: problem_description.clone(), + rework_count: count, + }); + } + } + } + } + app_log!( + "ui_separation.batch_patch.completed tree_index={} bound={} problematic={} pending_root_children={}", + tree_index, + state.bound.len(), + state.problematic_nodes.len(), + tree.root.children.len() + ); + Ok(()) +} + +fn append_rework_note(node: &mut SeparationNode, id: &NodeId, note: &str) -> bool { + if node.id == *id { + node.note.rework_notes.push(note.to_string()); + return true; + } + node.children + .iter_mut() + .any(|child| append_rework_note(child, id, note)) +} + +fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) { + if node.id == *id { + node.rework_count = count; + return; + } + for child in &mut node.children { + increment_rework_count(child, id, count); + } +} + +#[derive(Deserialize)] +struct RawEditResponse { + data: Vec, +} +#[derive(Deserialize)] +struct RawEditItem { + b64_json: String, +} + +async fn raw_image_edit( + session: &crate::platform_session::PlatformSessionSnapshot, + image_data_url: &str, + prompt: &str, + width: u32, + height: u32, +) -> Result { + let started = Instant::now(); + let result = raw_image_edit_inner(session, image_data_url, prompt, width, height).await; + app_log!( + "ui_separation.image_edit.timing outcome={} elapsed_ms={} width={} height={}", + if result.is_ok() { "ok" } else { "error" }, + started.elapsed().as_millis(), + width, + height + ); + result +} + +async fn raw_image_edit_inner( + session: &crate::platform_session::PlatformSessionSnapshot, + image_data_url: &str, + prompt: &str, + width: u32, + height: u32, +) -> Result { + app_log!( + "ui_separation.image_edit.start width={} height={} prompt_chars={}", + width, + height, + prompt.chars().count() + ); + let (mime, data) = image_data_url + .split_once(",") + .ok_or_else(|| "界面图 data URL 无效".to_string())?; + let mime = mime + .strip_prefix("data:") + .and_then(|v| v.strip_suffix(";base64")) + .unwrap_or("image/png"); + if !mime.eq_ignore_ascii_case("image/png") { + return Err("图片分离请求只支持 PNG 标记图".to_string()); + } + let image_bytes = base64::engine::general_purpose::STANDARD + .decode(data.trim()) + .map_err(|error| format!("解码标记图失败：{error}"))?; + if image_bytes.is_empty() { + return Err("标记图不能为空".to_string()); + } + let client = crate::http_client::agc_main_site_client_builder() + .build() + .map_err(|e| format!("创建图片编辑客户端失败：{e}"))?; + let url = format!( + "{}/api/raw/v1/images/edit", + session.api_base_url.trim_end_matches('/') + ); + let image_part = reqwest::multipart::Part::bytes(image_bytes) + .file_name("image.png") + .mime_str("image/png") + .map_err(|error| format!("构造图片编辑文件部件失败：{error}"))?; + let body = reqwest::multipart::Form::new() + .part("image", image_part) + .text("prompt", prompt.to_string()) + .text("width", width.to_string()) + .text("height", height.to_string()) + .text("output_format", "png") + .text("background", "transparent"); + let response = crate::http_client::with_agc_main_site_marker( + client + .post(url) + .bearer_auth(&session.access_token) + .multipart(body), + ) + .send() + .await + .map_err(|e| { + app_log!("ui_separation.error stage=image_edit reason=send error={e}"); + format!("图片分离请求失败：{e}") + })?; + if !response.status().is_success() { + app_log!( + "ui_separation.error stage=image_edit reason=http_status status={}", + response.status() + ); + return Err(format!("图片分离请求失败（HTTP {}）", response.status())); + } + let payload = response.json::().await.map_err(|e| { + app_log!("ui_separation.error stage=image_edit reason=parse_response error={e}"); + format!("解析图片分离响应失败：{e}") + })?; + let result = payload + .data + .into_iter() + .next() + .map(|item| format!("data:image/png;base64,{}", item.b64_json)) + .ok_or_else(|| "图片分离响应没有图像".to_string()); + match &result { + Ok(value) => app_log!( + "ui_separation.image_edit.completed data_url_chars={}", + value.chars().count() + ), + Err(error) => { + app_log!("ui_separation.error stage=image_edit reason=empty_result error={error}") + } + } + result +} + +async fn write_processed_image(processed_url: String, target: PathBuf) -> Result<(), String> { + let started = Instant::now(); + let result = write_processed_image_inner(processed_url, target).await; + app_log!( + "ui_separation.processed_image.write.timing outcome={} elapsed_ms={}", + if result.is_ok() { "ok" } else { "error" }, + started.elapsed().as_millis() + ); + result +} + +async fn write_processed_image_inner(processed_url: String, target: PathBuf) -> Result<(), String> { + app_log!( + "ui_separation.processed_image.write.start target_file={} data_url_chars={}", + target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + processed_url.chars().count() + ); + tokio::task::spawn_blocking(move || { + let encoded = processed_url + .split_once(',') + .map(|(_, data)| data) + .ok_or_else(|| "处理图 data URL 无效".to_string())?; + let processed_bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|error| format!("解析处理图失败：{error}"))?; + let byte_len = processed_bytes.len(); + fs::write(&target, processed_bytes) + .map_err(|error| format!("写入处理图失败：{}: {error}", target.display())) + .map(|_| { + app_log!( + "ui_separation.processed_image.write.completed target_file={} bytes={}", + target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + byte_len + ); + }) + }) + .await + .map_err(|error| format!("写入处理图任务失败：{error}"))? +} + +async fn visual_binding( + source_url: String, + processed_url: String, + nodes: &[&SeparationNode], +) -> Result { + let started = Instant::now(); + let result = visual_binding_inner(source_url, processed_url, nodes).await; + app_log!( + "ui_separation.visual_binding.timing outcome={} elapsed_ms={} nodes={}", + if result.is_ok() { "ok" } else { "error" }, + started.elapsed().as_millis(), + nodes.len() + ); + result +} + +async fn visual_binding_inner( + source_url: String, + processed_url: String, + nodes: &[&SeparationNode], +) -> Result { + app_log!( + "ui_separation.visual_binding.start nodes={} source_url_chars={} processed_url_chars={}", + nodes.len(), + source_url.chars().count(), + processed_url.chars().count() + ); + let llm_config = load_game_creator_app_config() + .map_err(|e| { + app_log!("ui_separation.error stage=visual_binding reason=load_config error={e}"); + e.to_string() + })? + .llm; + + let client = + build_game_creator_llm_client_from_llm_config(&llm_config, "llm").map_err(|e| { + app_log!("ui_separation.error stage=visual_binding reason=build_client error={e}"); + e.to_string() + })?; + let schema = strict_json_schema::().map_err(|error| { + app_log!("ui_separation.error stage=visual_binding reason=build_schema error={error}"); + error + })?; + let tool = LlmFunctionTool::new( + "bind_ui_elements", + "确认处理图中的区域对应哪些 UI 节点", + schema, + ) + .with_strict(true); + let initial_history = vec![ + LlmMessage::system(gen_binding_prompt(nodes.to_vec())), + LlmMessage::user_multimodal(vec![ + LlmMessageContentPart::InputText { + text: "processed image:".to_string(), + }, + LlmMessageContentPart::InputImage { + image_url: processed_url.clone(), + }, + LlmMessageContentPart::InputText { + text: "src image:".to_string(), + }, + LlmMessageContentPart::InputImage { + image_url: source_url.clone(), + }, + ]), + ]; + let result = run_with_repair_history( + 2, + initial_history, + |history| { + let tool = tool.clone(); + let client = client.clone(); + let llm_config = llm_config.clone(); + async move { + let request = LlmRunRequest::new(history) + .with_function_tools(vec![tool.clone()]) + .with_tool_choice(LlmToolChoice::Required); + let request_started = Instant::now(); + let response = request_ui_editor_llm(&client, &llm_config, request).await; + app_log!( + "ui_separation.llm.timing outcome={} elapsed_ms={}", + if response.is_ok() { "ok" } else { "error" }, + request_started.elapsed().as_millis() + ); + response + .map_err(|e| e.to_string()) + .and_then(|response| { + response + .tool_calls + .into_iter() + .find(|call| call.name == "bind_ui_elements") + .map(|call| call.arguments) + .ok_or_else(|| "视觉绑定模型未返回工具调用".to_string()) + }) + .and_then(|arguments| parse_limited_llm_tool_arguments(&arguments)) + .and_then(|args| { + serde_json::from_value::(args) + .map_err(|e| format!("视觉绑定结果无效：{e}")) + }) + .and_then(|parsed| Ok(parsed)) + } + }, + |value: &BindingResp| validate_binding_response(value, nodes), + ) + .await; + match &result { + Ok(value) => app_log!( + "ui_separation.visual_binding.completed nodes={} decisions={}", + nodes.len(), + value.decisions.len() + ), + Err(error) => app_log!( + "ui_separation.error stage=visual_binding reason=failed nodes={} error={error}", + nodes.len() + ), + } + result +} + +pub(crate) async fn separate_ui_impl( + project_path: String, + asset_id: String, + state: State, +) -> Result { + app_log!( + "ui_separation.start asset_id={} ui_trees={} ui_images={} sprites={}", + asset_id, + state.ui_trees.len(), + state.ui_design_images.len(), + state.sprite_assets.len() + ); + let session = current_platform_session().ok_or_else(|| "请先登录平台账号".to_string())?; + let root = Path::new(project_path.trim()); + let sidecar = separation_sidecar_dir(root, &asset_id).map_err(|error| { + app_log!( + "ui_separation.error stage=sidecar_dir asset_id={} error={error}", + asset_id + ); + error + })?; + fs::create_dir_all(&sidecar).map_err(|e| { + app_log!( + "ui_separation.error stage=sidecar_create asset_id={} error={e}", + asset_id + ); + format!("创建 separation sidecar 失败：{e}") + })?; + let state_path = separation_state_path(root, &asset_id)?; + let restored = state_path.exists(); + let mut separation = if restored { + app_log!("ui_separation.state_restore.start asset_id={}", asset_id); + read_separation_state(&state_path).map_err(|error| { + app_log!( + "ui_separation.error stage=state_restore asset_id={} error={error}", + asset_id + ); + error + })? + } else { + app_log!("ui_separation.state_construct.start asset_id={}", asset_id); + construct_separation_state(&state) + }; + app_log!( + "ui_separation.state_ready asset_id={} restored={} trees={} bound={} problematic={}", + asset_id, + restored, + separation.trees.len(), + separation.bound.len(), + separation.problematic_nodes.len() + ); + for tree_index in 0..separation.trees.len() { + let tree = &separation.trees[tree_index]; + let image_id = tree.src_ui_design.clone(); + let image = state + .ui_design_images + .get(&image_id) + .ok_or_else(|| "缺少源界面图".to_string())?; + let source_path = crate::project::resolve_local_project_path(root, &image.path)?; + let source_url = read_ui_reference_image_data_url(source_path) + .await + .map_err(|error| { + app_log!( + "ui_separation.error stage=read_source tree_index={} image_id={} error={error}", + tree_index, + image_id.as_str() + ); + error + })?; + app_log!( + "ui_separation.tree_start tree_index={} image_id={} width={} height={}", + tree_index, + image_id.as_str(), + image.pixel_size.x.round() as u32, + image.pixel_size.y.round() as u32 + ); + write_separation_state(&state_path, &separation).map_err(|error| { + app_log!( + "ui_separation.error stage=state_checkpoint tree_index={} error={error}", + tree_index + ); + error + })?; + let mut batch_index = 0usize; + loop { + let batch_started = Instant::now(); + let Some(current_tree) = separation.trees.get(tree_index) else { + break; + }; + let batch_nodes = next_leaf_batch(&separation, current_tree) + .into_iter() + .cloned() + .collect::>(); + if batch_nodes.is_empty() { + app_log!( + "ui_separation.tree_completed tree_index={} image_id={} bound={} problematic={}", + tree_index, + image_id.as_str(), + separation.bound.len(), + separation.problematic_nodes.len() + ); + break; + } + let batch = batch_nodes.iter().collect::>(); + let prompt = + gen_extract_prompt(batch.iter().map(|n| n.note.clone()).collect::>()); + app_log!( + "ui_separation.batch_start tree_index={} batch_index={} nodes={} prompt_chars={} rework_total={}", + tree_index, + batch_index, + batch.len(), + prompt.chars().count(), + batch.iter().map(|node| node.rework_count).sum::() + ); + let marker_path = sidecar.join(format!("marked-{}.png", separation.bound.len())); + let marked_url = match build_marked_image( + source_url.clone(), + batch_nodes.clone(), + marker_path, + ) + .await + { + Ok(value) => value, + Err(error) => { + app_log!( + "ui_separation.error stage=mark_image tree_index={} batch_index={} error={error}", + tree_index, + batch_index + ); + write_separation_state(&state_path, &separation)?; + return Err(error); + } + }; + let processed_url = match raw_image_edit( + &session, + &marked_url, + &prompt, + image.pixel_size.x as u32, + image.pixel_size.y as u32, + ) + .await + { + Ok(value) => value, + Err(error) => { + app_log!( + "ui_separation.error stage=image_edit tree_index={} batch_index={} error={error}", + tree_index, + batch_index + ); + write_separation_state(&state_path, &separation)?; + return Err(error); + } + }; + let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len())); + if let Err(error) = + write_processed_image(processed_url.clone(), processed_path.clone()).await + { + app_log!( + "ui_separation.error stage=processed_image_write tree_index={} batch_index={} error={error}", + tree_index, + batch_index + ); + write_separation_state(&state_path, &separation)?; + return Err(error); + } + let binding = match visual_binding(source_url.clone(), processed_url, &batch).await { + Ok(value) => value, + Err(error) => { + app_log!( + "ui_separation.error stage=visual_binding tree_index={} batch_index={} error={error}", + tree_index, + batch_index + ); + write_separation_state(&state_path, &separation)?; + return Err(error); + } + }; + app_log!( + "ui_separation.binding_decisions tree_index={} batch_index={} decisions={}", + tree_index, + batch_index, + binding.decisions.len() + ); + let mut cut_paths = std::collections::HashMap::new(); + let mut cut_error = None; + for decision in &binding.decisions { + if let BindingDecision::Ok { + to_node, + extracted_area, + } = decision + { + let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str())); + match cut_processed_image( + processed_path.clone(), + *extracted_area, + cut_path.clone(), + ) + .await + { + Ok(()) => { + cut_paths + .insert(to_node.clone(), project_relative_path(root, &cut_path)?); + } + Err(error) => { + app_log!( + "ui_separation.error stage=cut_image tree_index={} batch_index={} node_id={} error={error}", + tree_index, + batch_index, + to_node.as_str() + ); + cut_error = + Some(format!("节点 {} 的分离区域无效：{error}", to_node.as_str())); + break; + } + } + } + } + if let Some(error) = cut_error { + app_log!( + "ui_separation.error stage=cut_batch tree_index={} batch_index={} error={error}", + tree_index, + batch_index + ); + write_separation_state(&state_path, &separation)?; + return Err(error); + } + apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?; + write_separation_state(&state_path, &separation)?; + app_log!( + "ui_separation.batch_completed tree_index={} batch_index={} cuts={} bound={} problematic={} elapsed_ms={}", + tree_index, + batch_index, + cut_paths.len(), + separation.bound.len(), + separation.problematic_nodes.len(), + batch_started.elapsed().as_millis() + ); + batch_index += 1; + } + } + app_log!( + "ui_separation.completed asset_id={} bound_nodes={} problematic_nodes={}", + asset_id, + separation.bound.len(), + separation.problematic_nodes.len() + ); + Ok(separation_dto(&separation)) +} + +async fn cut_processed_image( + source: PathBuf, + area: BindingArea, + target: PathBuf, +) -> Result<(), String> { + let started = Instant::now(); + let result = cut_processed_image_inner(source, area, target).await; + app_log!( + "ui_separation.cut_image.timing outcome={} elapsed_ms={}", + if result.is_ok() { "ok" } else { "error" }, + started.elapsed().as_millis() + ); + result +} + +async fn cut_processed_image_inner( + source: PathBuf, + area: BindingArea, + target: PathBuf, +) -> Result<(), String> { + app_log!( + "ui_separation.cut_image.start source_file={} target_file={} area=({}, {}, {}, {})", + source + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + area.global_pos_x_px, + area.global_pos_y_px, + area.width_px, + area.height_px + ); + tokio::task::spawn_blocking(move || cut_processed_image_blocking(&source, &area, &target)) + .await + .map_err(|error| format!("裁切处理图任务失败：{error}"))? +} + +fn cut_processed_image_blocking( + source: &Path, + area: &BindingArea, + target: &Path, +) -> Result<(), String> { + let image = image::open(source) + .map_err(|e| format!("读取处理图失败：{e}"))? + .to_rgba8(); + let normalized = normalize_binding_area(&image, *area)?; + let original_area = *area; + let normalized_area = normalized.area; + app_log!( + "ui_separation.cut_image.normalized changed={} clamped={} transparent={} original_area=({}, {}, {}, {}) normalized_area=({}, {}, {}, {})", + normalized.changed, + normalized.clamped, + normalized.transparent, + original_area.global_pos_x_px, + original_area.global_pos_y_px, + original_area.width_px, + original_area.height_px, + normalized_area.global_pos_x_px, + normalized_area.global_pos_y_px, + normalized_area.width_px, + normalized_area.height_px + ); + let cropped = image::imageops::crop_imm( + &image, + normalized_area.global_pos_x_px, + normalized_area.global_pos_y_px, + normalized_area.width_px, + normalized_area.height_px, + ) + .to_image(); + cropped + .save_with_format(target, ImageFormat::Png) + .map_err(|e| format!("写入 cut 图片失败：{e}"))?; + app_log!( + "ui_separation.cut_image.completed target_file={} width={} height={}", + target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + normalized_area.width_px, + normalized_area.height_px + ); + Ok(()) +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs index 4faa40579..84f7fa178 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs @@ -1,9 +1,11 @@ use crate::agent::request_game_creator_llm_text; use crate::config::{apply_game_creator_llm_reasoning_effort, parse_game_creator_llm_api_kind}; use base64::Engine as _; -use platform_llm::{LlmClient, LlmError, LlmRunRequest, LlmRunResponse}; +use platform_llm::{LlmClient, LlmError, LlmMessage, LlmRunRequest, LlmRunResponse}; use schemars::JsonSchema; +use serde::Serialize; use std::fs::File; +use std::future::Future; use std::io::Read; use std::path::{Path, PathBuf}; @@ -25,6 +27,49 @@ pub(crate) async fn request_ui_editor_llm( request_game_creator_llm_text(client, llm, request).await } +/// 按 append-only history 重试结构化 LLM 请求；仅业务校验失败会追加反馈消息。 +pub(crate) async fn run_with_repair_history( + max_retries: usize, + initial_history: Vec, + requester: Requester, + validator: Validator, +) -> Result +where + T: Serialize, + Requester: Fn(Vec) -> Fut, + Fut: Future>, + Validator: Fn(&T) -> Result<(), String>, +{ + let mut history = initial_history; + for attempt in 0..=max_retries { + let value = match requester(history.clone()).await { + Ok(value) => value, + Err(error) if attempt < max_retries => { + app_log!( + "ui_editor.llm.retry request_error attempt={} max_retries={} error={}", + attempt + 1, + max_retries, + error + ); + continue; + } + Err(error) => return Err(error), + }; + match validator(&value) { + Ok(()) => return Ok(value), + Err(error) if attempt < max_retries => { + let serialized = serde_json::to_string(&value) + .map_err(|serialize_error| format!("序列化修复反馈失败：{serialize_error}"))?; + history.push(LlmMessage::system(format!( + "上一次模型输出：\n{serialized}\n\n业务校验失败：\n{error}\n\n请修正并完整返回。" + ))); + } + Err(error) => return Err(error), + } + } + unreachable!("repair history runner always returns within requested retries") +} + pub(crate) fn parse_limited_llm_tool_arguments( arguments: &str, ) -> Result { @@ -144,6 +189,107 @@ mod tests { ); } + #[tokio::test] + async fn repair_history_zero_retries_calls_once_with_initial_history() { + let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let seen = calls.clone(); + let initial_history = vec![LlmMessage::user("初始 prompt")]; + let result = run_with_repair_history( + 0, + initial_history.clone(), + move |history| { + let seen = seen.clone(); + async move { + seen.lock().unwrap().push(history); + Ok::<_, String>(serde_json::json!({"ok": true})) + } + }, + |_| Ok(()), + ) + .await + .expect("single turn should succeed"); + assert_eq!(result, serde_json::json!({"ok": true})); + assert_eq!(calls.lock().unwrap().as_slice(), &[initial_history]); + } + + #[tokio::test] + async fn repair_history_request_error_retries_without_appending_history() { + let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let attempts = std::sync::Arc::new(std::sync::Mutex::new(0usize)); + let seen_calls = calls.clone(); + let seen_attempts = attempts.clone(); + let initial_history = vec![LlmMessage::user("初始 prompt")]; + let result = run_with_repair_history( + 1, + initial_history.clone(), + move |history| { + seen_calls.lock().unwrap().push(history); + let attempt = { + let mut attempts = seen_attempts.lock().unwrap(); + let attempt = *attempts; + *attempts += 1; + attempt + }; + async move { + if attempt == 0 { + Err("网络错误".to_string()) + } else { + Ok::<_, String>(serde_json::json!({"ok": true})) + } + } + }, + |_| Ok(()), + ) + .await + .expect("retry-only error should recover"); + assert_eq!(result, serde_json::json!({"ok": true})); + assert_eq!( + calls.lock().unwrap().as_slice(), + &[initial_history.clone(), initial_history] + ); + } + + #[tokio::test] + async fn repair_history_business_failure_appends_serialized_value_and_error() { + let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let attempts = std::sync::Arc::new(std::sync::Mutex::new(0usize)); + let seen_calls = calls.clone(); + let seen_attempts = attempts.clone(); + let initial_history = vec![LlmMessage::user("初始 prompt")]; + let result = run_with_repair_history( + 1, + initial_history.clone(), + move |history| { + seen_calls.lock().unwrap().push(history); + let mut attempts = seen_attempts.lock().unwrap(); + let attempt = *attempts; + *attempts += 1; + async move { Ok::<_, String>(serde_json::json!({"attempt": attempt})) } + }, + |value: &serde_json::Value| { + if value["attempt"] == 0 { + Err("业务校验失败".to_string()) + } else { + Ok(()) + } + }, + ) + .await + .expect("business feedback should recover"); + assert_eq!(result, serde_json::json!({"attempt": 1})); + let calls = calls.lock().unwrap(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0], initial_history); + assert_eq!(calls[1].len(), 2); + assert_eq!(calls[1][0], LlmMessage::user("初始 prompt")); + assert_eq!( + calls[1][1], + LlmMessage::system( + "上一次模型输出：\n{\"attempt\":0}\n\n业务校验失败：\n业务校验失败\n\n请修正并完整返回。" + ) + ); + } + #[test] fn reference_image_rejects_file_over_five_mib_before_reading() { let directory = tempfile::tempdir().expect("reference image fixture"); diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/component/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/component/mod.rs index 14ab9c854..664f40e42 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/component/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/component/mod.rs @@ -9,3 +9,27 @@ pub enum Component { Image(image::ImageComponent), Text(text::TextComponent), } + +/// LLM 工具返回的节点组件载荷。 +/// +/// 这里不能直接使用 `Option`：部分模型在严格工具 schema 下不会稳定地产生 +/// `null`。用显式的 `PureNode` / `WithComponent` 外部枚举表达两种情况，既保留纯结构节点 +/// 的语义，也让工具调用始终返回一个可判别的对象；落入编辑器 `Node` 时再映射为 +/// `Option`。 +#[derive( + Clone, Debug, PartialEq, schemars::JsonSchema, serde::Deserialize, serde::Serialize, ts_rs::TS, +)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub enum NodeComponent { + PureNode, + WithComponent(Component), +} + +impl NodeComponent { + pub fn into_option(self) -> Option { + match self { + Self::PureNode => None, + Self::WithComponent(component) => Some(component), + } + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/html_renderer/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/html_renderer/mod.rs index e0c8199e7..e908a3117 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/html_renderer/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/html_renderer/mod.rs @@ -199,14 +199,13 @@ fn render_node_with_scale( json!({"childrenDisplayMode": "Exclusive", "childrenRendered": "all"}), ) }); - let components = node - .components - .iter() + let component = node + .component + .as_ref() .map(|component| render_component(state, component)) - .collect::, _>>()? - .into_iter() + .transpose()? .map(|fragment| fragment.into_string()) - .collect::>(); + .unwrap_or_default(); let children = node .children .iter() @@ -226,7 +225,7 @@ fn render_node_with_scale( (comment) @if let Some(group_comment) = exclusive_comment { (group_comment) } div ui-node-id=(node.id.as_str()) style=(style) { - (PreEscaped(components.concat())) + (PreEscaped(component)) (PreEscaped(children.concat())) } }) diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs index 9375e913f..be70cdca6 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs @@ -11,7 +11,7 @@ pub struct Node { pub id: NodeId, pub layout: ControlLayout, pub metadata: NodeMetadata, - pub components: Vec, + pub component: Option, pub children_display_mode: ChildrenDisplayMode, pub children: Vec, } @@ -50,7 +50,7 @@ pub struct NodeMetadata { pub name: String, pub description: String, pub layout_status: StageStatus, - pub components_status: StageStatus, + pub component_status: StageStatus, pub allow_llm_edit_layout: bool, pub allow_llm_edit_component: bool, pub source: NodeSource, diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs index 04b72c0b3..ab4c07c78 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs @@ -25,7 +25,6 @@ const UI_DESIGN_STATE_MAX_IMAGES: usize = 4; const UI_DESIGN_STATE_MAX_SPRITES: usize = 1_024; pub(crate) const UI_DESIGN_STATE_MAX_NODES: usize = 10_000; const UI_DESIGN_STATE_MAX_DEPTH: usize = 128; -pub(crate) const UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE: usize = 64; const UI_DESIGN_STATE_MAX_SAFE_REVISION: u64 = 9_007_199_254_740_991; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] @@ -196,7 +195,7 @@ pub(crate) fn generate_ui_design_code_at( }) } -fn generated_file_stem(asset_id: &str) -> String { +pub(crate) fn generated_file_stem(asset_id: &str) -> String { let mut stem = String::new(); for character in asset_id.chars() { if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') { @@ -747,12 +746,8 @@ fn validate_node( } } } - if node.components.len() > UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE { - return Err(format!( - "单个 UI 节点最多支持 {UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE} 个组件" - )); - } - for component in &node.components { + validate_component_status(node.component.as_ref(), &node.metadata.component_status)?; + if let Some(component) = &node.component { match component { Component::Image(image) => { if image @@ -778,6 +773,22 @@ fn validate_node( Ok(()) } +fn validate_component_status( + component: Option<&Component>, + status: &crate::ui_editor::layout::node::StageStatus, +) -> Result<(), String> { + if component.is_none() + && matches!( + status, + crate::ui_editor::layout::node::StageStatus::NeedReview(_) + | crate::ui_editor::layout::node::StageStatus::Blocked(_) + ) + { + return Err("纯结构节点的 component_status 必须为 NoProblem".to_string()); + } + Ok(()) +} + fn validate_id(value: &str, label: &str) -> Result<(), String> { if value.is_empty() || value.trim() != value || value.chars().any(char::is_control) { return Err(format!("{label} 无效")); @@ -881,12 +892,12 @@ mod tests { "name": "页面根节点", "description": "", "layout_status": "NoProblem", - "components_status": "NoProblem", + "component_status": "NoProblem", "allow_llm_edit_layout": true, "allow_llm_edit_component": true, "source": "System" }, - "components": [], + "component": null, "children_display_mode": "Stack", "children": [{ "id": "dragged-node", @@ -907,17 +918,17 @@ mod tests { "name": "拖拽节点", "description": "", "layout_status": "NoProblem", - "components_status": "NoProblem", + "component_status": "NoProblem", "allow_llm_edit_layout": true, "allow_llm_edit_component": true, "source": "Human" }, - "components": [{ + "component": { "Image": { "target_graphic": "spirit", "image_type": { "Simple": { "preserve_aspect": false } } } - }], + }, "children_display_mode": "Stack", "children": [] }] @@ -1095,12 +1106,12 @@ mod tests { "name": "根节点", "description": "", "layout_status": "NoProblem", - "components_status": "NoProblem", + "component_status": "NoProblem", "allow_llm_edit_layout": true, "allow_llm_edit_component": true, "source": "System" }, - "components": [], + "component": null, "children_display_mode": "Stack", "children": [] } @@ -1296,12 +1307,12 @@ mod tests { "name": "根节点", "description": "", "layout_status": "NoProblem", - "components_status": "NoProblem", + "component_status": "NoProblem", "allow_llm_edit_layout": false, "allow_llm_edit_component": false, "source": "System" }, - "components": [{ + "component": { "Text": { "content": "标题", "font": {"Bound": "missing-font"}, @@ -1313,7 +1324,7 @@ mod tests { "vertical_overflow": "Truncate", "line_spacing": 1.0 } - }], + }, "children_display_mode": "Stack", "children": [] } @@ -1340,4 +1351,30 @@ mod tests { .expect_err("missing Text font reference must be rejected"); assert!(error.contains("Text 组件引用了不存在的字体素材")); } + + #[test] + fn component_status_matrix_keeps_pure_nodes_unproblematic() { + use crate::ui_editor::component::image::ImageComponent; + + assert!(validate_component_status( + None, + &crate::ui_editor::layout::node::StageStatus::NoProblem, + ) + .is_ok()); + assert!(validate_component_status( + None, + &crate::ui_editor::layout::node::StageStatus::NeedReview("原因".to_string()), + ) + .is_err()); + assert!(validate_component_status( + None, + &crate::ui_editor::layout::node::StageStatus::Blocked("原因".to_string()), + ) + .is_err()); + assert!(validate_component_status( + Some(&Component::Image(ImageComponent::new())), + &crate::ui_editor::layout::node::StageStatus::NeedReview("等待素材".to_string()), + ) + .is_ok()); + } } diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/workflow.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/workflow.rs index 829782148..f07769b28 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/workflow.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/workflow.rs @@ -296,7 +296,7 @@ pub(crate) async fn run_ui_workflow_at_with_provider( ) { let route = UiWorkflowFinalStageRoute { resource_id: statuses[0].ui_asset_id.clone(), - initial_step: "visual-binding".to_string(), + initial_step: "asset-separation".to_string(), render_mode: "final-preview".to_string(), }; if finalized { @@ -1145,8 +1145,8 @@ fn apply_binding_changes( ) -> usize { let mut changed = 0; if let Some(change) = changes.get(&node.id) { - node.components = change.components.clone(); - node.metadata.components_status = change.components_status.clone(); + node.component = change.component.clone().into_option(); + node.metadata.component_status = change.component_status.clone(); changed += 1; } for child in &mut node.children { @@ -1163,7 +1163,7 @@ fn apply_binding_changes( fn state_has_renderable_component(state: &crate::ui_editor::state::State) -> bool { fn has_component(node: &Node) -> bool { - !node.components.is_empty() || node.children.iter().any(has_component) + node.component.is_some() || node.children.iter().any(has_component) } state.ui_trees.iter().any(|tree| has_component(&tree.root)) } @@ -1315,14 +1315,14 @@ fn derive_page_status( } fn collect_binding_blockers(node: &Node, component_count: &mut usize, blockers: &mut Vec) { - *component_count += node.components.len(); + *component_count += usize::from(node.component.is_some()); if let StageStatus::NeedReview(reason) | StageStatus::Blocked(reason) = &node.metadata.layout_status { blockers.push(format!("{} 布局未通过：{reason}", node.metadata.name)); } if let StageStatus::NeedReview(reason) | StageStatus::Blocked(reason) = - &node.metadata.components_status + &node.metadata.component_status { blockers.push(format!("{} 组件未通过：{reason}", node.metadata.name)); } diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/binding.ts b/apps/ai-game-creator-shell/src/features/ui-editor/binding.ts index 47a8a3a17..3f3422b18 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/binding.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/binding.ts @@ -7,8 +7,10 @@ function applyChanges(node: Node, result: BindingDTO): void { (candidate) => candidate.node_id === node.id, ); if (change) { - node.components = structuredClone(change.components); - node.metadata.components_status = structuredClone(change.components_status); + node.component = structuredClone( + change.component === 'PureNode' ? null : change.component.WithComponent, + ); + node.metadata.component_status = structuredClone(change.component_status); } for (const child of node.children) applyChanges(child, result); } diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/recognition.ts b/apps/ai-game-creator-shell/src/features/ui-editor/recognition.ts index 2238c6b40..68bd9e003 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/recognition.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/recognition.ts @@ -3,7 +3,7 @@ import type { State } from './types/State'; /** * 识别结果是整棵结构草稿树的替换结果，不与旧树逐节点合并。 - * 识别阶段有意不携带视觉组件；组件绑定由后续 visual-binding 阶段完成。 + * 识别阶段有意不携带 SpriteAsset；视觉素材由后续 asset-separation 阶段自动切分并回填。 */ export function applyRecognitionResult( state: State, diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/requisites.ts b/apps/ai-game-creator-shell/src/features/ui-editor/requisites.ts index af3e0c929..759792923 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/requisites.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/requisites.ts @@ -72,7 +72,7 @@ export function validateComponentRecognitionPrerequisites( return issues; } -export function validateAssetRecognitionPrerequisites( +export function validateAssetSeparationPrerequisites( state: State, ): UiEditorPrerequisiteIssue[] { const issues = validateComponentRecognitionPrerequisites(state); @@ -91,57 +91,9 @@ export function validateAssetRecognitionPrerequisites( }); } } - if (Object.keys(state.sprite_assets).length === 0) { - issues.push({ - code: 'missing-sprite-assets', - message: '请先导入独立素材', - }); - } return issues; } -export function validateLayoutGenerationPrerequisites( - state: State, -): UiEditorPrerequisiteIssue[] { - const issues = validateAssetRecognitionPrerequisites(state); - const visit = (nodes: State['ui_trees'][number]['root']['children']) => { - for (const node of nodes) { - for (const component of node.components) { - if ( - 'Image' in component && - component.Image.target_graphic !== null && - !(component.Image.target_graphic in state.sprite_assets) - ) { - issues.push({ - code: 'missing-target-graphic', - message: '图片组件引用的独立素材不存在', - resourceId: component.Image.target_graphic, - }); - } - if ('Text' in component) { - const font = component.Text.font; - if (typeof font !== 'string' && !(font.Bound in state.font_assets)) { - issues.push({ - code: 'missing-font', - message: '文本组件引用的字体不存在', - resourceId: font.Bound, - }); - } - } - } - visit(node.children); - } - }; - for (const tree of state.ui_trees) visit([tree.root]); - return issues; -} - -export function validateLayoutReviewPrerequisites( - state: State, -): UiEditorPrerequisiteIssue[] { - return validateLayoutGenerationPrerequisites(state); -} - function layoutStatusIssue( status: StageStatus, ): UiEditorPrerequisiteIssue | null { @@ -213,17 +165,43 @@ export function validateStructureRecognitionResult( return issues; } -export function validateVisualBindingResult( +export function validateAssetSeparationResult( state: State, ): UiEditorPrerequisiteIssue[] { const issues: UiEditorPrerequisiteIssue[] = []; for (const tree of state.ui_trees) { visitNodes([tree.root], (node) => { - if (node.components.length == 0) { + if (node.component === null) { return; } - const issue = componentStatusIssue(node.metadata.components_status); + const issue = componentStatusIssue(node.metadata.component_status); if (issue) issues.push(issue); + const component = node.component; + if ('Image' in component) { + const targetGraphic = component.Image.target_graphic; + if (targetGraphic === null) { + issues.push({ + code: 'missing-separated-image', + message: '图片组件尚未完成素材切分', + }); + } else if (!(targetGraphic in state.sprite_assets)) { + issues.push({ + code: 'missing-target-graphic', + message: '图片组件引用的切分素材不存在', + resourceId: targetGraphic, + }); + } + } + if ('Text' in component) { + const font = component.Text.font; + if (typeof font !== 'string' && !(font.Bound in state.font_assets)) { + issues.push({ + code: 'missing-font', + message: '文本组件引用的字体不存在', + resourceId: font.Bound, + }); + } + } }); } return issues; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/bindingOverview.ts b/apps/ai-game-creator-shell/src/features/ui-editor/separationOverview.ts similarity index 58% rename from apps/ai-game-creator-shell/src/features/ui-editor/bindingOverview.ts rename to apps/ai-game-creator-shell/src/features/ui-editor/separationOverview.ts index 84e6cc6ce..1330e2b7c 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/bindingOverview.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/separationOverview.ts @@ -8,27 +8,27 @@ import type { Component } from './types/Component'; import type { SpriteAsset } from './types/SpriteAsset'; import type { UITree } from './types/UITree'; -export type ComponentBindingCounts = { +export type ComponentSeparationCounts = { componentsNeedingAssets: number; assetSlots: number; boundSlots: number; pendingSlots: number; }; -export type BindingOverview = ComponentBindingCounts & { +export type SeparationOverview = ComponentSeparationCounts & { needsAttention: number; blocked: number; independentAssets: number; }; -const EMPTY_COMPONENT_BINDING_COUNTS: ComponentBindingCounts = { +const EMPTY_COMPONENT_SEPARATION_COUNTS: ComponentSeparationCounts = { componentsNeedingAssets: 0, assetSlots: 0, boundSlots: 0, pendingSlots: 0, }; -function countRequiredAssetSlot(isBound: boolean): ComponentBindingCounts { +function countRequiredAssetSlot(isBound: boolean): ComponentSeparationCounts { return { componentsNeedingAssets: 1, assetSlots: 1, @@ -37,40 +37,40 @@ function countRequiredAssetSlot(isBound: boolean): ComponentBindingCounts { }; } -export function getImageBindingCounts( +export function getImageSeparationCounts( component: Extract['Image'], -): ComponentBindingCounts { +): ComponentSeparationCounts { return countRequiredAssetSlot(component.target_graphic !== null); } -export function getTextBindingCounts( +export function getTextSeparationCounts( component: Extract['Text'], -): ComponentBindingCounts { +): ComponentSeparationCounts { if ( typeof component.font !== 'object' || component.font === null || !('Bound' in component.font) ) { - return EMPTY_COMPONENT_BINDING_COUNTS; + return EMPTY_COMPONENT_SEPARATION_COUNTS; } return countRequiredAssetSlot(true); } -export function getComponentBindingCounts( +export function getComponentSeparationCounts( component: Component, -): ComponentBindingCounts { +): ComponentSeparationCounts { if ('Image' in component) { - return getImageBindingCounts(component.Image); + return getImageSeparationCounts(component.Image); } if ('Text' in component) { - return getTextBindingCounts(component.Text); + return getTextSeparationCounts(component.Text); } - return EMPTY_COMPONENT_BINDING_COUNTS; + return EMPTY_COMPONENT_SEPARATION_COUNTS; } -function addBindingCounts( - overview: ComponentBindingCounts, - counts: ComponentBindingCounts, +function addSeparationCounts( + overview: ComponentSeparationCounts, + counts: ComponentSeparationCounts, ) { overview.componentsNeedingAssets += counts.componentsNeedingAssets; overview.assetSlots += counts.assetSlots; @@ -78,42 +78,46 @@ function addBindingCounts( overview.pendingSlots += counts.pendingSlots; } -export function getBindingOverview( +export function getSeparationOverview( uiTrees: UITree[], spriteAssets: Record, -): BindingOverview { - const overview: BindingOverview = { - ...EMPTY_COMPONENT_BINDING_COUNTS, +): SeparationOverview { + const overview: SeparationOverview = { + ...EMPTY_COMPONENT_SEPARATION_COUNTS, needsAttention: 0, blocked: 0, independentAssets: Object.keys(spriteAssets).length, }; for (const { node } of collectUiTreeNodeTargets(uiTrees)) { - const status = node.metadata.components_status; + const status = node.metadata.component_status; if (isBlocked(status)) overview.blocked += 1; if (isBlocked(status) || isNeedReview(status)) { overview.needsAttention += 1; } - for (const component of node.components) { - addBindingCounts(overview, getComponentBindingCounts(component)); + if (node.component) { + addSeparationCounts( + overview, + getComponentSeparationCounts(node.component), + ); } } return overview; } -export function nodeHasPendingBinding(target: UiTreeNodeTarget): boolean { - return target.node.components.some( - (component) => getComponentBindingCounts(component).pendingSlots > 0, +export function nodeHasPendingSeparation(target: UiTreeNodeTarget): boolean { + return ( + target.node.component !== null && + getComponentSeparationCounts(target.node.component).pendingSlots > 0 ); } export function nodeNeedsComponentReview(target: UiTreeNodeTarget): boolean { - const status = target.node.metadata.components_status; + const status = target.node.metadata.component_status; return isBlocked(status) || isNeedReview(status); } export function nodeHasBlockedComponents(target: UiTreeNodeTarget): boolean { - return isBlocked(target.node.metadata.components_status); + return isBlocked(target.node.metadata.component_status); } diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/stageStatusOverview.ts b/apps/ai-game-creator-shell/src/features/ui-editor/stageStatusOverview.ts index c2f9e9c00..80dc27fac 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/stageStatusOverview.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/stageStatusOverview.ts @@ -6,7 +6,7 @@ import type { UITree } from './types/UITree'; export type StageStatusField = Extract< keyof NodeMetadata, - 'layout_status' | 'components_status' + 'layout_status' | 'component_status' >; export type UiTreeNodeTarget = { diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/BindingChange.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/BindingChange.ts index ec71edbfd..3bb2c5c4d 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/types/BindingChange.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/BindingChange.ts @@ -1,6 +1,6 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Component } from "./Component"; +import type { NodeComponent } from "./NodeComponent"; import type { NodeId } from "./NodeId"; import type { StageStatus } from "./StageStatus"; -export type BindingChange = { node_id: NodeId, components: Array, components_status: StageStatus, }; +export type BindingChange = { node_id: NodeId, component: NodeComponent, component_status: StageStatus, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/BoundNode.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/BoundNode.ts new file mode 100644 index 000000000..b27bee935 --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/BoundNode.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { NodeId } from "./NodeId"; + +export type BoundNode = { node_id: NodeId, cut_image_path: string, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/Node.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/Node.ts index c05d8743a..bac60a9ca 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/types/Node.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/Node.ts @@ -5,4 +5,4 @@ import type { ControlLayout } from "./ControlLayout"; import type { NodeId } from "./NodeId"; import type { NodeMetadata } from "./NodeMetadata"; -export type Node = { id: NodeId, layout: ControlLayout, metadata: NodeMetadata, components: Array, children_display_mode: ChildrenDisplayMode, children: Array, }; +export type Node = { id: NodeId, layout: ControlLayout, metadata: NodeMetadata, component: Component | null, children_display_mode: ChildrenDisplayMode, children: Array, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeComponent.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeComponent.ts new file mode 100644 index 000000000..b3116576c --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeComponent.ts @@ -0,0 +1,12 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Component } from "./Component"; + +/** + * LLM 工具返回的节点组件载荷。 + * + * 这里不能直接使用 `Option`：部分模型在严格工具 schema 下不会稳定地产生 + * `null`。用显式的 `PureNode` / `WithComponent` 外部枚举表达两种情况，既保留纯结构节点 + * 的语义，也让工具调用始终返回一个可判别的对象；落入编辑器 `Node` 时再映射为 + * `Option`。 + */ +export type NodeComponent = "PureNode" | { "WithComponent": Component }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeMetadata.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeMetadata.ts index 43aee349e..f822b7d4b 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeMetadata.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeMetadata.ts @@ -2,4 +2,4 @@ import type { NodeSource } from "./NodeSource"; import type { StageStatus } from "./StageStatus"; -export type NodeMetadata = { name: string, description: string, layout_status: StageStatus, components_status: StageStatus, allow_llm_edit_layout: boolean, allow_llm_edit_component: boolean, source: NodeSource, }; +export type NodeMetadata = { name: string, description: string, layout_status: StageStatus, component_status: StageStatus, allow_llm_edit_layout: boolean, allow_llm_edit_component: boolean, source: NodeSource, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/ProblematicNode.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/ProblematicNode.ts new file mode 100644 index 000000000..c77e0a49b --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/ProblematicNode.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { NodeId } from "./NodeId"; + +export type ProblematicNode = { node_id: NodeId, problem_description: string, rework_count: number, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationDTO.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationDTO.ts new file mode 100644 index 000000000..7408ed0d0 --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationDTO.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BoundNode } from "./BoundNode"; +import type { ProblematicNode } from "./ProblematicNode"; + +export type SeparationDTO = { bound_nodes: Array, problematic_nodes: Array, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationNode.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationNode.ts new file mode 100644 index 000000000..09a0d8a53 --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationNode.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { NodeId } from "./NodeId"; +import type { SeparationNote } from "./SeparationNote"; +import type { TextMaskArea } from "./TextMaskArea"; + +export type SeparationNode = { id: NodeId, global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, note: SeparationNote, text_mask_areas: Array, children: Array, rework_count: number, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationNote.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationNote.ts new file mode 100644 index 000000000..0cb01f4ee --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationNote.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SeparationNote = { description: string, rework_notes: Array, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationRecoveryDTO.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationRecoveryDTO.ts new file mode 100644 index 000000000..38c5bb8b2 --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationRecoveryDTO.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SeparationRecoveryDTO = { exists: boolean, bound_node_count: number, problematic_node_count: number, has_pending_tree: boolean, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationState.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationState.ts new file mode 100644 index 000000000..fa180b51e --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationState.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BoundNode } from "./BoundNode"; +import type { ProblematicNode } from "./ProblematicNode"; +import type { SeparationTree } from "./SeparationTree"; + +export type SeparationState = { schema_version: string, trees: Array, bound: Array, problematic_nodes: Array, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationTree.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationTree.ts new file mode 100644 index 000000000..2196bac3b --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/SeparationTree.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { SeparationNode } from "./SeparationNode"; +import type { UIDesignImageId } from "./UIDesignImageId"; + +export type SeparationTree = { src_ui_design: UIDesignImageId, root: SeparationNode, root_extractable: boolean, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/TextMaskArea.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/TextMaskArea.ts new file mode 100644 index 000000000..5e450d070 --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/TextMaskArea.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type TextMaskArea = { global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/UIRect.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/UIRect.ts new file mode 100644 index 000000000..d1db67fd7 --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/UIRect.ts @@ -0,0 +1,9 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * UI 布局在父级坐标系中的轴对齐矩形。 + * + * `min` 是矩形的最小角，`size` 是沿两个坐标轴的尺寸。这里不规定 Y 轴方向， + * 因而既能用于 Y 轴向上的游戏坐标，也能用于 Y 轴向下的画布坐标。 + */ +export type UIRect = { min: [number, number], size: [number, number], }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/useUiEditorState.ts b/apps/ai-game-creator-shell/src/features/ui-editor/useUiEditorState.ts index a65ed50dc..a83ab93b4 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/useUiEditorState.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/useUiEditorState.ts @@ -64,14 +64,12 @@ export type NodeMetadataPatch = Partial< | 'name' | 'description' | 'layout_status' - | 'components_status' + | 'component_status' | 'allow_llm_edit_layout' | 'allow_llm_edit_component' > >; -export type ComponentIndex = number; - export type NodeTransformOptions = { keepChildrenUnchanged?: boolean; }; @@ -106,6 +104,12 @@ function visitNodes(node: Node, visit: (node: Node) => void): void { for (const child of node.children) visitNodes(child, visit); } +function isProblematicComponentStatus( + status: NodeMetadata['component_status'], +): boolean { + return typeof status !== 'string'; +} + function existingNodeIds(state: State): Set { const ids = new Set(); for (const tree of state.ui_trees) { @@ -144,12 +148,12 @@ function createPageRoot(state: State): Node { name: '页面根节点', description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'System', }, - components: [], + component: null, children_display_mode: 'Stack', children: [], }; @@ -175,12 +179,12 @@ function createHumanNode(state: State): Node { name: '新节点', description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'Human', }, - components: [], + component: null, children_display_mode: 'Stack', children: [], }; @@ -283,9 +287,7 @@ function sameResource( function visitComponents(nodes: Node[], visit: (component: Component) => void) { for (const node of nodes) { - for (const component of node.components) { - visit(component); - } + if (node.component) visit(node.component); visitComponents(node.children, visit); } } @@ -524,6 +526,40 @@ function spriteResourceValidationError(sprite: SpriteAsset) { return border.ok ? null : border.message; } +/** + * Apply the SpriteAsset resource checks without acquiring the editor mutation + * guard. Workflow adapters use this while a State lock is already held so + * resource insertion and component backfill can be committed atomically. + */ +export function addSpriteAssetsToState( + current: State, + assets: readonly SpriteAsset[], +): UiEditorOperationResult { + const unique = new Map(); + for (const asset of assets) { + const candidate = + unique.get(asset.asset_id) ?? current.sprite_assets[asset.asset_id]; + if (candidate && !sameResource(candidate, asset)) { + return { ok: false, reason: 'duplicate' }; + } + unique.set(asset.asset_id, asset); + } + const invalidSprite = [...unique.values()] + .map((asset) => ({ asset, error: spriteResourceValidationError(asset) })) + .find((item) => item.error); + if (invalidSprite?.error) { + return { + ok: false, + reason: `invalid:${invalidSprite.asset.asset_id}：${invalidSprite.error}`, + }; + } + const next = cloneState(current); + for (const asset of unique.values()) { + next.sprite_assets[asset.asset_id] = structuredClone(asset); + } + return { ok: true, value: next }; +} + function fontResourceValidationError(font: FontAsset) { if (font.asset_id.trim().length === 0) return '缺少字体 ID'; if (font.path.trim().length === 0) return '缺少字体路径'; @@ -782,32 +818,9 @@ export function useUiEditorState(initialState: State = EMPTY_UI_EDITOR_STATE) { const blocked = guard(); if (blocked) return blocked; const current = stateRef.current; - const unique = new Map(); - for (const asset of assets) { - const candidate = - unique.get(asset.asset_id) ?? current.sprite_assets[asset.asset_id]; - if (candidate && !sameResource(candidate, asset)) { - return { ok: false, reason: 'duplicate' }; - } - unique.set(asset.asset_id, asset); - } - const invalidSprite = [...unique.values()] - .map((asset) => ({ - asset, - error: spriteResourceValidationError(asset), - })) - .find((item) => item.error); - if (invalidSprite?.error) { - return { - ok: false, - reason: `invalid:${invalidSprite.asset.asset_id}：${invalidSprite.error}`, - }; - } - const next = cloneState(current); - for (const asset of unique.values()) { - next.sprite_assets[asset.asset_id] = structuredClone(asset); - } - commit(next); + const result = addSpriteAssetsToState(current, assets); + if (!result.ok) return result; + commit(result.value); return { ok: true, value: undefined }; }, [commit, guard], @@ -1110,15 +1123,15 @@ export function useUiEditorState(initialState: State = EMPTY_UI_EDITOR_STATE) { [commit, guard], ); - const setNodeComponents = useCallback( + const setNodeComponent = useCallback( ( treeId: UIDesignImageId, nodeId: NodeId, - components: Component[], + component: Component | null, ): UiEditorOperationResult => { const blocked = guard(); if (blocked) return blocked; - if (!components.every(isValidComponent)) { + if (component && !isValidComponent(component)) { return { ok: false, reason: 'invalid' }; } const current = stateRef.current; @@ -1132,126 +1145,9 @@ export function useUiEditorState(initialState: State = EMPTY_UI_EDITOR_STATE) { const nextTree = next.ui_trees.find( (candidate) => candidate.src_ui_design === treeId, )!; - findNodeLocation(nextTree.root, nodeId)!.node.components = - structuredClone(components); - commit(next); - return { ok: true, value: undefined }; - }, - [commit, guard], - ); - - const insertComponent = useCallback( - ( - treeId: UIDesignImageId, - nodeId: NodeId, - index: ComponentIndex, - component: Component, - ): UiEditorOperationResult => { - const blocked = guard(); - if (blocked) return blocked; - if ( - !isValidComponent(component) || - !Number.isInteger(index) || - index < 0 - ) { - return { ok: false, reason: 'invalid' }; - } - const current = stateRef.current; - const tree = current.ui_trees.find( - (candidate) => candidate.src_ui_design === treeId, - ); - if (!tree) return { ok: false, reason: 'missing' }; - const location = findNodeLocation(tree.root, nodeId); - if (!location) return { ok: false, reason: 'missing' }; - if (index > location.node.components.length) { - return { ok: false, reason: 'invalid' }; - } - const next = cloneState(current); - const nextNode = findNodeLocation( - next.ui_trees.find((candidate) => candidate.src_ui_design === treeId)! - .root, - nodeId, - )!.node; - nextNode.components.splice(index, 0, structuredClone(component)); - commit(next); - return { ok: true, value: undefined }; - }, - [commit, guard], - ); - - const deleteComponent = useCallback( - ( - treeId: UIDesignImageId, - nodeId: NodeId, - index: ComponentIndex, - ): UiEditorOperationResult => { - const blocked = guard(); - if (blocked) return blocked; - if (!Number.isInteger(index) || index < 0) - return { ok: false, reason: 'invalid' }; - const current = stateRef.current; - const tree = current.ui_trees.find( - (candidate) => candidate.src_ui_design === treeId, - ); - if (!tree) return { ok: false, reason: 'missing' }; - const location = findNodeLocation(tree.root, nodeId); - if (!location) return { ok: false, reason: 'missing' }; - if (index >= location.node.components.length) { - return { ok: false, reason: 'invalid' }; - } - const next = cloneState(current); - const nextNode = findNodeLocation( - next.ui_trees.find((candidate) => candidate.src_ui_design === treeId)! - .root, - nodeId, - )!.node; - nextNode.components.splice(index, 1); - commit(next); - return { ok: true, value: undefined }; - }, - [commit, guard], - ); - - const moveComponent = useCallback( - ( - treeId: UIDesignImageId, - nodeId: NodeId, - fromIndex: ComponentIndex, - toIndex: ComponentIndex, - ): UiEditorOperationResult => { - const blocked = guard(); - if (blocked) return blocked; - if ( - !Number.isInteger(fromIndex) || - !Number.isInteger(toIndex) || - fromIndex < 0 || - toIndex < 0 - ) { - return { ok: false, reason: 'invalid' }; - } - const current = stateRef.current; - const tree = current.ui_trees.find( - (candidate) => candidate.src_ui_design === treeId, - ); - if (!tree) return { ok: false, reason: 'missing' }; - const location = findNodeLocation(tree.root, nodeId); - if (!location) return { ok: false, reason: 'missing' }; - if ( - fromIndex >= location.node.components.length || - toIndex >= location.node.components.length - ) { - return { ok: false, reason: 'invalid' }; - } - if (fromIndex === toIndex) return { ok: true, value: undefined }; - const next = cloneState(current); - const nextNode = findNodeLocation( - next.ui_trees.find((candidate) => candidate.src_ui_design === treeId)! - .root, - nodeId, - )!.node; - const [component] = nextNode.components.splice(fromIndex, 1); - if (!component) return { ok: false, reason: 'invalid' }; - nextNode.components.splice(toIndex, 0, component); + const nextNode = findNodeLocation(nextTree.root, nodeId)!.node; + nextNode.component = structuredClone(component); + nextNode.metadata.component_status = 'NoProblem'; commit(next); return { ok: true, value: undefined }; }, @@ -1279,13 +1175,20 @@ export function useUiEditorState(initialState: State = EMPTY_UI_EDITOR_STATE) { .root, nodeId, )!.node; + if ( + patch.component_status !== undefined && + node.component === null && + isProblematicComponentStatus(patch.component_status) + ) { + return { ok: false, reason: 'invalid' }; + } if (patch.name !== undefined) node.metadata.name = patch.name; if (patch.description !== undefined) node.metadata.description = patch.description; if (patch.layout_status !== undefined) node.metadata.layout_status = patch.layout_status; - if (patch.components_status !== undefined) - node.metadata.components_status = patch.components_status; + if (patch.component_status !== undefined) + node.metadata.component_status = patch.component_status; if (patch.allow_llm_edit_layout !== undefined) node.metadata.allow_llm_edit_layout = patch.allow_llm_edit_layout; if (patch.allow_llm_edit_component !== undefined) @@ -1564,10 +1467,7 @@ export function useUiEditorState(initialState: State = EMPTY_UI_EDITOR_STATE) { deleteNode, setNodeTransform, setNodeLayout, - setNodeComponents, - insertComponent, - deleteComponent, - moveComponent, + setNodeComponent, setNodeMetadata, setNodeChildrenDisplayMode, moveNode, diff --git a/apps/ai-game-creator-shell/src/view/project-development/index.tsx b/apps/ai-game-creator-shell/src/view/project-development/index.tsx index f539fc02f..cc8d7ec43 100644 --- a/apps/ai-game-creator-shell/src/view/project-development/index.tsx +++ b/apps/ai-game-creator-shell/src/view/project-development/index.tsx @@ -3335,7 +3335,7 @@ export default function ProjectDevelopmentView({ resource.label, ...(result.asset.source.generationKind === 'ui-workflow.completed' ? { - initialStep: 'visual-binding', + initialStep: 'asset-separation', initialFurthestStepIndex: 2, } : {}), @@ -3376,7 +3376,7 @@ export default function ProjectDevelopmentView({ (asset) => asset.id === resource.manifestAssetId, )?.source.generationKind === 'ui-workflow.completed' ? { - initialStep: 'visual-binding' as const, + initialStep: 'asset-separation' as const, initialFurthestStepIndex: 2, } : {}), @@ -3445,7 +3445,7 @@ export default function ProjectDevelopmentView({ resourceLabel: completed.localPath.split(/[\\/]/u).filter(Boolean).pop() ?? 'UI 设计资源', - initialStep: 'visual-binding', + initialStep: 'asset-separation', initialFurthestStepIndex: 2, }); }, [advanceFocusGeneration, manifest.assets, uiEditorRoute]); diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/ImportOverview.tsx b/apps/ai-game-creator-shell/src/view/ui-editor/components/ImportOverview.tsx index cd76e9fec..0823299bd 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/ImportOverview.tsx +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/ImportOverview.tsx @@ -6,7 +6,7 @@ import type { UITree } from '../../../features/ui-editor/types/UITree'; function countUiComponents(nodes: UiNode[]): number { return nodes.reduce( (total, node) => - total + node.components.length + countUiComponents(node.children), + total + (node.component ? 1 : 0) + countUiComponents(node.children), 0, ); } diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/InputSidebar.tsx b/apps/ai-game-creator-shell/src/view/ui-editor/components/InputSidebar.tsx index 495db5bd9..b473accb2 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/InputSidebar.tsx +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/InputSidebar.tsx @@ -62,12 +62,12 @@ export function InputSidebar({ input }: { input: UiEditorInputProjection }) { name: 'UI Trees', description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: false, allow_llm_edit_component: false, source: 'System', }, - components: [], + component: null, children_display_mode: 'Stack', children: uiTrees.map((tree) => tree.root), }; diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/Components/ComponentPanel.tsx b/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/Components/ComponentPanel.tsx index ab030488d..89d20155c 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/Components/ComponentPanel.tsx +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/Components/ComponentPanel.tsx @@ -1,12 +1,5 @@ -import { - ArrowDown, - ArrowUp, - ChevronDown, - ChevronRight, - Plus, - Trash2, -} from 'lucide-react'; -import { useEffect, useState } from 'react'; +import { ChevronDown, ChevronRight, Plus, Trash2 } from 'lucide-react'; +import { useState } from 'react'; import type { Component } from '../../../../../features/ui-editor/types/Component'; import type { UiEditorOperationResult } from '../../../../../features/ui-editor/useUiEditorState'; @@ -18,111 +11,30 @@ import { TextPanel } from './TextPanel'; import { createDefaultTextComponent } from './TextPanelDefaults'; export function ComponentPanel(props: ComponentPanelProps) { - const { - components, - readOnly: propReadOnly, - onSetComponents, - onInsertComponent, - onDeleteComponent, - onMoveComponent, - } = props; + const { component, readOnly: propReadOnly, onSetComponent } = props; const inspectorReadOnly = useInspectorReadOnly(); const readOnly = propReadOnly || inspectorReadOnly; const [addKind, setAddKind] = useState<'Image' | 'Text'>('Image'); - const [expandedIndexes, setExpandedIndexes] = useState>( - () => new Set(), - ); + const [expanded, setExpanded] = useState(Boolean(component)); const [error, setError] = useState(null); - useEffect(() => { - setExpandedIndexes((current) => { - const next = new Set( - [...current].filter((index) => index >= 0 && index < components.length), - ); - if (next.size === current.size) return current; - return next; - }); - }, [components.length]); - - const updateComponent = (index: number, next: Component) => { + function setComponent(next: Component | null) { if (readOnly) return undefined; - const nextComponents = components.slice(); - nextComponents[index] = next; - const result = onSetComponents(nextComponents); - if (result && !result.ok) setError('组件字段无效，更新未应用。'); + const result = onSetComponent(next); + if (result && !result.ok) setError('组件更新失败。'); else setError(null); return result; - }; - - function addComponent() { - if (readOnly) return; - let component: Component; - switch (addKind) { - case 'Image': - component = { Image: createDefaultImageComponent() }; - break; - case 'Text': - component = { Text: createDefaultTextComponent() }; - break; - } - const result = onInsertComponent(components.length, component); - if (result?.ok) { - setExpandedIndexes((current) => new Set(current).add(components.length)); - setError(null); - } else if (result) { - setError('组件新增失败。'); - } } - function deleteComponentAt(index: number) { - if (readOnly) return; - const result = onDeleteComponent(index); - if (result?.ok) { - setExpandedIndexes((current) => { - const next = new Set(); - for (const expanded of current) { - if (expanded === index) continue; - if (expanded > index) next.add(expanded - 1); - else next.add(expanded); - } - return next; - }); - setError(null); - } else if (result) { - setError('组件删除失败。'); - } + function createComponent(): Component { + return addKind === 'Image' + ? { Image: createDefaultImageComponent() } + : { Text: createDefaultTextComponent() }; } - function moveComponent(index: number, direction: 'up' | 'down') { - if (readOnly) return; - // Components are rendered in array order. The last item therefore sits - // visually at the top of the stack. - let nextIndex: number; - switch (direction) { - case 'up': - nextIndex = index + 1; - break; - case 'down': - nextIndex = index - 1; - break; - } - if (nextIndex < 0 || nextIndex >= components.length) return; - const result = onMoveComponent(index, nextIndex); - if (result?.ok) { - setExpandedIndexes((current) => { - const next = new Set(current); - const wasCurrentExpanded = next.has(index); - const wasTargetExpanded = next.has(nextIndex); - next.delete(index); - next.delete(nextIndex); - if (wasCurrentExpanded) next.add(nextIndex); - if (wasTargetExpanded) next.add(index); - return next; - }); - setError(null); - } else if (result) { - setError('组件顺序更新失败。'); - } + function replaceComponent() { + const result = setComponent(createComponent()); + if (result?.ok) setExpanded(true); } return ( @@ -130,7 +42,7 @@ export function ComponentPanel(props: ComponentPanelProps) {

组件

- {components.length} 个 + {component ? componentKind(component) : '无'}
@@ -142,7 +54,7 @@ export function ComponentPanel(props: ComponentPanelProps) { onChange={(event) => setAddKind(event.target.value as 'Image' | 'Text') } - aria-label="新增组件类型" + aria-label="组件类型" > @@ -151,93 +63,49 @@ export function ComponentPanel(props: ComponentPanelProps) { type="button" className="flex h-8 items-center gap-1 rounded-lg bg-blue-600 px-3 text-xs font-semibold text-white disabled:cursor-not-allowed disabled:opacity-40" disabled={readOnly} - onClick={addComponent} - aria-label="新增组件" - title="新增组件" + onClick={replaceComponent} + aria-label={component ? '替换组件' : '新增组件'} + title={component ? '替换组件' : '新增组件'} > - {componentKindLabel(addKind)} + {component ? '替换' : '新增'} - {components.length > 0 && ( -
- {[...components].reverse().map((component, reverseIndex) => { - const index = components.length - reverseIndex - 1; - const expanded = expandedIndexes.has(index); - return ( -
-
- - - - -
- {expanded && ( -
- {renderComponentEditor( - component, - index, - props, - readOnly, - updateComponent, - )} -
- )} -
- ); - })} + {component ? ( +
+
+ + +
+ {expanded && ( +
+ {renderComponentEditor(component, props, readOnly, setComponent)} +
+ )}
- )} - {components.length === 0 && ( + ) : (

当前节点没有组件。

@@ -248,72 +116,45 @@ export function ComponentPanel(props: ComponentPanelProps) { } function componentKind(component: Component): string { - switch (true) { - case 'Image' in component: - return '图片'; - case 'Text' in component: - return '文本'; - default: - return '未知'; - } -} - -function componentKindLabel(kind: 'Image' | 'Text'): string { - switch (kind) { - case 'Image': - return '图片'; - case 'Text': - return '文本'; - } -} - -function componentLayerLabel(index: number, count: number): string { - if (index === count - 1) return '顶部'; - return `层级 ${index + 1}`; -} - -function ExpandIcon({ expanded }: { expanded: boolean }) { - if (expanded) return ; - return ; + if ('Image' in component) return '图片'; + if ('Text' in component) return '文本'; + return '未知'; } function renderComponentEditor( component: Component, - index: number, props: ComponentPanelProps, readOnly: boolean, updateComponent: ( - index: number, - next: Component, + next: Component | null, ) => UiEditorOperationResult | undefined, ) { - switch (true) { - case 'Image' in component: - return ( - updateComponent(index, { Image: next })} - /> - ); - case 'Text' in component: - return ( - updateComponent(index, { Text: next })} - /> - ); - default: - return ( -

- 当前组件类型暂不支持编辑。 -

- ); + if ('Image' in component) { + return ( + updateComponent({ Image: next })} + /> + ); } + if ('Text' in component) { + return ( + updateComponent({ Text: next })} + /> + ); + } + return ( +

+ 当前组件类型暂不支持编辑。 +

+ ); } diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/Components/componentEditorTypes.ts b/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/Components/componentEditorTypes.ts index 9d3adc06a..0319745b5 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/Components/componentEditorTypes.ts +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/Components/componentEditorTypes.ts @@ -7,24 +7,15 @@ import type { UiEditorFontFaceState } from '../../../../../features/ui-editor/us import type { UiEditorOperationResult } from '../../../../../features/ui-editor/useUiEditorState'; export type ComponentPanelProps = { - components: Component[]; + component: Component | null; sprites: Record; previewUrls: Record; fonts: Record; fontFaces: Record; projectPath: string; readOnly: boolean; - onSetComponents: ( - components: Component[], - ) => UiEditorOperationResult | undefined; - onInsertComponent: ( - index: number, - component: Component, - ) => UiEditorOperationResult | undefined; - onDeleteComponent: (index: number) => UiEditorOperationResult | undefined; - onMoveComponent: ( - fromIndex: number, - toIndex: number, + onSetComponent: ( + component: Component | null, ) => UiEditorOperationResult | undefined; }; diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/InspectorSidebar.tsx b/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/InspectorSidebar.tsx index 7557c3727..97c962104 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/InspectorSidebar.tsx +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/Inspector/InspectorSidebar.tsx @@ -115,10 +115,7 @@ export function InspectorSidebar({ fonts={inspector.fonts} fontFaces={inspector.fontFaces} projectPath={inspector.projectPath} - onSetComponents={inspector.setNodeComponents} - onInsertComponent={inspector.insertNodeComponent} - onDeleteComponent={inspector.deleteNodeComponent} - onMoveComponent={inspector.moveNodeComponent} + onSetComponent={inspector.setNodeComponent} onDeleteNode={() => inspector.deleteNode(view.node.id)} deleteDisabled={ inspector.isLocked || view.node.id === inspector.tree?.root.id @@ -293,10 +290,7 @@ function NodeInspector({ fonts, fontFaces, projectPath, - onSetComponents, - onInsertComponent, - onDeleteComponent, - onMoveComponent, + onSetComponent, onDeleteNode, deleteDisabled, }: { @@ -323,10 +317,7 @@ function NodeInspector({ fonts: UiEditorInspectorProjection['fonts']; fontFaces: UiEditorInspectorProjection['fontFaces']; projectPath: string; - onSetComponents: UiEditorInspectorProjection['setNodeComponents']; - onInsertComponent: UiEditorInspectorProjection['insertNodeComponent']; - onDeleteComponent: UiEditorInspectorProjection['deleteNodeComponent']; - onMoveComponent: UiEditorInspectorProjection['moveNodeComponent']; + onSetComponent: UiEditorInspectorProjection['setNodeComponent']; onDeleteNode: () => void; deleteDisabled: boolean; }) { @@ -414,7 +405,7 @@ function NodeInspector({ 来源：{node.metadata.source}
- 组件：{node.components.length} + 组件：{node.component ? 1 : 0}
@@ -434,18 +425,18 @@ function NodeInspector({ }} /> { - if (!isReadOnly) onMetadataChange({ components_status }); + onChange={(component_status) => { + if (!isReadOnly) onMetadataChange({ component_status }); }} />
@@ -498,17 +489,14 @@ function NodeInspector({ onChange={onLayoutChange} /> void; }) { const overview = useMemo( - () => getBindingOverview(uiTrees, sprites), + () => getSeparationOverview(uiTrees, sprites), [sprites, uiTrees], ); const attentionCycle = useUiTreeNodeCycle({ @@ -38,20 +38,20 @@ export function BindingOverview({ return (
Overview -

绑定概览

+

自动切分素材概览

- - + +
+ +

+ 发现未完成的自动切分素材 +

+

+ 上次自动切分素材留下了可恢复状态（已登记{' '} + {workflow.separationRecovery?.bound_node_count ?? 0}{' '} + 个节点）。请选择继续上次自动切分素材，或开始新的自动切分素材。 +

+
+ + + +
+
); } @@ -76,8 +117,8 @@ function getStepAction(workflow: UiEditorWorkflowProjection) { }; } return { - label: '绑定视觉素材', - runningLabel: '绑定中…', - action: workflow.bindComponents, + label: '自动切分素材', + runningLabel: '素材切分中…', + action: workflow.separateUi, }; } diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/WorkflowChecks.ts b/apps/ai-game-creator-shell/src/view/ui-editor/components/WorkflowChecks.ts index 54b77ef1f..fbf4d1582 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/WorkflowChecks.ts +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/WorkflowChecks.ts @@ -1,11 +1,10 @@ import { type UiEditorPrerequisiteIssue, - validateAssetRecognitionPrerequisites, + validateAssetSeparationPrerequisites, + validateAssetSeparationResult, validateComponentRecognitionPrerequisites, - validateLayoutReviewPrerequisites, validateReferenceAnalysisResult, validateStructureRecognitionResult, - validateVisualBindingResult, } from '../../../features/ui-editor/requisites'; import type { State } from '../../../features/ui-editor/types/State'; import type { UiEditorStepId } from '../model'; @@ -21,8 +20,8 @@ export function prerequisiteIssuesForStep( return []; case 'structure-recognition': return validateComponentRecognitionPrerequisites(state); - case 'visual-binding': - return validateAssetRecognitionPrerequisites(state); + case 'asset-separation': + return validateAssetSeparationPrerequisites(state); } } @@ -35,8 +34,8 @@ export function postCheckIssuesForStep( return validateReferenceAnalysisResult(state); case 'structure-recognition': return validateStructureRecognitionResult(state); - case 'visual-binding': - return validateVisualBindingResult(state); + case 'asset-separation': + return validateAssetSeparationResult(state); } } @@ -46,7 +45,7 @@ export function postCheckIssuesForSave( return [ ...validateReferenceAnalysisResult(state), ...validateStructureRecognitionResult(state), - ...validateVisualBindingResult(state), + ...validateAssetSeparationResult(state), ]; } @@ -58,8 +57,8 @@ export function activeStepPrerequisiteIssues( case 'reference-analysis': return validateComponentRecognitionPrerequisites(state); case 'structure-recognition': - return validateAssetRecognitionPrerequisites(state); - case 'visual-binding': - return validateLayoutReviewPrerequisites(state); + return validateComponentRecognitionPrerequisites(state); + case 'asset-separation': + return validateAssetSeparationPrerequisites(state); } } diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/WorkflowCompletionModal.tsx b/apps/ai-game-creator-shell/src/view/ui-editor/components/WorkflowCompletionModal.tsx index fbda3f214..c96e8d670 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/WorkflowCompletionModal.tsx +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/WorkflowCompletionModal.tsx @@ -23,7 +23,7 @@ export function WorkflowCompletionModal({ {stepLabel} {outcomeLabel} -

+

{notice.message}

diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/PreviewWorkspace.tsx b/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/PreviewWorkspace.tsx index c923ca875..8df4e4adf 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/PreviewWorkspace.tsx +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/PreviewWorkspace.tsx @@ -1,10 +1,12 @@ import { CANVAS_ZOOM_IN_FACTOR, CANVAS_ZOOM_OUT_FACTOR, + canvasDisplayScaleToViewportScale, type CanvasViewport, createPanDragState, type DragState, fitViewportToBounds, + formatCanvasDisplayScalePercent, moveViewportFromPan, resolveViewportFromWheel, scaleViewportFromScreenPoint, @@ -12,7 +14,6 @@ import { import { CanvasViewport as SharedCanvasViewport, CanvasWorld, - ZoomControls, } from '@genarrative/image-canvas-react'; import { Image as ImageIcon, Minus, Plus } from 'lucide-react'; import { @@ -37,7 +38,6 @@ import { } from './previewZoomKeyboard'; import { type UiEditorRenderMode, UiTreeRenderer } from './UiTreeRenderer'; import { useNodeTransformInteraction } from './useNodeTransformInteraction'; -import { ZoomPercentageInput } from './ZoomPercentageInput'; export function PreviewWorkspace({ canvas, @@ -140,26 +140,6 @@ export function PreviewWorkspace({ setViewportState(next); }, []); - const fitToCanvas = useCallback(() => { - if (!logicalSize) return; - const element = viewportElementRef.current; - const size = { - width: element?.clientWidth || 900, - height: element?.clientHeight || 640, - }; - setViewport( - fitViewportToBounds({ - bounds: { - x: 0, - y: 0, - width: logicalSize.width, - height: logicalSize.height, - }, - canvasSize: size, - }), - ); - }, [logicalSize, setViewport]); - const scaleViewportFromCenter = useCallback( (nextScale: number) => { const element = viewportElementRef.current; @@ -188,6 +168,15 @@ export function PreviewWorkspace({ scaleViewportFromCenter(viewportRef.current.scale * CANVAS_ZOOM_OUT_FACTOR); }, [scaleViewportFromCenter]); + const displayPercent = formatCanvasDisplayScalePercent(viewport.scale); + + const zoomToDisplayScale = useCallback( + (displayScale: number) => { + scaleViewportFromCenter(canvasDisplayScaleToViewportScale(displayScale)); + }, + [scaleViewportFromCenter], + ); + useEffect(() => { const element = viewportElementRef.current; if (!element) return; @@ -203,12 +192,6 @@ export function PreviewWorkspace({ return () => observer.disconnect(); }, []); - useEffect(() => { - fitToCanvas(); - // This effect intentionally follows the active image, not every controller render. - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [activeImageId, fitToCanvas]); - useEffect(() => { const request = canvas.focusRequest; if ( @@ -282,7 +265,6 @@ export function PreviewWorkspace({ usesMetaModifier, }, { - fit: fitToCanvas, resetToActualSize, zoomIn, zoomOut, @@ -301,7 +283,7 @@ export function PreviewWorkspace({ window.removeEventListener('keyup', onKeyUp); window.removeEventListener('blur', onWindowBlur); }; - }, [fitToCanvas, logicalSize, resetToActualSize, zoomIn, zoomOut]); + }, [logicalSize, resetToActualSize, zoomIn, zoomOut]); const handlePointerDown = (event: ReactPointerEvent) => { if (event.button === 0 && !isPreviewZoomInteractiveTarget(event.target)) { @@ -507,70 +489,52 @@ export function PreviewWorkspace({ onDelete={(nodeId) => canvas.deleteNode(nodeId)} /> ) : null} - { + if ( + event.target instanceof Element && + event.target.closest('button') + ) { + event.preventDefault(); + } + }} > - {(actions) => ( -
{ - if ( - event.target instanceof Element && - event.target.closest('button') - ) { - event.preventDefault(); - } - }} - > - - - actions.zoomToDisplayScale(Number(event.target.value) / 100) - } - /> - - actions.zoomToDisplayScale(percent / 100) - } - /> - - -
- )} - + + + zoomToDisplayScale(Number(event.target.value) / 100) + } + /> + + {displayPercent} + + +
) : (
diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/UiTreeRenderer.tsx b/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/UiTreeRenderer.tsx index b8873be1a..c4ca88a09 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/UiTreeRenderer.tsx +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/UiTreeRenderer.tsx @@ -191,13 +191,9 @@ function RenderNode({ {node.metadata.name} ) : null} - {node.components.map((component, index) => ( - - ))} + {node.component ? ( + + ) : null} {renderMode === 'final-preview' && selectedNodeId === node.id ? ( void; -}) { - const [draft, setDraft] = useState(() => - displayPercentToValue(displayPercent), - ); - const draftRef = useRef(draft); - const isEditingRef = useRef(false); - - useEffect(() => { - if (!isEditingRef.current) { - setDraft(displayPercentToValue(displayPercent)); - draftRef.current = displayPercentToValue(displayPercent); - } - }, [displayPercent]); - - const commit = () => { - const currentPercent = Number.parseFloat( - displayPercentToValue(displayPercent), - ); - const parsed = Number.parseFloat(draftRef.current); - const nextPercent = Number.isFinite(parsed) - ? Math.min(MAX_ZOOM_PERCENT, Math.max(MIN_ZOOM_PERCENT, parsed)) - : currentPercent; - setDraft(String(nextPercent)); - draftRef.current = String(nextPercent); - isEditingRef.current = false; - if (nextPercent !== currentPercent) { - onCommit(nextPercent); - } - }; - - return ( - - ); -} diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/previewZoomKeyboard.ts b/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/previewZoomKeyboard.ts index 85941ca1e..ce09b1495 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/previewZoomKeyboard.ts +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/preview/previewZoomKeyboard.ts @@ -1,8 +1,4 @@ -export type PreviewZoomShortcut = - | 'fit' - | 'actual-size' - | 'zoom-in' - | 'zoom-out'; +export type PreviewZoomShortcut = 'actual-size' | 'zoom-in' | 'zoom-out'; export type PreviewZoomKeyboardContext = { hasZoomableViewport: boolean; @@ -12,7 +8,6 @@ export type PreviewZoomKeyboardContext = { }; export type PreviewZoomKeyboardActions = { - fit: () => void; resetToActualSize: () => void; zoomIn: () => void; zoomOut: () => void; @@ -61,7 +56,6 @@ export function resolvePreviewZoomShortcut( event: KeyboardEvent, ): PreviewZoomShortcut | null { if (event.altKey) return null; - if (event.key === '0') return 'fit'; if (event.key === '1') return 'actual-size'; if (event.code === 'NumpadAdd' || event.key === '+' || event.key === '=') { return 'zoom-in'; @@ -92,8 +86,7 @@ export function handlePreviewZoomKeyDown( event.preventDefault(); event.stopPropagation(); - if (shortcut === 'fit') actions.fit(); - else if (shortcut === 'actual-size') actions.resetToActualSize(); + if (shortcut === 'actual-size') actions.resetToActualSize(); else if (shortcut === 'zoom-in') actions.zoomIn(); else actions.zoomOut(); return true; diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/components/workflowCompletionNotice.ts b/apps/ai-game-creator-shell/src/view/ui-editor/components/workflowCompletionNotice.ts index ff90b6dd5..02c5b317a 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/components/workflowCompletionNotice.ts +++ b/apps/ai-game-creator-shell/src/view/ui-editor/components/workflowCompletionNotice.ts @@ -18,8 +18,8 @@ export function workflowStepLabel(step: UiEditorStepId): string { return '分析参考图'; case 'structure-recognition': return '识别界面结构'; - case 'visual-binding': - return '绑定视觉素材'; + case 'asset-separation': + return '自动切分素材'; } const exhaustiveCheck: never = step; return exhaustiveCheck; diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/index.tsx b/apps/ai-game-creator-shell/src/view/ui-editor/index.tsx index e60169b1f..2ea43b868 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/index.tsx +++ b/apps/ai-game-creator-shell/src/view/ui-editor/index.tsx @@ -7,13 +7,13 @@ import { type IUiDesignStateStore, uiDesignStateStore, } from '../../features/ui-editor/uiDesignStateStore'; -import { BindingOverview } from './components/BindingOverview'; import { EditorDialogs } from './components/EditorDialogs'; import { ImportOverview } from './components/ImportOverview'; import { InputSidebar } from './components/InputSidebar'; import { InspectorSidebar } from './components/Inspector/InspectorSidebar'; import { PreviewWorkspace } from './components/preview/PreviewWorkspace'; import { RecognitionOverview } from './components/RecognitionOverview'; +import { SeparationOverview } from './components/SeparationOverview'; import { ToolNavigation } from './components/ToolNavigation'; import { WorkflowActionCard } from './components/WorkflowActionCard'; import { WorkflowCompletionModal } from './components/WorkflowCompletionModal'; @@ -265,13 +265,13 @@ export default function UiEditorPage({ session.input.highlightStatusField(nodeId, 'layout_status'); }} /> - ) : session.input.activeStep === 'visual-binding' ? ( - { session.input.focusNode(treeId, nodeId); - session.input.highlightStatusField(nodeId, 'components_status'); + session.input.highlightStatusField(nodeId, 'component_status'); }} /> ) : ( diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/model.ts b/apps/ai-game-creator-shell/src/view/ui-editor/model.ts index 13ec54cc7..f03f6ef47 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/model.ts +++ b/apps/ai-game-creator-shell/src/view/ui-editor/model.ts @@ -6,7 +6,7 @@ import type { RemovalImpact } from '../../features/ui-editor/useUiEditorState'; export type UiEditorStepId = | 'reference-analysis' | 'structure-recognition' - | 'visual-binding'; + | 'asset-separation'; export type UiEditorImportKind = 'design-image' | 'font' | 'sprite'; export type UiEditorNodeFocusRequest = { @@ -27,7 +27,7 @@ export const UI_EDITOR_STEPS: Array<{ }> = [ { id: 'reference-analysis', label: '分析参考图' }, { id: 'structure-recognition', label: '识别界面结构' }, - { id: 'visual-binding', label: '绑定视觉素材' }, + { id: 'asset-separation', label: '自动切分素材' }, ]; export const UI_DESIGN_IMAGE_ROLES: Array<{ diff --git a/apps/ai-game-creator-shell/src/view/ui-editor/useUiEditorPage.ts b/apps/ai-game-creator-shell/src/view/ui-editor/useUiEditorPage.ts index a580a652d..39726526d 100644 --- a/apps/ai-game-creator-shell/src/view/ui-editor/useUiEditorPage.ts +++ b/apps/ai-game-creator-shell/src/view/ui-editor/useUiEditorPage.ts @@ -2,7 +2,6 @@ import { invoke } from '@tauri-apps/api/core'; import { useCallback, useEffect, useMemo, useState } from 'react'; import type { ImportedAsset } from '../../components/AssetImporter'; -import { applyBindingResult } from '../../features/ui-editor/binding'; import { prepareDesignImageBatch, prepareFontAssetBatch, @@ -15,7 +14,6 @@ import { type StageStatusField, } from '../../features/ui-editor/stageStatusOverview'; import { collectUiNodeIds } from '../../features/ui-editor/treeUtils'; -import type { BindingDTO } from '../../features/ui-editor/types/BindingDTO'; import type { ChildrenDisplayMode } from '../../features/ui-editor/types/ChildrenDisplayMode'; import type { Component } from '../../features/ui-editor/types/Component'; import type { FontAssetId } from '../../features/ui-editor/types/FontAssetId'; @@ -23,6 +21,8 @@ import type { MergeDTO } from '../../features/ui-editor/types/MergeDTO'; import type { Node as UiNode } from '../../features/ui-editor/types/Node'; import type { NodeId } from '../../features/ui-editor/types/NodeId'; import type { RecognitionDTO } from '../../features/ui-editor/types/RecognitionDTO'; +import type { SeparationDTO } from '../../features/ui-editor/types/SeparationDTO'; +import type { SeparationRecoveryDTO } from '../../features/ui-editor/types/SeparationRecoveryDTO'; import type { SpriteAssetId } from '../../features/ui-editor/types/SpriteAssetId'; import type { SpriteBorder } from '../../features/ui-editor/types/SpriteBorder'; import type { State } from '../../features/ui-editor/types/State'; @@ -37,6 +37,7 @@ import { } from '../../features/ui-editor/uiDesignStateStore'; import { applyUiDesignSuggestions } from '../../features/ui-editor/uiDesignSuggestions'; import { useUiEditorFontFaces } from '../../features/ui-editor/useUiEditorFontFaces'; +import { addSpriteAssetsToState } from '../../features/ui-editor/useUiEditorState'; import { EMPTY_UI_EDITOR_STATE, type NodeLayoutPatch, @@ -70,7 +71,15 @@ import { } from './model'; import { useUiEditorNodeFocus } from './useUiEditorNodeFocus'; -const ASSET_BATCH_SIZE = 5; +const SEPARATION_IMPORT_BATCH_SIZE = 100; + +type LocalImageImportResponse = { + assets: Array<{ id: string; localPath: string; assetKind?: string | null }>; +}; + +function normalizeProjectRelativePath(path: string): string { + return path.replaceAll('\\', '/').replace(/^\/+/, ''); +} type StatusFieldHighlight = { nodeId: NodeId; @@ -250,11 +259,13 @@ export function useUiEditorSession( ); const [isMerging, setIsMerging] = useState(false); const [mergeStatus, setMergeStatus] = useState(null); - const [isBinding, setIsBinding] = useState(false); - const [bindingStatus, setBindingStatus] = useState(null); + const [isSeparating, setIsSeparating] = useState(false); + const [separationStatus, setSeparationStatus] = useState(null); + const [separationRecovery, setSeparationRecovery] = + useState(null); const [hasSuggested, setHasSuggested] = useState(false); const [hasRecognized, setHasRecognized] = useState(false); - const [hasBound, setHasBound] = useState(false); + const [hasSeparated, setHasSeparated] = useState(false); const [completionNotice, setCompletionNotice] = useState(null); @@ -393,7 +404,8 @@ export function useUiEditorSession( image.metadata.role === 'Page' && !isSlaveToDescendant(images, id as UIDesignImageId, activeImageId), ); - const isAiRunning = isSuggesting || isRecognizing || isBinding || isMerging; + const isAiRunning = + isSuggesting || isRecognizing || isMerging || isSeparating; const isWorkflowBusy = isAiRunning || isSaving || isGenerating || isLoading || editor.isLocked; const stateSignature = JSON.stringify(editor.state); @@ -405,14 +417,15 @@ export function useUiEditorSession( activeStep === 'reference-analysis' ? 'structure-recognition' : activeStep === 'structure-recognition' - ? 'visual-binding' + ? 'asset-separation' : null; const spriteReferenceCounts = useMemo(() => { const counts: Record = {}; function visit(nodes: UiNode[]) { for (const node of nodes) { - for (const component of node.components) { + const component = node.component; + if (component) { if ('Image' in component && component.Image.target_graphic) { counts[component.Image.target_graphic] = (counts[component.Image.target_graphic] ?? 0) + 1; @@ -429,7 +442,8 @@ export function useUiEditorSession( const counts: Record = {}; function visit(nodes: UiNode[]) { for (const node of nodes) { - for (const component of node.components) { + const component = node.component; + if (component) { if ('Text' in component && typeof component.Text.font !== 'string') { const id = component.Text.font.Bound; counts[id] = (counts[id] ?? 0) + 1; @@ -787,7 +801,7 @@ export function useUiEditorSession( if ( result.ok && (patch.layout_status !== undefined || - patch.components_status !== undefined) + patch.component_status !== undefined) ) { setHighlightedStatusField(null); } @@ -826,45 +840,14 @@ export function useUiEditorSession( return result; } - function setNodeComponents(components: Component[]) { + function setNodeComponent(component: Component | null) { if (!activeImageId || !selectedNodeId) return; - const result = editor.setNodeComponents( + const result = editor.setNodeComponent( activeImageId, selectedNodeId, - components, - ); - if (!result.ok) setStatus('组件更新失败。'); - return result; - } - - function insertNodeComponent(index: number, component: Component) { - if (!activeImageId || !selectedNodeId) return; - const result = editor.insertComponent( - activeImageId, - selectedNodeId, - index, component, ); - if (!result.ok) setStatus('组件新增失败。'); - return result; - } - - function deleteNodeComponent(index: number) { - if (!activeImageId || !selectedNodeId) return; - const result = editor.deleteComponent(activeImageId, selectedNodeId, index); - if (!result.ok) setStatus('组件删除失败。'); - return result; - } - - function moveNodeComponent(fromIndex: number, toIndex: number) { - if (!activeImageId || !selectedNodeId) return; - const result = editor.moveComponent( - activeImageId, - selectedNodeId, - fromIndex, - toIndex, - ); - if (!result.ok) setStatus('组件顺序更新失败。'); + if (!result.ok) setStatus('组件更新失败。'); return result; } @@ -1061,53 +1044,255 @@ export function useUiEditorSession( } } - async function bindComponents() { - if (isBinding || isWorkflowBusy) return; + async function runSeparationWorkflow() { + if (!resourceId) return; + setIsSeparating(true); + setSeparationStatus(null); setCompletionNotice(null); - setBindingStatus(null); - setIsBinding(true); + let preparedSprites: Awaited> = + []; try { + let backfillErrors: string[] = []; + let separationResult: SeparationDTO | null = null; await editor.runWithStateLocked(async (snapshot) => { - const allSpriteIds = Object.keys(snapshot.sprite_assets); - const batches: string[][] = []; + const result = await invoke('separate_ui', { + projectPath, + assetId: resourceId, + state: snapshot, + }); + separationResult = result; + + const uniquePaths = [ + ...new Set( + result.bound_nodes.map((bound) => + normalizeProjectRelativePath(bound.cut_image_path), + ), + ), + ]; + const importedByPath = new Map< + string, + { id: string; localPath: string; assetKind: string | null } + >(); for ( let index = 0; - index < allSpriteIds.length; - index += ASSET_BATCH_SIZE + index < uniquePaths.length; + index += SEPARATION_IMPORT_BATCH_SIZE ) { - batches.push(allSpriteIds.slice(index, index + ASSET_BATCH_SIZE)); + const relativePaths = uniquePaths.slice( + index, + index + SEPARATION_IMPORT_BATCH_SIZE, + ); + const imported = await invoke( + 'import_local_project_image_assets', + { projectPath, relativePaths }, + ); + for (const [assetIndex, asset] of imported.assets.entries()) { + const normalizedAsset = { + id: asset.id, + localPath: normalizeProjectRelativePath(asset.localPath), + assetKind: asset.assetKind ?? null, + }; + // The importer may copy a sidecar file into assets/uploads and + // therefore return a different localPath. Keep both identities: + // the cut path is the separation contract, while the returned + // path is the SpriteAsset resource path. + importedByPath.set(normalizedAsset.localPath, normalizedAsset); + const requestedPath = relativePaths[assetIndex]; + if (requestedPath) { + importedByPath.set( + normalizeProjectRelativePath(requestedPath), + normalizedAsset, + ); + } + } } - if (batches.length === 0) batches.push([]); - let current = snapshot; - for (const [index, spriteIds] of batches.entries()) { - setBindingStatus(`绑定组件中（${index + 1}/${batches.length}）…`); - const result = await invoke('bind_components', { - projectPath, - state: current, - spriteIds, - }); - current = applyBindingResult(current, result); - editor.replaceState(current, { - history: index < batches.length - 1 ? 'skip' : 'record', - }); - } - reportWorkflowCompletion( - 'visual-binding', - 'success', - `视觉素材绑定完成：已处理 ${batches.length}/${batches.length} 个批次`, - setBindingStatus, + + const missingImports = uniquePaths.filter( + (path) => !importedByPath.has(path), ); - setHasBound(true); + backfillErrors = missingImports.map( + (path) => `未能登记自动切分素材图片：${path}`, + ); + const importedAssets: ImportedAsset[] = [ + ...new Map( + [...importedByPath.values()].map((asset) => [asset.id, asset]), + ).values(), + ]; + preparedSprites = await prepareSpriteAssetBatch( + projectPath, + importedAssets, + ); + const spriteById = new Map( + preparedSprites.map((item) => [ + item.resource.asset_id, + item.resource, + ]), + ); + const spriteByPath = new Map( + [...importedByPath.entries()].flatMap(([path, asset]) => { + const sprite = spriteById.get(asset.id); + return sprite ? [[path, sprite] as const] : []; + }), + ); + const added = addSpriteAssetsToState( + snapshot, + preparedSprites.map((item) => item.resource), + ); + if (!added.ok) { + throw new Error(uiEditorOperationError(added.reason)); + } + const next = added.value; + for (const bound of result.bound_nodes) { + const path = normalizeProjectRelativePath(bound.cut_image_path); + const sprite = spriteByPath.get(path); + if (!sprite) { + backfillErrors.push( + `节点 ${bound.node_id} 缺少已登记的自动切分素材图片：${path}`, + ); + continue; + } + const location = next.ui_trees + .map((tree) => findUiNodeLocation(tree.root, bound.node_id)) + .find((candidate) => candidate !== null); + if (!location) { + backfillErrors.push(`节点 ${bound.node_id} 已不存在，素材已保留`); + continue; + } + const imageComponent = location.node.component; + if ( + !imageComponent || + !('Image' in imageComponent) || + imageComponent.Image.target_graphic !== null + ) { + backfillErrors.push( + `节点 ${bound.node_id} 没有可回填的未绑定 Image 组件，素材已保留`, + ); + continue; + } + imageComponent.Image.target_graphic = sprite.asset_id; + } + editor.replaceState(next); }); + + setPreviewUrls((current) => ({ + ...current, + ...Object.fromEntries( + preparedSprites.map((item) => [ + item.resource.asset_id, + item.previewUrl, + ]), + ), + })); + if (separationResult === null) + throw new Error('自动切分素材没有返回结果'); + const completedResult = separationResult; + if (!(await save())) { + throw new Error( + '自动切分素材结果已写入编辑器，但 State 保存失败；sidecar 已保留，可继续恢复。', + ); + } + if (backfillErrors.length > 0) { + reportWorkflowCompletion( + 'asset-separation', + 'failure', + `自动切分素材已完成，但有 ${backfillErrors.length} 项未能回填；已登记素材并保留恢复状态。\n${backfillErrors.join('\n')}`, + setSeparationStatus, + ); + return; + } + await invoke('finalize_separation', { + projectPath, + assetId: resourceId, + }); + setHasSeparated(true); + reportWorkflowCompletion( + 'asset-separation', + 'success', + `自动切分素材完成：${completedResult.bound_nodes.length} 个已切分并回填，${completedResult.problematic_nodes.length} 个待处理。`, + setSeparationStatus, + ); } catch (cause) { reportWorkflowCompletion( - 'visual-binding', + 'asset-separation', 'failure', cause instanceof Error ? cause.message : String(cause), - setBindingStatus, + setSeparationStatus, ); } finally { - setIsBinding(false); + setIsSeparating(false); + } + } + + async function separateUi() { + if ( + isSeparating || + isWorkflowBusy || + separationRecovery !== null || + !resourceId + ) { + return; + } + try { + const recovery = await invoke( + 'inspect_separation_recovery', + { projectPath, assetId: resourceId }, + ); + if (recovery.exists) { + setSeparationRecovery(recovery); + return; + } + const prerequisiteIssues = prerequisiteIssuesForStep( + editor.state, + 'asset-separation', + ); + if (prerequisiteIssues.length > 0) { + reportWorkflowCompletion( + 'asset-separation', + 'failure', + prerequisiteIssues.map((issue) => issue.message).join('；'), + setSeparationStatus, + ); + return; + } + await runSeparationWorkflow(); + } catch (cause) { + setSeparationStatus( + cause instanceof Error ? cause.message : String(cause), + ); + } + } + + async function continueSeparation() { + setSeparationRecovery(null); + await runSeparationWorkflow(); + } + + async function restartSeparation() { + if (!resourceId) return; + setSeparationRecovery(null); + const prerequisiteIssues = prerequisiteIssuesForStep( + editor.state, + 'asset-separation', + ); + if (prerequisiteIssues.length > 0) { + reportWorkflowCompletion( + 'asset-separation', + 'failure', + prerequisiteIssues.map((issue) => issue.message).join('；'), + setSeparationStatus, + ); + return; + } + try { + await invoke('discard_separation_recovery', { + projectPath, + assetId: resourceId, + }); + await runSeparationWorkflow(); + } catch (cause) { + setSeparationStatus( + cause instanceof Error ? cause.message : String(cause), + ); } } @@ -1244,17 +1429,15 @@ export function useUiEditorSession( selectedNodeId, focusRequest, operations: { - isBinding, isMerging, isRecognizing, isSuggesting, - bindingStatus, mergeStatus, recognitionStatus, suggestionStatus, }, checkPrerequisites, - bindComponents, + separateUi, mergeUi, recognizeUi, suggestUiDesignSemantics, @@ -1336,10 +1519,7 @@ export function useUiEditorSession( setNodeMetadata, setNodeTransform, setNodeLayout, - setNodeComponents, - insertNodeComponent, - deleteNodeComponent, - moveNodeComponent, + setNodeComponent, deleteNode, setSpriteName, setSpriteAssetType, @@ -1367,11 +1547,15 @@ export function useUiEditorSession( hasRecognized, recognitionStatus, recognizeUi, - isBinding, - hasBound, - bindingStatus, + hasSeparated, completionNotice, - bindComponents, + separateUi, + isSeparating, + separationStatus, + separationRecovery, + continueSeparation, + restartSeparation, + cancelSeparationRecovery: () => setSeparationRecovery(null), requestStepChange, continueToNextStep: () => { if (nextStep) requestStepChange(nextStep); diff --git a/apps/ai-game-creator-shell/tests/ZoomPercentageInput.test.tsx b/apps/ai-game-creator-shell/tests/ZoomPercentageInput.test.tsx deleted file mode 100644 index f02a71e3d..000000000 --- a/apps/ai-game-creator-shell/tests/ZoomPercentageInput.test.tsx +++ /dev/null @@ -1,67 +0,0 @@ -// @vitest-environment jsdom - -import { fireEvent, render, screen } from '@testing-library/react'; -import { describe, expect, it, vi } from 'vitest'; - -import { ZoomPercentageInput } from '../src/view/ui-editor/components/preview/ZoomPercentageInput'; - -describe('ZoomPercentageInput', () => { - it('edits the displayed percentage and commits on blur without fitting', () => { - const onCommit = vi.fn(); - render(); - - const input = screen.getByRole('spinbutton', { name: /画布缩放/ }); - expect((input as HTMLInputElement).value).toBe('50'); - - fireEvent.focus(input); - fireEvent.change(input, { target: { value: '125' } }); - expect(onCommit).not.toHaveBeenCalled(); - fireEvent.blur(input); - - expect(onCommit).toHaveBeenCalledWith(125); - expect((input as HTMLInputElement).value).toBe('125'); - }); - - it.each([ - { value: '0', expected: 25 }, - { value: '999', expected: 200 }, - ])('clamps $value to $expected on blur', ({ value, expected }) => { - const onCommit = vi.fn(); - render(); - - const input = screen.getByRole('spinbutton', { name: /画布缩放/ }); - fireEvent.focus(input); - fireEvent.change(input, { target: { value } }); - fireEvent.blur(input); - - expect(onCommit).toHaveBeenCalledWith(expected); - expect((input as HTMLInputElement).value).toBe(String(expected)); - }); - - it('restores the current percentage when the draft is invalid', () => { - const onCommit = vi.fn(); - render(); - - const input = screen.getByRole('spinbutton', { name: /画布缩放/ }); - fireEvent.focus(input); - fireEvent.change(input, { target: { value: '' } }); - fireEvent.blur(input); - - expect(onCommit).not.toHaveBeenCalled(); - expect((input as HTMLInputElement).value).toBe('80'); - }); - - it('tracks viewport updates while not editing', () => { - const onCommit = vi.fn(); - const view = render( - , - ); - const input = screen.getByRole('spinbutton', { name: /画布缩放/ }); - - view.rerender( - , - ); - - expect((input as HTMLInputElement).value).toBe('140'); - }); -}); diff --git a/apps/ai-game-creator-shell/tests/nodeTransformGeometry.test.ts b/apps/ai-game-creator-shell/tests/nodeTransformGeometry.test.ts index cb414da56..e0ad8356b 100644 --- a/apps/ai-game-creator-shell/tests/nodeTransformGeometry.test.ts +++ b/apps/ai-game-creator-shell/tests/nodeTransformGeometry.test.ts @@ -23,7 +23,7 @@ function node(id: string, transform: Transform, children: Node[] = []): Node { id, layout: { transform } as Node['layout'], metadata: {} as Node['metadata'], - components: [], + component: null, children_display_mode: 'Stack', children, }; diff --git a/apps/ai-game-creator-shell/tests/previewWorkspaceZoom.test.tsx b/apps/ai-game-creator-shell/tests/previewWorkspaceZoom.test.tsx index 908086f1d..1159785c3 100644 --- a/apps/ai-game-creator-shell/tests/previewWorkspaceZoom.test.tsx +++ b/apps/ai-game-creator-shell/tests/previewWorkspaceZoom.test.tsx @@ -33,12 +33,12 @@ const root: UiNode = { name: '根节点', description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'System', }, - components: [], + component: null, children_display_mode: 'Stack', children: [], }; @@ -160,10 +160,9 @@ describe('PreviewWorkspace quick zoom', () => { expect(renderedScale(rendered.container)).toBeCloseTo(initial * 0.86); }); - it('keeps actual-size and fit shortcuts within the preview scope', () => { + it('keeps the actual-size shortcut within the preview scope', () => { const rendered = render(); const preview = screen.getByRole('region', { name: 'UI 预览画布' }); - const fitted = renderedScale(rendered.container); fireEvent.focus(preview); fireEvent.keyDown(window, { @@ -172,13 +171,6 @@ describe('PreviewWorkspace quick zoom', () => { cancelable: true, }); expect(renderedScale(rendered.container)).toBe(1); - - fireEvent.keyDown(window, { - key: '0', - ctrlKey: true, - cancelable: true, - }); - expect(renderedScale(rendered.container)).toBe(fitted); }); it('leaves browser zoom untouched when the preview has no content', () => { diff --git a/apps/ai-game-creator-shell/tests/previewZoomKeyboard.test.ts b/apps/ai-game-creator-shell/tests/previewZoomKeyboard.test.ts index 5ef72b7db..519c3247f 100644 --- a/apps/ai-game-creator-shell/tests/previewZoomKeyboard.test.ts +++ b/apps/ai-game-creator-shell/tests/previewZoomKeyboard.test.ts @@ -12,7 +12,6 @@ import { function createActions(): PreviewZoomKeyboardActions { return { - fit: vi.fn(), resetToActualSize: vi.fn(), zoomIn: vi.fn(), zoomOut: vi.fn(), @@ -96,15 +95,12 @@ describe('preview zoom keyboard shortcuts', () => { expect(keyboardEvent?.defaultPrevented).toBe(false); }); - it.each([ - { key: '0', action: 'fit' as const }, - { key: '1', action: 'resetToActualSize' as const }, - ])('keeps the existing $key shortcut', ({ key, action }) => { + it('keeps the existing actual-size shortcut', () => { const { actions } = dispatchShortcut({ - event: { key, ctrlKey: true, cancelable: true }, + event: { key: '1', ctrlKey: true, cancelable: true }, }); - expect(actions[action]).toHaveBeenCalledTimes(1); + expect(actions.resetToActualSize).toHaveBeenCalledTimes(1); }); it('uses Cmd on Apple platforms and Ctrl elsewhere', () => { diff --git a/apps/ai-game-creator-shell/tests/bindingOverview.test.ts b/apps/ai-game-creator-shell/tests/separationOverview.test.ts similarity index 76% rename from apps/ai-game-creator-shell/tests/bindingOverview.test.ts rename to apps/ai-game-creator-shell/tests/separationOverview.test.ts index 314d95655..91b39ca72 100644 --- a/apps/ai-game-creator-shell/tests/bindingOverview.test.ts +++ b/apps/ai-game-creator-shell/tests/separationOverview.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it } from 'vitest'; import { - getBindingOverview, + getSeparationOverview, nodeHasBlockedComponents, - nodeHasPendingBinding, + nodeHasPendingSeparation, nodeNeedsComponentReview, -} from '../src/features/ui-editor/bindingOverview'; +} from '../src/features/ui-editor/separationOverview'; import { getNextMatchingUiTreeNodeTarget } from '../src/features/ui-editor/stageStatusOverview'; import type { Component } from '../src/features/ui-editor/types/Component'; import type { Node } from '../src/features/ui-editor/types/Node'; @@ -39,8 +39,8 @@ function text(font: 'SystemFont' | { Bound: string }): Component { function node( id: string, - components: Component[], - componentsStatus: Node['metadata']['components_status'] = 'NoProblem', + component: Component | null, + componentStatus: Node['metadata']['component_status'] = 'NoProblem', children: Node[] = [], ): Node { return { @@ -62,12 +62,12 @@ function node( name: id, description: '', layout_status: 'NoProblem', - components_status: componentsStatus, + component_status: componentStatus, allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'System', }, - components, + component, children, }; } @@ -90,24 +90,19 @@ const sprites = { const trees: UITree[] = [ { src_ui_design: 'page-a', - root: node( - 'root', - [image('validSprite'), text({ Bound: 'font-id' })], - 'NoProblem', - [ - node('review', [image(null)], { NeedReview: '确认素材' }), - node('blocked', [text('SystemFont')], { Blocked: '素材绑定失败' }), - ], - ), + root: node('root', image('validSprite'), 'NoProblem', [ + node('review', image(null), { NeedReview: '确认素材' }), + node('blocked', text('SystemFont'), { Blocked: '素材绑定失败' }), + ]), }, ]; -describe('getBindingOverview', () => { +describe('getSeparationOverview', () => { it('uses per-component helpers to include both image and text slots', () => { - expect(getBindingOverview(trees, sprites)).toEqual({ - componentsNeedingAssets: 3, - assetSlots: 3, - boundSlots: 2, + expect(getSeparationOverview(trees, sprites)).toEqual({ + componentsNeedingAssets: 2, + assetSlots: 2, + boundSlots: 1, pendingSlots: 1, needsAttention: 2, blocked: 1, @@ -115,15 +110,15 @@ describe('getBindingOverview', () => { }); }); - it('reuses the common preorder next-target search for every binding queue', () => { + it('reuses the common preorder next-target search for every separation queue', () => { expect( getNextMatchingUiTreeNodeTarget(trees, null, (target) => - nodeHasPendingBinding(target), + nodeHasPendingSeparation(target), )?.node.id, ).toBe('review'); expect( getNextMatchingUiTreeNodeTarget(trees, 'review', (target) => - nodeHasPendingBinding(target), + nodeHasPendingSeparation(target), )?.node.id, ).toBe('review'); expect( diff --git a/apps/ai-game-creator-shell/tests/stageStatusOverview.test.ts b/apps/ai-game-creator-shell/tests/stageStatusOverview.test.ts index 9c4122367..8e7389b02 100644 --- a/apps/ai-game-creator-shell/tests/stageStatusOverview.test.ts +++ b/apps/ai-game-creator-shell/tests/stageStatusOverview.test.ts @@ -32,12 +32,12 @@ function node(id: string, status: StageStatus, children: Node[] = []): Node { name: id, description: '', layout_status: status, - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'System', }, - components: [], + component: null, children_display_mode: 'Stack', children, }; diff --git a/apps/ai-game-creator-shell/tests/uiEditorPage.test.ts b/apps/ai-game-creator-shell/tests/uiEditorPage.test.ts index 2c4206167..f9615a790 100644 --- a/apps/ai-game-creator-shell/tests/uiEditorPage.test.ts +++ b/apps/ai-game-creator-shell/tests/uiEditorPage.test.ts @@ -72,12 +72,12 @@ function node(id: string, children: UiNode[] = []): UiNode { name: id, description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'System', }, - components: [], + component: null, children_display_mode: undefined, children, }; @@ -347,12 +347,14 @@ describe('UiEditorPage', () => { fireEvent.click(screen.getByRole('button', { name: '仍然继续' })); expect(screen.getByRole('heading', { name: '识别概览' })).toBeTruthy(); - fireEvent.click(screen.getByRole('button', { name: /绑定视觉素材/ })); + fireEvent.click(screen.getByRole('button', { name: /自动切分素材/ })); fireEvent.click(screen.getByRole('button', { name: '仍然继续' })); - expect(screen.getByRole('heading', { name: '绑定概览' })).toBeTruthy(); + expect( + screen.getByRole('heading', { name: '自动切分素材概览' }), + ).toBeTruthy(); }); - it('opens a completed workflow directly at the visual binding review stage', async () => { + it('opens a completed workflow directly at the asset separation review stage', async () => { const stateStore: IUiDesignStateStore = { load: vi.fn().mockResolvedValue({ revision: 3, @@ -367,25 +369,25 @@ describe('UiEditorPage', () => { projectPath: '/tmp/ui-editor-final-review', resourceId: 'ui-resource', stateStore, - initialStep: 'visual-binding', + initialStep: 'asset-separation', initialFurthestStepIndex: 2, }), ); expect( - await screen.findByRole('heading', { name: '绑定概览' }), + await screen.findByRole('heading', { name: '自动切分素材概览' }), ).toBeTruthy(); expect( screen .getByRole('navigation', { name: 'UI 编辑流程' }) .querySelector('button[aria-current="step"]')?.textContent, - ).toContain('绑定视觉素材'); + ).toContain('自动切分素材'); }); it('keeps the pending binding count informational instead of navigable', () => { render(createElement(UiEditorPage, { projectPath: '/tmp/ui-editor' })); - fireEvent.click(screen.getByRole('button', { name: /绑定视觉素材/ })); + fireEvent.click(screen.getByRole('button', { name: /自动切分素材/ })); fireEvent.click(screen.getByRole('button', { name: '仍然继续' })); expect( @@ -396,7 +398,7 @@ describe('UiEditorPage', () => { it('switches tools freely without inventing completed workflow state', () => { render(createElement(UiEditorPage, { projectPath: '/tmp/ui-editor' })); - fireEvent.click(screen.getByRole('button', { name: /绑定视觉素材/ })); + fireEvent.click(screen.getByRole('button', { name: /自动切分素材/ })); expect(screen.getByRole('heading', { name: '检查发现问题' })).toBeTruthy(); }); @@ -572,10 +574,10 @@ describe('UiEditorPage', () => { act(() => { result.current.input.highlightStatusField( otherNodeId!, - 'components_status', + 'component_status', ); result.current.inspector.setNodeMetadata({ - components_status: 'NoProblem', + component_status: 'NoProblem', }); }); expect(result.current.inspector.highlightedStatusField).toBeNull(); diff --git a/apps/ai-game-creator-shell/tests/uiEditorPreview.test.tsx b/apps/ai-game-creator-shell/tests/uiEditorPreview.test.tsx index 6f79be210..487ea4fa4 100644 --- a/apps/ai-game-creator-shell/tests/uiEditorPreview.test.tsx +++ b/apps/ai-game-creator-shell/tests/uiEditorPreview.test.tsx @@ -33,12 +33,12 @@ function node(id: string, children: UiNode[] = []): UiNode { name: id, description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'System', }, - components: [], + component: null, children_display_mode: 'Stack', children, }; diff --git a/apps/ai-game-creator-shell/tests/uiEditorState.test.ts b/apps/ai-game-creator-shell/tests/uiEditorState.test.ts index 89c4cdf3e..248ea7335 100644 --- a/apps/ai-game-creator-shell/tests/uiEditorState.test.ts +++ b/apps/ai-game-creator-shell/tests/uiEditorState.test.ts @@ -70,19 +70,17 @@ function nodeWithSprite(id: string): Node { name: 'Image', description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'Llm', }, - components: [ - { - Image: { - target_graphic: id, - image_type: { Simple: { preserve_aspect: false } }, - }, + component: { + Image: { + target_graphic: id, + image_type: { Simple: { preserve_aspect: false } }, }, - ], + }, children: [], }; } @@ -107,12 +105,12 @@ function pageRoot(id: string, children: Node[] = []): Node { name: id, description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'System', }, - components: [], + component: null, children_display_mode: 'Stack', children, }; @@ -251,7 +249,7 @@ describe('useUiEditorState', () => { root: expect.objectContaining({ metadata: expect.objectContaining({ layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', }), }), }), @@ -280,7 +278,7 @@ describe('useUiEditorState', () => { result.current.state.ui_trees[0]!.root.children[0]?.metadata, ).toMatchObject({ layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', }); }); @@ -428,12 +426,12 @@ describe('useUiEditorState', () => { name: '页面根节点', description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'System', }, - components: [], + component: null, children_display_mode: 'Stack', children: [nodeWithSprite('panel')], }, @@ -521,21 +519,19 @@ describe('useUiEditorState', () => { root: { ...nodeWithSprite('unused'), id: 'text-root', - components: [ - { - Text: { - content: '你好', - font: { Bound: 'body' }, - font_style: 'Normal', - font_sizing: { Fixed: 14 }, - color: [255, 255, 255, 255], - alignment: 'UpperLeft', - horizontal_overflow: 'Wrap', - vertical_overflow: 'Truncate', - line_spacing: 1, - }, + component: { + Text: { + content: '你好', + font: { Bound: 'body' }, + font_style: 'Normal', + font_sizing: { Fixed: 14 }, + color: [255, 255, 255, 255], + alignment: 'UpperLeft', + horizontal_overflow: 'Wrap', + vertical_overflow: 'Truncate', + line_spacing: 1, }, - ], + }, }, }, ], @@ -544,14 +540,14 @@ describe('useUiEditorState', () => { act(() => { expect( - result.current.setNodeComponents('page', 'text-root', [ - { - Text: { - ...initial.ui_trees[0]!.root.components[0]!.Text!, - font: { Bound: 'body' }, - }, + result.current.setNodeComponent('page', 'text-root', { + Text: { + ...('Text' in initial.ui_trees[0]!.root.component! + ? initial.ui_trees[0]!.root.component!.Text + : {}), + font: { Bound: 'body' }, }, - ]), + }), ).toEqual({ ok: true, value: undefined }); expect(result.current.removeFontAsset('body', { dryRun: true })).toEqual({ ok: true, @@ -566,7 +562,7 @@ describe('useUiEditorState', () => { result.current.removeFontAsset('body', { dryRun: false }); }); expect(result.current.state.font_assets.body).toBeUndefined(); - expect(result.current.state.ui_trees[0]?.root.components[0]).toMatchObject({ + expect(result.current.state.ui_trees[0]?.root.component).toMatchObject({ Text: { font: 'SystemFont' }, }); }); diff --git a/apps/ai-game-creator-shell/tests/uiTreeUtils.test.ts b/apps/ai-game-creator-shell/tests/uiTreeUtils.test.ts index 196077400..2795e71f1 100644 --- a/apps/ai-game-creator-shell/tests/uiTreeUtils.test.ts +++ b/apps/ai-game-creator-shell/tests/uiTreeUtils.test.ts @@ -11,7 +11,7 @@ function node(id: string, children: Node[] = []): Node { id, layout: {} as Node['layout'], metadata: {} as Node['metadata'], - components: [], + component: null, children_display_mode: 'Stack', children, }; diff --git a/apps/ai-game-creator-shell/tests/useNodeTransformInteraction.test.tsx b/apps/ai-game-creator-shell/tests/useNodeTransformInteraction.test.tsx index 7434c355e..e2f8339f9 100644 --- a/apps/ai-game-creator-shell/tests/useNodeTransformInteraction.test.tsx +++ b/apps/ai-game-creator-shell/tests/useNodeTransformInteraction.test.tsx @@ -28,12 +28,12 @@ function node(id: string, children: UiNode[] = []): UiNode { name: id, description: '', layout_status: 'NoProblem', - components_status: 'NoProblem', + component_status: 'NoProblem', allow_llm_edit_layout: true, allow_llm_edit_component: true, source: 'System', }, - components: [], + component: null, children, }; } diff --git a/apps/ai-game-creator-shell/tests/workflowCompletionNotice.test.ts b/apps/ai-game-creator-shell/tests/workflowCompletionNotice.test.ts index 01016c4ae..af3f2d1cc 100644 --- a/apps/ai-game-creator-shell/tests/workflowCompletionNotice.test.ts +++ b/apps/ai-game-creator-shell/tests/workflowCompletionNotice.test.ts @@ -15,6 +15,6 @@ describe('workflow completion notice helpers', () => { it('maps every workflow step to a user-facing label', () => { expect(workflowStepLabel('reference-analysis')).toBe('分析参考图'); expect(workflowStepLabel('structure-recognition')).toBe('识别界面结构'); - expect(workflowStepLabel('visual-binding')).toBe('绑定视觉素材'); + expect(workflowStepLabel('asset-separation')).toBe('自动切分素材'); }); }); diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 00092043a..5e45daf45 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -7903,7 +7903,7 @@ CI 上 `background_agent_runtime_recovers_stale_running_before_pending_task` 在 ## 2026-08-24 AGC UI 原型桥接与自主 UI workflow - 决策：`ui-prototype` 图片与 `UI` JSON 编辑资源保持两种正式类型。Agent 通过受控 `ui.workflow.run` 按 `prepare -> recognize -> status -> finalize` 创建页面资源、关联源图、持久化 UI State 和 manifest 阶段；`recognize` 直接复用 UI Editor 的 provider-backed 结构识别、多树合并与组件绑定命令，按 `reference-ready -> structure-ready -> merge-ready -> binding-ready` 逐阶段写入并推进项目 revision。页面可显式关联已登记图片/图标和字体，图片/图标按 5 项一批绑定，字体安全元数据进入绑定上下文且未知引用失败关闭。Runtime 回执携带 `revisionAdvanceCount`；Provider 未配置、请求失败、工具调用缺失、结果不匹配、未产出可渲染组件或仍有待审节点时保留最近真实阶段，禁止用 deterministic seed 冒充语义处理完成。 -- 客户端：画布点击 `ui-prototype` 先幂等桥接到 `UI` JSON，并立即刷新 manifest；关联查找按 canonical resource identity 且优先已完成 workflow 资源。全部页面完成后，工作台自动打开首个页面的 UI 编辑器 `visual-binding` 最终阶段。 +- 客户端：画布点击 `ui-prototype` 先幂等桥接到 `UI` JSON，并立即刷新 manifest；关联查找按 canonical resource identity 且优先已完成 workflow 资源。全部页面完成后，工作台自动打开首个页面的 UI 编辑器 `asset-separation` 最终阶段。 - 完成门：`finalize` 必须为每个页面提供 `game/` 下真实 UTF-8 应用文件并安装当前 UI State revision 标记；缺少结构、组件、页面或标记时拒绝完成。详细输入、阶段与恢复契约见 [`docs/【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md`](../../【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md)。 - 验证：前端 bridge 6/6、资源实时集成 19/19、AppSurface 410/410、AGC typecheck、Rust workflow 定向测试覆盖 provider 前的 reference 阶段与真实调用失败关闭、Rust bridge 1/1、编码、格式和 diff 门禁通过；认证登录与真实 Provider 生成的桌面端 E2E 尚未具备可用会话，保持未验证。 @@ -8034,6 +8034,12 @@ CI 上 `background_agent_runtime_recovers_stale_running_before_pending_task` 在 - AGC LLM 对话入口在解析 Router 凭据和访问上游前先读取用户 `wallet_balance`。余额为 `0` 时直接返回 `409 MUD_POINTS_INSUFFICIENT`，客户端显示“泥点余额不足”；不创建、续期或使用 Router 账号。余额读取失败同样失败关闭，返回“泥点余额暂时不可用”。 - 余额大于 `0` 的请求继续走 Router，成功后仍按 best-effort 后置结算；退款占用、冻结或扣费时余额不足的处理继续由钱包事务和既有结算规则负责。 +## 2026-09-09 UI Editor 结构化请求 repair history + +- UI Editor 的结构化 LLM repair 由 `run_with_repair_history` 统一维护 append-only `LlmMessage` history；调用方只构造初始 prompt 并提供 `requester(history) -> Result`。 +- `validater(&T) -> Result<(), String>` 只负责业务校验。网络、模型、tool 缺失、JSON 或反序列化错误只按原 history 重试；只有业务校验失败才把序列化后的响应和校验错误合并为一条 system message 追加到 history。 +- history 仅存在本次请求内存中，不重复图片、不截断、不扩展 `platform-llm` 消息协议；重试次数参数统一使用 `max_retries`。 + ## 2026-08-29 DirectProject 受控联网搜索默认与边界 - 正式产品本次只覆盖 `DirectProject` 单 Codex Agent。`Provider`、`ToolHost`、`DirectHome` 不是 Agent，也不是本次联网主链路；不新增全路由联网或工具桥。唯一受控联网工具为 `agc_tools.agc_web_search`，链路固定为 Codex MCP 工具目录 -> 客户端 loopback `DirectToolBridge` -> 有界 Bing RSS HTTPS -> 过滤 / 脱敏 -> MCP 结果回传。 diff --git a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md index 2f63f00e6..8f5321635 100644 --- a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md +++ b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md @@ -106,7 +106,7 @@ UI Editor Inspector 的全局只读状态唯一来源是 `controller.editor.isLo ## 2026-08-18 UI Editor 结构识别、合并与增量导入边界 -UI Editor 当前把“识别界面结构”定义为结构草稿阶段，而不是完整视觉还原阶段。识别 DTO 只负责输出节点层级、几何、名称、描述和置信度；节点组件暂为空，由后续“绑定视觉素材”阶段补齐 Image / Text 组件。`applyRecognitionResult` 可以整体替换当前 `ui_trees`，但该替换只代表结构结果，不能宣称已经保留截图中的视觉内容；组件状态使用 `NoProblem`，前置检查仍会根据空组件和素材绑定情况阻止跳过绑定阶段。 +UI Editor 当前把“识别界面结构”定义为结构草稿阶段，而不是完整视觉还原阶段。识别 DTO 只负责输出节点层级、几何、名称、描述和置信度；节点组件暂为空，由后续“自动切分素材”阶段补齐 Image / Text 组件。`applyRecognitionResult` 可以整体替换当前 `ui_trees`，但该替换只代表结构结果，不能宣称已经保留截图中的视觉内容；组件状态使用 `NoProblem`，前置检查仍会根据空组件和素材切分情况阻止跳过自动切分阶段。 结构识别、界面语义建议、多图合并和组件绑定只接受不超过 `1 MiB` 的 LLM 工具调用 arguments，并在递归业务类型反序列化前先解析为通用 JSON、迭代检查结构预算。结构识别按每棵返回树独立限制为最多 `512` 个 LLM 节点和 `32` 层，不汇总多棵树的节点数，也不计 Rust 自动补建的页面根；界面语义建议最多 `4` 个节点和 `4` 层；合并计划最多 `512` 个计划节点和 `32` 层，`Simple.children` 与 `Merged.merged_from` 使用同一计数和深度口径；组件绑定 `changes` 不得超过当前可编辑节点数且绝对上限为 `10,000`，每个 change 的完整组件栈最多 `64` 个组件。任何超限结果均整次拒绝，不截断、不返回部分结果，也不把工具 arguments 正文写入日志。 @@ -1286,7 +1286,7 @@ game-project/ DirectProject 使用 `approvalPolicy=never`，避免每次原生调用再经过泛化 ToolHost 包装；原生命令网络保持关闭，联网资料继续走受控 `agc_web_search`。多 Agent、Apps、完整插件 Runtime、hooks、Goals、Workspace Dependencies、Tool Suggestion 和原生浏览器/电脑控制仍关闭，避免绕过 AGC durable delegation、浏览器证据和副作用审计；图片生成通过客户端审核的 `agc_tools.agc_generate_image` 暴露普通单图、角色图、视觉规范图和 UI 设计图，完整游戏美术包继续使用 `agc_tools.taonier_prepare_game_art`，两者都复用同一客户端登录态、幂等账本、下载校验和 manifest/revision 投影，不开放 Codex 原生 image tool。app-server 使用隔离 `CODEX_HOME`：内置 `agc_tools` 由客户端启动参数注入，用户在客户端扩展列表启用的独立第三方 MCP 以原生配置写入该次隔离 home；全局 Codex MCP、禁用项、Plugin hooks/apps 和其它插件能力不进入 DirectProject。第三方项固定非 required，配置或启动失败只记录该项，不替换 `agc_tools`；provider session token、工具桥地址和受控搜索标记不得通过第三方 MCP 的环境转发字段泄露。配置了 AGC LLM Key 或可解析的 `OPENAI_API_KEY` 登录态时，真实 provider 凭据只由 AGC 本地 provider proxy 持有，Codex 仅使用连接级随机代理令牌；无法安全代理的 OAuth `auth.json` 继续关闭 native shell/unified exec。`agc_tools` 的平台授权由 AGC 客户端当前登录会话和受控后端完成，普通客户端不得把 DirectProject 请求改成外部 API Key 请求；401/403 只投影为客户端登录或权限异常，不向用户索要凭据或暴露内部 URL。shell 子进程采用 `shell_environment_policy` core 继承及 secret/proxy/bridge 排除，provider key 和桥接凭据不得进入命令环境。系统提示词不再预注入项目源码快照或 Skill 正文，Codex 按需读取当前 cwd 文件。 ## 2026-08-24 AGC UI 原型桥接与自主 UI workflow -- 2026-08-24 起，`ui-prototype` 与 UI 编辑器的 `UI` JSON 资源明确分离。设计图生成后必须由白名单 `ui.workflow.run` 按页面执行 `prepare → recognize → status → finalize`：为每个功能页面创建并关联 `UI` JSON，载入页面设计图和已登记图片/图标/字体，调用 UI Editor 的 provider-backed 结构识别、多树合并与分批组件绑定，持久化 State/revision，写入 `game/` 应用标记，并把 `reference-ready → structure-ready → merge-ready → binding-ready → application-ready → completed` 各阶段的 `generationKind` 和 manifest revision 投影给客户端。Provider 未配置、请求失败、工具调用缺失、结果不匹配、未知字体引用、未产出可渲染组件或仍有待审节点时保留最近真实阶段并返回 blocker，不得使用 deterministic seed 冒充完成。工作台点击 `ui-prototype` 时通过 `ensure_ui_design_resource_for_prototype` 幂等补齐关联资源；工作流完成后自动打开首个页面的 UI 编辑器 `visual-binding` 最终阶段，交给用户检查和手动调整。UI 编辑器独立的语义建议请求也必须复用统一 LLM 传输选择，`llm.stream=true` 时发送 `stream=true` 并聚合完整工具调用后再校验结果。只生成图片、登记空 JSON 或进入普通图片画布均不构成 UI 工作流完成，详见 [`【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md`](../【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md)。 +- 2026-08-24 起，`ui-prototype` 与 UI 编辑器的 `UI` JSON 资源明确分离。设计图生成后必须由白名单 `ui.workflow.run` 按页面执行 `prepare → recognize → status → finalize`：为每个功能页面创建并关联 `UI` JSON，载入页面设计图和已登记图片/图标/字体，调用 UI Editor 的 provider-backed 结构识别、多树合并与分批自动切分素材，持久化 State/revision，写入 `game/` 应用标记，并把 `reference-ready → structure-ready → merge-ready → binding-ready → application-ready → completed` 各阶段的 `generationKind` 和 manifest revision 投影给客户端。Provider 未配置、请求失败、工具调用缺失、结果不匹配、未知字体引用、未产出可渲染组件或仍有待审节点时保留最近真实阶段并返回 blocker，不得使用 deterministic seed 冒充完成。工作台点击 `ui-prototype` 时通过 `ensure_ui_design_resource_for_prototype` 幂等补齐关联资源；工作流完成后自动打开首个页面的 UI 编辑器 `asset-separation` 最终阶段，交给用户检查和手动调整。UI 编辑器独立的语义建议请求也必须复用统一 LLM 传输选择，`llm.stream=true` 时发送 `stream=true` 并聚合完整工具调用后再校验结果。只生成图片、登记空 JSON 或进入普通图片画布均不构成 UI 工作流完成，详见 [`【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md`](../【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md)。 ## 2026-08-28 AGC 自主构建 relaxed 编排覆盖 diff --git a/docs/technical/【技术方案】Raw GPT Image 2图片编辑代理-2026-09-07.md b/docs/technical/【技术方案】Raw GPT Image 2图片编辑代理-2026-09-07.md new file mode 100644 index 000000000..ed130ed2a --- /dev/null +++ b/docs/technical/【技术方案】Raw GPT Image 2图片编辑代理-2026-09-07.md @@ -0,0 +1,105 @@ +# Raw GPT Image 2 图片编辑代理 + +更新时间：`2026-09-08` + +## 目标 + +提供一个由主站客户端调用的独立同步图片编辑代理： + +```text +POST /api/raw/v1/images/edit +``` + +该入口使用登录态 Bearer access token，不进入 External v1 / MCP OpenAPI，不读取或写入画布、项目资源、素材库、OSS 结果或 `external_generation_job`。 + +## 请求合同 + +请求使用 `multipart/form-data`，不再接受 JSON/base64 入站格式。图片直接作为文件字段上传，避免 base64 膨胀和入站解码；服务端仍在扣费前完成 PNG 完整解码与资源限制校验。 + +```text +image: +mask: +prompt: 修改图片 +quality: auto +background: auto +output_format: png +width: 1536 +height: 1024 +``` + +`image` 和 `mask` 必须是 `image/png` 文件字段；服务端不信任客户端文件名，转发时使用固定文件名。空文件、非 PNG 字节、MIME 不匹配或 mask 与 image 尺寸不一致均在扣费前返回 400。`prompt` 必填，UTF-8 原始字节长度不得超过 `4 KiB`；超限在扣费前返回 400。`quality`、`background` 和 `output_format` 采用 GPT Image 模型支持的值。字段不能重复，未知字段拒绝；缺失的必填字段拒绝。 + +`width`、`height` 使用严格输出尺寸规则，均在扣费前校验： + +1. 单边最大值为 `3840px`； +2. 宽、高均为 `16px` 的倍数； +3. 长边 / 短边不超过 `3:1`； +4. 总像素范围为 `655360` 至 `8294400`（含边界）。 + +校验通过后按整数尺寸发送给 provider，不静默 clamp 或改写调用者尺寸。 + +Raw 路由的 multipart body limit 为 `64 MiB`，覆盖图片和文本字段；文件字段由 multipart 解析器直接收集为字节，随后在阻塞线程中完成 PNG 解码。PNG 解码使用与输出合同一致的资源上限：宽高各不超过 `3840`，解码分配不超过 `8294400 × 4` 字节；不再执行 base64 入站解码。 + +服务端发送给 `platform-image` 时固定注入： + +```text +model = gpt-image-2 +n = 1 +``` + +请求不暴露 `model`、`n`、`response_format`、`style`、`user` 或 `output_compression`。 + +## 成功响应合同 + +响应始终为 JSON，响应只保留 `data` 字段，图片内容只以 base64 返回： + +```json +{ + "data": [ + { + "b64_json": "" + } + ] +} +``` + +`data` 保持数组形状，即使服务端固定 `n=1`。响应不重复返回请求参数，不返回 URL、资源 ID、任务 ID、provider 原始 JSON 或 editor 字段。 + +## 预检查与计费事务 + +所有 multipart 字段、图片结构和 provider 参数检查必须在扣费前完成。预检查失败直接返回 4xx，不产生钱包流水，也不调用 provider。 + +检查通过后，api-server 进入现有资产操作计费边界，通过 SpacetimeDB 钱包事务 procedure 原子完成： + +1. 按现有图片编辑算法解析价格：GPT Image 2 长边不超过 1536 使用 1K 价格，否则使用 2K 价格；当前默认价格为 3 / 5 泥点； +2. 以认证后的用户、`raw-image-edit` 命名空间和请求 ID 组成幂等扣费流水 ID； +3. 原子扣除用户泥点并写入 `asset_operation_consume` 流水。 + +provider 调用在 SpacetimeDB 事务之外执行。失败时由现有计费边界把幂等退款事实写入 SpacetimeDB refund outbox，再由 worker 完成退款。 + +TODO：新增 raw 操作持久化状态，将“创建 raw 操作事实 + 扣费”收入同一事务，并由恢复 worker 对“已扣费但未收口”状态自动退款，填补进程在扣费后、写入 refund outbox 前崩溃的窗口。 + +raw 操作使用独立的 operation / ledger 命名空间，例如 `raw-image-edit`，不能复用编辑器资源 ID、编辑器任务 ID 或 `external_generation_job`。 + +## Provider 边界 + +`platform-image` 保留 VectorEngine 协议细节。raw handler 只负责：认证、multipart 字段解析、PNG 预检查、计费编排和响应映射。provider 请求仍由 `platform-image` 统一构造，并携带 `model`、`n`、`quality`、`background`、`output_format`、尺寸及图片参考字节。 + +provider 响应只提取并透传 `data[].b64_json` 字符串，不在服务端解码图片 base64，也不读取或回传 provider 的 `output_format`（该字段只是请求参数回显）。发送、响应读取、上游状态、响应解析和缺图失败必须生成 `PlatformImageFailureAudit`，由 api-server 写入现有外部 API 失败审计链；成功结果同时写入统一的 `external_generation_run` 追踪事件。raw handler 只将上游 `b64_json` 原样写入 `data[].b64_json`。 + +## 代码拆分 + +- `server-rs/crates/api-server/src/raw_image.rs`：独立路由 handler、multipart 字段解析、请求/响应 DTO、PNG 输入校验、预检查和 raw billing 编排。 +- `server-rs/crates/platform-image/src/vector_engine/raw_edit.rs`：raw 编辑选项、严格尺寸校验、独立 provider 请求映射和 `b64_json` 响应透传；不复用现有 editor 图片编辑 client 或其 multipart transport。 +- `server-rs/crates/api-server/src/modules/raw.rs`：只注册 `/api/raw/v1/images/edit` 并挂载 Bearer middleware。 + +不修改 External v1 OpenAPI；不在 `external_editor_api.rs`、编辑器项目模块或外部生成 worker 中增加 raw 分支。 + +## 验收 + +- 未认证请求被 Bearer middleware 拒绝。 +- 预检查失败时钱包无扣费、provider 无请求。 +- 成功响应严格只包含 `data[].b64_json`。 +- provider 失败时 raw 操作失败事务产生可恢复退款事实。 +- raw 请求不创建 `external_generation_job`，不写 editor project/resource/asset/OSS。 +- 运行 api-server 与 platform-image 定向测试、`npm run check:encoding` 和 `git diff --check`。 diff --git a/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md b/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md new file mode 100644 index 000000000..4f19e0e78 --- /dev/null +++ b/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md @@ -0,0 +1,114 @@ +# UI 编辑器自动切分素材工作流 + +更新时间：`2026-09-08` + +## 目标 + +将 UI 编辑器现有“用户先提供独立图片/图标，再执行组件绑定”的入口替换为自动切分素材：结构识别阶段直接返回可渲染组件草稿，切分阶段按整页叶节点批次调用图片编辑模型，再由视觉模型确认处理图中的区域与目标节点。 + +## 识别结果 + +- `recognize` 返回完整 `Node.component` 草稿，不再要求用户先导入独立素材。 +- `component = null` 表示纯节点。 +- `ImageComponent.target_graphic = None` 表示图片组件等待分离结果回填；它不是“明确没有图片”。 +- 一个 Node 最多承载一个 `Component`；需要多个视觉层时使用多个 Node 表达。 +- 组件草稿直接保存在正式 UI Node 中；临时 separation tree 不复制组件。 + +### LLM 工具中的组件载荷 + +正式 `Node` 使用 `component: Option`，因为 Rust/前端状态需要直接表达“纯结构节点”或唯一组件；但识别和绑定 LLM 工具不使用可空字段。部分模型在严格工具 schema 下不会稳定地产生 `null`，因此工具返回显式外部枚举： + +- `"PureNode"` 表示纯结构节点； +- `{ "WithComponent": <完整 Component> }` 表示该节点拥有一个组件。 + +Rust 在接收并校验工具结果后，才把这两个枚举变体映射为正式 Node 的 `None` / `Some(Component)`。`PureNode + NeedReview/Blocked` 是非法组合；`PureNode + NoProblem` 合法，带组件节点可以处于 `NoProblem`、`NeedReview` 或 `Blocked`。该枚举只解决工具调用的可判别性，不引入旧 `null` 格式兼容或迁移。 + +## Separation tree + +- recognition 完成后由 UI tree 构造临时 separation tree；每棵树保留原 UI tree 的 `src_ui_design` 与真实 `root`，不生成 synthetic root。 +- 非 root 的纯容器、纯 Text 节点和不需要切图的节点在构造时过滤，被过滤节点的 children 向上透传。真实 root 始终保留；`root_extractable` 表示 root 是否含未绑定图片组件并可作为候选。 +- 节点的 `children` 在整个 workflow 中始终保留，不能因处理成功或失败而从树上删除。节点终态由 `bound`、`problematic_nodes` 反查；两者均不存在时仍待处理，`rework_count` 仅记录视觉模型返工次数。 +- 逻辑叶必须是未终止、可处理且所有 children 都已终止的节点；root 在 `root_extractable=true` 时按普通节点参与，否则只递归其 children。 +- 候选按 DFS 和 children 原顺序遍历，使用简单贪心选择与已选矩形无正面积交集的节点组成 batch；边或角接触不算重叠，不做面积或偏差排序检查。正常树结构下有候选时至少选中一个。 +- 一个 batch 是当前树中整批互不重叠的逻辑叶节点。 +- 一个 batch 的最小处理单元是：一次 image-edit + 一次 visual binding。 +- batch 成功后只把结果追加到 bound 容器，失败节点在达到返工上限后追加到 problematic 容器；树拓扑不变，流程继续消费剩余树。 +- 不额外维护节点状态枚举；节点是否仍在 pending tree、`rework_count` 和 problematic 容器共同表达状态。 + +### 普通文字遮罩 + +- 普通 `Text` 仍是正式 UI tree 中的独立 UI 元素，不进入 separation tree，也不参与 batch、绿色框、visual binding 或 bound 结果。 +- 构造 separation tree 时，把 Text 节点的布局矩形转换为页面像素坐标，挂到最近的未绑定图片节点（`ImageComponent.target_graphic == None`）的 `text_mask_areas`。纯容器只透传；嵌套图片下归最近图片；没有可切图片祖先的 Text 直接忽略。 +- `text_mask_areas` 只保存 `global_pos_x_px`、`global_pos_y_px`、`width_px`、`height_px`。不保存 NodeId、父节点、文字内容、字体样式，也不做 OCR、字形估算、偏差检查、合并或去重。 +- marker 阶段在 image-edit 前把当前 batch 节点自身的文字矩形填充为紫色；它与子图片区域一起绘制，绿色框随后绘制并位于最上层。文字遮罩不递归读取后代节点的 mask。 +- mask 仅是 image-edit 输入标记，未对 image-edit 残留文字增加 OCR 或视觉复核；正式文字语义仍由 `TextComponent` 保持。 + +## 图片编辑与视觉绑定 + +- image-edit 使用源 UI design 图片及由 Rust 生成的绿色标记/紫色重建输入。处理父节点时，紫色填充其 children 的矩形区域（包括已 problematic 的 children），再在父节点自身外围绘制绿色框和角到角的绿色交叉线；绿色标记覆盖在紫色之上。叶节点只绘制绿色框和角到角的绿色交叉线，不填充自身。 +- 请求尺寸始终使用源 UI design 尺寸；Raw GPT Image 2 API 保证返回相同尺寸，客户端不额外做尺寸拒绝检查。 +- 标记图构建、处理图解码/写入和 cut 裁切属于本地 CPU/文件操作，放入独立的 + `spawn_blocking` 任务；image-edit 与 visual binding 网络请求仍运行在 async future 中。 +- 视觉 binding 输入源图与处理图，必须为当前 batch 每个节点恰好返回一次 `Ok` 或 `NeedRework`。 +- `Ok` 返回 `NodeId + BindingArea`；Rust 仅校验 NodeId、区域边界和非零尺寸，不检查与原节点框的偏差，也不要求区域不重叠。 +- cut 前会对视觉模型返回的 `BindingArea` 做本地像素边界归一化。处理图是透明 PNG，有效像素定义为 `alpha > 0`。四条边以模型 area 为起点，每条边根据首次扫描结果固定方向：边上无有效像素则只向内收缩，边上有有效像素则只向外扩展；四边每轮从同一矩形快照同时逐像素推进，直到达到“内侧有像素、外侧无像素”的分界、图像边界或每条边相对原始 area 的位移上限。该上限由模块级常量 `MAX_BINDING_AREA_EDGE_ADJUSTMENT_PERCENT` 定义。方向固定用于避免稀疏像素造成边界来回振荡；没有理想分界时使用受限范围内的最终 area，不重新请求视觉模型，也不转 problematic。全透明处理图不走特殊错误分支，仍沿同一规则得到最终 area 后裁切。归一化只影响本地 cut，不改写原始 `BindingDecision`、sidecar 或 DTO；日志记录原始 area、最终 area、是否变更，以及仍需移动时是否受到该常量上限、图像边界或非零尺寸约束。性能优化列 TODO。 +- `NeedRework` 携带短问题描述（最多 512 个 Unicode 字符）；通过校验后按产生顺序追加到目标 `SeparationNode.note.rework_notes`，下一次该节点进入 image-edit 时全部意见会注入提取 prompt。结构化工具调用失败时使用可复用 repair harness，把错误反馈给模型并额外请求一次；image-edit 不使用该 harness。 +- 达到返工上限时仍先保留最后一条视觉模型意见，再把节点追加到 problematic；网络、IO、裁切等基础设施错误不写入节点意见。 +- 达到模块级重做常量后，节点移入 problematic；不中断整条工作流，最终统一通知用户。image-edit、图像写入或裁切失败保留当前 state 并返回错误，不自动把整批标记为 problematic。 +- 父节点背景重建由 image-edit 模型完成，不由 Rust 硬编码重建算法完成。 +- 性能观测沿用 `app_log!`：marker 记录端到端与 `spawn_blocking` 耗时，area 记录 `elapsed_us` 与扫描轮数，image-edit 与 visual binding 记录整个请求耗时，处理图写入、cut 和 batch 记录阶段耗时；日志不写入 prompt、图片内容、绝对路径或模型原文。 + +## 临时 sidecar + +- separation 状态不写入 UI JSON，也不进入 manifest。 +- sidecar 目录按 UI manifest `asset_id` 生成，复用 `generated_file_stem(asset_id)` 的安全字符替换和 SHA-256 摘要规则，位于项目 `ui/` 下。 +- 目录只保存一份当前 separation state，而不是每 batch 一个状态文件。 +- state 文件只保留 `schema_version`、separation trees、bound 结果和 problematic 节点，不重复保存 `projectId / assetId / uiStateRevision`。 +- `SeparationNode.text_mask_areas` 是 separation tree 的必选字段，当前开发阶段继续使用 `ui-editor-separation-state.v1`，不提供旧 sidecar 迁移或回退。 +- sidecar 只在 separation 未完成期间存在；完成后删除 state JSON。 +- 当前只持久化已经完成的 batch；正在执行 batch 的恢复语义列 TODO。 +- 临时图片可跨重启保留。raw image-edit 返回图、绿色/紫色标记图、处理图和 cut 图片当前都保留用于 debug；理论上只应在内存中，清理/归档策略列 TODO。 +- 并发边界：当前由前端 `isSeparating` 与 `runWithStateLocked` 保证同一 UI 编辑会话 + 同时只有一次 separation。sidecar 是临时恢复状态，不是正式 UI 资产真相，不参与 + manifest 或项目 revision，因此当前不额外持有项目写锁；若未来支持多窗口/多进程并发， + 再增加按 UI asset 的 sidecar 进程级锁。 + +## bound 与 problematic + +- bound 结果仅保存 `NodeId + cut_image_path`，不保存 `BindingArea` 或 component kind。 +- problematic 记录原始 NodeId、问题描述和 `rework_count`；原始 UI Node 保留不变。 +- `SeparationDTO` 不返回计数字段，只返回 `bound_nodes` 与 `problematic_nodes`。 +- separation Rust 流程不自动登记项目级 SpriteAsset。 +- 前端调用方消费 `SeparationDTO.bound_nodes`，复制/登记 cut 图片为项目级 SpriteAsset，再回填对应 Node 的唯一 Image component。 +- 每次重做产生新的 SpriteAssetId，不假设 NodeId 到 SpriteAssetId 的稳定映射。 +- sidecar 中的图片保留，正式 SpriteAsset 的最终清理策略列 TODO。 + +## 前端正式接入 + +- UI 编辑器点击“自动切分素材”时，前端只调用一次 `separate_ui`，消费完整 `SeparationDTO`；separation 内部 batch 不向前端暴露，也不在 UI 中显示 batch 进度。 +- 自动切分的前置检查只要求界面图及对应 UI tree 有效，不要求用户预先导入 SpriteAsset；结果检查负责报告未回填的 Image、丢失的切分素材引用、丢失的字体引用及组件审阅状态。新建和重新开始切分时执行前置检查，继续已有 sidecar 时直接按恢复状态尽力完成。 +- 如果 sidecar 已存在未完成的 `state.json`，点击入口时先打开独立弹窗，由用户选择“继续上次自动切分素材”或“开始新的自动切分素材”。继续复用 sidecar 的 pending tree；重新开始只替换当前 `state.json`，不删除 sidecar 图片。 +- `BoundNode.cut_image_path` 必须是项目根相对路径。前端使用现有 `import_local_project_image_assets` 登记 cut 图片；由于该通用命令单次最多 100 个路径，前端可以在资源登记阶段按 100 条分组调用，但这不属于 separation batch，也不向用户展示。 +- 现有本地资源导入按清洗后的文件名 stem 与内容摘要生成目标路径；相同目标路径直接复用已有 manifest asset ID，内容不同则拒绝覆盖或生成不同摘要路径。前端不自行猜测 SpriteAsset 是否存在，也不从 NodeId 派生 SpriteAssetId。 +- 全部可登记图片完成导入后，前端在一个 `runWithStateLocked` 中复用 `addSpriteAssets` 的内部 State 变换逻辑，加入返回的 SpriteAsset 并回填仍匹配 Node 的唯一未绑定 Image component，最后一次性提交 State。公开 `addSpriteAssets` 的普通 mutation guard 不放宽。 +- UI tree 在 separation 期间发生变化时，已登记的 cut 图片和可匹配节点的回填保留；找不到 Node 或没有未绑定 Image 的结果产生明确问题提示，不回滚已登记资源，不静默跳过。 +- State 保存成功后才调用 `finalize_separation` 删除 sidecar `state.json`；登记、回填或保存失败时保留 sidecar，允许下次选择继续。sidecar 图片按当前 debug 策略保留。 +- UI 编辑器独立页面的 `bindComponents` 前端入口、`binding.rs`、`bind_components` Tauri 命令及 Runtime `workflow.rs` 统一使用当前单组件模型；LLM 工具载荷使用 `NodeComponent`，正式 Node 仍使用 `Option`。 + +## 重启与 Raw GPT Image 2 + +- 已保存的 separation state 是跨重启继续工作的最小单位；重启后从上一个已保存 batch 的状态继续。 +- 当前执行中的 batch 是否持久化、以及如何避免 image-edit 成功后在 patch 前崩溃导致重复调用，列为 TODO。 +- Raw endpoint 每次 HTTP 调用都是一次新操作；客户端不保存或复用 raw operation ID，不实现第二套本地幂等账本。 +- 图片编辑调用当前 Raw GPT Image 2 multipart 合同：`image`（PNG 文件）、`prompt`、`width`、`height`、`output_format=png`、`background=transparent`；不再发送旧 JSON/base64 请求体。 +- 后端 raw operation 的持久状态与扣费后崩溃恢复窗口，遵循 Raw GPT Image 2 方案中的独立 TODO。 + +## TODO + +- 正在执行 batch 的持久化和恢复。 +- 前端自动切分素材接入已实现；仍需补齐真实 Tauri/前端联调回归测试与失败注入测试。 +- 临时图片清理/归档策略。 +- 手动抠图能力。 +- problematic 对更高层 workflow 完成门禁的最终定义。 +- separation workflow 与 manifest/stage 的接入。 +- Raw GPT Image 2 后端 raw operation 持久状态及恢复 worker。 diff --git a/docs/technical/【设计】UI编辑器工作流完成通知弹窗-2026-09-04.md b/docs/technical/【设计】UI编辑器工作流完成通知弹窗-2026-09-04.md index 5cdc52832..385dcea54 100644 --- a/docs/technical/【设计】UI编辑器工作流完成通知弹窗-2026-09-04.md +++ b/docs/technical/【设计】UI编辑器工作流完成通知弹窗-2026-09-04.md @@ -2,7 +2,7 @@ ## 目标 -UI 编辑器的“分析参考图”“识别界面结构”“绑定视觉素材”三个工作流动作在每次运行结束后，用独立的阻塞通知弹窗明确反馈结果，避免仅依赖卡片内一行状态文本而被忽略。 +UI 编辑器的“分析参考图”“识别界面结构”“自动切分素材”三个工作流动作在每次运行结束后，用独立的阻塞通知弹窗明确反馈结果，避免仅依赖卡片内一行状态文本而被忽略。 ## 交互约定 @@ -15,13 +15,13 @@ UI 编辑器的“分析参考图”“识别界面结构”“绑定视觉素 ## 文案 -弹窗标题由步骤名和结果态组成，例如“识别界面结构完成”或“绑定视觉素材失败”。正文复用卡片状态文本，并逐条扩展结果信息，统一以“请检查”收尾。 +弹窗标题由步骤名和结果态组成，例如“识别界面结构完成”或“自动切分素材失败”。正文复用卡片状态文本，并逐条扩展结果信息，统一以“请检查”收尾。 成功状态的基线文案： - 分析参考图：保留已应用的语义建议数量；若现有状态可可靠取得问题/待确认数量，则一并展示。 - 识别界面结构：保留替换的界面树数量，并展示识别结果中的待检查/必须修复数量（若可取得）。 -- 绑定视觉素材：保留现有 `B/B` 批次计数，改为用户可读的绑定结果。 +- 自动切分素材：保留现有 `B/B` 批次计数，改为用户可读的切分结果。 失败状态保留实际错误文本，仅在弹窗标题中补充步骤和失败上下文，正文同样以“请检查”收尾。 diff --git a/docs/【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md b/docs/【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md index fe5c971a3..61258c68e 100644 --- a/docs/【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md +++ b/docs/【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md @@ -38,7 +38,7 @@ Agent 通过白名单工具 `ui.workflow.run` 发起工作流。项目路径由 1. `prepare` 为每个页面创建确定性的 `kind=UI` JSON 资源，引用源 `ui-prototype` 和页面设计图，载入设计图尺寸与相对路径到 `ui_design_images`，并保存 State。 2. `recognize` 依次执行 Provider 多模态结构识别、现有多树合并器、最多每批 5 项的图片/图标组件绑定，并把已登记字体的安全元数据提供给绑定器；阶段分别持久化为 `structure-ready`、`merge-ready`、`binding-ready`，重复执行从最近真实阶段恢复。 3. `status` 只回读 State、页面阶段和 blockers，不推进项目 revision。 -4. `finalize` 只接受 `game/` 下的真实 UTF-8 文件，写入与 UI State revision 绑定的应用标记；所有页面通过应用门禁后才返回 `visual-binding` 最终阶段路由。缺少页面、资源、组件或应用标记时拒绝伪造完成。 +4. `finalize` 只接受 `game/` 下的真实 UTF-8 文件，写入与 UI State revision 绑定的应用标记；所有页面通过应用门禁后才返回 `asset-separation` 最终阶段路由。缺少页面、资源、组件或应用标记时拒绝伪造完成。 每次 State 或 manifest 阶段变化都推进项目 revision。Runtime 回执带有 `revisionAdvanceCount`，用于并发项目 revision 门禁；manifest 资产的 `source.generationKind` 依次记录： @@ -67,7 +67,7 @@ ui-workflow.completed - 没有关联时原子创建 `ui/UI 设计 N.json`，登记 `kind=UI`、`application/json`，并把原型图作为首张页面设计图载入 State。 - 成功后通过 `onManifestChange` 更新客户端资源投影，再打开 UI 编辑器；普通桥接从 `reference-analysis` 开始。 -点击已有 `UI` 资源直接打开 UI 编辑器。若 manifest 阶段为 `ui-workflow.completed`，工作台自动打开该资源的 `visual-binding` 阶段（最远步骤为 2），交给用户做最终检查和手动调整。 +点击已有 `UI` 资源直接打开 UI 编辑器。若 manifest 阶段为 `ui-workflow.completed`，工作台自动打开该资源的 `asset-separation` 阶段（最远步骤为 2），交给用户做最终检查和手动调整。 ## 诚实完成门禁 diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index 1acba75f0..b77676b63 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -470,6 +470,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "serde_core", @@ -2924,6 +2925,23 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "naga" version = "27.0.3" @@ -4074,6 +4092,7 @@ dependencies = [ "image", "platform-oss", "reqwest", + "serde", "serde_json", "tokio", "tracing", @@ -5628,6 +5647,12 @@ dependencies = [ "tokio-tungstenite 0.27.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "sse-stream" version = "0.2.5" diff --git a/server-rs/crates/api-server/Cargo.toml b/server-rs/crates/api-server/Cargo.toml index 78cfb8bab..0a166cd5f 100644 --- a/server-rs/crates/api-server/Cargo.toml +++ b/server-rs/crates/api-server/Cargo.toml @@ -7,7 +7,7 @@ license.workspace = true [dependencies] aes = { workspace = true } async-stream = { workspace = true } -axum = { workspace = true, features = ["ws"] } +axum = { workspace = true, features = ["ws", "multipart"] } base64 = { workspace = true } cbc = { workspace = true } bytes = { workspace = true } diff --git a/server-rs/crates/api-server/src/app.rs b/server-rs/crates/api-server/src/app.rs index d898bdbf9..a3a761b3f 100644 --- a/server-rs/crates/api-server/src/app.rs +++ b/server-rs/crates/api-server/src/app.rs @@ -50,6 +50,7 @@ pub fn build_router(state: AppState) -> Router { .merge(modules::platform::router(state.clone())) .merge(modules::external_generation::router(state.clone())) .merge(modules::platform_support::router(state.clone())) + .merge(modules::raw::router(state.clone())) .merge(crate::error_reports::router(state.clone())) .route( "/api/profile/recharge/wechat/notify", diff --git a/server-rs/crates/api-server/src/main.rs b/server-rs/crates/api-server/src/main.rs index f40e6309e..25e42cd28 100644 --- a/server-rs/crates/api-server/src/main.rs +++ b/server-rs/crates/api-server/src/main.rs @@ -67,6 +67,7 @@ mod profile_identity; mod profile_recharge_expiration_listener; mod profile_recharge_refund_reconciliation; mod prompt; +mod raw_image; mod refresh_session; mod registration_reward; mod request_context; diff --git a/server-rs/crates/api-server/src/modules.rs b/server-rs/crates/api-server/src/modules.rs index 558c37c2b..76f5958b7 100644 --- a/server-rs/crates/api-server/src/modules.rs +++ b/server-rs/crates/api-server/src/modules.rs @@ -10,3 +10,4 @@ pub mod internal; pub mod platform; pub mod platform_support; pub mod profile; +pub mod raw; diff --git a/server-rs/crates/api-server/src/modules/raw.rs b/server-rs/crates/api-server/src/modules/raw.rs new file mode 100644 index 000000000..8307da70c --- /dev/null +++ b/server-rs/crates/api-server/src/modules/raw.rs @@ -0,0 +1,14 @@ +use axum::{Router, extract::DefaultBodyLimit, middleware, routing::post}; + +use crate::{auth::require_bearer_auth, raw_image::edit_raw_image, state::AppState}; + +const RAW_IMAGE_EDIT_BODY_LIMIT_BYTES: usize = 64 * 1024 * 1024; + +pub fn router(state: AppState) -> Router { + Router::new().route( + "/api/raw/v1/images/edit", + post(edit_raw_image) + .route_layer(middleware::from_fn_with_state(state, require_bearer_auth)) + .layer(DefaultBodyLimit::max(RAW_IMAGE_EDIT_BODY_LIMIT_BYTES)), + ) +} diff --git a/server-rs/crates/api-server/src/openai_image_generation.rs b/server-rs/crates/api-server/src/openai_image_generation.rs index e3e381eb5..f0fa7591c 100644 --- a/server-rs/crates/api-server/src/openai_image_generation.rs +++ b/server-rs/crates/api-server/src/openai_image_generation.rs @@ -414,7 +414,7 @@ impl OpenAiImageSettings { self } - fn provider_settings(&self) -> VectorEngineImageSettings { + pub(crate) fn provider_settings(&self) -> VectorEngineImageSettings { VectorEngineImageSettings { base_url: self.base_url.clone(), api_key: self.api_key.clone(), diff --git a/server-rs/crates/api-server/src/raw_image.rs b/server-rs/crates/api-server/src/raw_image.rs new file mode 100644 index 000000000..fc0d33a6d --- /dev/null +++ b/server-rs/crates/api-server/src/raw_image.rs @@ -0,0 +1,573 @@ +use axum::{ + Json, + extract::{Extension, Multipart, State}, + http::StatusCode, +}; +use image::{GenericImageView, ImageFormat, ImageReader}; +use platform_image::{ + RAW_IMAGE_MAX_EDGE, RAW_IMAGE_MAX_PIXELS, RawImageEditOptions, ReferenceImage, + create_vector_engine_raw_image_edit, validate_raw_image_edit_dimensions, +}; +use serde::Serialize; +use serde_json::json; +use std::io::Cursor; + +use crate::{ + asset_billing::{ + execute_billable_asset_operation_with_cost, with_editor_generation_durable_billing_boundary, + }, + auth::AuthenticatedAccessToken, + http_error::AppError, + openai_image_generation::{ + map_platform_image_error, record_openai_image_failure_if_configured, + require_openai_image_settings, + }, + request_context::RequestContext, + state::AppState, + tracking::record_external_generation_run_after_success, +}; +use time::OffsetDateTime; + +#[derive(Debug)] +struct RawImageData { + pub(crate) bytes: Vec, + pub(crate) mime_type: String, + pub(crate) file_name: String, +} + +#[derive(Debug)] +struct RawImageEditRequest { + pub(crate) image: RawImageData, + pub(crate) mask: Option, + pub(crate) prompt: String, + pub(crate) quality: Option, + pub(crate) background: Option, + pub(crate) output_format: Option, + pub(crate) width: u32, + pub(crate) height: u32, +} + +#[derive(Debug, Serialize)] +pub(crate) struct RawImageEditItem { + pub(crate) b64_json: String, +} + +#[derive(Debug, Serialize)] +pub(crate) struct RawImageEditResponse { + pub(crate) data: Vec, +} + +const RAW_IMAGE_MAX_PROMPT_BYTES: usize = 4 * 1024; + +pub(crate) async fn edit_raw_image( + State(state): State, + Extension(request_context): Extension, + Extension(authenticated): Extension, + multipart: Multipart, +) -> Result, AppError> { + let payload = parse_multipart_request(multipart).await?; + let prepared = tokio::task::spawn_blocking(move || prepare_request(payload)) + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_message(error.to_string()) + })??; + let settings = require_openai_image_settings(&state)?.with_external_api_audit_context( + &request_context, + Some(authenticated.claims().user_id().to_string()), + None, + ); + let provider_settings = settings.provider_settings(); + let user_id = authenticated.claims().user_id().to_string(); + let request_id = request_context.request_id().to_string(); + let points_cost = raw_image_edit_price(&state, prepared.width, prepared.height).await?; + let audit_settings = settings.clone(); + let tracking_state = audit_settings.external_api_audit_state.clone(); + let tracking_payload = json!({ + "width": prepared.width, + "height": prepared.height, + "promptChars": prepared.prompt.chars().count(), + "hasMask": prepared.options.mask.is_some(), + "quality": prepared.options.quality.as_deref(), + "background": prepared.options.background.as_deref(), + "outputFormat": prepared.options.output_format.as_deref(), + }); + let started_at_micros = (OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000) as i64; + let operation = async move { + let generated = match create_vector_engine_raw_image_edit( + &provider_settings, + prepared.prompt.as_str(), + prepared.image, + prepared.options, + "raw_image_edit", + ) + .await + { + Ok(generated) => generated, + Err(error) => { + record_openai_image_failure_if_configured(&audit_settings, &error).await; + return Err(map_platform_image_error(error)); + } + }; + let data: Vec = generated + .b64_images + .into_iter() + .map(|b64_json| RawImageEditItem { b64_json }) + .collect(); + if let Some(state) = tracking_state.as_ref() { + record_external_generation_run_after_success( + state, + platform_image::VECTOR_ENGINE_PROVIDER, + "raw_image_edit", + "raw_image_edit", + tracking_payload, + started_at_micros, + true, + None, + Some("raw-image-edit".to_string()), + Some(json!({ "imageCount": data.len() })), + ) + .await; + } + Ok::<_, AppError>(RawImageEditResponse { data }) + }; + let result = with_editor_generation_durable_billing_boundary( + execute_billable_asset_operation_with_cost( + &state, + user_id.as_str(), + "raw-image-edit", + request_id.as_str(), + u64::from(points_cost), + operation, + ), + ) + .await?; + Ok(Json(result)) +} + +struct PreparedRawImageEdit { + image: ReferenceImage, + prompt: String, + options: RawImageEditOptions, + width: u32, + height: u32, +} + +async fn parse_multipart_request( + mut multipart: Multipart, +) -> Result { + let mut image = None; + let mut mask = None; + let mut prompt = None; + let mut quality = None; + let mut background = None; + let mut output_format = None; + let mut width = None; + let mut height = None; + + while let Some(field) = multipart + .next_field() + .await + .map_err(|error| bad_request(format!("multipart 字段读取失败：{error}")))? + { + let name = field + .name() + .ok_or_else(|| bad_request("multipart 字段缺少名称"))? + .to_string(); + match name.as_str() { + "image" => { + if image.is_some() { + return Err(bad_request("image 字段不能重复")); + } + image = Some(read_multipart_image(field, "image").await?); + } + "mask" => { + if mask.is_some() { + return Err(bad_request("mask 字段不能重复")); + } + mask = Some(read_multipart_image(field, "mask").await?); + } + "prompt" => set_text_field(&mut prompt, field, "prompt").await?, + "quality" => set_text_field(&mut quality, field, "quality").await?, + "background" => set_text_field(&mut background, field, "background").await?, + "output_format" => set_text_field(&mut output_format, field, "output_format").await?, + "width" => set_text_field(&mut width, field, "width").await?, + "height" => set_text_field(&mut height, field, "height").await?, + _ => return Err(bad_request(format!("不支持的 multipart 字段：{name}"))), + } + } + + let image = image.ok_or_else(|| bad_request("image 字段不能为空"))?; + let prompt = prompt.ok_or_else(|| bad_request("prompt 字段不能为空"))?; + let width = parse_multipart_u32(width, "width")?; + let height = parse_multipart_u32(height, "height")?; + + Ok(RawImageEditRequest { + image, + mask, + prompt, + quality, + background, + output_format, + width, + height, + }) +} + +async fn read_multipart_image( + field: axum::extract::multipart::Field<'_>, + name: &str, +) -> Result { + let mime_type = field.content_type().unwrap_or_default().to_string(); + if !mime_type.eq_ignore_ascii_case("image/png") { + return Err(bad_request(format!("{name} 必须为 image/png"))); + } + let bytes = field + .bytes() + .await + .map_err(|error| bad_request(format!("{name} 文件读取失败：{error}")))?; + if bytes.is_empty() { + return Err(bad_request(format!("{name} 文件不能为空"))); + } + Ok(RawImageData { + bytes: bytes.to_vec(), + mime_type: "image/png".to_string(), + file_name: format!("{name}.png"), + }) +} + +async fn set_text_field( + target: &mut Option, + field: axum::extract::multipart::Field<'_>, + name: &str, +) -> Result<(), AppError> { + if target.is_some() { + return Err(bad_request(format!("{name} 字段不能重复"))); + } + *target = Some( + field + .text() + .await + .map_err(|error| bad_request(format!("{name} 字段读取失败：{error}")))?, + ); + Ok(()) +} + +fn parse_multipart_u32(value: Option, field: &str) -> Result { + let value = value.ok_or_else(|| bad_request(format!("{field} 字段不能为空")))?; + value + .trim() + .parse::() + .map_err(|_| bad_request(format!("{field} 必须为有效整数"))) +} + +fn prepare_request(payload: RawImageEditRequest) -> Result { + if payload.prompt.trim().is_empty() { + return Err(bad_request("prompt 不能为空")); + } + if payload.prompt.len() > RAW_IMAGE_MAX_PROMPT_BYTES { + return Err(bad_request(format!( + "prompt 不能超过 {RAW_IMAGE_MAX_PROMPT_BYTES} 字节" + ))); + } + validate_raw_image_edit_dimensions(payload.width, payload.height) + .map_err(|error| bad_request(error.to_string()))?; + validate_optional_value( + payload.quality.as_deref(), + "quality", + ["low", "medium", "high", "auto"], + )?; + validate_optional_value( + payload.background.as_deref(), + "background", + ["transparent", "opaque", "auto"], + )?; + validate_optional_value( + payload.output_format.as_deref(), + "output_format", + ["png", "webp", "jpeg"], + )?; + let quality = normalize_optional(payload.quality); + let background = normalize_optional(payload.background); + let output_format = normalize_optional(payload.output_format); + let (image, image_width, image_height) = decode_image(payload.image, "image")?; + let mask = payload + .mask + .map(|value| { + let (mask, mask_width, mask_height) = decode_image(value, "mask")?; + if mask_width != image_width || mask_height != image_height { + return Err(bad_request("mask 尺寸必须与 image 一致")); + } + Ok(mask) + }) + .transpose()?; + Ok(PreparedRawImageEdit { + image, + prompt: payload.prompt, + options: RawImageEditOptions { + quality, + background, + output_format, + width: payload.width, + height: payload.height, + mask, + }, + width: payload.width, + height: payload.height, + }) +} + +fn normalize_optional(value: Option) -> Option { + value + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +fn validate_optional_value( + value: Option<&str>, + field: &str, + allowed: [&str; N], +) -> Result<(), AppError> { + let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else { + return Ok(()); + }; + if allowed.contains(&value) { + return Ok(()); + } + Err(bad_request(format!("{field} 值无效"))) +} + +fn decode_image(value: RawImageData, field: &str) -> Result<(ReferenceImage, u32, u32), AppError> { + let mime_type = value.mime_type.trim().to_string(); + if !mime_type.eq_ignore_ascii_case("image/png") { + return Err(bad_request(format!( + "{field} Content-Type 必须为 image/png" + ))); + } + let bytes = value.bytes; + if bytes.is_empty() { + return Err(bad_request(format!("{field} 文件不能为空"))); + } + let mut reader = ImageReader::new(Cursor::new(bytes.as_slice())) + .with_guessed_format() + .map_err(|_| bad_request(format!("{field} 文件必须是有效 PNG 文件")))?; + let mut limits = image::Limits::default(); + limits.max_image_width = Some(RAW_IMAGE_MAX_EDGE); + limits.max_image_height = Some(RAW_IMAGE_MAX_EDGE); + limits.max_alloc = Some(RAW_IMAGE_MAX_PIXELS.saturating_mul(4)); + reader.limits(limits); + if reader.format() != Some(ImageFormat::Png) { + return Err(bad_request(format!("{field} 文件必须是有效 PNG 文件"))); + } + let decoded = reader + .decode() + .map_err(|error| map_decode_image_error(field, error))?; + let (width, height) = decoded.dimensions(); + Ok(( + ReferenceImage { + bytes, + file_name: value.file_name, + mime_type, + }, + width, + height, + )) +} + +fn map_decode_image_error(field: &str, error: image::ImageError) -> AppError { + let message = match error { + image::ImageError::Limits(_) => { + format!("{field} 文件超出 PNG 尺寸或解码资源上限（单边不超过 {RAW_IMAGE_MAX_EDGE}px）") + } + _ => format!("{field} 文件必须是有效 PNG 文件"), + }; + bad_request(message) +} + +async fn raw_image_edit_price(state: &AppState, width: u32, height: u32) -> Result { + let tier = if width.max(height) > 1536 { "2K" } else { "1K" }; + state + .editor_generation_pricing() + .await + .map(|pricing| { + pricing.image_generation_mud_points(Some("quick-edit"), Some("gpt-image-2"), Some(tier)) + }) + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "provider": "editor-generation-pricing", + "message": error.to_string(), + })) + }) +} + +fn bad_request(message: impl Into) -> AppError { + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": "raw-image-edit", + "message": message.into(), + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{body::Body, extract::FromRequest, http::Request}; + use image::{ImageFormat, Rgba, RgbaImage}; + use std::io::Cursor; + + fn png_bytes(width: u32, height: u32) -> Vec { + let image = RgbaImage::from_pixel(width, height, Rgba([255, 0, 0, 255])); + let mut bytes = Vec::new(); + image + .write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png) + .expect("test PNG should encode"); + bytes + } + + fn request(image: Vec, mask: Option>) -> RawImageEditRequest { + RawImageEditRequest { + image: RawImageData { + bytes: image, + mime_type: "image/png".to_string(), + file_name: "image.png".to_string(), + }, + mask: mask.map(|bytes| RawImageData { + bytes, + mime_type: "image/png".to_string(), + file_name: "mask.png".to_string(), + }), + prompt: "edit".to_string(), + width: 1024, + height: 1024, + quality: None, + background: None, + output_format: None, + } + } + + fn multipart_body(boundary: &str, image: &[u8]) -> Vec { + let mut body = Vec::new(); + let add_text = |body: &mut Vec, name: &str, value: &str| { + body.extend_from_slice(format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n" + ).as_bytes()); + }; + add_text(&mut body, "prompt", "edit"); + add_text(&mut body, "width", "1024"); + add_text(&mut body, "height", "1024"); + body.extend_from_slice( + format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"image\"; filename=\"ignored.png\"\r\nContent-Type: image/png\r\n\r\n" + ) + .as_bytes(), + ); + body.extend_from_slice(image); + body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes()); + body + } + + #[tokio::test] + async fn multipart_parser_accepts_binary_image_and_text_fields() { + let boundary = "raw-test-boundary"; + let image = png_bytes(1, 1); + let body = multipart_body(boundary, &image); + let request = Request::builder() + .header( + "content-type", + format!("multipart/form-data; boundary={boundary}"), + ) + .body(Body::from(body)) + .expect("multipart request"); + let multipart = Multipart::from_request(request, &()) + .await + .expect("multipart"); + let parsed = parse_multipart_request(multipart) + .await + .expect("multipart fields should parse"); + + assert_eq!(parsed.prompt, "edit"); + assert_eq!(parsed.width, 1024); + assert_eq!(parsed.height, 1024); + assert!(parsed.image.bytes.starts_with(b"\x89PNG\r\n\x1a\n")); + } + + #[test] + fn response_contains_only_data_b64_json() { + let response = serde_json::to_value(RawImageEditResponse { + data: vec![RawImageEditItem { + b64_json: "aGVsbG8=".to_string(), + }], + }) + .expect("response should serialize"); + assert_eq!( + response, + serde_json::json!({"data": [{"b64_json": "aGVsbG8="}]}) + ); + } + + #[test] + fn dimensions_follow_strict_raw_image_contract() { + assert!(validate_raw_image_edit_dimensions(1024, 1024).is_ok()); + assert!(validate_raw_image_edit_dimensions(3840, 1280).is_ok()); + assert!(validate_raw_image_edit_dimensions(3839, 1280).is_err()); + assert!(validate_raw_image_edit_dimensions(3840, 1264).is_err()); + assert!(validate_raw_image_edit_dimensions(1024, 1000).is_err()); + assert!(validate_raw_image_edit_dimensions(16, 16).is_err()); + assert!(validate_raw_image_edit_dimensions(3840, 3840).is_err()); + } + + #[test] + fn input_requires_decodable_png_and_png_mime() { + assert!(prepare_request(request(png_bytes(1, 1), None)).is_ok()); + + let invalid_bytes = RawImageEditRequest { + image: RawImageData { + bytes: b"hello".to_vec(), + mime_type: "image/png".to_string(), + file_name: "image.png".to_string(), + }, + ..request(png_bytes(1, 1), None) + }; + assert!(prepare_request(invalid_bytes).is_err()); + + let invalid_mime = RawImageEditRequest { + image: RawImageData { + bytes: png_bytes(1, 1), + mime_type: "image/jpeg".to_string(), + file_name: "image.jpg".to_string(), + }, + ..request(png_bytes(1, 1), None) + }; + assert!(prepare_request(invalid_mime).is_err()); + } + + #[test] + fn oversized_valid_png_reports_resource_limit() { + let error = match prepare_request(request(png_bytes(RAW_IMAGE_MAX_EDGE + 1, 1), None)) { + Ok(_) => panic!("oversized PNG should fail"), + Err(error) => error, + }; + + assert!(format!("{error:?}").contains("超出 PNG 尺寸或解码资源上限")); + } + + #[test] + fn mask_must_match_source_image_dimensions() { + let error = match prepare_request(request(png_bytes(2, 1), Some(png_bytes(1, 1)))) { + Ok(_) => panic!("mismatched mask should fail before billing"), + Err(error) => error, + }; + + assert!(format!("{error:?}").contains("mask 尺寸必须与 image 一致")); + } + + #[test] + fn prompt_uses_raw_utf8_byte_limit() { + let mut parsed = request(png_bytes(1, 1), None); + parsed.prompt = "a".repeat(RAW_IMAGE_MAX_PROMPT_BYTES + 1); + let error = match prepare_request(parsed) { + Ok(_) => panic!("oversized prompt should fail before image decode and billing"), + Err(error) => error, + }; + + assert!(format!("{error:?}").contains("prompt 不能超过 4096 字节")); + } +} diff --git a/server-rs/crates/platform-image/Cargo.toml b/server-rs/crates/platform-image/Cargo.toml index 9da088343..2030dab06 100644 --- a/server-rs/crates/platform-image/Cargo.toml +++ b/server-rs/crates/platform-image/Cargo.toml @@ -9,6 +9,7 @@ base64 = { workspace = true } curl = { workspace = true } image = { workspace = true, features = ["jpeg", "png", "webp"] } reqwest = { workspace = true, features = ["json", "multipart", "rustls-tls"] } +serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["io-util", "macros", "net", "time"] } tracing = { workspace = true } diff --git a/server-rs/crates/platform-image/src/lib.rs b/server-rs/crates/platform-image/src/lib.rs index 8db6a9c97..c527d4c7d 100644 --- a/server-rs/crates/platform-image/src/lib.rs +++ b/server-rs/crates/platform-image/src/lib.rs @@ -10,14 +10,16 @@ pub use pixel_art_snapper::{ }; pub use vector_engine::{ DownloadedImage, GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, GeneratedImages, NANOBANANA_2_MODEL, - PlatformImageError, PlatformImageFailureAudit, PlatformImageStatusHint, ReferenceImage, + PlatformImageError, PlatformImageFailureAudit, PlatformImageStatusHint, + RAW_IMAGE_DIMENSION_ALIGNMENT, RAW_IMAGE_MAX_EDGE, RAW_IMAGE_MAX_PIXELS, RAW_IMAGE_MIN_PIXELS, + RawImageEditDimensionError, RawImageEditOptions, RawImageEditResult, ReferenceImage, VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER, VectorEngineImageSettings, build_vector_engine_image_http_client, build_vector_engine_image_request_body, build_vector_engine_nanobanana_generate_content_request_body, create_vector_engine_image_edit, create_vector_engine_image_edit_with_references, create_vector_engine_image_edit_with_references_and_model, create_vector_engine_image_generation, create_vector_engine_image_generation_with_model, - create_vector_engine_nanobanana_generate_content, download_remote_image, - vector_engine_images_edit_url, vector_engine_images_generation_url, - vector_engine_nanobanana_generate_content_url, + create_vector_engine_nanobanana_generate_content, create_vector_engine_raw_image_edit, + download_remote_image, validate_raw_image_edit_dimensions, vector_engine_images_edit_url, + vector_engine_images_generation_url, vector_engine_nanobanana_generate_content_url, }; diff --git a/server-rs/crates/platform-image/src/vector_engine/constants.rs b/server-rs/crates/platform-image/src/vector_engine/constants.rs index 6480fba73..dfb70f0f5 100644 --- a/server-rs/crates/platform-image/src/vector_engine/constants.rs +++ b/server-rs/crates/platform-image/src/vector_engine/constants.rs @@ -5,3 +5,8 @@ pub const VECTOR_ENGINE_GPT_IMAGE_2_MODEL: &str = GPT_IMAGE_2_MODEL; pub const VECTOR_ENGINE_PROVIDER: &str = "vector-engine"; pub const VECTOR_ENGINE_IMAGE_EDIT_MAX_REFERENCE_IMAGES: usize = 5; pub const VECTOR_ENGINE_NANOBANANA_MAX_REFERENCE_IMAGES: usize = 14; + +pub(crate) const GPT_IMAGE_2_MIN_PIXELS: u64 = 655_360; +pub(crate) const GPT_IMAGE_2_MAX_PIXELS: u64 = 8_294_400; +pub(crate) const GPT_IMAGE_2_MAX_EDGE: u32 = 3_840; +pub(crate) const GPT_IMAGE_2_DIMENSION_ALIGNMENT: u32 = 16; diff --git a/server-rs/crates/platform-image/src/vector_engine/mod.rs b/server-rs/crates/platform-image/src/vector_engine/mod.rs index f64cba54a..351244650 100644 --- a/server-rs/crates/platform-image/src/vector_engine/mod.rs +++ b/server-rs/crates/platform-image/src/vector_engine/mod.rs @@ -6,6 +6,7 @@ mod curl_transport; mod error; mod image_source; mod payload; +mod raw_edit; mod request; mod response; mod transport; @@ -25,6 +26,11 @@ pub use constants::{ }; pub use error::{PlatformImageError, PlatformImageStatusHint}; pub use image_source::download_remote_image; +pub use raw_edit::{ + RAW_IMAGE_DIMENSION_ALIGNMENT, RAW_IMAGE_MAX_EDGE, RAW_IMAGE_MAX_PIXELS, RAW_IMAGE_MIN_PIXELS, + RawImageEditDimensionError, RawImageEditOptions, create_vector_engine_raw_image_edit, + validate_raw_image_edit_dimensions, +}; pub use request::{ build_vector_engine_image_request_body, build_vector_engine_image_request_body_with_model, build_vector_engine_nanobanana_generate_content_request_body, normalize_image_size_for_model, @@ -32,4 +38,6 @@ pub use request::{ vector_engine_nanobanana_generate_content_url, }; pub use transport::build_vector_engine_image_http_client; -pub use types::{DownloadedImage, GeneratedImages, ReferenceImage, VectorEngineImageSettings}; +pub use types::{ + DownloadedImage, GeneratedImages, RawImageEditResult, ReferenceImage, VectorEngineImageSettings, +}; diff --git a/server-rs/crates/platform-image/src/vector_engine/raw_edit.rs b/server-rs/crates/platform-image/src/vector_engine/raw_edit.rs new file mode 100644 index 000000000..984d37add --- /dev/null +++ b/server-rs/crates/platform-image/src/vector_engine/raw_edit.rs @@ -0,0 +1,473 @@ +use std::time::{Duration, Instant}; + +use reqwest::multipart::{Form, Part}; +use serde::Deserialize; + +use super::{ + audit::build_failure_audit, + budget::{effective_request_timeout_ms, request_budget_exhausted_error}, + constants::{ + GPT_IMAGE_2_DIMENSION_ALIGNMENT, GPT_IMAGE_2_MAX_EDGE, GPT_IMAGE_2_MAX_PIXELS, + GPT_IMAGE_2_MIN_PIXELS, GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER, + }, + error::PlatformImageError, + request::vector_engine_images_edit_url, + types::{RawImageEditResult, ReferenceImage, VectorEngineImageSettings}, + util::truncate_raw, +}; + +#[derive(Clone, Debug)] +pub struct RawImageEditOptions { + pub quality: Option, + pub background: Option, + pub output_format: Option, + pub width: u32, + pub height: u32, + pub mask: Option, +} + +#[derive(Debug, Deserialize)] +struct RawImageEditResponsePayload { + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct RawImageEditResponseEntry { + b64_json: String, +} + +pub const RAW_IMAGE_MAX_EDGE: u32 = GPT_IMAGE_2_MAX_EDGE; +pub const RAW_IMAGE_DIMENSION_ALIGNMENT: u32 = GPT_IMAGE_2_DIMENSION_ALIGNMENT; +pub const RAW_IMAGE_MIN_PIXELS: u64 = GPT_IMAGE_2_MIN_PIXELS; +pub const RAW_IMAGE_MAX_PIXELS: u64 = GPT_IMAGE_2_MAX_PIXELS; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RawImageEditDimensionError { + Zero, + MaxEdge, + Alignment, + AspectRatio, + PixelCount, +} + +impl std::fmt::Display for RawImageEditDimensionError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Zero => formatter.write_str("width 和 height 必须为正整数"), + Self::MaxEdge => write!( + formatter, + "width 和 height 的单边最大值为 {RAW_IMAGE_MAX_EDGE}px" + ), + Self::Alignment => write!( + formatter, + "width 和 height 必须是 {RAW_IMAGE_DIMENSION_ALIGNMENT}px 的倍数" + ), + Self::AspectRatio => formatter.write_str("长边与短边的比例不能超过 3:1"), + Self::PixelCount => write!( + formatter, + "总像素必须在 {RAW_IMAGE_MIN_PIXELS} 至 {RAW_IMAGE_MAX_PIXELS} 之间" + ), + } + } +} + +impl std::error::Error for RawImageEditDimensionError {} + +pub fn validate_raw_image_edit_dimensions( + width: u32, + height: u32, +) -> Result<(), RawImageEditDimensionError> { + if width == 0 || height == 0 { + return Err(RawImageEditDimensionError::Zero); + } + if width > RAW_IMAGE_MAX_EDGE || height > RAW_IMAGE_MAX_EDGE { + return Err(RawImageEditDimensionError::MaxEdge); + } + if !width.is_multiple_of(RAW_IMAGE_DIMENSION_ALIGNMENT) + || !height.is_multiple_of(RAW_IMAGE_DIMENSION_ALIGNMENT) + { + return Err(RawImageEditDimensionError::Alignment); + } + let long_edge = u64::from(width.max(height)); + let short_edge = u64::from(width.min(height)); + if long_edge > short_edge.saturating_mul(3) { + return Err(RawImageEditDimensionError::AspectRatio); + } + let pixels = u64::from(width) * u64::from(height); + if !(RAW_IMAGE_MIN_PIXELS..=RAW_IMAGE_MAX_PIXELS).contains(&pixels) { + return Err(RawImageEditDimensionError::PixelCount); + } + Ok(()) +} + +/// Independent raw GPT Image 2 proxy; it does not call the editor image-edit client. +pub async fn create_vector_engine_raw_image_edit( + settings: &VectorEngineImageSettings, + prompt: &str, + image: ReferenceImage, + options: RawImageEditOptions, + failure_context: &str, +) -> Result { + validate_raw_image_edit_dimensions(options.width, options.height) + .map_err(|error| invalid_input(failure_context, error.to_string()))?; + let url = vector_engine_images_edit_url(settings); + let started_at = Instant::now(); + let prompt_chars = Some(prompt.chars().count()); + let reference_image_count = Some(1_usize + usize::from(options.mask.is_some())); + let Some(request_timeout_ms) = + effective_request_timeout_ms(settings.request_timeout_ms, settings.request_deadline) + else { + return Err(request_budget_exhausted_error( + url.as_str(), + failure_context, + Some(GPT_IMAGE_2_MODEL), + Some(started_at.elapsed().as_millis() as u64), + prompt_chars, + reference_image_count, + )); + }; + let ReferenceImage { + bytes: image_bytes, + file_name: image_file_name, + mime_type: image_mime_type, + } = image; + let mut form = Form::new() + .text("model", GPT_IMAGE_2_MODEL.to_string()) + .text("n", "1".to_string()) + .text("prompt", prompt.to_string()) + .text("size", format!("{}x{}", options.width, options.height)) + .part( + "image", + Part::bytes(image_bytes) + .file_name(image_file_name) + .mime_str(image_mime_type.as_str()) + .map_err(|error| invalid_request(failure_context, error.to_string()))?, + ); + if let Some(value) = options.quality { + form = form.text("quality", value); + } + if let Some(value) = options.background { + form = form.text("background", value); + } + if let Some(value) = options.output_format { + form = form.text("output_format", value); + } + if let Some(mask) = options.mask { + form = form.part( + "mask", + Part::bytes(mask.bytes) + .file_name(mask.file_name) + .mime_str(mask.mime_type.as_str()) + .map_err(|error| invalid_request(failure_context, error.to_string()))?, + ); + } + + let client = reqwest::Client::builder() + .timeout(Duration::from_millis(request_timeout_ms)) + .http1_only() + .build() + .map_err(|error| invalid_config(failure_context, error.to_string()))?; + let response = client + .post(url.as_str()) + .bearer_auth(settings.api_key.as_str()) + .multipart(form) + .send() + .await + .map_err(|error| { + request_error( + &url, + failure_context, + "request_send", + error, + started_at, + prompt_chars, + reference_image_count, + ) + })?; + let status = response.status(); + tracing::info!( + provider = VECTOR_ENGINE_PROVIDER, + endpoint = %url, + status = status.as_u16(), + prompt_chars, + reference_image_count, + elapsed_ms = started_at.elapsed().as_millis() as u64, + failure_context, + "VectorEngine Raw 图片编辑 HTTP 返回" + ); + let body = response.text().await.map_err(|error| { + request_error( + &url, + failure_context, + "response_read", + error, + started_at, + prompt_chars, + reference_image_count, + ) + })?; + if !status.is_success() { + let message = format!( + "{failure_context}：上游图片编辑失败（HTTP {}）", + status.as_u16() + ); + let raw_excerpt = truncate_raw(body.as_str()); + let audit = build_failure_audit( + url.as_str(), + failure_context, + "upstream_status", + Some(status.as_u16()), + Some(status_class(status.as_u16())), + false, + false, + message.as_str(), + None, + Some(raw_excerpt.clone()), + Some(started_at.elapsed().as_millis() as u64), + prompt_chars, + reference_image_count, + Some(GPT_IMAGE_2_MODEL), + ); + return Err(PlatformImageError::Upstream { + provider: VECTOR_ENGINE_PROVIDER, + message, + upstream_status: status.as_u16(), + raw_excerpt, + audit: Some(audit), + }); + } + let payload: RawImageEditResponsePayload = match serde_json::from_str(body.as_str()) { + Ok(payload) => payload, + Err(error) => { + let message = format!("{failure_context}：上游响应不是 JSON：{error}"); + let audit = build_failure_audit( + url.as_str(), + failure_context, + "response_parse", + Some(status.as_u16()), + Some(status_class(status.as_u16())), + false, + false, + message.as_str(), + Some(error.to_string()), + Some(truncate_raw(body.as_str())), + Some(started_at.elapsed().as_millis() as u64), + prompt_chars, + reference_image_count, + Some(GPT_IMAGE_2_MODEL), + ); + return Err(PlatformImageError::ResponseParse { + provider: VECTOR_ENGINE_PROVIDER, + message, + raw_excerpt: truncate_raw(body.as_str()), + audit: Some(audit), + }); + } + }; + let b64_images = collect_b64_images(payload.data); + if b64_images.is_empty() { + let message = format!("{failure_context}：上游未返回 b64_json 图片"); + let audit = build_failure_audit( + url.as_str(), + failure_context, + "missing_image", + Some(status.as_u16()), + Some(status_class(status.as_u16())), + false, + false, + message.as_str(), + None, + Some(truncate_raw(body.as_str())), + Some(started_at.elapsed().as_millis() as u64), + prompt_chars, + reference_image_count, + Some(GPT_IMAGE_2_MODEL), + ); + return Err(PlatformImageError::MissingImage { + provider: VECTOR_ENGINE_PROVIDER, + message, + audit: Some(audit), + }); + } + Ok(RawImageEditResult { + b64_images, + recovered_failure_audits: Vec::new(), + }) +} + +fn collect_b64_images(data: Vec) -> Vec { + data.into_iter() + .map(|entry| entry.b64_json) + .filter(|value| !value.is_empty()) + .collect() +} + +fn invalid_request(context: &str, message: String) -> PlatformImageError { + PlatformImageError::InvalidRequest { + provider: VECTOR_ENGINE_PROVIDER, + message: format!("{context}：构造上游请求失败：{message}"), + } +} + +fn invalid_input(context: &str, message: String) -> PlatformImageError { + PlatformImageError::InvalidRequest { + provider: VECTOR_ENGINE_PROVIDER, + message: format!("{context}：请求参数无效：{message}"), + } +} + +fn invalid_config(context: &str, message: String) -> PlatformImageError { + PlatformImageError::InvalidConfig { + provider: VECTOR_ENGINE_PROVIDER, + message: format!("{context}：构造请求客户端失败：{message}"), + } +} + +fn request_error( + url: &str, + context: &str, + failure_stage: &'static str, + error: reqwest::Error, + started_at: Instant, + prompt_chars: Option, + reference_image_count: Option, +) -> PlatformImageError { + let timeout = error.is_timeout(); + let connect = error.is_connect(); + let source = error.to_string(); + let message = format!("{context}：上游请求失败：{source}"); + tracing::warn!( + provider = VECTOR_ENGINE_PROVIDER, + endpoint = %url, + failure_stage, + timeout, + connect, + prompt_chars, + reference_image_count, + elapsed_ms = started_at.elapsed().as_millis() as u64, + failure_context = context, + error = %source, + "VectorEngine Raw 图片编辑请求失败" + ); + let audit = build_failure_audit( + url, + context, + failure_stage, + None, + Some("transport"), + timeout, + connect, + message.as_str(), + Some(source.clone()), + None, + Some(started_at.elapsed().as_millis() as u64), + prompt_chars, + reference_image_count, + Some(GPT_IMAGE_2_MODEL), + ); + PlatformImageError::Request { + provider: VECTOR_ENGINE_PROVIDER, + message, + endpoint: Some(url.to_string()), + timeout, + connect, + request: failure_stage == "request_send", + body: failure_stage == "response_read", + status_code: None, + source: Some(source), + audit: Some(audit), + } +} + +fn status_class(status: u16) -> &'static str { + match status { + 100..=199 => "1xx", + 200..=299 => "2xx", + 300..=399 => "3xx", + 400..=499 => "4xx", + _ => "5xx", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn raw_result_forwards_b64_without_decoding_or_using_output_format() { + let payload = r#"{ + "output_format": "png", + "data": [{"b64_json": "not-base64-but-forwarded", "output_format": "jpeg"}] + }"#; + let payload: RawImageEditResponsePayload = + serde_json::from_str(payload).expect("response envelope"); + + assert_eq!( + payload + .data + .into_iter() + .map(|entry| entry.b64_json) + .collect::>(), + vec!["not-base64-but-forwarded".to_string()] + ); + } + + #[test] + fn raw_success_response_requires_typed_data_and_b64_json_fields() { + assert!(serde_json::from_str::(r#"{}"#).is_err()); + assert!(serde_json::from_str::(r#"{"data":[{}]}"#).is_err()); + } + + #[test] + fn raw_success_response_discards_empty_b64_json_entries() { + let payload: RawImageEditResponsePayload = serde_json::from_str( + r#"{"data":[{"b64_json":""},{"b64_json":"valid"}]}"#, + ) + .expect("response envelope"); + + assert_eq!(collect_b64_images(payload.data), vec!["valid"]); + } + + #[test] + fn raw_image_edit_dimensions_enforce_strict_contract() { + assert!(validate_raw_image_edit_dimensions(1024, 640).is_ok()); + assert!(validate_raw_image_edit_dimensions(3840, 2160).is_ok()); + assert_eq!( + validate_raw_image_edit_dimensions(3856, 2160), + Err(RawImageEditDimensionError::MaxEdge) + ); + assert_eq!( + validate_raw_image_edit_dimensions(1024, 1000), + Err(RawImageEditDimensionError::Alignment) + ); + assert_eq!( + validate_raw_image_edit_dimensions(1936, 640), + Err(RawImageEditDimensionError::AspectRatio) + ); + assert_eq!( + validate_raw_image_edit_dimensions(1024, 624), + Err(RawImageEditDimensionError::PixelCount) + ); + assert_eq!( + validate_raw_image_edit_dimensions(3840, 2176), + Err(RawImageEditDimensionError::PixelCount) + ); + } + + #[test] + fn dimension_validation_error_identifies_client_input() { + let error = invalid_input("raw_image_edit", "尺寸无效".to_string()); + + assert_eq!(error.to_string(), "raw_image_edit：请求参数无效：尺寸无效"); + } + + #[test] + fn invalid_config_error_keeps_operation_context() { + let error = invalid_config("raw_image_edit", "builder failed".to_string()); + + assert_eq!( + error.to_string(), + "raw_image_edit：构造请求客户端失败：builder failed" + ); + } +} diff --git a/server-rs/crates/platform-image/src/vector_engine/request.rs b/server-rs/crates/platform-image/src/vector_engine/request.rs index af232dbc8..4dc53443e 100644 --- a/server-rs/crates/platform-image/src/vector_engine/request.rs +++ b/server-rs/crates/platform-image/src/vector_engine/request.rs @@ -1,7 +1,10 @@ use serde_json::{Map, Value, json}; use super::{ - constants::{GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL}, + constants::{ + GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_DIMENSION_ALIGNMENT, GPT_IMAGE_2_MAX_EDGE, + GPT_IMAGE_2_MAX_PIXELS, GPT_IMAGE_2_MIN_PIXELS, GPT_IMAGE_2_MODEL, + }, types::{ReferenceImage, VectorEngineImageSettings}, }; @@ -129,10 +132,10 @@ fn normalize_explicit_pixel_size(value: &str) -> String { } fn clamp_gpt_image_2_pixel_size(size: &str) -> String { - const MIN_PIXELS: u64 = 655_360; - const MAX_PIXELS: u64 = 8_294_400; - const MAX_EDGE: u32 = 3_840; - const DIMENSION_ALIGNMENT: u32 = 16; + const MIN_PIXELS: u64 = GPT_IMAGE_2_MIN_PIXELS; + const MAX_PIXELS: u64 = GPT_IMAGE_2_MAX_PIXELS; + const MAX_EDGE: u32 = GPT_IMAGE_2_MAX_EDGE; + const DIMENSION_ALIGNMENT: u32 = GPT_IMAGE_2_DIMENSION_ALIGNMENT; const MAX_ASPECT_RATIO: f64 = 3.0; // 中文注释：这里是 VectorEngine 的共享发送边界，只处理 gpt-image-2 的显式像素尺寸。 diff --git a/server-rs/crates/platform-image/src/vector_engine/types.rs b/server-rs/crates/platform-image/src/vector_engine/types.rs index 77fbd19f9..d25ad5c12 100644 --- a/server-rs/crates/platform-image/src/vector_engine/types.rs +++ b/server-rs/crates/platform-image/src/vector_engine/types.rs @@ -16,6 +16,12 @@ pub struct GeneratedImages { pub recovered_failure_audits: Vec, } +#[derive(Clone, Debug)] +pub struct RawImageEditResult { + pub b64_images: Vec, + pub recovered_failure_audits: Vec, +} + #[derive(Clone, Debug)] pub struct DownloadedImage { pub bytes: Vec,