WIP: UI编辑器自动分图层切图标 #304

Draft
k88936 wants to merge 100 commits from feat/ui-editor-auto-seperation into master
98 changed files with 5177 additions and 1142 deletions
@@ -108,6 +108,8 @@ const rustSharedContractSource = fs.readFileSync(
);
const allowedUncalledTauriCommands = [
'append_direct_project_conversation_message',
// TODO: Remove the retired binding command after the legacy runtime path is removed.
'bind_components',
'chat_with_game_creator_agent',
'check_ui_editor_font_glyph_coverage',
'create_ui_design_resource',
@@ -1832,7 +1832,7 @@ mod tests {
assert!(with_canvas.contains("由 Runtime 在收束门内同时核对固定 owner 文档"));
assert!(!with_canvas.contains("成功动作本身就是当前 revision 的验证"));
assert!(with_canvas.contains("调用 ui.workflow.run"));
assert!(with_canvas.contains("visual-binding 最终编辑器路由"));
assert!(with_canvas.contains("asset-separation 最终编辑器路由"));
assert!(with_canvas.contains("每个功能页面各写一行 @genarrative-ui-page"));
assert!(with_canvas.contains("ui.workflow.run 的 discover"));
assert!(with_canvas.contains("assets/ui-pages/{pageId}.png"));
@@ -801,7 +801,7 @@ fn agent_runtime_action_receipt_safe_detail_with_owner(
let initial_step = route.get("initialStep")?.as_str()?;
let render_mode = route.get("renderMode")?.as_str()?;
if resource_id.is_empty()
|| initial_step != "visual-binding"
|| initial_step != "asset-separation"
|| render_mode != "final-preview"
{
return None;
@@ -328,6 +328,39 @@ async fn recognize_ui(
ui_editor::commands::recognize_ui_impl(project_path, state).await
}
#[tauri::command]
async fn separate_ui(
project_path: String,
asset_id: String,
state: ui_editor::state::State,
) -> Result<ui_editor::commands::SeparationDTO, String> {
ui_editor::commands::separate_ui_impl(project_path, asset_id, state).await
}
#[tauri::command]
fn inspect_separation_recovery(
project_path: String,
asset_id: String,
) -> Result<ui_editor::commands::SeparationRecoveryDTO, String> {
let root = Path::new(project_path.trim());
enforce_project_permission_policy(root, "asset.list")?;
ui_editor::commands::separation::inspect_separation_recovery(root, &asset_id)
}
#[tauri::command]
fn finalize_separation(project_path: String, asset_id: String) -> Result<(), String> {
let root = Path::new(project_path.trim());
enforce_project_permission_policy(root, "asset.register")?;
ui_editor::commands::separation::finalize_separation(root, &asset_id)
}
#[tauri::command]
fn discard_separation_recovery(project_path: String, asset_id: String) -> Result<(), String> {
let root = Path::new(project_path.trim());
enforce_project_permission_policy(root, "asset.register")?;
ui_editor::commands::separation::finalize_separation(root, &asset_id)
}
#[tauri::command]
async fn merge_ui(state: ui_editor::state::State) -> Result<ui_editor::commands::MergeDTO, String> {
ui_editor::commands::merge_ui_impl(state).await
@@ -2559,6 +2592,10 @@ fn main() {
check_ui_editor_font_glyph_coverage,
suggest_ui_design_semantic,
recognize_ui,
separate_ui,
inspect_separation_recovery,
finalize_separation,
discard_separation_recovery,
merge_ui,
bind_components,
load_ui_design_state,
@@ -5,11 +5,9 @@ use crate::ui_editor::commands::utils::{
strict_json_schema,
};
use crate::ui_editor::component::text::FontSource;
use crate::ui_editor::component::Component;
use crate::ui_editor::component::{Component, NodeComponent};
use crate::ui_editor::layout::node::{Node, StageStatus};
use crate::ui_editor::persistence::{
UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE, UI_DESIGN_STATE_MAX_NODES,
};
use crate::ui_editor::persistence::UI_DESIGN_STATE_MAX_NODES;
use crate::ui_editor::state::State;
use crate::ui_editor::utils::{FontAssetId, NodeId, SpriteAssetId};
use platform_llm::{
@@ -31,10 +29,10 @@ const SYSTEM_PROMPT: &str = r#"
你是游戏 UI 组件绑定器。你会看到全部 UI 参考图、可编辑节点说明,以及本批独立素材的真实像素。
* 只对视觉上确实需要改变组件的节点返回 changes;
* 每个 change 的 components 是该节点完整的新渲染栈,空数组表示明确清空。数组顺序从底到顶渲染
* 对每个 Component,直接完整返回其全部参数.
* 每个 change 的 component 是该节点完整的新组件;纯结构节点返回 "PureNode",有组件返回 {"WithComponent": <完整 Component>}
* 对 Component,直接完整返回其全部参数.
* 有任何困难或者不确定把状态设为 NeedReview,说明中文原因。
* 纯结构节点可以返回空数组并标为 NoProblem。
* 纯结构节点可以返回 "PureNode" 并标为 NoProblem。
* 容器背景等推荐使用Simple + preserve_aspect: false 实现与node大小一致
* 面向用户的 reason 使用中文。
@@ -53,8 +51,8 @@ enum DraftStatus {
#[schemars(deny_unknown_fields)]
struct BindingChangeDraft {
node_id: NodeId,
components: Vec<Component>,
components_status: DraftStatus,
component: NodeComponent,
component_status: DraftStatus,
}
#[derive(Clone, Debug, Deserialize, JsonSchema)]
@@ -68,8 +66,8 @@ struct BindingResponse {
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct BindingChange {
pub node_id: NodeId,
pub components: Vec<Component>,
pub components_status: StageStatus,
pub component: NodeComponent,
pub component_status: StageStatus,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
@@ -83,7 +81,7 @@ struct EditableNodeContext<'a> {
node_id: &'a NodeId,
name: &'a str,
description: &'a str,
components: &'a [Component],
component: Option<&'a Component>,
}
#[derive(Debug, Serialize)]
@@ -106,7 +104,7 @@ fn collect_editable_nodes<'a>(node: &'a Node, output: &mut Vec<EditableNodeConte
node_id: &node.id,
name: &node.metadata.name,
description: &node.metadata.description,
components: &node.components,
component: node.component.as_ref(),
});
}
for child in &node.children {
@@ -172,14 +170,11 @@ fn validate_binding_response_shape(
return Err(format!("组件绑定 changes 不能超过 {max_changes}"));
}
for change in changes {
let components = change
.get("components")
.and_then(serde_json::Value::as_array)
.ok_or_else(|| "组件绑定 change 缺少 components 数组".to_string())?;
if components.len() > UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE {
return Err(format!(
"单个组件绑定栈不能超过 {UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE} 个组件"
));
if !change
.as_object()
.is_some_and(|object| object.contains_key("component"))
{
return Err("组件绑定 change 缺少 component 字段".to_string());
}
}
Ok(())
@@ -212,7 +207,7 @@ fn validate_and_materialize(
if !changed_ids.insert(change.node_id.clone()) {
return Err(format!("组件绑定重复返回节点:{}", change.node_id.as_str()));
}
for component in &change.components {
if let NodeComponent::WithComponent(component) = &change.component {
match component {
Component::Image(image) => {
if image
@@ -232,18 +227,22 @@ fn validate_and_materialize(
}
}
}
let components = change.components;
let components_status = match change.components_status {
let component_status = match change.component_status {
DraftStatus::NoProblem => StageStatus::NoProblem,
DraftStatus::NeedReview(reason) if reason.trim().is_empty() => {
return Err("组件待审状态必须包含原因".to_string())
}
DraftStatus::NeedReview(reason) => StageStatus::NeedReview(reason),
DraftStatus::NeedReview(reason) => {
if matches!(&change.component, NodeComponent::PureNode) {
return Err("纯结构节点不能标记为组件待审".to_string());
}
StageStatus::NeedReview(reason)
}
};
materialized.push(BindingChange {
node_id: change.node_id,
components,
components_status,
component: change.component,
component_status,
});
}
Ok(BindingDTO {
@@ -440,8 +439,8 @@ mod tests {
]);
let unapproved = BindingChangeDraft {
node_id: id("other"),
components: Vec::new(),
components_status: DraftStatus::NoProblem,
component: NodeComponent::PureNode,
component_status: DraftStatus::NoProblem,
};
assert!(
validate_and_materialize(vec![unapproved], &editable, &known, &HashSet::new()).is_err()
@@ -450,7 +449,7 @@ mod tests {
// References to sprites from another batch are allowed once they exist in the project.
let other_batch = BindingChangeDraft {
node_id: id("editable"),
components: vec![Component::Image(
component: NodeComponent::WithComponent(Component::Image(
crate::ui_editor::component::image::ImageComponent {
target_graphic: Some(
SpriteAssetId::new("other-batch-sprite").expect("valid sprite"),
@@ -459,8 +458,8 @@ mod tests {
preserve_aspect: false,
},
},
)],
components_status: DraftStatus::NoProblem,
)),
component_status: DraftStatus::NoProblem,
};
assert!(
validate_and_materialize(vec![other_batch], &editable, &known, &HashSet::new()).is_ok()
@@ -469,15 +468,15 @@ mod tests {
// References to sprites that do not exist in the project at all are still rejected.
let unknown = BindingChangeDraft {
node_id: id("editable"),
components: vec![Component::Image(
component: NodeComponent::WithComponent(Component::Image(
crate::ui_editor::component::image::ImageComponent {
target_graphic: Some(SpriteAssetId::new("unknown").expect("valid sprite")),
image_type: crate::ui_editor::component::image::ImageType::Simple {
preserve_aspect: false,
},
},
)],
components_status: DraftStatus::NoProblem,
)),
component_status: DraftStatus::NoProblem,
};
assert!(
validate_and_materialize(vec![unknown], &editable, &known, &HashSet::new()).is_err()
@@ -491,8 +490,8 @@ mod tests {
text.font = FontSource::Bound(FontAssetId::new("unknown-font").expect("valid font"));
let change = BindingChangeDraft {
node_id: id("editable"),
components: vec![Component::Text(text)],
components_status: DraftStatus::NoProblem,
component: NodeComponent::WithComponent(Component::Text(text)),
component_status: DraftStatus::NoProblem,
};
let error = validate_and_materialize(
@@ -506,13 +505,13 @@ mod tests {
}
#[test]
fn materialization_preserves_changed_only_empty_component_lists() {
fn materialization_preserves_pure_node_change() {
let editable = HashSet::from([id("editable")]);
let result = validate_and_materialize(
vec![BindingChangeDraft {
node_id: id("editable"),
components: Vec::new(),
components_status: DraftStatus::NoProblem,
component: NodeComponent::PureNode,
component_status: DraftStatus::NoProblem,
}],
&editable,
&HashSet::new(),
@@ -520,8 +519,28 @@ mod tests {
)
.expect("valid changed-only clear");
assert_eq!(result.changes.len(), 1);
assert!(result.changes[0].components.is_empty());
assert_eq!(result.changes[0].components_status, StageStatus::NoProblem);
assert!(matches!(
result.changes[0].component,
NodeComponent::PureNode
));
assert_eq!(result.changes[0].component_status, StageStatus::NoProblem);
}
#[test]
fn materialization_rejects_problematic_pure_node() {
let editable = HashSet::from([id("editable")]);
let error = validate_and_materialize(
vec![BindingChangeDraft {
node_id: id("editable"),
component: NodeComponent::PureNode,
component_status: DraftStatus::NeedReview("缺少可确认的组件".to_string()),
}],
&editable,
&HashSet::new(),
&HashSet::new(),
)
.expect_err("pure node cannot carry a component review status");
assert!(error.contains("纯结构节点"));
}
#[test]
@@ -575,20 +594,26 @@ mod tests {
}
#[test]
fn binding_response_bounds_changes_and_each_component_stack() {
fn binding_response_bounds_changes_and_uses_single_component_shape() {
let too_many_changes = serde_json::json!({
"changes": [{"components": []}, {"components": []}]
"changes": [{"component": "PureNode"}, {"component": "PureNode"}]
});
assert!(validate_binding_response_shape(&too_many_changes, 1).is_err());
let too_many_components = serde_json::json!({
let one_component = serde_json::json!({
"changes": [{
"components": (0..=UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE)
.map(|_| serde_json::Value::Null)
.collect::<Vec<_>>()
"node_id": "editable",
"component": "PureNode",
"component_status": "NoProblem"
}]
});
assert!(validate_binding_response_shape(&too_many_components, 1).is_err());
assert!(validate_binding_response_shape(&one_component, 1).is_ok());
let parsed = parse_binding_response(&one_component.to_string(), 1)
.expect("explicit PureNode payload should parse");
assert!(matches!(
parsed.changes[0].component,
NodeComponent::PureNode
));
}
#[tokio::test]
@@ -295,12 +295,12 @@ mod materialize {
name: container_name,
description: container_description,
layout_status: StageStatus::NoProblem,
components_status: StageStatus::NoProblem,
component_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::Llm,
},
components: Vec::new(),
component: None,
children_display_mode: ChildrenDisplayMode::Exclusive,
children: members.into_iter().map(|member| member.node).collect(),
},
@@ -542,12 +542,12 @@ mod tests {
name: id.to_string(),
description: String::new(),
layout_status: StageStatus::NoProblem,
components_status: StageStatus::NoProblem,
component_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::Human,
},
components: Vec::<Component>::new(),
component: None,
children_display_mode: ChildrenDisplayMode::Stack,
children,
}
@@ -586,7 +586,7 @@ mod tests {
ChildrenDisplayMode::Exclusive
);
assert_eq!(
result.root.metadata.components_status,
result.root.metadata.component_status,
StageStatus::NoProblem
);
assert_eq!(result.root.children.len(), 2);
@@ -1,6 +1,7 @@
pub mod binding;
pub mod merge;
pub mod recognition;
pub mod separation;
pub mod ui_design_suggestion;
pub mod utils;
@@ -10,5 +11,7 @@ pub use merge::MergeDTO;
pub(crate) use merge::{merge_ui_impl, merge_ui_impl_with_provider};
pub use recognition::RecognitionDTO;
pub(crate) use recognition::{recognize_ui_impl, recognize_ui_impl_with_provider};
pub(crate) use separation::separate_ui_impl;
pub use separation::{SeparationDTO, SeparationRecoveryDTO};
pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl;
pub use ui_design_suggestion::UIDesignSuggestionTreeNode;
@@ -4,6 +4,7 @@ use crate::ui_editor::commands::utils::{
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm,
strict_json_schema,
};
use crate::ui_editor::component::{Component, NodeComponent};
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
use crate::ui_editor::layout::control_layout::ControlLayout;
use crate::ui_editor::layout::dimension::UIRect;
@@ -27,16 +28,13 @@ const MAX_RECOGNITION_TREE_NODES: usize = 512;
const MAX_RECOGNITION_TREE_DEPTH: usize = 32;
const SYSTEM_PROMPT: &str = r#"
角色:
你是游戏 UI 多图结构识别器。
任务:
同时分析同一 UI 系统的全部参考图,建立UI树
用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构
识别规则:
* 只识别 UI,不识别场景人物、地形、建筑、光影和背景装饰。
* 只识别 UI元素. 要区分动态内容, 不要白费力气识别应该由程序生成/绘制的内容.(此类内容应该用一个整体节点+自然语言描述) 除此之外必须完整包含所有元素,结构.
* 无法确定类型、层级、关系时,在 UnSure 中写明原因。
* 返回的 trees 必须与输入图片一一对应,每张输入图片只能有一棵树,不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id。
* 每棵树必须使用自己的输入图片原始像素坐标系(0,0 as left top)输出
@@ -45,8 +43,15 @@ const SYSTEM_PROMPT: &str = r#"
* 面向用户的字段如名称描述等请用中文
* 由于每个截图未必是完整的, 可能是局部的, 每棵树描述清楚每个截图上UI的层次结构即可
* 不同树的共用框架/层次/...请使用使用相同的名称描述. 不同状态/变体名称使用相同的前缀, 用后缀区别
* 粒度要求: 尽可能细致, 最小单元举例: 进度条的底槽、填充和外框; slider的底槽, dragger等
* 粒度要求: 尽可能细致, 以可交互,方便程序化控制的最小单位为准. 包括不限于: icon, 进度条的底槽、填充和外框; slider的底槽, dragger等.
* 为每个节点直接返回 component.
无背景的逻辑容器返回 "PureNode",不要返回 null.
有背景的容器推荐使用Simple+不锁定宽高比的Image component.
目前我们只做识别, 不要求图片字体参数.
每个节点最多返回一个 component;需要多个视觉层时拆成多个节点。
文字组件要求: 艺术字等作为图片组件, 其余正常文字要作为单独的节点识别.
* 不鼓励兄弟节点相互重叠.
* 对于面板等容器的背景等, 必须作为父节点的组件, 禁止新增冗余的所谓"背景节点".
"#;
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
@@ -109,6 +114,7 @@ struct RecognitionNode {
description: String,
children: Vec<RecognitionNode>,
confidence: Confidence,
component: NodeComponent,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
@@ -307,15 +313,12 @@ fn convert_node(
name: source.name.clone(),
description: source.description.clone(),
layout_status: status,
components_status: StageStatus::NoProblem,
component_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::Llm,
},
// V1 有意把识别结果限定为“结构草稿”:组件绑定属于后续独立阶段。
// 因此空组件不是丢失数据,而是等待 visual-binding 阶段补齐 Image/Text。
// 约定见 docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md。
components: Vec::new(),
component: source.component.clone().into_option(),
children_display_mode: ChildrenDisplayMode::Stack,
children,
})
@@ -323,6 +326,26 @@ fn convert_node(
fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> {
for node in nodes {
if let NodeComponent::WithComponent(component) = &node.component {
if matches!(
component,
Component::Image(crate::ui_editor::component::image::ImageComponent {
target_graphic: Some(_),
..
})
) {
return Err("识别阶段不能返回已绑定的 SpriteAssetId".to_string());
}
if matches!(
component,
Component::Text(crate::ui_editor::component::text::TextComponent {
font: crate::ui_editor::component::text::FontSource::Bound(_),
..
})
) {
return Err("识别阶段不能返回已绑定的字体素材".to_string());
}
}
if let Confidence::UnSure(reason) = &node.confidence {
if reason.trim().is_empty() {
return Err("UnSure 必须包含审阅原因".to_string());
@@ -387,6 +410,7 @@ mod tests {
description: String::new(),
children: Vec::new(),
confidence: Confidence::Confident,
component: NodeComponent::PureNode,
}
}
@@ -490,6 +514,31 @@ mod tests {
description: String::new(),
children: Vec::new(),
confidence: Confidence::UnSure(String::new()),
component: NodeComponent::PureNode,
};
assert!(validate_confidence(&[node]).is_err());
}
#[test]
fn recognition_rejects_bound_font_references() {
let mut text = crate::ui_editor::component::text::TextComponent::default();
text.font = crate::ui_editor::component::text::FontSource::Bound(
crate::ui_editor::utils::FontAssetId::new("font").expect("valid font id"),
);
let node = RecognitionNode {
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
local_anchor: Anchor::Preset(PresetAnchor {
horizontal: HorizontalAnchor::Left,
vertical: VerticalAnchor::Top,
}),
name: "文本".to_string(),
description: String::new(),
children: Vec::new(),
confidence: Confidence::Confident,
component: NodeComponent::WithComponent(Component::Text(text)),
};
assert!(validate_confidence(&[node]).is_err());
}
@@ -506,7 +555,7 @@ mod tests {
converted.layout.transform.resolve(&root_rect),
UIRect::new(Point2::new(50.0, 25.0), Vector2::new(100.0, 50.0)),
);
assert_eq!(converted.metadata.components_status, StageStatus::NoProblem);
assert_eq!(converted.metadata.component_status, StageStatus::NoProblem);
}
#[test]
@@ -787,12 +836,12 @@ pub(crate) async fn recognize_ui_impl_with_provider(
name: "页面根节点".to_string(),
description: String::new(),
layout_status: StageStatus::NoProblem,
components_status: StageStatus::NoProblem,
component_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::System,
},
components: Vec::new(),
component: None,
children_display_mode: ChildrenDisplayMode::Stack,
children,
};
@@ -0,0 +1,433 @@
use super::model::BindingArea;
use image::RgbaImage;
use std::time::Instant;
/// Each edge may move by at most this percentage of the corresponding area
/// dimension returned by the visual model. Keep this policy explicit so
/// changing it is an intentional workflow decision rather than a scattered
/// numeric literal.
pub(crate) const MAX_BINDING_AREA_EDGE_ADJUSTMENT_PERCENT: u32 = 100;
#[derive(Clone, Copy, Debug, PartialEq)]
pub(crate) struct NormalizedBindingArea {
pub(crate) area: BindingArea,
pub(crate) changed: bool,
pub(crate) clamped: bool,
pub(crate) transparent: bool,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum EdgeDirection {
Inward,
Outward,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct Rect {
left: u32,
top: u32,
right: u32,
bottom: u32,
}
impl Rect {
fn from_area(area: BindingArea) -> Self {
Self {
left: area.global_pos_x_px,
top: area.global_pos_y_px,
right: area.global_pos_x_px + area.width_px,
bottom: area.global_pos_y_px + area.height_px,
}
}
fn into_area(self) -> BindingArea {
BindingArea {
global_pos_x_px: self.left,
global_pos_y_px: self.top,
width_px: self.right - self.left,
height_px: self.bottom - self.top,
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum Edge {
Left,
Right,
Top,
Bottom,
}
impl Edge {
const ALL: [Self; 4] = [Self::Left, Self::Right, Self::Top, Self::Bottom];
}
fn edge_has_visible_pixel(image: &RgbaImage, rect: Rect, edge: Edge) -> bool {
match edge {
Edge::Left | Edge::Right => {
let x = if edge == Edge::Left {
rect.left
} else {
rect.right - 1
};
(rect.top..rect.bottom).any(|y| image.get_pixel(x, y).0[3] > 0)
}
Edge::Top | Edge::Bottom => {
let y = if edge == Edge::Top {
rect.top
} else {
rect.bottom - 1
};
(rect.left..rect.right).any(|x| image.get_pixel(x, y).0[3] > 0)
}
}
}
fn rect_has_visible_pixel(image: &RgbaImage, rect: Rect) -> bool {
(rect.top..rect.bottom).any(|y| (rect.left..rect.right).any(|x| image.get_pixel(x, y).0[3] > 0))
}
fn max_edge_adjustment(dimension: u32) -> u32 {
((u64::from(dimension) * u64::from(MAX_BINDING_AREA_EDGE_ADJUSTMENT_PERCENT)) / 100)
.min(u64::from(u32::MAX)) as u32
}
fn edge_direction(image: &RgbaImage, rect: Rect, edge: Edge) -> EdgeDirection {
if edge_has_visible_pixel(image, rect, edge) {
EdgeDirection::Outward
} else {
EdgeDirection::Inward
}
}
fn move_edge(rect: &mut Rect, edge: Edge, direction: EdgeDirection) {
match (edge, direction) {
(Edge::Left, EdgeDirection::Inward) => rect.left += 1,
(Edge::Left, EdgeDirection::Outward) => rect.left -= 1,
(Edge::Right, EdgeDirection::Inward) => rect.right -= 1,
(Edge::Right, EdgeDirection::Outward) => rect.right += 1,
(Edge::Top, EdgeDirection::Inward) => rect.top += 1,
(Edge::Top, EdgeDirection::Outward) => rect.top -= 1,
(Edge::Bottom, EdgeDirection::Inward) => rect.bottom -= 1,
(Edge::Bottom, EdgeDirection::Outward) => rect.bottom += 1,
}
}
fn edge_coordinate(rect: Rect, edge: Edge) -> u32 {
match edge {
Edge::Left => rect.left,
Edge::Right => rect.right,
Edge::Top => rect.top,
Edge::Bottom => rect.bottom,
}
}
fn edge_displacement(original: Rect, current: Rect, edge: Edge) -> u32 {
edge_coordinate(original, edge).abs_diff(edge_coordinate(current, edge))
}
fn edge_adjustment_limit(original: Rect, edge: Edge) -> u32 {
max_edge_adjustment(match edge {
Edge::Left | Edge::Right => original.right - original.left,
Edge::Top | Edge::Bottom => original.bottom - original.top,
})
}
fn reached_adjustment_limit(original: Rect, current: Rect, edge: Edge) -> bool {
edge_displacement(original, current, edge) >= edge_adjustment_limit(original, edge)
}
fn can_move_geometrically(
image: &RgbaImage,
current: Rect,
edge: Edge,
direction: EdgeDirection,
) -> bool {
match (edge, direction) {
(Edge::Left, EdgeDirection::Inward) => current.left + 1 < current.right,
(Edge::Left, EdgeDirection::Outward) => current.left > 0,
(Edge::Right, EdgeDirection::Inward) => current.right > current.left + 1,
(Edge::Right, EdgeDirection::Outward) => current.right < image.width(),
(Edge::Top, EdgeDirection::Inward) => current.top + 1 < current.bottom,
(Edge::Top, EdgeDirection::Outward) => current.top > 0,
(Edge::Bottom, EdgeDirection::Inward) => current.bottom > current.top + 1,
(Edge::Bottom, EdgeDirection::Outward) => current.bottom < image.height(),
}
}
fn next_edge_rect(rect: Rect, edge: Edge, direction: EdgeDirection) -> Option<Rect> {
let mut next = rect;
match (edge, direction) {
(Edge::Left, EdgeDirection::Inward) if rect.left + 1 < rect.right => next.left += 1,
(Edge::Left, EdgeDirection::Outward) if rect.left > 0 => next.left -= 1,
(Edge::Right, EdgeDirection::Inward) if rect.right > rect.left + 1 => next.right -= 1,
(Edge::Right, EdgeDirection::Outward) => next.right = next.right.checked_add(1)?,
(Edge::Top, EdgeDirection::Inward) if rect.top + 1 < rect.bottom => next.top += 1,
(Edge::Top, EdgeDirection::Outward) if rect.top > 0 => next.top -= 1,
(Edge::Bottom, EdgeDirection::Inward) if rect.bottom > rect.top + 1 => next.bottom -= 1,
(Edge::Bottom, EdgeDirection::Outward) => next.bottom = next.bottom.checked_add(1)?,
_ => return None,
}
Some(next)
}
fn edge_requires_move(image: &RgbaImage, rect: Rect, edge: Edge, direction: EdgeDirection) -> bool {
match direction {
EdgeDirection::Inward => !edge_has_visible_pixel(image, rect, edge),
EdgeDirection::Outward => {
if !edge_has_visible_pixel(image, rect, edge) {
return false;
}
if !can_move_geometrically(image, rect, edge, direction) {
return true;
}
next_edge_rect(rect, edge, direction)
.is_some_and(|next| edge_has_visible_pixel(image, next, edge))
}
}
}
fn apply_edge_step(
image: &RgbaImage,
original: Rect,
current: Rect,
edge: Edge,
direction: EdgeDirection,
) -> (Rect, bool, bool) {
if !edge_requires_move(image, current, edge, direction) {
return (current, false, false);
}
if reached_adjustment_limit(original, current, edge)
|| !can_move_geometrically(image, current, edge, direction)
{
return (current, false, true);
}
let mut next = current;
move_edge(&mut next, edge, direction);
(next, true, false)
}
/// Normalizes a model-provided area using visible pixels on the processed
/// transparent image. Each edge chooses inward/outward direction once from
/// its initial scan and then moves monotonically, so sparse pixels cannot make
/// the boundary oscillate. The four edge steps are calculated from the same
/// rectangle on each round.
pub(crate) fn normalize_binding_area(
image: &RgbaImage,
original_area: BindingArea,
) -> Result<NormalizedBindingArea, String> {
let started = Instant::now();
if let Err(error) = original_area.validate_in(image.width(), image.height()) {
app_log!(
"ui_separation.area.timing outcome=error elapsed_us={} rounds=0 image_width={} image_height={} area=({}, {}, {}, {})",
started.elapsed().as_micros(),
image.width(),
image.height(),
original_area.global_pos_x_px,
original_area.global_pos_y_px,
original_area.width_px,
original_area.height_px
);
return Err(error);
}
let original = Rect::from_area(original_area);
let directions = Edge::ALL.map(|edge| edge_direction(image, original, edge));
let mut current = original;
let mut clamped = false;
let mut active = [true; 4];
let mut rounds = 0u32;
// TODO: Replace the deliberately simple pixel-by-pixel scan if real UI
// design sizes show this path to be a measurable bottleneck.
while active.iter().any(|value| *value) {
rounds = rounds.saturating_add(1);
let before = current;
let mut next = current;
let mut moved = [false; 4];
for (index, edge) in Edge::ALL.into_iter().enumerate() {
if !active[index] {
continue;
}
let (candidate, did_move, reached_limit) =
apply_edge_step(image, original, current, edge, directions[index]);
if reached_limit {
clamped = true;
active[index] = false;
} else if !did_move {
active[index] = false;
}
moved[index] = did_move;
match edge {
Edge::Left => next.left = candidate.left,
Edge::Right => next.right = candidate.right,
Edge::Top => next.top = candidate.top,
Edge::Bottom => next.bottom = candidate.bottom,
}
}
if next.left >= next.right {
clamped = true;
if moved[0] {
active[0] = false;
}
if moved[1] {
active[1] = false;
}
next.left = current.left;
next.right = current.right;
}
if next.top >= next.bottom {
clamped = true;
if moved[2] {
active[2] = false;
}
if moved[3] {
active[3] = false;
}
next.top = current.top;
next.bottom = current.bottom;
}
current = next;
if current == before {
break;
}
}
let area = current.into_area();
let result = Ok(NormalizedBindingArea {
changed: area != original_area,
area,
clamped,
transparent: !rect_has_visible_pixel(image, current),
});
app_log!(
"ui_separation.area.timing outcome=ok elapsed_us={} rounds={} image_width={} image_height={} area=({}, {}, {}, {}) changed={} clamped={} transparent={}",
started.elapsed().as_micros(),
rounds,
image.width(),
image.height(),
original_area.global_pos_x_px,
original_area.global_pos_y_px,
original_area.width_px,
original_area.height_px,
result.as_ref().expect("normalization result exists").changed,
result.as_ref().expect("normalization result exists").clamped,
result.as_ref().expect("normalization result exists").transparent
);
result
}
#[cfg(test)]
mod tests {
use super::*;
use image::{Rgba, RgbaImage};
fn image_with_rect(
width: u32,
height: u32,
left: u32,
top: u32,
right: u32,
bottom: u32,
) -> RgbaImage {
let mut image = RgbaImage::from_pixel(width, height, Rgba([0, 0, 0, 0]));
for y in top..bottom {
for x in left..right {
image.put_pixel(x, y, Rgba([255, 255, 255, 255]));
}
}
image
}
fn area(x: u32, y: u32, width: u32, height: u32) -> BindingArea {
BindingArea {
global_pos_x_px: x,
global_pos_y_px: y,
width_px: width,
height_px: height,
}
}
#[test]
fn shrinks_empty_edges_to_visible_bounds() {
let image = image_with_rect(32, 32, 10, 11, 16, 18);
let result = normalize_binding_area(&image, area(6, 7, 14, 16)).unwrap();
assert_eq!(result.area, area(10, 11, 6, 7));
assert!(result.changed);
assert!(!result.clamped);
assert!(!result.transparent);
}
#[test]
fn expands_visible_edges_to_cover_the_element() {
let image = image_with_rect(32, 32, 10, 11, 16, 18);
let result = normalize_binding_area(&image, area(11, 12, 4, 5)).unwrap();
assert_eq!(result.area, area(10, 11, 6, 7));
assert!(result.changed);
assert!(!result.clamped);
assert!(!result.transparent);
}
#[test]
fn adjusts_each_edge_independently() {
let image = image_with_rect(32, 32, 10, 11, 16, 18);
let result = normalize_binding_area(&image, area(10, 12, 10, 3)).unwrap();
assert_eq!(result.area, area(10, 11, 6, 7));
}
#[test]
fn keeps_nonzero_alpha_antialias_pixels() {
let mut image = RgbaImage::from_pixel(16, 16, Rgba([0, 0, 0, 0]));
image.put_pixel(5, 6, Rgba([255, 255, 255, 1]));
image.put_pixel(7, 8, Rgba([255, 255, 255, 255]));
let result = normalize_binding_area(&image, area(4, 5, 5, 5)).unwrap();
assert_eq!(result.area, area(5, 6, 3, 3));
}
#[test]
fn fully_transparent_image_uses_the_same_path() {
let image = RgbaImage::from_pixel(32, 32, Rgba([0, 0, 0, 0]));
let result = normalize_binding_area(&image, area(10, 10, 10, 10)).unwrap();
assert_eq!(result.area, area(14, 14, 2, 2));
assert!(result.changed);
assert!(result.transparent);
}
#[test]
fn caps_each_edge_at_original_dimension() {
let image = image_with_rect(64, 64, 0, 0, 64, 64);
let result = normalize_binding_area(&image, area(16, 16, 8, 8)).unwrap();
assert_eq!(result.area, area(8, 8, 24, 24));
assert!(result.clamped);
}
#[test]
fn clamps_expansion_to_image_edges() {
let image = image_with_rect(16, 16, 0, 0, 4, 4);
let result = normalize_binding_area(&image, area(1, 1, 2, 2)).unwrap();
assert_eq!(result.area, area(0, 0, 4, 4));
assert!(result.clamped);
}
#[test]
fn exact_split_at_adjustment_limit_is_not_clamped() {
let image = image_with_rect(16, 16, 4, 4, 8, 8);
let result = normalize_binding_area(&image, area(5, 5, 2, 2)).unwrap();
assert_eq!(result.area, area(4, 4, 4, 4));
assert!(!result.clamped);
}
#[test]
fn one_pixel_area_expands_with_configured_adjustment_limit() {
let image = image_with_rect(8, 8, 2, 2, 5, 5);
let result = normalize_binding_area(&image, area(3, 3, 1, 1)).unwrap();
assert_eq!(result.area, area(2, 2, 3, 3));
assert!(!result.clamped);
}
#[test]
fn rejects_zero_sized_or_out_of_bounds_model_areas() {
let image = RgbaImage::from_pixel(16, 16, Rgba([0, 0, 0, 0]));
assert!(normalize_binding_area(&image, area(0, 0, 0, 1)).is_err());
assert!(normalize_binding_area(&image, area(15, 15, 2, 2)).is_err());
}
}
@@ -0,0 +1,364 @@
use super::model::SeparationNode;
use base64::Engine as _;
use std::path::Path;
use std::path::PathBuf;
use std::time::Instant;
const MARKER_LINE_WIDTH: u32 = 2;
const PURPLE_FILL: image::Rgba<u8> = image::Rgba([180, 0, 180, 120]);
pub async fn build_marked_image(
source_url: String,
nodes: Vec<SeparationNode>,
target: PathBuf,
) -> Result<String, String> {
let node_count = nodes.len();
let started = Instant::now();
let result = match tokio::task::spawn_blocking(move || {
let blocking_started = Instant::now();
let result = build_marked_image_blocking(&source_url, &nodes, &target);
app_log!(
"ui_separation.marker.blocking_timing outcome={} elapsed_ms={} nodes={}",
if result.is_ok() { "ok" } else { "error" },
blocking_started.elapsed().as_millis(),
node_count
);
result
})
.await
{
Ok(result) => result,
Err(error) => Err(format!("构建标记图任务失败:{error}")),
};
app_log!(
"ui_separation.marker.timing outcome={} elapsed_ms={} nodes={}",
if result.is_ok() { "ok" } else { "error" },
started.elapsed().as_millis(),
node_count
);
result
}
fn build_marked_image_blocking(
source_url: &str,
nodes: &[SeparationNode],
target: &Path,
) -> Result<String, String> {
let encoded = source_url
.split_once(',')
.map(|(_, d)| d)
.ok_or_else(|| "源图 data URL 无效".to_string())?;
let bytes = base64::engine::general_purpose::STANDARD
.decode(encoded)
.map_err(|e| format!("解码源图失败:{e}"))?;
let mut image = image::load_from_memory(&bytes)
.map_err(|e| format!("读取源图失败:{e}"))?
.to_rgba8();
let width = image.width();
let height = image.height();
// 先填充紫色重建区域,随后绘制绿色框,确保绿色框位于最上层。
for node in nodes {
for child in &node.children {
fill_rect(
&mut image,
child.global_pos_x_px,
child.global_pos_y_px,
child.width_px,
child.height_px,
PURPLE_FILL,
width,
height,
);
}
for mask in &node.text_mask_areas {
fill_rect(
&mut image,
mask.global_pos_x_px,
mask.global_pos_y_px,
mask.width_px,
mask.height_px,
PURPLE_FILL,
width,
height,
);
}
}
for node in nodes {
draw_frame(
&mut image,
node.global_pos_x_px,
node.global_pos_y_px,
node.width_px,
node.height_px,
width,
height,
);
}
let image = image::DynamicImage::ImageRgba8(image);
image
.save_with_format(target, image::ImageFormat::Png)
.map_err(|e| format!("写入标记图失败:{e}"))?;
let mut png = Vec::new();
image
.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
.map_err(|e| format!("编码标记图失败:{e}"))?;
Ok(format!(
"data:image/png;base64,{}",
base64::engine::general_purpose::STANDARD.encode(png)
))
}
fn clipped_rect(
x: u32,
y: u32,
w: u32,
h: u32,
width: u32,
height: u32,
) -> Option<(u32, u32, u32, u32)> {
if width == 0 || height == 0 || w == 0 || h == 0 {
return None;
}
if x >= width || y >= height {
return None;
}
let x0 = x;
let y0 = y;
let x1 = x.saturating_add(w).min(width).saturating_sub(1);
let y1 = y.saturating_add(h).min(height).saturating_sub(1);
(x0 <= x1 && y0 <= y1).then_some((x0, y0, x1, y1))
}
fn fill_rect(
image: &mut image::RgbaImage,
x: u32,
y: u32,
w: u32,
h: u32,
color: image::Rgba<u8>,
width: u32,
height: u32,
) {
let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else {
return;
};
for yy in y0..=y1 {
for xx in x0..=x1 {
image.put_pixel(xx, yy, color);
}
}
}
fn draw_frame(
image: &mut image::RgbaImage,
x: u32,
y: u32,
w: u32,
h: u32,
width: u32,
height: u32,
) {
let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else {
return;
};
let green = image::Rgba([0, 255, 0, 255]);
draw_line(
image,
(x0, y0),
(x1, y0),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x0, y1),
(x1, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x0, y0),
(x0, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x1, y0),
(x1, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x0, y0),
(x1, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x1, y0),
(x0, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
}
fn draw_line(
image: &mut image::RgbaImage,
start: (u32, u32),
end: (u32, u32),
color: image::Rgba<u8>,
line_width: u32,
bounds: (u32, u32, u32, u32),
) {
let mut x = start.0 as i64;
let mut y = start.1 as i64;
let target_x = end.0 as i64;
let target_y = end.1 as i64;
let dx = (target_x - x).abs();
let sx = if x < target_x { 1 } else { -1 };
let dy = -(target_y - y).abs();
let sy = if y < target_y { 1 } else { -1 };
let mut error = dx + dy;
loop {
draw_brush(image, x, y, color, line_width, bounds);
if x == target_x && y == target_y {
break;
}
let twice_error = error * 2;
if twice_error >= dy {
error += dy;
x += sx;
}
if twice_error <= dx {
error += dx;
y += sy;
}
}
}
fn draw_brush(
image: &mut image::RgbaImage,
x: i64,
y: i64,
color: image::Rgba<u8>,
line_width: u32,
bounds: (u32, u32, u32, u32),
) {
let (x0, y0, x1, y1) = bounds;
let line_width = line_width.max(1) as i64;
let before = (line_width - 1) / 2;
let after = line_width / 2;
let min_x = (x - before).max(x0 as i64);
let max_x = (x + after).min(x1 as i64);
let min_y = (y - before).max(y0 as i64);
let max_y = (y + after).min(y1 as i64);
for yy in min_y..=max_y {
for xx in min_x..=max_x {
image.put_pixel(xx as u32, yy as u32, color);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ui_editor::commands::separation::{SeparationNote, TextMaskArea};
#[test]
fn draw_frame_adds_green_cross_corner_lines() {
let mut image = image::RgbaImage::from_pixel(8, 6, image::Rgba([1, 2, 3, 255]));
draw_frame(&mut image, 1, 1, 5, 3, 8, 6);
let green = image::Rgba([0, 255, 0, 255]);
for &(x, y) in &[(1, 1), (5, 1), (1, 3), (5, 3), (3, 2)] {
assert_eq!(*image.get_pixel(x, y), green, "pixel ({x}, {y})");
}
assert_eq!(*image.get_pixel(3, 1), green);
assert_eq!(*image.get_pixel(3, 3), green);
assert_eq!(*image.get_pixel(2, 2), green);
assert_eq!(*image.get_pixel(4, 2), green);
assert_eq!(*image.get_pixel(0, 0), image::Rgba([1, 2, 3, 255]));
}
#[test]
fn draw_frame_keeps_cross_inside_clipped_rect() {
let mut image = image::RgbaImage::from_pixel(4, 4, image::Rgba([1, 2, 3, 255]));
draw_frame(&mut image, 2, 2, 4, 4, 4, 4);
let green = image::Rgba([0, 255, 0, 255]);
for y in 2..4 {
for x in 2..4 {
assert_eq!(*image.get_pixel(x, y), green);
}
}
assert_eq!(*image.get_pixel(1, 1), image::Rgba([1, 2, 3, 255]));
}
#[test]
fn clipped_rect_ignores_rectangles_starting_outside_image() {
assert_eq!(clipped_rect(4, 0, 1, 1, 4, 4), None);
assert_eq!(clipped_rect(0, 4, 1, 1, 4, 4), None);
}
#[test]
fn text_mask_is_purple_before_green_frame() {
let mut image = image::RgbaImage::from_pixel(8, 8, image::Rgba([1, 2, 3, 255]));
let node = SeparationNode {
id: crate::ui_editor::utils::NodeId::new("image").unwrap(),
global_pos_x_px: 1,
global_pos_y_px: 1,
width_px: 6,
height_px: 6,
note: SeparationNote::default(),
text_mask_areas: vec![
TextMaskArea {
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
},
TextMaskArea {
global_pos_x_px: 1,
global_pos_y_px: 1,
width_px: 1,
height_px: 1,
},
],
children: vec![],
rework_count: 0,
};
for mask in &node.text_mask_areas {
fill_rect(
&mut image,
mask.global_pos_x_px,
mask.global_pos_y_px,
mask.width_px,
mask.height_px,
PURPLE_FILL,
8,
8,
);
}
draw_frame(
&mut image,
node.global_pos_x_px,
node.global_pos_y_px,
node.width_px,
node.height_px,
8,
8,
);
assert_eq!(*image.get_pixel(0, 0), PURPLE_FILL);
assert_eq!(*image.get_pixel(1, 1), image::Rgba([0, 255, 0, 255]));
}
}
@@ -0,0 +1,389 @@
mod area;
mod marker;
mod model;
mod persistence;
mod prompt;
mod tree;
mod workflow;
pub(crate) use marker::build_marked_image;
pub use model::*;
pub use persistence::*;
pub use tree::*;
pub use workflow::apply_batch_patch;
pub(crate) use workflow::separate_ui_impl;
#[cfg(test)]
mod tests {
use super::*;
use crate::ui_editor::component::image::{ImageComponent, ImageType};
use crate::ui_editor::component::text::TextComponent;
use crate::ui_editor::component::Component;
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
use crate::ui_editor::layout::control_layout::ControlLayout;
use crate::ui_editor::layout::node::Node;
use crate::ui_editor::layout::node::{NodeMetadata, NodeSource, StageStatus};
use crate::ui_editor::resource::ui_design_image::UIDesignImage;
use crate::ui_editor::state::{State, UITree};
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
use nalgebra::Vector2;
use std::collections::HashMap;
use std::path::Path;
use typed_floats::tf32::StrictlyPositiveFinite;
fn node(id: &str, component: Option<Component>, children: Vec<Node>) -> Node {
Node {
id: NodeId::new(id).unwrap(),
layout: ControlLayout::default(),
metadata: NodeMetadata {
name: id.to_string(),
description: String::new(),
layout_status: StageStatus::NoProblem,
component_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::Llm,
},
component,
children_display_mode: ChildrenDisplayMode::Stack,
children,
}
}
fn state(root: Node) -> State {
let image_id = UIDesignImageId::new("page").unwrap();
State {
ui_trees: vec![UITree {
src_ui_design: image_id.clone(),
root,
}],
ui_design_images: HashMap::from([(
image_id,
UIDesignImage {
metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata {
name: "page".to_string(),
description: String::new(),
role: None,
slave_to: None,
},
path: "page.png".to_string(),
pixel_size: Vector2::new(100.0, 100.0),
pixels_per_unit: StrictlyPositiveFinite::new(1.0).unwrap(),
},
)]),
sprite_assets: HashMap::new(),
font_assets: HashMap::new(),
}
}
#[test]
fn construction_filters_pure_nodes_and_passes_children_through() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let root = node(
"root",
None,
vec![node(
"container",
None,
vec![node("image", Some(image), vec![])],
)],
);
let result = construct_separation_state(&state(root));
assert_eq!(result.trees[0].root.children[0].id.as_str(), "image");
}
#[test]
fn construction_keeps_real_root_for_root_image() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let root = node("root-image", Some(image), vec![]);
let result = construct_separation_state(&state(root));
let tree = &result.trees[0];
assert_eq!(tree.root.id.as_str(), "root-image");
assert!(tree.root_extractable);
}
#[test]
fn construction_attaches_text_mask_to_nearest_unbound_image() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let text = Component::Text(TextComponent::new("按钮"));
let root = node(
"root",
None,
vec![node(
"outer-image",
Some(image.clone()),
vec![node("text", Some(text.clone()), vec![])],
)],
);
let result = construct_separation_state(&state(root));
let outer = &result.trees[0].root.children[0];
assert_eq!(outer.id.as_str(), "outer-image");
assert_eq!(outer.text_mask_areas.len(), 1);
assert!(outer.children.is_empty());
let nested_root = node(
"root",
None,
vec![node(
"outer-image",
Some(image.clone()),
vec![node(
"inner-image",
Some(image),
vec![node("text", Some(text), vec![])],
)],
)],
);
let nested = construct_separation_state(&state(nested_root));
let inner = &nested.trees[0].root.children[0].children[0];
assert_eq!(inner.text_mask_areas.len(), 1);
assert!(nested.trees[0].root.children[0].text_mask_areas.is_empty());
}
#[test]
fn root_image_receives_text_mask() {
let root = node(
"root-image",
Some(Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
})),
vec![node(
"text",
Some(Component::Text(TextComponent::new("标题"))),
vec![],
)],
);
let result = construct_separation_state(&state(root));
assert_eq!(result.trees[0].root.text_mask_areas.len(), 1);
}
#[test]
fn binding_validation_requires_exact_batch_coverage() {
let node = SeparationNode {
id: NodeId::new("image").unwrap(),
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
note: SeparationNote {
description: "image".to_string(),
rework_notes: Vec::new(),
},
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err());
}
#[test]
fn separation_note_prompt_keeps_rework_notes_in_order() {
let without_notes = SeparationNote {
description: "按钮".to_string(),
rework_notes: Vec::new(),
};
assert_eq!(without_notes.as_prompt(), "desc: 按钮");
let with_notes = SeparationNote {
description: "按钮".to_string(),
rework_notes: vec!["保留圆角".to_string(), "去掉阴影".to_string()],
};
assert_eq!(
with_notes.as_prompt(),
"desc: 按钮\nprevious rework notes:\n- 保留圆角\n- 去掉阴影"
);
}
#[test]
fn need_rework_appends_note_and_final_attempt_becomes_problematic() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let mut separation = construct_separation_state(&state(node(
"root",
None,
vec![node("image", Some(image), vec![])],
)));
let id = NodeId::new("image").unwrap();
let paths = HashMap::new();
for note in ["第一次意见", "第二次意见", "最后一次意见"] {
apply_batch_patch(
&mut separation,
0,
&[BindingDecision::NeedRework {
to_node: id.clone(),
advice: note.to_string(),
}],
&paths,
)
.unwrap();
}
let node = &separation.trees[0].root.children[0];
assert_eq!(
node.note.rework_notes,
["第一次意见", "第二次意见", "最后一次意见"]
);
assert_eq!(separation.problematic_nodes.len(), 1);
assert_eq!(
separation.problematic_nodes[0].rework_count,
MAX_REWORK_COUNT
);
assert_eq!(node.rework_count, MAX_REWORK_COUNT);
assert!(next_leaf_batch(&separation, &separation.trees[0]).is_empty());
}
#[test]
fn next_extract_prompt_contains_previous_rework_notes() {
let note = SeparationNote {
description: "图标".to_string(),
rework_notes: vec!["不要带父背景".to_string()],
};
let prompt = super::prompt::gen_extract_prompt(vec![note]);
assert!(prompt.contains("previous rework notes:\n- 不要带父背景"));
}
#[test]
fn binding_validation_rejects_overlong_rework_note() {
let node = SeparationNode {
id: NodeId::new("image").unwrap(),
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
let decision = BindingDecision::NeedRework {
to_node: node.id.clone(),
advice: "x".repeat(MAX_REWORK_NOTE_CHARS + 1),
};
assert!(validate_binding_response(
&BindingResp {
decisions: vec![decision]
},
&[&node]
)
.is_err());
}
#[test]
fn sidecar_name_uses_asset_id_digest() {
let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap();
assert!(dir.to_string_lossy().contains("ui_1-"));
assert!(dir.to_string_lossy().ends_with("-separation"));
}
#[test]
fn patch_collects_bound_and_keeps_tree_topology() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let mut state = construct_separation_state(&state(node(
"root",
None,
vec![node("image", Some(image), vec![])],
)));
let id = NodeId::new("image").unwrap();
let decisions = vec![BindingDecision::Ok {
to_node: id.clone(),
extracted_area: BindingArea {
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
},
}];
let paths = HashMap::from([(id.clone(), "ui/.sidecar/cut.png".to_string())]);
apply_batch_patch(&mut state, 0, &decisions, &paths).unwrap();
assert_eq!(state.bound[0].node_id, id);
assert_eq!(state.trees[0].root.children.len(), 1);
}
#[test]
fn batch_selection_greedily_skips_overlapping_leaves() {
let a = SeparationNode {
id: NodeId::new("a").unwrap(),
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 10,
height_px: 10,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
let b = SeparationNode {
id: NodeId::new("b").unwrap(),
global_pos_x_px: 5,
global_pos_y_px: 5,
width_px: 10,
height_px: 10,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
let c = SeparationNode {
id: NodeId::new("c").unwrap(),
global_pos_x_px: 20,
global_pos_y_px: 0,
width_px: 5,
height_px: 5,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
let tree = SeparationTree {
src_ui_design: UIDesignImageId::new("page").unwrap(),
root: SeparationNode {
id: NodeId::new("root").unwrap(),
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 100,
height_px: 100,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![a, b, c],
rework_count: 0,
},
root_extractable: false,
};
let state = SeparationState {
schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(),
trees: vec![tree.clone()],
bound: vec![],
problematic_nodes: vec![],
};
let batch = next_leaf_batch(&state, &tree);
assert_eq!(
batch
.iter()
.map(|node| node.id.as_str())
.collect::<Vec<_>>(),
vec!["a", "c"]
);
}
}
@@ -0,0 +1,50 @@
use crate::ui_editor::utils::NodeId;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub struct BindingArea {
pub global_pos_x_px: u32,
pub global_pos_y_px: u32,
pub width_px: u32,
pub height_px: u32,
}
impl BindingArea {
pub fn validate_in(&self, w: u32, h: u32) -> Result<(), String> {
if self.width_px == 0 || self.height_px == 0 {
return Err("BindingArea 宽度和高度必须大于 0".into());
}
if self
.global_pos_x_px
.checked_add(self.width_px)
.is_none_or(|v| v > w)
|| self
.global_pos_y_px
.checked_add(self.height_px)
.is_none_or(|v| v > h)
{
return Err("BindingArea 超出处理图边界".into());
}
Ok(())
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub enum BindingDecision {
Ok {
extracted_area: BindingArea,
to_node: NodeId,
},
NeedRework {
advice: String,
to_node: NodeId,
},
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
pub struct BindingResp {
pub decisions: Vec<BindingDecision>,
}
@@ -0,0 +1,13 @@
mod binding;
mod node;
mod note;
mod result;
pub use binding::*;
pub use node::*;
pub use note::*;
pub use result::*;
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1";
pub const MAX_REWORK_COUNT: u32 = 3;
pub const MAX_REWORK_NOTE_CHARS: usize = 512;
@@ -0,0 +1,44 @@
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct TextMaskArea {
pub global_pos_x_px: u32,
pub global_pos_y_px: u32,
pub width_px: u32,
pub height_px: u32,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationNode {
pub id: NodeId,
pub global_pos_x_px: u32,
pub global_pos_y_px: u32,
pub width_px: u32,
pub height_px: u32,
pub note: super::SeparationNote,
pub text_mask_areas: Vec<TextMaskArea>,
pub children: Vec<SeparationNode>,
pub rework_count: u32,
}
impl SeparationNode {
pub fn as_prompt(&self) -> String {
format!(
"node_id={} note: {}",
self.id.as_str(),
self.note.as_prompt()
)
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationTree {
pub src_ui_design: UIDesignImageId,
pub root: SeparationNode,
pub root_extractable: bool,
}
@@ -0,0 +1,23 @@
use serde::{Deserialize, Serialize};
use ts_rs::TS;
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationNote {
pub description: String,
pub rework_notes: Vec<String>,
}
impl SeparationNote {
pub fn as_prompt(&self) -> String {
let mut prompt = format!("desc: {}", self.description);
if !self.rework_notes.is_empty() {
prompt.push_str("\nprevious rework notes:");
for note in &self.rework_notes {
prompt.push_str("\n- ");
prompt.push_str(note);
}
}
prompt
}
}
@@ -0,0 +1,43 @@
use crate::ui_editor::utils::NodeId;
use serde::{Deserialize, Serialize};
use ts_rs::TS;
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct BoundNode {
pub node_id: NodeId,
pub cut_image_path: String,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct ProblematicNode {
pub node_id: NodeId,
pub problem_description: String,
pub rework_count: u32,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationState {
pub schema_version: String,
pub trees: Vec<super::SeparationTree>,
pub bound: Vec<BoundNode>,
pub problematic_nodes: Vec<ProblematicNode>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationDTO {
pub bound_nodes: Vec<BoundNode>,
pub problematic_nodes: Vec<ProblematicNode>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationRecoveryDTO {
pub exists: bool,
pub bound_node_count: usize,
pub problematic_node_count: usize,
pub has_pending_tree: bool,
}
@@ -0,0 +1,167 @@
use super::model::*;
use crate::ui_editor::commands::separation::*;
use std::fs;
use std::path::{Path, PathBuf};
pub fn separation_sidecar_dir(root: &Path, asset_id: &str) -> Result<PathBuf, String> {
if asset_id.trim().is_empty() || asset_id.trim() != asset_id {
app_log!("ui_separation.error stage=sidecar_dir reason=invalid_asset_id");
return Err("UI 资源 ID 无效".to_string());
}
let dir = root.join("ui").join(format!(
".{}-separation",
crate::ui_editor::persistence::generated_file_stem(asset_id)
));
if !dir.starts_with(root) {
app_log!("ui_separation.error stage=sidecar_dir reason=path_escape");
return Err("separation sidecar 路径越界".to_string());
}
app_log!(
"ui_separation.sidecar_resolved asset_id={} directory={}",
asset_id,
dir.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>")
);
Ok(dir)
}
pub fn separation_state_path(root: &Path, asset_id: &str) -> Result<PathBuf, String> {
Ok(separation_sidecar_dir(root, asset_id)?.join("state.json"))
}
pub fn project_relative_path(root: &Path, path: &Path) -> Result<String, String> {
let relative = path
.strip_prefix(root)
.map_err(|_| "separation 产物必须位于项目目录内".to_string())?;
let value = relative.to_string_lossy().replace('\\', "/");
if value.is_empty() || value.starts_with('/') || value.split('/').any(|part| part == "..") {
return Err("separation 产物相对路径无效".to_string());
}
Ok(value)
}
pub fn write_separation_state(path: &Path, state: &SeparationState) -> Result<(), String> {
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
app_log!("ui_separation.error stage=state_write reason=schema_mismatch");
return Err("不支持的 separation state schema".to_string());
}
app_log!(
"ui_separation.state_write.start file={} trees={} bound={} problematic={}",
path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>"),
state.trees.len(),
state.bound.len(),
state.problematic_nodes.len()
);
let bytes = serde_json::to_vec_pretty(state).map_err(|error| {
app_log!("ui_separation.error stage=state_write reason=serialize error={error}");
format!("序列化 separation state 失败:{error}")
})?;
let parent = path.parent().ok_or_else(|| {
app_log!("ui_separation.error stage=state_write reason=missing_parent");
"separation state 路径缺少父目录".to_string()
})?;
fs::create_dir_all(parent).map_err(|error| {
app_log!("ui_separation.error stage=state_write reason=create_parent error={error}");
format!("创建 separation sidecar 失败:{error}")
})?;
let temporary = path.with_extension("json.tmp");
fs::write(&temporary, bytes).map_err(|error| {
app_log!("ui_separation.error stage=state_write reason=write_temp error={error}");
format!("写入 separation state 失败:{error}")
})?;
fs::rename(&temporary, path).map_err(|error| {
app_log!("ui_separation.error stage=state_write reason=install error={error}");
format!("安装 separation state 失败:{error}")
})?;
app_log!(
"ui_separation.state_write.completed file={} bytes={} trees={} bound={} problematic={}",
path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>"),
fs::metadata(path)
.map(|metadata| metadata.len())
.unwrap_or(0),
state.trees.len(),
state.bound.len(),
state.problematic_nodes.len()
);
Ok(())
}
pub fn read_separation_state(path: &Path) -> Result<SeparationState, String> {
app_log!(
"ui_separation.state_read.start file={}",
path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>")
);
let bytes = fs::read(path).map_err(|error| {
app_log!("ui_separation.error stage=state_read reason=read error={error}");
format!("读取 separation state 失败:{error}")
})?;
let state: SeparationState = serde_json::from_slice(&bytes).map_err(|error| {
app_log!("ui_separation.error stage=state_read reason=parse error={error}");
format!("解析 separation state 失败:{error}")
})?;
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
app_log!("ui_separation.error stage=state_read reason=schema_mismatch");
return Err("不支持的 separation state schema".to_string());
}
app_log!(
"ui_separation.state_read.completed bytes={} trees={} bound={} problematic={}",
bytes.len(),
state.trees.len(),
state.bound.len(),
state.problematic_nodes.len()
);
Ok(state)
}
pub fn separation_dto(state: &SeparationState) -> SeparationDTO {
app_log!(
"ui_separation.dto bound_nodes={} problematic_nodes={} remaining_trees={}",
state.bound.len(),
state.problematic_nodes.len(),
state.trees.len()
);
SeparationDTO {
bound_nodes: state.bound.clone(),
problematic_nodes: state.problematic_nodes.clone(),
}
}
pub fn inspect_separation_recovery(
root: &Path,
asset_id: &str,
) -> Result<SeparationRecoveryDTO, String> {
let state_path = separation_state_path(root, asset_id)?;
if !state_path.exists() {
return Ok(SeparationRecoveryDTO {
exists: false,
bound_node_count: 0,
problematic_node_count: 0,
has_pending_tree: false,
});
}
let state = read_separation_state(&state_path)?;
Ok(SeparationRecoveryDTO {
exists: true,
bound_node_count: state.bound.len(),
problematic_node_count: state.problematic_nodes.len(),
has_pending_tree: state
.trees
.iter()
.any(|tree| !tree.root.children.is_empty() || tree.root_extractable),
})
}
pub fn finalize_separation(root: &Path, asset_id: &str) -> Result<(), String> {
let state_path = separation_state_path(root, asset_id)?;
match fs::remove_file(&state_path) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(format!("删除 separation state 失败:{error}")),
}
}
@@ -0,0 +1,70 @@
use crate::ui_editor::commands::separation::{SeparationNode, SeparationNote};
const SHARED_SEPARATION_REQ: &str = r#"
MUST hard edges; preserve no glow/blur beyond the exact visible shape.
NEVER keep its parent's background with it.
NEVER include any text unless requested.
UI elements that needs to extract has been marked with GREEN line frames box with crossline inside. (only for mark purpose, NEVER wrap a frame in your extraction).
On some UI elements, there is some PURPLE filled area, they were removed UI elements, reconstruct the background under where they were.
MUST extract exactly these marked UI elements area.
"#;
pub(super) fn gen_extract_prompt(separation_notes: Vec<SeparationNote>) -> String {
let extract_system_prompt = format!(
r#"
This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction.
Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element.
MUST keep each element at its original position on a transparent canvas.
{SHARED_SEPARATION_REQ}
here are UI elements to extract:
"#
);
let mut result = extract_system_prompt;
result.reserve(512);
for elem in separation_notes {
result.push_str(&elem.as_prompt());
result.push('\n');
}
result
}
pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String {
let binding_system_prompt = format!(
r#"
You will be given a src UI design image and a processed image, where some ui elements are separated.
You need to recognize and review the separation using the given tool.
field notes:
* extracted_area MUST be the recognized area from the processed image, INSTEAD OF from the src image.
The processed image is the only authoritative image for extracted_area.
Return the pixel bounding box of the extracted element as it appears in the processed image.
Do not copy, infer, or reuse the source node rectangle.
The src image is only for identifying which semantic UI element belongs to to_node.
Here were the separation requirements:
```
{SHARED_SEPARATION_REQ}
```
And you should also review if the extracted's successfully meet the src image:
* shape
* color
* style
* edge process
...
if not, use `NeedRework` data structure in the tool to indicate the (it should be from which) node id and advice.
your advice (less than 20 words) will be used to improve the separation in the next time.
these node need handle:
"#
);
let mut result = binding_system_prompt;
result.reserve(512);
for elem in nodes {
result.push_str(&elem.as_prompt());
result.push('\n');
}
result
}
@@ -0,0 +1,273 @@
use super::model::*;
use crate::ui_editor::component::{image::ImageComponent, Component};
use crate::ui_editor::layout::node::Node;
use crate::ui_editor::state::State;
use crate::ui_editor::utils::NodeId;
use std::collections::HashMap;
use std::collections::HashSet;
fn is_unbound_image(node: &Node) -> bool {
matches!(
node.component.as_ref(),
Some(Component::Image(ImageComponent {
target_graphic: None,
..
}))
)
}
fn has_image_component(node: &Node) -> bool {
matches!(node.component.as_ref(), Some(Component::Image(_)))
}
fn has_text_component(node: &Node) -> bool {
matches!(node.component.as_ref(), Some(Component::Text(_)))
}
fn node_pixel_rect(
node: &Node,
parent: &crate::ui_editor::layout::dimension::UIRect,
ppu: f32,
) -> (u32, u32, u32, u32) {
let rect = node.layout.transform.resolve(parent);
(
(rect.min.x * ppu).max(0.0).round() as u32,
(rect.min.y * ppu).max(0.0).round() as u32,
(rect.size.x * ppu).max(0.0).round() as u32,
(rect.size.y * ppu).max(0.0).round() as u32,
)
}
fn node_description(node: &Node) -> String {
let name = node.metadata.name.trim();
let description = node.metadata.description.trim();
match (name.is_empty(), description.is_empty()) {
(true, true) => "未命名 UI 图片元素".to_string(),
(false, true) => name.to_string(),
(true, false) => description.to_string(),
(false, false) => format!("{name}{description}"),
}
}
fn collect_todo_nodes(
node: &Node,
parent: &crate::ui_editor::layout::dimension::UIRect,
ppu: f32,
output: &mut Vec<SeparationNode>,
text_masks: &mut HashMap<NodeId, Vec<TextMaskArea>>,
) {
let rect = node.layout.transform.resolve(parent);
let mut children = Vec::new();
for child in &node.children {
collect_todo_nodes(child, &rect, ppu, &mut children, text_masks);
}
if is_unbound_image(node) {
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
output.push(SeparationNode {
id: node.id.clone(),
global_pos_x_px: x,
global_pos_y_px: y,
width_px: w,
height_px: h,
note: SeparationNote {
description: node_description(node),
rework_notes: Vec::new(),
},
text_mask_areas: text_masks.remove(&node.id).unwrap_or_default(),
children,
rework_count: 0,
});
} else {
output.extend(children);
}
}
fn collect_text_masks(
node: &Node,
parent: &crate::ui_editor::layout::dimension::UIRect,
ppu: f32,
nearest_image: Option<NodeId>,
output: &mut HashMap<NodeId, Vec<TextMaskArea>>,
) {
let node_is_image = is_unbound_image(node);
let nearest_image = if node_is_image {
Some(node.id.clone())
} else {
nearest_image
};
if has_text_component(node) && !has_image_component(node) {
if let Some(image_id) = nearest_image.clone() {
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
output.entry(image_id).or_default().push(TextMaskArea {
global_pos_x_px: x,
global_pos_y_px: y,
width_px: w,
height_px: h,
});
}
}
let rect = node.layout.transform.resolve(parent);
for child in &node.children {
collect_text_masks(child, &rect, ppu, nearest_image.clone(), output);
}
}
pub fn construct_separation_state(state: &State) -> SeparationState {
let trees = state
.ui_trees
.iter()
.filter_map(|tree| {
let image = state.ui_design_images.get(&tree.src_ui_design)?;
let ppu = image.pixels_per_unit.get();
let size = image.pixel_size / ppu;
let root_rect =
crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size);
let mut text_masks = HashMap::new();
collect_text_masks(&tree.root, &root_rect, ppu, None, &mut text_masks);
let mut children = Vec::new();
for child in &tree.root.children {
collect_todo_nodes(child, &root_rect, ppu, &mut children, &mut text_masks);
}
let root_extractable = is_unbound_image(&tree.root);
if !root_extractable && children.is_empty() {
return None;
}
let (x, y, w, h) = node_pixel_rect(&tree.root, &root_rect, ppu);
Some(SeparationTree {
src_ui_design: tree.src_ui_design.clone(),
root: SeparationNode {
id: tree.root.id.clone(),
global_pos_x_px: x,
global_pos_y_px: y,
width_px: w,
height_px: h,
note: SeparationNote {
description: node_description(&tree.root),
rework_notes: Vec::new(),
},
text_mask_areas: text_masks.remove(&tree.root.id).unwrap_or_default(),
children,
rework_count: 0,
},
root_extractable,
})
})
.collect();
SeparationState {
schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(),
trees,
bound: Vec::new(),
problematic_nodes: Vec::new(),
}
}
fn terminal_ids(state: &SeparationState) -> HashSet<NodeId> {
state
.bound
.iter()
.map(|n| n.node_id.clone())
.chain(state.problematic_nodes.iter().map(|n| n.node_id.clone()))
.collect()
}
fn logical_leaves<'a>(
node: &'a SeparationNode,
extractable: bool,
terminal: &HashSet<NodeId>,
output: &mut Vec<&'a SeparationNode>,
) {
let is_terminal = terminal.contains(&node.id);
let children_terminal = node
.children
.iter()
.all(|child| terminal.contains(&child.id));
if extractable && !is_terminal && children_terminal {
output.push(node);
return;
}
for child in &node.children {
logical_leaves(child, true, terminal, output);
}
}
fn overlaps(a: &SeparationNode, b: &SeparationNode) -> bool {
let ax1 = a.global_pos_x_px as u64 + a.width_px as u64;
let ay1 = a.global_pos_y_px as u64 + a.height_px as u64;
let bx1 = b.global_pos_x_px as u64 + b.width_px as u64;
let by1 = b.global_pos_y_px as u64 + b.height_px as u64;
let width = ax1
.min(bx1)
.saturating_sub(a.global_pos_x_px.max(b.global_pos_x_px) as u64);
let height = ay1
.min(by1)
.saturating_sub(a.global_pos_y_px.max(b.global_pos_y_px) as u64);
width > 0 && height > 0
}
pub fn next_leaf_batch<'a>(
state: &SeparationState,
tree: &'a SeparationTree,
) -> Vec<&'a SeparationNode> {
let terminal = terminal_ids(state);
let mut candidates = Vec::new();
logical_leaves(
&tree.root,
tree.root_extractable,
&terminal,
&mut candidates,
);
let mut selected: Vec<&'a SeparationNode> = Vec::new();
for candidate in candidates {
if selected.iter().all(|other| !overlaps(candidate, other)) {
selected.push(candidate);
}
}
app_log!(
"ui_separation.batch_selected image_id={} leaf_nodes={}",
tree.src_ui_design.as_str(),
selected.len()
);
selected
}
pub fn validate_binding_response(
response: &BindingResp,
batch: &[&SeparationNode],
) -> Result<(), String> {
let expected = batch
.iter()
.map(|node| node.id.clone())
.collect::<HashSet<_>>();
let mut seen = HashSet::new();
for decision in &response.decisions {
let node_id = match decision {
BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => {
to_node
}
};
if !expected.contains(node_id) {
return Err(format!("视觉绑定返回了未知节点:{}", node_id.as_str()));
}
if !seen.insert(node_id.clone()) {
return Err(format!("视觉绑定重复返回节点:{}", node_id.as_str()));
}
if let BindingDecision::NeedRework {
advice,
..
} = decision
{
if advice.trim().is_empty() {
return Err("NeedRework 必须包含问题描述".to_string());
}
if advice.chars().count() > MAX_REWORK_NOTE_CHARS {
return Err(format!(
"NeedRework 问题描述不能超过 {MAX_REWORK_NOTE_CHARS} 个字符"
));
}
}
}
if seen.len() != expected.len() {
return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string());
}
Ok(())
}
@@ -1,9 +1,11 @@
use crate::agent::request_game_creator_llm_text;
use crate::config::{apply_game_creator_llm_reasoning_effort, parse_game_creator_llm_api_kind};
use base64::Engine as _;
use platform_llm::{LlmClient, LlmError, LlmRunRequest, LlmRunResponse};
use platform_llm::{LlmClient, LlmError, LlmMessage, LlmRunRequest, LlmRunResponse};
use schemars::JsonSchema;
use serde::Serialize;
use std::fs::File;
use std::future::Future;
use std::io::Read;
use std::path::{Path, PathBuf};
@@ -25,6 +27,49 @@ pub(crate) async fn request_ui_editor_llm(
request_game_creator_llm_text(client, llm, request).await
}
/// 按 append-only history 重试结构化 LLM 请求;仅业务校验失败会追加反馈消息。
pub(crate) async fn run_with_repair_history<T, Requester, Fut, Validator>(
max_retries: usize,
initial_history: Vec<LlmMessage>,
requester: Requester,
validator: Validator,
) -> Result<T, String>
where
T: Serialize,
Requester: Fn(Vec<LlmMessage>) -> Fut,
Fut: Future<Output = Result<T, String>>,
Validator: Fn(&T) -> Result<(), String>,
{
let mut history = initial_history;
for attempt in 0..=max_retries {
let value = match requester(history.clone()).await {
Ok(value) => value,
Err(error) if attempt < max_retries => {
app_log!(
"ui_editor.llm.retry request_error attempt={} max_retries={} error={}",
attempt + 1,
max_retries,
error
);
continue;
}
Err(error) => return Err(error),
};
match validator(&value) {
Ok(()) => return Ok(value),
Err(error) if attempt < max_retries => {
let serialized = serde_json::to_string(&value)
.map_err(|serialize_error| format!("序列化修复反馈失败:{serialize_error}"))?;
history.push(LlmMessage::system(format!(
"上一次模型输出:\n{serialized}\n\n业务校验失败:\n{error}\n\n请修正并完整返回。"
)));
}
Err(error) => return Err(error),
}
}
unreachable!("repair history runner always returns within requested retries")
}
pub(crate) fn parse_limited_llm_tool_arguments(
arguments: &str,
) -> Result<serde_json::Value, String> {
@@ -144,6 +189,107 @@ mod tests {
);
}
#[tokio::test]
async fn repair_history_zero_retries_calls_once_with_initial_history() {
let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let seen = calls.clone();
let initial_history = vec![LlmMessage::user("初始 prompt")];
let result = run_with_repair_history(
0,
initial_history.clone(),
move |history| {
let seen = seen.clone();
async move {
seen.lock().unwrap().push(history);
Ok::<_, String>(serde_json::json!({"ok": true}))
}
},
|_| Ok(()),
)
.await
.expect("single turn should succeed");
assert_eq!(result, serde_json::json!({"ok": true}));
assert_eq!(calls.lock().unwrap().as_slice(), &[initial_history]);
}
#[tokio::test]
async fn repair_history_request_error_retries_without_appending_history() {
let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let attempts = std::sync::Arc::new(std::sync::Mutex::new(0usize));
let seen_calls = calls.clone();
let seen_attempts = attempts.clone();
let initial_history = vec![LlmMessage::user("初始 prompt")];
let result = run_with_repair_history(
1,
initial_history.clone(),
move |history| {
seen_calls.lock().unwrap().push(history);
let attempt = {
let mut attempts = seen_attempts.lock().unwrap();
let attempt = *attempts;
*attempts += 1;
attempt
};
async move {
if attempt == 0 {
Err("网络错误".to_string())
} else {
Ok::<_, String>(serde_json::json!({"ok": true}))
}
}
},
|_| Ok(()),
)
.await
.expect("retry-only error should recover");
assert_eq!(result, serde_json::json!({"ok": true}));
assert_eq!(
calls.lock().unwrap().as_slice(),
&[initial_history.clone(), initial_history]
);
}
#[tokio::test]
async fn repair_history_business_failure_appends_serialized_value_and_error() {
let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let attempts = std::sync::Arc::new(std::sync::Mutex::new(0usize));
let seen_calls = calls.clone();
let seen_attempts = attempts.clone();
let initial_history = vec![LlmMessage::user("初始 prompt")];
let result = run_with_repair_history(
1,
initial_history.clone(),
move |history| {
seen_calls.lock().unwrap().push(history);
let mut attempts = seen_attempts.lock().unwrap();
let attempt = *attempts;
*attempts += 1;
async move { Ok::<_, String>(serde_json::json!({"attempt": attempt})) }
},
|value: &serde_json::Value| {
if value["attempt"] == 0 {
Err("业务校验失败".to_string())
} else {
Ok(())
}
},
)
.await
.expect("business feedback should recover");
assert_eq!(result, serde_json::json!({"attempt": 1}));
let calls = calls.lock().unwrap();
assert_eq!(calls.len(), 2);
assert_eq!(calls[0], initial_history);
assert_eq!(calls[1].len(), 2);
assert_eq!(calls[1][0], LlmMessage::user("初始 prompt"));
assert_eq!(
calls[1][1],
LlmMessage::system(
"上一次模型输出:\n{\"attempt\":0}\n\n业务校验失败:\n业务校验失败\n\n请修正并完整返回。"
)
);
}
#[test]
fn reference_image_rejects_file_over_five_mib_before_reading() {
let directory = tempfile::tempdir().expect("reference image fixture");
@@ -9,3 +9,27 @@ pub enum Component {
Image(image::ImageComponent),
Text(text::TextComponent),
}
/// LLM 工具返回的节点组件载荷。
///
/// 这里不能直接使用 `Option<Component>`:部分模型在严格工具 schema 下不会稳定地产生
/// `null`。用显式的 `PureNode` / `WithComponent` 外部枚举表达两种情况,既保留纯结构节点
/// 的语义,也让工具调用始终返回一个可判别的对象;落入编辑器 `Node` 时再映射为
/// `Option<Component>`。
#[derive(
Clone, Debug, PartialEq, schemars::JsonSchema, serde::Deserialize, serde::Serialize, ts_rs::TS,
)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub enum NodeComponent {
PureNode,
WithComponent(Component),
}
impl NodeComponent {
pub fn into_option(self) -> Option<Component> {
match self {
Self::PureNode => None,
Self::WithComponent(component) => Some(component),
}
}
}
@@ -199,14 +199,13 @@ fn render_node_with_scale(
json!({"childrenDisplayMode": "Exclusive", "childrenRendered": "all"}),
)
});
let components = node
.components
.iter()
let component = node
.component
.as_ref()
.map(|component| render_component(state, component))
.collect::<Result<Vec<_>, _>>()?
.into_iter()
.transpose()?
.map(|fragment| fragment.into_string())
.collect::<Vec<_>>();
.unwrap_or_default();
let children = node
.children
.iter()
@@ -226,7 +225,7 @@ fn render_node_with_scale(
(comment)
@if let Some(group_comment) = exclusive_comment { (group_comment) }
div ui-node-id=(node.id.as_str()) style=(style) {
(PreEscaped(components.concat()))
(PreEscaped(component))
(PreEscaped(children.concat()))
}
})
@@ -11,7 +11,7 @@ pub struct Node {
pub id: NodeId,
pub layout: ControlLayout,
pub metadata: NodeMetadata,
pub components: Vec<Component>,
pub component: Option<Component>,
pub children_display_mode: ChildrenDisplayMode,
pub children: Vec<Node>,
}
@@ -50,7 +50,7 @@ pub struct NodeMetadata {
pub name: String,
pub description: String,
pub layout_status: StageStatus,
pub components_status: StageStatus,
pub component_status: StageStatus,
pub allow_llm_edit_layout: bool,
pub allow_llm_edit_component: bool,
pub source: NodeSource,
@@ -25,7 +25,6 @@ const UI_DESIGN_STATE_MAX_IMAGES: usize = 4;
const UI_DESIGN_STATE_MAX_SPRITES: usize = 1_024;
pub(crate) const UI_DESIGN_STATE_MAX_NODES: usize = 10_000;
const UI_DESIGN_STATE_MAX_DEPTH: usize = 128;
pub(crate) const UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE: usize = 64;
const UI_DESIGN_STATE_MAX_SAFE_REVISION: u64 = 9_007_199_254_740_991;
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
@@ -196,7 +195,7 @@ pub(crate) fn generate_ui_design_code_at(
})
}
fn generated_file_stem(asset_id: &str) -> String {
pub(crate) fn generated_file_stem(asset_id: &str) -> String {
let mut stem = String::new();
for character in asset_id.chars() {
if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') {
@@ -747,12 +746,8 @@ fn validate_node(
}
}
}
if node.components.len() > UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE {
return Err(format!(
"单个 UI 节点最多支持 {UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE} 个组件"
));
}
for component in &node.components {
validate_component_status(node.component.as_ref(), &node.metadata.component_status)?;
if let Some(component) = &node.component {
match component {
Component::Image(image) => {
if image
@@ -778,6 +773,22 @@ fn validate_node(
Ok(())
}
fn validate_component_status(
component: Option<&Component>,
status: &crate::ui_editor::layout::node::StageStatus,
) -> Result<(), String> {
if component.is_none()
&& matches!(
status,
crate::ui_editor::layout::node::StageStatus::NeedReview(_)
| crate::ui_editor::layout::node::StageStatus::Blocked(_)
)
{
return Err("纯结构节点的 component_status 必须为 NoProblem".to_string());
}
Ok(())
}
fn validate_id(value: &str, label: &str) -> Result<(), String> {
if value.is_empty() || value.trim() != value || value.chars().any(char::is_control) {
return Err(format!("{label} 无效"));
@@ -881,12 +892,12 @@ mod tests {
"name": "页面根节点",
"description": "",
"layout_status": "NoProblem",
"components_status": "NoProblem",
"component_status": "NoProblem",
"allow_llm_edit_layout": true,
"allow_llm_edit_component": true,
"source": "System"
},
"components": [],
"component": null,
"children_display_mode": "Stack",
"children": [{
"id": "dragged-node",
@@ -907,17 +918,17 @@ mod tests {
"name": "拖拽节点",
"description": "",
"layout_status": "NoProblem",
"components_status": "NoProblem",
"component_status": "NoProblem",
"allow_llm_edit_layout": true,
"allow_llm_edit_component": true,
"source": "Human"
},
"components": [{
"component": {
"Image": {
"target_graphic": "spirit",
"image_type": { "Simple": { "preserve_aspect": false } }
}
}],
},
"children_display_mode": "Stack",
"children": []
}]
@@ -1095,12 +1106,12 @@ mod tests {
"name": "根节点",
"description": "",
"layout_status": "NoProblem",
"components_status": "NoProblem",
"component_status": "NoProblem",
"allow_llm_edit_layout": true,
"allow_llm_edit_component": true,
"source": "System"
},
"components": [],
"component": null,
"children_display_mode": "Stack",
"children": []
}
@@ -1296,12 +1307,12 @@ mod tests {
"name": "根节点",
"description": "",
"layout_status": "NoProblem",
"components_status": "NoProblem",
"component_status": "NoProblem",
"allow_llm_edit_layout": false,
"allow_llm_edit_component": false,
"source": "System"
},
"components": [{
"component": {
"Text": {
"content": "标题",
"font": {"Bound": "missing-font"},
@@ -1313,7 +1324,7 @@ mod tests {
"vertical_overflow": "Truncate",
"line_spacing": 1.0
}
}],
},
"children_display_mode": "Stack",
"children": []
}
@@ -1340,4 +1351,30 @@ mod tests {
.expect_err("missing Text font reference must be rejected");
assert!(error.contains("Text 组件引用了不存在的字体素材"));
}
#[test]
fn component_status_matrix_keeps_pure_nodes_unproblematic() {
use crate::ui_editor::component::image::ImageComponent;
assert!(validate_component_status(
None,
&crate::ui_editor::layout::node::StageStatus::NoProblem,
)
.is_ok());
assert!(validate_component_status(
None,
&crate::ui_editor::layout::node::StageStatus::NeedReview("原因".to_string()),
)
.is_err());
assert!(validate_component_status(
None,
&crate::ui_editor::layout::node::StageStatus::Blocked("原因".to_string()),
)
.is_err());
assert!(validate_component_status(
Some(&Component::Image(ImageComponent::new())),
&crate::ui_editor::layout::node::StageStatus::NeedReview("等待素材".to_string()),
)
.is_ok());
}
}

Some files were not shown because too many files have changed in this diff Show More