Compare commits

..

1 Commits

Author SHA1 Message Date
k88936 bd94db73c9 修复资源卡片 Lucide 图标描边宽度
Project CI / Repository checks (pull_request) Failing after 13s
Project CI / Backend tests (pull_request) Failing after 9s
Project CI / Frontend tests (pull_request) Successful in 3m14s
Project CI / Native shell tests (pull_request) Successful in 18m2s
恢复 CanvasWorld 超采样下资源卡片 SVG 图标的 inverse-scale 描边补偿

新增资源工作台 CSS 回归断言,防止局部图标规则被误删

同步 CanvasWorld 超采样常量与图标描边约定文档
2026-09-12 15:04:08 +08:00
106 changed files with 1217 additions and 5972 deletions
@@ -108,8 +108,6 @@ const rustSharedContractSource = fs.readFileSync(
);
const allowedUncalledTauriCommands = [
'append_direct_project_conversation_message',
// TODO: Remove the retired binding command after the legacy runtime path is removed.
'bind_components',
'chat_with_game_creator_agent',
'check_ui_editor_font_glyph_coverage',
'create_ui_design_resource',
@@ -1156,10 +1156,6 @@ mod tests {
assert!(with_canvas.contains("根据当前玩法需求编写规格和界面建议"));
assert!(with_canvas.contains("用途、数量、输出路径、尺寸、参考资源和是否需要 spritesheet"));
assert!(with_canvas.contains("再调用 canvas.asset_generate"));
assert!(with_canvas.contains("调用 canvas.asset_generate"));
assert!(with_canvas.contains("不要使用固定图片合同"));
assert!(with_canvas.contains("不修改 game/index.html"));
assert!(!with_canvas.contains("不调用 canvas.asset_generate"));
}
#[test]
@@ -801,7 +801,7 @@ fn agent_runtime_action_receipt_safe_detail_with_owner(
let initial_step = route.get("initialStep")?.as_str()?;
let render_mode = route.get("renderMode")?.as_str()?;
if resource_id.is_empty()
|| initial_step != "asset-separation"
|| initial_step != "visual-binding"
|| render_mode != "final-preview"
{
return None;
@@ -328,41 +328,6 @@ async fn recognize_ui(
ui_editor::commands::recognize_ui_impl(project_path, state).await
}
#[tauri::command]
async fn separate_ui(
project_path: String,
asset_id: String,
state: ui_editor::state::State,
) -> Result<ui_editor::commands::SeparationDTO, String> {
let root = Path::new(project_path.trim());
enforce_project_permission_policy(root, "asset.register")?;
ui_editor::commands::separate_ui_impl(project_path, asset_id, state).await
}
#[tauri::command]
fn inspect_separation_recovery(
project_path: String,
asset_id: String,
) -> Result<ui_editor::commands::SeparationRecoveryDTO, String> {
let root = Path::new(project_path.trim());
enforce_project_permission_policy(root, "asset.list")?;
ui_editor::commands::separation::inspect_separation_recovery(root, &asset_id)
}
#[tauri::command]
fn finalize_separation(project_path: String, asset_id: String) -> Result<(), String> {
let root = Path::new(project_path.trim());
enforce_project_permission_policy(root, "asset.register")?;
ui_editor::commands::separation::finalize_separation(root, &asset_id)
}
#[tauri::command]
fn discard_separation_recovery(project_path: String, asset_id: String) -> Result<(), String> {
let root = Path::new(project_path.trim());
enforce_project_permission_policy(root, "asset.register")?;
ui_editor::commands::separation::discard_separation_recovery(root, &asset_id)
}
#[tauri::command]
async fn merge_ui(state: ui_editor::state::State) -> Result<ui_editor::commands::MergeDTO, String> {
ui_editor::commands::merge_ui_impl(state).await
@@ -2664,10 +2629,6 @@ fn main() {
check_ui_editor_font_glyph_coverage,
suggest_ui_design_semantic,
recognize_ui,
separate_ui,
inspect_separation_recovery,
finalize_separation,
discard_separation_recovery,
merge_ui,
bind_components,
load_ui_design_state,
@@ -5,9 +5,11 @@ use crate::ui_editor::commands::utils::{
strict_json_schema,
};
use crate::ui_editor::component::text::FontSource;
use crate::ui_editor::component::{Component, NodeComponent};
use crate::ui_editor::component::Component;
use crate::ui_editor::layout::node::{Node, StageStatus};
use crate::ui_editor::persistence::UI_DESIGN_STATE_MAX_NODES;
use crate::ui_editor::persistence::{
UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE, UI_DESIGN_STATE_MAX_NODES,
};
use crate::ui_editor::state::State;
use crate::ui_editor::utils::{FontAssetId, NodeId, SpriteAssetId};
use platform_llm::{
@@ -29,10 +31,10 @@ const SYSTEM_PROMPT: &str = r#"
你是游戏 UI 组件绑定器。你会看到全部 UI 参考图、可编辑节点说明,以及本批独立素材的真实像素。
* 只对视觉上确实需要改变组件的节点返回 changes;
* 每个 change 的 component 是该节点完整的新组件;纯结构节点返回 "PureNode",有组件返回 {"WithComponent": <完整 Component>}
* 对 Component,直接完整返回其全部参数.
* 每个 change 的 components 是该节点完整的新渲染栈,空数组表示明确清空。数组顺序从底到顶渲染
* 对每个 Component,直接完整返回其全部参数.
* 有任何困难或者不确定把状态设为 NeedReview,说明中文原因。
* 纯结构节点可以返回 "PureNode" 并标为 NoProblem。
* 纯结构节点可以返回空数组并标为 NoProblem。
* 容器背景等推荐使用Simple + preserve_aspect: false 实现与node大小一致
* 面向用户的 reason 使用中文。
@@ -51,8 +53,8 @@ enum DraftStatus {
#[schemars(deny_unknown_fields)]
struct BindingChangeDraft {
node_id: NodeId,
component: NodeComponent,
component_status: DraftStatus,
components: Vec<Component>,
components_status: DraftStatus,
}
#[derive(Clone, Debug, Deserialize, JsonSchema)]
@@ -66,8 +68,8 @@ struct BindingResponse {
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct BindingChange {
pub node_id: NodeId,
pub component: NodeComponent,
pub component_status: StageStatus,
pub components: Vec<Component>,
pub components_status: StageStatus,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
@@ -81,7 +83,7 @@ struct EditableNodeContext<'a> {
node_id: &'a NodeId,
name: &'a str,
description: &'a str,
component: Option<&'a Component>,
components: &'a [Component],
}
#[derive(Debug, Serialize)]
@@ -104,7 +106,7 @@ fn collect_editable_nodes<'a>(node: &'a Node, output: &mut Vec<EditableNodeConte
node_id: &node.id,
name: &node.metadata.name,
description: &node.metadata.description,
component: node.component.as_ref(),
components: &node.components,
});
}
for child in &node.children {
@@ -170,21 +172,14 @@ fn validate_binding_response_shape(
return Err(format!("组件绑定 changes 不能超过 {max_changes}"));
}
for change in changes {
let Some(object) = change.as_object() else {
return Err("组件绑定 change 缺少 component 字段".to_string());
};
let Some(component) = object.get("component") else {
return Err("组件绑定 change 缺少 component 字段".to_string());
};
let valid_component = component == "PureNode"
|| component
.as_object()
.and_then(|value| value.get("WithComponent"))
.is_some_and(serde_json::Value::is_object);
if !valid_component {
return Err(
"组件绑定 change 的 component 必须是 PureNode 或 WithComponent 对象".to_string(),
);
let components = change
.get("components")
.and_then(serde_json::Value::as_array)
.ok_or_else(|| "组件绑定 change 缺少 components 数组".to_string())?;
if components.len() > UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE {
return Err(format!(
"单个组件绑定栈不能超过 {UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE} 个组件"
));
}
}
Ok(())
@@ -217,7 +212,7 @@ fn validate_and_materialize(
if !changed_ids.insert(change.node_id.clone()) {
return Err(format!("组件绑定重复返回节点:{}", change.node_id.as_str()));
}
if let NodeComponent::WithComponent(component) = &change.component {
for component in &change.components {
match component {
Component::Image(image) => {
if image
@@ -237,22 +232,18 @@ fn validate_and_materialize(
}
}
}
let component_status = match change.component_status {
let components = change.components;
let components_status = match change.components_status {
DraftStatus::NoProblem => StageStatus::NoProblem,
DraftStatus::NeedReview(reason) if reason.trim().is_empty() => {
return Err("组件待审状态必须包含原因".to_string())
}
DraftStatus::NeedReview(reason) => {
if matches!(&change.component, NodeComponent::PureNode) {
return Err("纯结构节点不能标记为组件待审".to_string());
}
StageStatus::NeedReview(reason)
}
DraftStatus::NeedReview(reason) => StageStatus::NeedReview(reason),
};
materialized.push(BindingChange {
node_id: change.node_id,
component: change.component,
component_status,
components,
components_status,
});
}
Ok(BindingDTO {
@@ -449,8 +440,8 @@ mod tests {
]);
let unapproved = BindingChangeDraft {
node_id: id("other"),
component: NodeComponent::PureNode,
component_status: DraftStatus::NoProblem,
components: Vec::new(),
components_status: DraftStatus::NoProblem,
};
assert!(
validate_and_materialize(vec![unapproved], &editable, &known, &HashSet::new()).is_err()
@@ -459,7 +450,7 @@ mod tests {
// References to sprites from another batch are allowed once they exist in the project.
let other_batch = BindingChangeDraft {
node_id: id("editable"),
component: NodeComponent::WithComponent(Component::Image(
components: vec![Component::Image(
crate::ui_editor::component::image::ImageComponent {
target_graphic: Some(
SpriteAssetId::new("other-batch-sprite").expect("valid sprite"),
@@ -468,8 +459,8 @@ mod tests {
preserve_aspect: false,
},
},
)),
component_status: DraftStatus::NoProblem,
)],
components_status: DraftStatus::NoProblem,
};
assert!(
validate_and_materialize(vec![other_batch], &editable, &known, &HashSet::new()).is_ok()
@@ -478,15 +469,15 @@ mod tests {
// References to sprites that do not exist in the project at all are still rejected.
let unknown = BindingChangeDraft {
node_id: id("editable"),
component: NodeComponent::WithComponent(Component::Image(
components: vec![Component::Image(
crate::ui_editor::component::image::ImageComponent {
target_graphic: Some(SpriteAssetId::new("unknown").expect("valid sprite")),
image_type: crate::ui_editor::component::image::ImageType::Simple {
preserve_aspect: false,
},
},
)),
component_status: DraftStatus::NoProblem,
)],
components_status: DraftStatus::NoProblem,
};
assert!(
validate_and_materialize(vec![unknown], &editable, &known, &HashSet::new()).is_err()
@@ -500,8 +491,8 @@ mod tests {
text.font = FontSource::Bound(FontAssetId::new("unknown-font").expect("valid font"));
let change = BindingChangeDraft {
node_id: id("editable"),
component: NodeComponent::WithComponent(Component::Text(text)),
component_status: DraftStatus::NoProblem,
components: vec![Component::Text(text)],
components_status: DraftStatus::NoProblem,
};
let error = validate_and_materialize(
@@ -515,13 +506,13 @@ mod tests {
}
#[test]
fn materialization_preserves_pure_node_change() {
fn materialization_preserves_changed_only_empty_component_lists() {
let editable = HashSet::from([id("editable")]);
let result = validate_and_materialize(
vec![BindingChangeDraft {
node_id: id("editable"),
component: NodeComponent::PureNode,
component_status: DraftStatus::NoProblem,
components: Vec::new(),
components_status: DraftStatus::NoProblem,
}],
&editable,
&HashSet::new(),
@@ -529,28 +520,8 @@ mod tests {
)
.expect("valid changed-only clear");
assert_eq!(result.changes.len(), 1);
assert!(matches!(
result.changes[0].component,
NodeComponent::PureNode
));
assert_eq!(result.changes[0].component_status, StageStatus::NoProblem);
}
#[test]
fn materialization_rejects_problematic_pure_node() {
let editable = HashSet::from([id("editable")]);
let error = validate_and_materialize(
vec![BindingChangeDraft {
node_id: id("editable"),
component: NodeComponent::PureNode,
component_status: DraftStatus::NeedReview("缺少可确认的组件".to_string()),
}],
&editable,
&HashSet::new(),
&HashSet::new(),
)
.expect_err("pure node cannot carry a component review status");
assert!(error.contains("纯结构节点"));
assert!(result.changes[0].components.is_empty());
assert_eq!(result.changes[0].components_status, StageStatus::NoProblem);
}
#[test]
@@ -604,26 +575,20 @@ mod tests {
}
#[test]
fn binding_response_bounds_changes_and_uses_single_component_shape() {
fn binding_response_bounds_changes_and_each_component_stack() {
let too_many_changes = serde_json::json!({
"changes": [{"component": "PureNode"}, {"component": "PureNode"}]
"changes": [{"components": []}, {"components": []}]
});
assert!(validate_binding_response_shape(&too_many_changes, 1).is_err());
let one_component = serde_json::json!({
let too_many_components = serde_json::json!({
"changes": [{
"node_id": "editable",
"component": "PureNode",
"component_status": "NoProblem"
"components": (0..=UI_DESIGN_STATE_MAX_COMPONENTS_PER_NODE)
.map(|_| serde_json::Value::Null)
.collect::<Vec<_>>()
}]
});
assert!(validate_binding_response_shape(&one_component, 1).is_ok());
let parsed = parse_binding_response(&one_component.to_string(), 1)
.expect("explicit PureNode payload should parse");
assert!(matches!(
parsed.changes[0].component,
NodeComponent::PureNode
));
assert!(validate_binding_response_shape(&too_many_components, 1).is_err());
}
#[tokio::test]
@@ -295,12 +295,12 @@ mod materialize {
name: container_name,
description: container_description,
layout_status: StageStatus::NoProblem,
component_status: StageStatus::NoProblem,
components_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::Llm,
},
component: None,
components: Vec::new(),
children_display_mode: ChildrenDisplayMode::Exclusive,
children: members.into_iter().map(|member| member.node).collect(),
},
@@ -525,6 +525,7 @@ mod tests {
materialize, validate_merge_input_state, validate_merge_plan_shape, MAX_MERGE_INPUT_DEPTH,
MAX_MERGE_INPUT_NODES, MAX_MERGE_PLAN_DEPTH, MAX_MERGE_PLAN_NODES,
};
use crate::ui_editor::component::Component;
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
use crate::ui_editor::layout::control_layout::ControlLayout;
use crate::ui_editor::layout::node::{Node, NodeMetadata, NodeSource, StageStatus};
@@ -541,12 +542,12 @@ mod tests {
name: id.to_string(),
description: String::new(),
layout_status: StageStatus::NoProblem,
component_status: StageStatus::NoProblem,
components_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::Human,
},
component: None,
components: Vec::<Component>::new(),
children_display_mode: ChildrenDisplayMode::Stack,
children,
}
@@ -585,7 +586,7 @@ mod tests {
ChildrenDisplayMode::Exclusive
);
assert_eq!(
result.root.metadata.component_status,
result.root.metadata.components_status,
StageStatus::NoProblem
);
assert_eq!(result.root.children.len(), 2);
@@ -1,7 +1,6 @@
pub mod binding;
pub mod merge;
pub mod recognition;
pub mod separation;
pub mod ui_design_suggestion;
pub mod utils;
@@ -11,7 +10,5 @@ pub use merge::MergeDTO;
pub(crate) use merge::{merge_ui_impl, merge_ui_impl_with_provider};
pub use recognition::RecognitionDTO;
pub(crate) use recognition::{recognize_ui_impl, recognize_ui_impl_with_provider};
pub(crate) use separation::separate_ui_impl;
pub use separation::{SeparationDTO, SeparationRecoveryDTO};
pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl;
pub use ui_design_suggestion::UIDesignSuggestionTreeNode;
@@ -4,7 +4,6 @@ use crate::ui_editor::commands::utils::{
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm,
strict_json_schema,
};
use crate::ui_editor::component::{Component, NodeComponent};
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
use crate::ui_editor::layout::control_layout::ControlLayout;
use crate::ui_editor::layout::dimension::UIRect;
@@ -28,13 +27,16 @@ const MAX_RECOGNITION_TREE_NODES: usize = 512;
const MAX_RECOGNITION_TREE_DEPTH: usize = 32;
const SYSTEM_PROMPT: &str = r#"
角色:
你是游戏 UI 多图结构识别器。
任务:
同时分析同一 UI 系统的全部参考图,建立UI树
用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构
识别规则:
* 只识别 UI元素. 要区分动态内容, 不要白费力气识别应该由程序生成/绘制的内容.(此类内容应该用一个整体节点+自然语言描述) 除此之外必须完整包含所有元素,结构.
* 只识别 UI,不识别场景人物、地形、建筑、光影和背景装饰。
* 无法确定类型、层级、关系时,在 UnSure 中写明原因。
* 返回的 trees 必须与输入图片一一对应,每张输入图片只能有一棵树,不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id。
* 每棵树必须使用自己的输入图片原始像素坐标系(0,0 as left top)输出
@@ -43,15 +45,8 @@ const SYSTEM_PROMPT: &str = r#"
* 面向用户的字段如名称描述等请用中文
* 由于每个截图未必是完整的, 可能是局部的, 每棵树描述清楚每个截图上UI的层次结构即可
* 不同树的共用框架/层次/...请使用使用相同的名称描述. 不同状态/变体名称使用相同的前缀, 用后缀区别
* 粒度要求: 尽可能细致, 以可交互,方便程序化控制的最小单位为准. 包括不限于: icon, 进度条的底槽、填充和外框; slider的底槽, dragger等.
* 为每个节点直接返回 component.
无背景的逻辑容器返回 "PureNode",不要返回 null.
有背景的容器推荐使用Simple+不锁定宽高比的Image component.
目前我们只做识别, 不要求图片字体的具体绑定参数.
文字组件要求: 艺术字等作为图片组件, 其余正常文字要作为单独的节点识别.
* 多行文本只使用一个节点.
* 不鼓励兄弟节点相互重叠.
* 对于面板等容器的背景等, 必须作为父节点的组件, 禁止新增冗余的所谓"背景节点".
* 粒度要求: 尽可能细致, 最小单元举例: 进度条的底槽、填充和外框; slider的底槽, dragger等
"#;
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
@@ -114,7 +109,6 @@ struct RecognitionNode {
description: String,
children: Vec<RecognitionNode>,
confidence: Confidence,
component: NodeComponent,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
@@ -313,12 +307,15 @@ fn convert_node(
name: source.name.clone(),
description: source.description.clone(),
layout_status: status,
component_status: StageStatus::NoProblem,
components_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::Llm,
},
component: source.component.clone().into_option(),
// V1 有意把识别结果限定为“结构草稿”:组件绑定属于后续独立阶段。
// 因此空组件不是丢失数据,而是等待 visual-binding 阶段补齐 Image/Text。
// 约定见 docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md。
components: Vec::new(),
children_display_mode: ChildrenDisplayMode::Stack,
children,
})
@@ -326,26 +323,6 @@ fn convert_node(
fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> {
for node in nodes {
if let NodeComponent::WithComponent(component) = &node.component {
if matches!(
component,
Component::Image(crate::ui_editor::component::image::ImageComponent {
target_graphic: Some(_),
..
})
) {
return Err("识别阶段不能返回已绑定的 SpriteAssetId".to_string());
}
if matches!(
component,
Component::Text(crate::ui_editor::component::text::TextComponent {
font: crate::ui_editor::component::text::FontSource::Bound(_),
..
})
) {
return Err("识别阶段不能返回已绑定的字体素材".to_string());
}
}
if let Confidence::UnSure(reason) = &node.confidence {
if reason.trim().is_empty() {
return Err("UnSure 必须包含审阅原因".to_string());
@@ -410,7 +387,6 @@ mod tests {
description: String::new(),
children: Vec::new(),
confidence: Confidence::Confident,
component: NodeComponent::PureNode,
}
}
@@ -514,31 +490,6 @@ mod tests {
description: String::new(),
children: Vec::new(),
confidence: Confidence::UnSure(String::new()),
component: NodeComponent::PureNode,
};
assert!(validate_confidence(&[node]).is_err());
}
#[test]
fn recognition_rejects_bound_font_references() {
let mut text = crate::ui_editor::component::text::TextComponent::default();
text.font = crate::ui_editor::component::text::FontSource::Bound(
crate::ui_editor::utils::FontAssetId::new("font").expect("valid font id"),
);
let node = RecognitionNode {
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
local_anchor: Anchor::Preset(PresetAnchor {
horizontal: HorizontalAnchor::Left,
vertical: VerticalAnchor::Top,
}),
name: "文本".to_string(),
description: String::new(),
children: Vec::new(),
confidence: Confidence::Confident,
component: NodeComponent::WithComponent(Component::Text(text)),
};
assert!(validate_confidence(&[node]).is_err());
}
@@ -555,7 +506,7 @@ mod tests {
converted.layout.transform.resolve(&root_rect),
UIRect::new(Point2::new(50.0, 25.0), Vector2::new(100.0, 50.0)),
);
assert_eq!(converted.metadata.component_status, StageStatus::NoProblem);
assert_eq!(converted.metadata.components_status, StageStatus::NoProblem);
}
#[test]
@@ -836,12 +787,12 @@ pub(crate) async fn recognize_ui_impl_with_provider(
name: "页面根节点".to_string(),
description: String::new(),
layout_status: StageStatus::NoProblem,
component_status: StageStatus::NoProblem,
components_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::System,
},
component: None,
components: Vec::new(),
children_display_mode: ChildrenDisplayMode::Stack,
children,
};
@@ -1,480 +0,0 @@
use super::model::BindingArea;
use image::RgbaImage;
use std::time::Instant;
/// Each edge may move by at most this many pixels from the area returned by
/// the visual model. Keep this policy explicit so changing it is an
/// intentional workflow decision rather than a scattered numeric literal.
pub(crate) const MAX_BINDING_AREA_EDGE_ADJUSTMENT_PX: u32 = 32;
/// Alpha values below this threshold are treated as transparent for boundary
/// detection. The cropped pixels themselves are preserved unchanged.
pub(crate) const MIN_VISIBLE_ALPHA: u8 = 16;
/// An edge needs this many consecutive visible pixels to count as supported.
/// The requirement is reduced to the edge length for one-pixel-wide elements.
pub(crate) const MIN_CONSECUTIVE_VISIBLE_EDGE_PIXELS: usize = 2;
#[derive(Clone, Copy, Debug, PartialEq)]
pub(crate) struct NormalizedBindingArea {
pub(crate) area: BindingArea,
pub(crate) changed: bool,
pub(crate) clamped: bool,
pub(crate) transparent: bool,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum EdgeDirection {
Inward,
Outward,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct Rect {
left: u32,
top: u32,
right: u32,
bottom: u32,
}
impl Rect {
fn from_area(area: BindingArea) -> Self {
Self {
left: area.global_pos_x_px,
top: area.global_pos_y_px,
right: area.global_pos_x_px + area.width_px,
bottom: area.global_pos_y_px + area.height_px,
}
}
fn into_area(self) -> BindingArea {
BindingArea {
global_pos_x_px: self.left,
global_pos_y_px: self.top,
width_px: self.right - self.left,
height_px: self.bottom - self.top,
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum Edge {
Left,
Right,
Top,
Bottom,
}
impl Edge {
const ALL: [Self; 4] = [Self::Left, Self::Right, Self::Top, Self::Bottom];
}
fn pixel_is_visible(alpha: u8) -> bool {
alpha >= MIN_VISIBLE_ALPHA
}
fn has_consecutive_visible_pixels<I>(alphas: I, required: usize) -> bool
where
I: IntoIterator<Item = u8>,
{
let required = required.max(1);
let mut consecutive = 0usize;
for alpha in alphas {
if pixel_is_visible(alpha) {
consecutive = consecutive.saturating_add(1);
if consecutive >= required {
return true;
}
} else {
consecutive = 0;
}
}
false
}
fn edge_has_visible_pixel(image: &RgbaImage, rect: Rect, edge: Edge) -> bool {
let edge_length = match edge {
Edge::Left | Edge::Right => rect.bottom - rect.top,
Edge::Top | Edge::Bottom => rect.right - rect.left,
} as usize;
let required = MIN_CONSECUTIVE_VISIBLE_EDGE_PIXELS.max(1).min(edge_length);
match edge {
Edge::Left | Edge::Right => {
let x = if edge == Edge::Left {
rect.left
} else {
rect.right - 1
};
has_consecutive_visible_pixels(
(rect.top..rect.bottom).map(|y| image.get_pixel(x, y).0[3]),
required,
)
}
Edge::Top | Edge::Bottom => {
let y = if edge == Edge::Top {
rect.top
} else {
rect.bottom - 1
};
has_consecutive_visible_pixels(
(rect.left..rect.right).map(|x| image.get_pixel(x, y).0[3]),
required,
)
}
}
}
fn rect_has_visible_pixel(image: &RgbaImage, rect: Rect) -> bool {
(rect.top..rect.bottom)
.any(|y| (rect.left..rect.right).any(|x| pixel_is_visible(image.get_pixel(x, y).0[3])))
}
fn edge_direction(image: &RgbaImage, rect: Rect, edge: Edge) -> EdgeDirection {
if edge_has_visible_pixel(image, rect, edge) {
EdgeDirection::Outward
} else {
EdgeDirection::Inward
}
}
fn move_edge(rect: &mut Rect, edge: Edge, direction: EdgeDirection) {
match (edge, direction) {
(Edge::Left, EdgeDirection::Inward) => rect.left += 1,
(Edge::Left, EdgeDirection::Outward) => rect.left -= 1,
(Edge::Right, EdgeDirection::Inward) => rect.right -= 1,
(Edge::Right, EdgeDirection::Outward) => rect.right += 1,
(Edge::Top, EdgeDirection::Inward) => rect.top += 1,
(Edge::Top, EdgeDirection::Outward) => rect.top -= 1,
(Edge::Bottom, EdgeDirection::Inward) => rect.bottom -= 1,
(Edge::Bottom, EdgeDirection::Outward) => rect.bottom += 1,
}
}
fn edge_coordinate(rect: Rect, edge: Edge) -> u32 {
match edge {
Edge::Left => rect.left,
Edge::Right => rect.right,
Edge::Top => rect.top,
Edge::Bottom => rect.bottom,
}
}
fn edge_displacement(original: Rect, current: Rect, edge: Edge) -> u32 {
edge_coordinate(original, edge).abs_diff(edge_coordinate(current, edge))
}
fn edge_adjustment_limit() -> u32 {
MAX_BINDING_AREA_EDGE_ADJUSTMENT_PX
}
fn reached_adjustment_limit(original: Rect, current: Rect, edge: Edge) -> bool {
edge_displacement(original, current, edge) >= edge_adjustment_limit()
}
fn can_move_geometrically(
image: &RgbaImage,
current: Rect,
edge: Edge,
direction: EdgeDirection,
) -> bool {
match (edge, direction) {
(Edge::Left, EdgeDirection::Inward) => current.left + 1 < current.right,
(Edge::Left, EdgeDirection::Outward) => current.left > 0,
(Edge::Right, EdgeDirection::Inward) => current.right > current.left + 1,
(Edge::Right, EdgeDirection::Outward) => current.right < image.width(),
(Edge::Top, EdgeDirection::Inward) => current.top + 1 < current.bottom,
(Edge::Top, EdgeDirection::Outward) => current.top > 0,
(Edge::Bottom, EdgeDirection::Inward) => current.bottom > current.top + 1,
(Edge::Bottom, EdgeDirection::Outward) => current.bottom < image.height(),
}
}
fn next_edge_rect(rect: Rect, edge: Edge, direction: EdgeDirection) -> Option<Rect> {
let mut next = rect;
match (edge, direction) {
(Edge::Left, EdgeDirection::Inward) if rect.left + 1 < rect.right => next.left += 1,
(Edge::Left, EdgeDirection::Outward) if rect.left > 0 => next.left -= 1,
(Edge::Right, EdgeDirection::Inward) if rect.right > rect.left + 1 => next.right -= 1,
(Edge::Right, EdgeDirection::Outward) => next.right = next.right.checked_add(1)?,
(Edge::Top, EdgeDirection::Inward) if rect.top + 1 < rect.bottom => next.top += 1,
(Edge::Top, EdgeDirection::Outward) if rect.top > 0 => next.top -= 1,
(Edge::Bottom, EdgeDirection::Inward) if rect.bottom > rect.top + 1 => next.bottom -= 1,
(Edge::Bottom, EdgeDirection::Outward) => next.bottom = next.bottom.checked_add(1)?,
_ => return None,
}
Some(next)
}
fn edge_requires_move(image: &RgbaImage, rect: Rect, edge: Edge, direction: EdgeDirection) -> bool {
match direction {
EdgeDirection::Inward => !edge_has_visible_pixel(image, rect, edge),
EdgeDirection::Outward => {
if !edge_has_visible_pixel(image, rect, edge) {
return false;
}
if !can_move_geometrically(image, rect, edge, direction) {
return true;
}
next_edge_rect(rect, edge, direction)
.is_some_and(|next| edge_has_visible_pixel(image, next, edge))
}
}
}
fn apply_edge_step(
image: &RgbaImage,
original: Rect,
current: Rect,
edge: Edge,
direction: EdgeDirection,
) -> (Rect, bool, bool) {
if !edge_requires_move(image, current, edge, direction) {
return (current, false, false);
}
if reached_adjustment_limit(original, current, edge)
|| !can_move_geometrically(image, current, edge, direction)
{
return (current, false, true);
}
let mut next = current;
move_edge(&mut next, edge, direction);
(next, true, false)
}
/// Normalizes a model-provided area using visible pixels on the processed
/// transparent image. Each edge chooses inward/outward direction once from
/// its initial scan and then moves monotonically, so sparse pixels cannot make
/// the boundary oscillate. The four edge steps are calculated from the same
/// rectangle on each round.
pub(crate) fn normalize_binding_area(
image: &RgbaImage,
original_area: BindingArea,
) -> Result<NormalizedBindingArea, String> {
let started = Instant::now();
if let Err(error) = original_area.validate_in(image.width(), image.height()) {
app_log!(
"ui_separation.area.timing outcome=error elapsed_us={} rounds=0 image_width={} image_height={} area=({}, {}, {}, {})",
started.elapsed().as_micros(),
image.width(),
image.height(),
original_area.global_pos_x_px,
original_area.global_pos_y_px,
original_area.width_px,
original_area.height_px
);
return Err(error.to_string());
}
let original = Rect::from_area(original_area);
let directions = Edge::ALL.map(|edge| edge_direction(image, original, edge));
let mut current = original;
let mut clamped = false;
let mut active = [true; 4];
let mut rounds = 0u32;
// TODO: Replace the deliberately simple pixel-by-pixel scan if real UI
// design sizes show this path to be a measurable bottleneck.
while active.iter().any(|value| *value) {
rounds = rounds.saturating_add(1);
let before = current;
let mut next = current;
let mut moved = [false; 4];
for (index, edge) in Edge::ALL.into_iter().enumerate() {
if !active[index] {
continue;
}
let (candidate, did_move, reached_limit) =
apply_edge_step(image, original, current, edge, directions[index]);
if reached_limit {
clamped = true;
active[index] = false;
} else if !did_move {
active[index] = false;
}
moved[index] = did_move;
match edge {
Edge::Left => next.left = candidate.left,
Edge::Right => next.right = candidate.right,
Edge::Top => next.top = candidate.top,
Edge::Bottom => next.bottom = candidate.bottom,
}
}
if next.left >= next.right {
clamped = true;
if moved[0] {
active[0] = false;
}
if moved[1] {
active[1] = false;
}
next.left = current.left;
next.right = current.right;
}
if next.top >= next.bottom {
clamped = true;
if moved[2] {
active[2] = false;
}
if moved[3] {
active[3] = false;
}
next.top = current.top;
next.bottom = current.bottom;
}
current = next;
if current == before {
break;
}
}
let area = current.into_area();
let normalized = NormalizedBindingArea {
changed: area != original_area,
area,
clamped,
transparent: !rect_has_visible_pixel(image, current),
};
app_log!(
"ui_separation.area.timing outcome=ok elapsed_us={} rounds={} image_width={} image_height={} area=({}, {}, {}, {}) changed={} clamped={} transparent={}",
started.elapsed().as_micros(),
rounds,
image.width(),
image.height(),
original_area.global_pos_x_px,
original_area.global_pos_y_px,
original_area.width_px,
original_area.height_px,
normalized.changed,
normalized.clamped,
normalized.transparent
);
Ok(normalized)
}
#[cfg(test)]
mod tests {
use super::*;
use image::{Rgba, RgbaImage};
fn image_with_rect(
width: u32,
height: u32,
left: u32,
top: u32,
right: u32,
bottom: u32,
) -> RgbaImage {
let mut image = RgbaImage::from_pixel(width, height, Rgba([0, 0, 0, 0]));
for y in top..bottom {
for x in left..right {
image.put_pixel(x, y, Rgba([255, 255, 255, 255]));
}
}
image
}
fn area(x: u32, y: u32, width: u32, height: u32) -> BindingArea {
BindingArea {
global_pos_x_px: x,
global_pos_y_px: y,
width_px: width,
height_px: height,
}
}
#[test]
fn shrinks_empty_edges_to_visible_bounds() {
let image = image_with_rect(32, 32, 10, 11, 16, 18);
let result = normalize_binding_area(&image, area(6, 7, 14, 16)).unwrap();
assert_eq!(result.area, area(10, 11, 6, 7));
assert!(result.changed);
assert!(!result.clamped);
assert!(!result.transparent);
}
#[test]
fn expands_visible_edges_to_cover_the_element() {
let image = image_with_rect(32, 32, 10, 11, 16, 18);
let result = normalize_binding_area(&image, area(11, 12, 4, 5)).unwrap();
assert_eq!(result.area, area(10, 11, 6, 7));
assert!(result.changed);
assert!(!result.clamped);
assert!(!result.transparent);
}
#[test]
fn adjusts_each_edge_independently() {
let image = image_with_rect(32, 32, 10, 11, 16, 18);
let result = normalize_binding_area(&image, area(10, 12, 10, 3)).unwrap();
assert_eq!(result.area, area(10, 11, 6, 7));
}
#[test]
fn ignores_low_alpha_halo_while_preserving_visible_bounds() {
let mut image = RgbaImage::from_pixel(16, 16, Rgba([0, 0, 0, 0]));
for y in 6..10 {
for x in 5..9 {
image.put_pixel(x, y, Rgba([255, 255, 255, 255]));
}
}
image.put_pixel(4, 7, Rgba([255, 255, 255, 1]));
image.put_pixel(9, 8, Rgba([255, 255, 255, 8]));
let result = normalize_binding_area(&image, area(4, 5, 6, 6)).unwrap();
assert_eq!(result.area, area(5, 6, 4, 4));
}
#[test]
fn ignores_isolated_visible_edge_pixel() {
let mut image = image_with_rect(16, 16, 4, 4, 6, 8);
image.put_pixel(6, 4, Rgba([255, 255, 255, 255]));
let result = normalize_binding_area(&image, area(4, 4, 2, 4)).unwrap();
assert_eq!(result.area, area(4, 4, 2, 4));
}
#[test]
fn fully_transparent_image_uses_the_same_path() {
let image = RgbaImage::from_pixel(32, 32, Rgba([0, 0, 0, 0]));
let result = normalize_binding_area(&image, area(10, 10, 10, 10)).unwrap();
assert_eq!(result.area, area(14, 14, 2, 2));
assert!(result.changed);
assert!(result.transparent);
}
#[test]
fn caps_each_edge_at_absolute_pixel_limit() {
let image = image_with_rect(128, 128, 0, 0, 128, 128);
let result = normalize_binding_area(&image, area(48, 48, 8, 8)).unwrap();
assert_eq!(result.area, area(16, 16, 72, 72));
assert!(result.clamped);
}
#[test]
fn clamps_expansion_to_image_edges() {
let image = image_with_rect(16, 16, 0, 0, 4, 4);
let result = normalize_binding_area(&image, area(1, 1, 2, 2)).unwrap();
assert_eq!(result.area, area(0, 0, 4, 4));
assert!(result.clamped);
}
#[test]
fn exact_split_at_adjustment_limit_is_not_clamped() {
let image = image_with_rect(16, 16, 4, 4, 8, 8);
let result = normalize_binding_area(&image, area(5, 5, 2, 2)).unwrap();
assert_eq!(result.area, area(4, 4, 4, 4));
assert!(!result.clamped);
}
#[test]
fn one_pixel_area_expands_with_configured_adjustment_limit() {
let image = image_with_rect(8, 8, 2, 2, 5, 5);
let result = normalize_binding_area(&image, area(3, 3, 1, 1)).unwrap();
assert_eq!(result.area, area(2, 2, 3, 3));
assert!(!result.clamped);
}
#[test]
fn rejects_zero_sized_or_out_of_bounds_model_areas() {
let image = RgbaImage::from_pixel(16, 16, Rgba([0, 0, 0, 0]));
assert!(normalize_binding_area(&image, area(0, 0, 0, 1)).is_err());
assert!(normalize_binding_area(&image, area(15, 15, 2, 2)).is_err());
}
}
@@ -1,106 +0,0 @@
use super::area::MIN_VISIBLE_ALPHA;
use base64::Engine as _;
use image::{ImageFormat, Rgba, RgbaImage};
use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
pub(crate) const VISUAL_BINDING_TRANSPARENT_MARKER_RGBA: [u8; 4] = [255, 0, 255, 255];
pub(crate) async fn preprocess_for_visual_binding(
processed_url: String,
sidecar: PathBuf,
) -> Result<String, String> {
tokio::task::spawn_blocking(move || {
preprocess_for_visual_binding_blocking(&processed_url, &sidecar)
})
.await
.map_err(|error| format!("视觉绑定预处理任务失败:{error}"))?
}
fn preprocess_for_visual_binding_blocking(
processed_url: &str,
sidecar: &Path,
) -> Result<String, String> {
let encoded = processed_url
.split_once(',')
.map(|(_, data)| data)
.ok_or_else(|| "处理图 data URL 无效".to_string())?;
let bytes = base64::engine::general_purpose::STANDARD
.decode(encoded.trim())
.map_err(|error| format!("解析处理图失败:{error}"))?;
let mut image = image::load_from_memory(&bytes)
.map_err(|error| format!("解码处理图失败:{error}"))?
.to_rgba8();
for pixel in image.pixels_mut() {
if pixel.0[3] < MIN_VISIBLE_ALPHA {
*pixel = Rgba(VISUAL_BINDING_TRANSPARENT_MARKER_RGBA);
} else {
pixel.0[3] = 255;
}
}
let mut png = Vec::new();
image::DynamicImage::ImageRgba8(image)
.write_to(&mut Cursor::new(&mut png), ImageFormat::Png)
.map_err(|error| format!("编码视觉绑定预览失败:{error}"))?;
let debug_name = format!("binding-{}.png", uuid::Uuid::new_v4().simple());
if let Err(error) = fs::write(sidecar.join(&debug_name), &png) {
app_log!(
"ui_separation.warning stage=visual_binding_preview_write file={} error={error}",
debug_name
);
}
Ok(format!(
"data:image/png;base64,{}",
base64::engine::general_purpose::STANDARD.encode(png)
))
}
#[cfg(test)]
mod tests {
use super::*;
use image::Rgba;
use tempfile::tempdir;
fn data_url(image: RgbaImage) -> String {
let mut bytes = Vec::new();
image::DynamicImage::ImageRgba8(image)
.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)
.expect("encode fixture");
format!(
"data:image/png;base64,{}",
base64::engine::general_purpose::STANDARD.encode(bytes)
)
}
#[test]
fn preprocesses_alpha_using_existing_visibility_threshold() {
let mut image = RgbaImage::from_pixel(4, 1, Rgba([10, 20, 30, 255]));
image.put_pixel(0, 0, Rgba([1, 2, 3, 0]));
image.put_pixel(1, 0, Rgba([4, 5, 6, MIN_VISIBLE_ALPHA - 1]));
image.put_pixel(2, 0, Rgba([7, 8, 9, MIN_VISIBLE_ALPHA]));
image.put_pixel(3, 0, Rgba([11, 12, 13, 254]));
let directory = tempdir().expect("create sidecar fixture");
let url = preprocess_for_visual_binding_blocking(&data_url(image), directory.path())
.expect("preprocess fixture");
let encoded = url.split_once(',').expect("data URL").1;
let bytes = base64::engine::general_purpose::STANDARD
.decode(encoded)
.expect("decode output");
let output = image::load_from_memory(&bytes)
.expect("decode output png")
.to_rgba8();
assert_eq!(
output.get_pixel(0, 0).0,
VISUAL_BINDING_TRANSPARENT_MARKER_RGBA
);
assert_eq!(
output.get_pixel(1, 0).0,
VISUAL_BINDING_TRANSPARENT_MARKER_RGBA
);
assert_eq!(output.get_pixel(2, 0).0, [7, 8, 9, 255]);
assert_eq!(output.get_pixel(3, 0).0, [11, 12, 13, 255]);
}
}
File diff suppressed because it is too large Load Diff
@@ -1,97 +0,0 @@
use crate::ui_editor::utils::NodeId;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum BindingAreaValidationError {
ZeroDimension,
OutOfBounds,
}
impl std::fmt::Display for BindingAreaValidationError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::ZeroDimension => "BindingArea 宽度和高度必须大于 0",
Self::OutOfBounds => "BindingArea 超出处理图边界",
})
}
}
impl std::error::Error for BindingAreaValidationError {}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub struct BindingArea {
pub global_pos_x_px: u32,
pub global_pos_y_px: u32,
pub width_px: u32,
pub height_px: u32,
}
impl BindingArea {
pub fn validate_in(&self, w: u32, h: u32) -> Result<(), BindingAreaValidationError> {
if self.width_px == 0 || self.height_px == 0 {
return Err(BindingAreaValidationError::ZeroDimension);
}
if self
.global_pos_x_px
.checked_add(self.width_px)
.is_none_or(|v| v > w)
|| self
.global_pos_y_px
.checked_add(self.height_px)
.is_none_or(|v| v > h)
{
return Err(BindingAreaValidationError::OutOfBounds);
}
Ok(())
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub enum BindingDecision {
Ok {
extracted_area: BindingArea,
to_node: NodeId,
},
NeedRework {
advice: String,
to_node: NodeId,
},
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
pub struct BindingResp {
pub decisions: Vec<BindingDecision>,
}
#[cfg(test)]
mod tests {
use super::{BindingArea, BindingAreaValidationError};
#[test]
fn validates_binding_area_with_typed_errors() {
let zero = BindingArea {
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 0,
height_px: 1,
};
assert_eq!(
zero.validate_in(10, 10),
Err(BindingAreaValidationError::ZeroDimension)
);
let outside = BindingArea {
global_pos_x_px: 10,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
};
assert_eq!(
outside.validate_in(10, 10),
Err(BindingAreaValidationError::OutOfBounds)
);
}
}
@@ -1,20 +0,0 @@
mod binding;
mod node;
mod note;
mod result;
pub use binding::*;
pub use node::*;
pub use note::*;
pub use result::*;
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v2";
pub const MAX_REWORK_COUNT: u32 = 3;
pub const MAX_REWORK_NOTE_CHARS: usize = 512;
pub const IMAGE_EDIT_MAX_DIMENSION_PX: u64 = 2880;
pub const IMAGE_EDIT_MIN_DIMENSION_PX: u64 = 816;
pub const IMAGE_EDIT_DIMENSION_ALIGNMENT_PX: u64 = 16;
pub const IMAGE_EDIT_AREA_UTILIZATION_PERCENT: u64 = 80;
pub const IMAGE_EDIT_AREA_LIMIT_PX: u64 =
IMAGE_EDIT_MAX_DIMENSION_PX * IMAGE_EDIT_MAX_DIMENSION_PX * IMAGE_EDIT_AREA_UTILIZATION_PERCENT
/ 100;
@@ -1,43 +0,0 @@
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub enum SeparationNodeKind {
ImageTarget,
TextRemovalOnly,
PureContainer,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationNode {
pub id: NodeId,
pub kind: SeparationNodeKind,
pub global_pos_x_px: u32,
pub global_pos_y_px: u32,
pub width_px: u32,
pub height_px: u32,
pub note: super::SeparationNote,
pub children: Vec<SeparationNode>,
pub rework_count: u32,
}
impl SeparationNode {
pub fn as_prompt(&self) -> String {
format!(
"node_id={} note: {}",
super::sanitize_prompt_text(self.id.as_str()),
self.note.as_prompt()
)
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationTree {
pub src_ui_design: UIDesignImageId,
pub root: SeparationNode,
pub root_extractable: bool,
}
@@ -1,39 +0,0 @@
use serde::{Deserialize, Serialize};
use ts_rs::TS;
pub(crate) fn sanitize_prompt_text(value: &str) -> String {
value
.chars()
.filter_map(|character| {
if character == '`' {
Some('\'')
} else if character.is_control() {
Some(' ')
} else {
Some(character)
}
})
.take(super::MAX_REWORK_NOTE_CHARS)
.collect()
}
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationNote {
pub description: String,
pub rework_notes: Vec<String>,
}
impl SeparationNote {
pub fn as_prompt(&self) -> String {
let mut prompt = format!("desc: {}", sanitize_prompt_text(&self.description));
if !self.rework_notes.is_empty() {
prompt.push_str("\nprevious rework notes:");
for note in &self.rework_notes {
prompt.push_str("\n- ");
prompt.push_str(&sanitize_prompt_text(note));
}
}
prompt
}
}
@@ -1,43 +0,0 @@
use crate::ui_editor::utils::NodeId;
use serde::{Deserialize, Serialize};
use ts_rs::TS;
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct BoundNode {
pub node_id: NodeId,
pub cut_image_path: String,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct ProblematicNode {
pub node_id: NodeId,
pub problem_description: String,
pub rework_count: u32,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationState {
pub schema_version: String,
pub trees: Vec<super::SeparationTree>,
pub bound: Vec<BoundNode>,
pub problematic_nodes: Vec<ProblematicNode>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationDTO {
pub bound_nodes: Vec<BoundNode>,
pub problematic_nodes: Vec<ProblematicNode>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationRecoveryDTO {
pub exists: bool,
pub bound_node_count: usize,
pub problematic_node_count: usize,
pub has_pending_tree: bool,
}
@@ -1,214 +0,0 @@
use super::model::*;
use crate::ui_editor::commands::separation::*;
use std::fs;
use std::path::{Path, PathBuf};
const SEPARATION_STATE_MAX_BYTES: usize = 8 * 1024 * 1024;
pub fn separation_sidecar_dir(root: &Path, asset_id: &str) -> Result<PathBuf, String> {
if asset_id.trim().is_empty() || asset_id.trim() != asset_id {
app_log!("ui_separation.error stage=sidecar_dir reason=invalid_asset_id");
return Err("UI 资源 ID 无效".to_string());
}
let dir = root.join("ui").join(format!(
".{}-separation",
crate::ui_editor::persistence::generated_file_stem(asset_id)
));
if !dir.starts_with(root) {
app_log!("ui_separation.error stage=sidecar_dir reason=path_escape");
return Err("separation sidecar 路径越界".to_string());
}
app_log!(
"ui_separation.sidecar_resolved asset_id={} directory={}",
asset_id,
dir.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>")
);
Ok(dir)
}
pub fn separation_state_path(root: &Path, asset_id: &str) -> Result<PathBuf, String> {
Ok(separation_sidecar_dir(root, asset_id)?.join("state.json"))
}
pub fn project_relative_path(root: &Path, path: &Path) -> Result<String, String> {
let relative = path
.strip_prefix(root)
.map_err(|_| "separation 产物必须位于项目目录内".to_string())?;
let value = relative.to_string_lossy().replace('\\', "/");
if value.is_empty() || value.starts_with('/') || value.split('/').any(|part| part == "..") {
return Err("separation 产物相对路径无效".to_string());
}
Ok(value)
}
pub async fn write_separation_state(path: PathBuf, state: &SeparationState) -> Result<(), String> {
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
app_log!("ui_separation.error stage=state_write reason=schema_mismatch");
return Err("不支持的 separation state schema".to_string());
}
let state = serde_json::to_vec_pretty(state)
.map_err(|error| format!("序列化 separation state 失败:{error}"))?;
tokio::task::spawn_blocking(move || write_separation_state_blocking(&path, &state))
.await
.map_err(|error| format!("写入 separation state 任务失败:{error}"))?
}
fn write_separation_state_blocking(path: &Path, bytes: &[u8]) -> Result<(), String> {
if bytes.len() > SEPARATION_STATE_MAX_BYTES {
app_log!(
"ui_separation.error stage=state_write reason=too_large bytes={} max_bytes={}",
bytes.len(),
SEPARATION_STATE_MAX_BYTES
);
return Err(format!(
"separation state 超过 {} 字节上限",
SEPARATION_STATE_MAX_BYTES
));
}
app_log!(
"ui_separation.state_write.start file={} bytes={}",
path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>"),
bytes.len()
);
let parent = path.parent().ok_or_else(|| {
app_log!("ui_separation.error stage=state_write reason=missing_parent");
"separation state 路径缺少父目录".to_string()
})?;
fs::create_dir_all(parent).map_err(|error| {
app_log!("ui_separation.error stage=state_write reason=create_parent error={error}");
format!("创建 separation sidecar 失败:{error}")
})?;
let temporary = path.with_extension("json.tmp");
fs::write(&temporary, bytes).map_err(|error| {
app_log!("ui_separation.error stage=state_write reason=write_temp error={error}");
format!("写入 separation state 失败:{error}")
})?;
fs::rename(&temporary, path).map_err(|error| {
app_log!("ui_separation.error stage=state_write reason=install error={error}");
format!("安装 separation state 失败:{error}")
})?;
app_log!(
"ui_separation.state_write.completed file={} bytes={}",
path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>"),
fs::metadata(path)
.map(|metadata| metadata.len())
.unwrap_or(0)
);
Ok(())
}
pub fn read_separation_state(path: &Path) -> Result<SeparationState, String> {
app_log!(
"ui_separation.state_read.start file={}",
path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>")
);
let metadata = fs::metadata(path).map_err(|error| {
app_log!("ui_separation.error stage=state_read reason=metadata error={error}");
format!("读取 separation state 信息失败:{error}")
})?;
if metadata.len() > SEPARATION_STATE_MAX_BYTES as u64 {
app_log!(
"ui_separation.error stage=state_read reason=too_large bytes={} max_bytes={}",
metadata.len(),
SEPARATION_STATE_MAX_BYTES
);
return Err(format!(
"separation state 超过 {} 字节上限",
SEPARATION_STATE_MAX_BYTES
));
}
let bytes = fs::read(path).map_err(|error| {
app_log!("ui_separation.error stage=state_read reason=read error={error}");
format!("读取 separation state 失败:{error}")
})?;
if bytes.len() > SEPARATION_STATE_MAX_BYTES {
return Err(format!(
"separation state 超过 {} 字节上限",
SEPARATION_STATE_MAX_BYTES
));
}
let state: SeparationState = serde_json::from_slice(&bytes).map_err(|error| {
app_log!("ui_separation.error stage=state_read reason=parse error={error}");
format!("解析 separation state 失败:{error}")
})?;
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
app_log!("ui_separation.error stage=state_read reason=schema_mismatch");
return Err("不支持的 separation state schema".to_string());
}
app_log!(
"ui_separation.state_read.completed bytes={} trees={} bound={} problematic={}",
bytes.len(),
state.trees.len(),
state.bound.len(),
state.problematic_nodes.len()
);
Ok(state)
}
pub async fn read_separation_state_async(path: PathBuf) -> Result<SeparationState, String> {
tokio::task::spawn_blocking(move || read_separation_state(&path))
.await
.map_err(|error| format!("读取 separation state 任务失败:{error}"))?
}
pub fn separation_dto(state: &SeparationState) -> SeparationDTO {
app_log!(
"ui_separation.dto bound_nodes={} problematic_nodes={} remaining_trees={}",
state.bound.len(),
state.problematic_nodes.len(),
state.trees.len()
);
SeparationDTO {
bound_nodes: state.bound.clone(),
problematic_nodes: state.problematic_nodes.clone(),
}
}
pub fn inspect_separation_recovery(
root: &Path,
asset_id: &str,
) -> Result<SeparationRecoveryDTO, String> {
let state_path = separation_state_path(root, asset_id)?;
if !state_path.exists() {
return Ok(SeparationRecoveryDTO {
exists: false,
bound_node_count: 0,
problematic_node_count: 0,
has_pending_tree: false,
});
}
let state = read_separation_state(&state_path)?;
Ok(SeparationRecoveryDTO {
exists: true,
bound_node_count: state.bound.len(),
problematic_node_count: state.problematic_nodes.len(),
has_pending_tree: state
.trees
.iter()
.any(|tree| !tree.root.children.is_empty() || tree.root_extractable),
})
}
pub fn finalize_separation(root: &Path, asset_id: &str) -> Result<(), String> {
remove_separation_state(root, asset_id)
}
pub fn discard_separation_recovery(root: &Path, asset_id: &str) -> Result<(), String> {
remove_separation_state(root, asset_id)
}
fn remove_separation_state(root: &Path, asset_id: &str) -> Result<(), String> {
let state_path = separation_state_path(root, asset_id)?;
match fs::remove_file(&state_path) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(format!("删除 separation state 失败:{error}")),
}
}
@@ -1,46 +0,0 @@
use crate::ui_editor::commands::separation::image_preprocess::VISUAL_BINDING_TRANSPARENT_MARKER_RGBA;
use crate::ui_editor::commands::separation::SeparationNode;
pub(crate) fn gen_binding_prompt(nodes: &[&SeparationNode]) -> String {
let [marker_red, marker_green, marker_blue, marker_alpha] =
VISUAL_BINDING_TRANSPARENT_MARKER_RGBA;
let marker_color = format!("rgba({marker_red}, {marker_green}, {marker_blue}, {marker_alpha})");
let binding_system_prompt = format!(
r#"
You will be given a src UI design image and a processed image, where some ui elements are separated.
You need to recognize and review the separation using the given tool.
field notes:
* extracted_area MUST be the recognized area from the processed image, INSTEAD OF from the src image.
The processed image is the only authoritative image for extracted_area.
Return the pixel bounding box of the extracted element as it appears in the processed image.
Do not copy, infer, or reuse the source node rectangle.
The src image is only for identifying which semantic UI element belongs to to_node.
Here are the separation requirements:
Preserve hard edges and the exact visible shape.
The processed image is an opaque visual-binding preview containing the requested image layers.
The solid color {marker_color} is an intentional transparency marker added by this workflow before this request.
It is not an image-edit defect and is not part of any UI element.
Do not include this marker color in the extracted area.
Do not use the source node rectangle as the extracted area.
And you should also review if the extracted's successfully meet the src image:
* shape
* color
* style
* edge process
...
if not, use the `NeedRework` data structure in the tool to indicate the node id and advice.
your advice (less than 20 words) will be used to improve the separation next time.
these nodes need handling:
"#
);
let mut result = binding_system_prompt;
result.reserve(512);
for elem in nodes {
result.push_str(&elem.as_prompt());
result.push('\n');
}
result
}
@@ -1,167 +0,0 @@
use crate::ui_editor::commands::separation::model::{
sanitize_prompt_text, SeparationState, SeparationTree,
};
use crate::ui_editor::commands::separation::workflow::batch::terminal_node_ids;
use crate::ui_editor::commands::separation::{SeparationNode, SeparationNodeKind};
use crate::ui_editor::utils::NodeId;
use serde::Serialize;
use std::collections::HashSet;
#[derive(Serialize)]
struct ExtractPromptDocument {
ui_layer_tree: ExtractPromptNode,
}
#[derive(Serialize)]
struct ExtractPromptNode {
index: usize,
status: ExtractPromptStatus,
rect: ExtractPromptRect,
description: String,
#[serde(skip_serializing_if = "Vec::is_empty")]
rework_notes: Vec<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
children: Vec<ExtractPromptNode>,
}
#[derive(Serialize)]
enum ExtractPromptStatus {
#[serde(rename = "OUTPUT_THIS_TURN")]
OutputThisTurn,
#[serde(rename = "DONE")]
Done,
#[serde(rename = "CONTEXT_ONLY")]
ContextOnly,
#[serde(rename = "REMOVE_ONLY")]
RemoveOnly,
}
#[derive(Serialize)]
struct ExtractPromptRect {
x: u32,
y: u32,
width: u32,
height: u32,
}
pub(crate) fn gen_extract_prompt(
state: &SeparationState,
tree: &SeparationTree,
batch: &[&SeparationNode],
) -> Result<String, String> {
let mut result = r#"
This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction.
Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element.
Generate one transparent atlas at the requested canvas size. You may move or scale output layers so they do not cover one another.
Preserve hard edges and the exact visible shape. Never split a scene/background into multiple scene layers.
Ordinary text is editable UI text: remove it from its parent image/background and do not generate a text raster layer.
Extract only the image nodes marked OUTPUT_THIS_TURN. Reconstruct every child/text layer that is listed under a parent but is not an output target.
UI layer tree:
"#
.to_string();
result.reserve(2048);
let target_ids = batch
.iter()
.map(|node| node.id.clone())
.collect::<HashSet<_>>();
let terminal_ids = terminal_node_ids(state);
let mut index = 1;
let document = ExtractPromptDocument {
ui_layer_tree: project_node(&tree.root, &target_ids, &terminal_ids, &mut index),
};
let yaml = serde_yaml::to_string(&document)
.map_err(|error| format!("UI separation extract prompt projection failed: {error}"))?;
result.push_str("```yaml\n");
result.push_str(&yaml);
result.push_str("```\n");
Ok(result)
}
fn project_node(
node: &SeparationNode,
target_ids: &HashSet<NodeId>,
terminal_ids: &HashSet<NodeId>,
index: &mut usize,
) -> ExtractPromptNode {
let current_index = *index;
*index += 1;
let status = match node.kind {
SeparationNodeKind::TextRemovalOnly => ExtractPromptStatus::RemoveOnly,
SeparationNodeKind::PureContainer => ExtractPromptStatus::ContextOnly,
SeparationNodeKind::ImageTarget if target_ids.contains(&node.id) => {
ExtractPromptStatus::OutputThisTurn
}
SeparationNodeKind::ImageTarget if terminal_ids.contains(&node.id) => {
ExtractPromptStatus::Done
}
SeparationNodeKind::ImageTarget => ExtractPromptStatus::ContextOnly,
};
let children = node
.children
.iter()
.map(|child| project_node(child, target_ids, terminal_ids, index))
.collect();
ExtractPromptNode {
index: current_index,
status,
rect: ExtractPromptRect {
x: node.global_pos_x_px,
y: node.global_pos_y_px,
width: node.width_px,
height: node.height_px,
},
description: sanitize_prompt_text(&node.note.description),
rework_notes: node
.note
.rework_notes
.iter()
.map(|note| sanitize_prompt_text(note))
.collect(),
children,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ui_editor::commands::separation::model::SeparationNote;
#[test]
fn projects_source_node_to_prompt_view() {
let node = SeparationNode {
id: NodeId::new("image").unwrap(),
kind: SeparationNodeKind::ImageTarget,
global_pos_x_px: 1,
global_pos_y_px: 2,
width_px: 3,
height_px: 4,
note: SeparationNote {
description: "按钮".to_string(),
rework_notes: vec!["保留圆角".to_string()],
},
children: vec![],
rework_count: 0,
};
let mut index = 1;
let target_ids = HashSet::from([node.id.clone()]);
let projected = project_node(&node, &target_ids, &HashSet::new(), &mut index);
assert_eq!(projected.index, 1);
assert!(matches!(
projected.status,
ExtractPromptStatus::OutputThisTurn
));
assert_eq!(projected.rect.x, 1);
assert_eq!(projected.rect.y, 2);
assert_eq!(projected.rect.width, 3);
assert_eq!(projected.rect.height, 4);
assert_eq!(projected.description, "按钮");
assert_eq!(projected.rework_notes, vec!["保留圆角".to_string()]);
assert!(projected.children.is_empty());
}
}
@@ -1,5 +0,0 @@
mod binding;
mod extract;
pub(super) use binding::gen_binding_prompt;
pub(super) use extract::gen_extract_prompt;
@@ -1,191 +0,0 @@
use super::model::*;
use crate::ui_editor::component::{image::ImageComponent, Component};
use crate::ui_editor::layout::node::Node;
use crate::ui_editor::state::State;
use std::collections::HashSet;
fn is_unbound_image(node: &Node) -> bool {
matches!(
node.component.as_ref(),
Some(Component::Image(ImageComponent {
target_graphic: None,
..
}))
)
}
fn has_image_component(node: &Node) -> bool {
matches!(node.component.as_ref(), Some(Component::Image(_)))
}
fn has_text_component(node: &Node) -> bool {
matches!(node.component.as_ref(), Some(Component::Text(_)))
}
fn node_pixel_rect(
node: &Node,
parent: &crate::ui_editor::layout::dimension::UIRect,
ppu: f32,
) -> (u32, u32, u32, u32) {
let rect = node.layout.transform.resolve(parent);
(
(rect.min.x * ppu).max(0.0).round() as u32,
(rect.min.y * ppu).max(0.0).round() as u32,
(rect.size.x * ppu).max(0.0).round() as u32,
(rect.size.y * ppu).max(0.0).round() as u32,
)
}
fn node_description(node: &Node) -> String {
let name = node.metadata.name.trim();
let description = node.metadata.description.trim();
match (name.is_empty(), description.is_empty()) {
(true, true) => "未命名 UI 图片元素".to_string(),
(false, true) => name.to_string(),
(true, false) => description.to_string(),
(false, false) => format!("{name}{description}"),
}
}
fn collect_todo_nodes(
node: &Node,
parent: &crate::ui_editor::layout::dimension::UIRect,
ppu: f32,
output: &mut Vec<SeparationNode>,
) {
let rect = node.layout.transform.resolve(parent);
let mut children = Vec::new();
for child in &node.children {
collect_todo_nodes(child, &rect, ppu, &mut children);
}
let kind = if is_unbound_image(node) {
Some(SeparationNodeKind::ImageTarget)
} else if has_text_component(node) && !has_image_component(node) {
Some(SeparationNodeKind::TextRemovalOnly)
} else {
None
};
if let Some(kind) = kind {
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
if w > 0 && h > 0 {
output.push(SeparationNode {
id: node.id.clone(),
kind,
global_pos_x_px: x,
global_pos_y_px: y,
width_px: w,
height_px: h,
note: SeparationNote {
description: node_description(node),
rework_notes: Vec::new(),
},
children,
rework_count: 0,
});
} else {
output.extend(children);
}
} else {
output.extend(children);
}
}
pub fn construct_separation_state(state: &State) -> SeparationState {
let trees = state
.ui_trees
.iter()
.filter_map(|tree| {
let image = state.ui_design_images.get(&tree.src_ui_design)?;
let ppu = image.pixels_per_unit.get();
let size = image.pixel_size / ppu;
let root_rect =
crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size);
let root_resolved = tree.root.layout.transform.resolve(&root_rect);
let mut children = Vec::new();
for child in &tree.root.children {
collect_todo_nodes(child, &root_resolved, ppu, &mut children);
}
let root_extractable = is_unbound_image(&tree.root);
if !root_extractable && children.is_empty() {
return None;
}
let (x, y, w, h) = node_pixel_rect(&tree.root, &root_rect, ppu);
Some(SeparationTree {
src_ui_design: tree.src_ui_design.clone(),
root: SeparationNode {
id: tree.root.id.clone(),
kind: if root_extractable {
SeparationNodeKind::ImageTarget
} else if has_text_component(&tree.root) && !has_image_component(&tree.root) {
SeparationNodeKind::TextRemovalOnly
} else {
SeparationNodeKind::PureContainer
},
global_pos_x_px: x,
global_pos_y_px: y,
width_px: w,
height_px: h,
note: SeparationNote {
description: node_description(&tree.root),
rework_notes: Vec::new(),
},
children,
rework_count: 0,
},
root_extractable,
})
})
.collect();
SeparationState {
schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(),
trees,
bound: Vec::new(),
problematic_nodes: Vec::new(),
}
}
pub fn validate_binding_response(
response: &BindingResp,
batch: &[&SeparationNode],
processed_dimensions: (u32, u32),
) -> Result<(), String> {
let expected = batch
.iter()
.filter(|node| matches!(node.kind, SeparationNodeKind::ImageTarget))
.map(|node| node.id.clone())
.collect::<HashSet<_>>();
let mut seen = HashSet::new();
for decision in &response.decisions {
let node_id = match decision {
BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => {
to_node
}
};
if !expected.contains(node_id) {
return Err(format!("视觉绑定返回了未知节点:{}", node_id.as_str()));
}
if !seen.insert(node_id.clone()) {
return Err(format!("视觉绑定重复返回节点:{}", node_id.as_str()));
}
if let BindingDecision::Ok { extracted_area, .. } = decision {
extracted_area
.validate_in(processed_dimensions.0, processed_dimensions.1)
.map_err(|error| format!("节点 {} 的分离区域无效:{error}", node_id.as_str()))?;
}
if let BindingDecision::NeedRework { advice, .. } = decision {
if advice.trim().is_empty() {
return Err("NeedRework 必须包含问题描述".to_string());
}
if advice.chars().count() > MAX_REWORK_NOTE_CHARS {
// TODO add this back in prompt
return Err(format!(
"NeedRework 问题描述不能超过 {MAX_REWORK_NOTE_CHARS} 个字符"
));
}
}
}
if seen.len() != expected.len() {
return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string());
}
Ok(())
}
@@ -1,146 +0,0 @@
use crate::ui_editor::commands::separation::model::*;
use crate::ui_editor::utils::NodeId;
use std::collections::HashSet;
#[derive(Debug)]
pub struct ImageBatch<'a> {
pub nodes: Vec<&'a SeparationNode>,
pub area_px: u64,
pub image_edit_dimension_px: u32,
}
/// Derive the square raw image-edit canvas from the selected source area.
/// The calculation lives beside batch selection so the area budget and
/// request size cannot drift apart.
pub fn image_edit_dimension_for_area(area_px: u64) -> u32 {
let max = u128::from(IMAGE_EDIT_MAX_DIMENSION_PX);
let limit = u128::from(IMAGE_EDIT_AREA_LIMIT_PX);
let area = u128::from(area_px);
let raw = if area >= limit {
IMAGE_EDIT_MAX_DIMENSION_PX
} else if area == 0 {
0
} else {
// Find floor(max * sqrt(area / limit)) without floating-point rounding.
let target = max * max * area;
let mut low = 0u64;
let mut high = IMAGE_EDIT_MAX_DIMENSION_PX;
while low < high {
let mid = low + (high - low + 1) / 2;
if u128::from(mid) * u128::from(mid) * limit <= target {
low = mid;
} else {
high = mid - 1;
}
}
low
};
let alignment = IMAGE_EDIT_DIMENSION_ALIGNMENT_PX;
let aligned = raw / alignment * alignment;
aligned.clamp(IMAGE_EDIT_MIN_DIMENSION_PX, IMAGE_EDIT_MAX_DIMENSION_PX) as u32
}
pub(crate) fn terminal_node_ids(state: &SeparationState) -> HashSet<NodeId> {
state
.bound
.iter()
.map(|n| n.node_id.clone())
.chain(state.problematic_nodes.iter().map(|n| n.node_id.clone()))
.collect()
}
fn collect_dfs_batch<'a>(
node: &'a SeparationNode,
is_root: bool,
root_extractable: bool,
terminal: &HashSet<NodeId>,
selected: &mut Vec<&'a SeparationNode>,
area: &mut u64,
) -> bool {
let is_target = matches!(node.kind, SeparationNodeKind::ImageTarget)
&& (!is_root || root_extractable)
&& !terminal.contains(&node.id);
if is_target {
let node_area = u64::from(node.width_px).saturating_mul(u64::from(node.height_px));
let would_exceed = area.saturating_add(node_area) > IMAGE_EDIT_AREA_LIMIT_PX;
if selected.is_empty() || !would_exceed {
selected.push(node);
*area = area.saturating_add(node_area);
} else {
return true;
}
}
for child in &node.children {
if collect_dfs_batch(child, false, root_extractable, terminal, selected, area) {
return true;
}
}
false
}
pub fn next_image_batch_with_size<'a>(
state: &SeparationState,
tree: &'a SeparationTree,
) -> ImageBatch<'a> {
let terminal = terminal_node_ids(state);
let mut selected = Vec::new();
let mut area = 0;
collect_dfs_batch(
&tree.root,
true,
tree.root_extractable,
&terminal,
&mut selected,
&mut area,
);
let image_edit_dimension_px = image_edit_dimension_for_area(area);
app_log!(
"ui_separation.batch_selected image_id={} image_nodes={} area_px={} image_edit_dimension_px={}",
tree.src_ui_design.as_str(),
selected.len(),
area,
image_edit_dimension_px
);
ImageBatch {
nodes: selected,
area_px: area,
image_edit_dimension_px,
}
}
pub fn next_image_batch<'a>(
state: &SeparationState,
tree: &'a SeparationTree,
) -> Vec<&'a SeparationNode> {
next_image_batch_with_size(state, tree).nodes
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn image_edit_dimension_uses_minimum_and_alignment() {
assert_eq!(image_edit_dimension_for_area(0), 816);
assert_eq!(image_edit_dimension_for_area(1), 816);
assert_eq!(
image_edit_dimension_for_area(IMAGE_EDIT_AREA_LIMIT_PX / 16),
816
);
assert_eq!(
image_edit_dimension_for_area(IMAGE_EDIT_AREA_LIMIT_PX),
2880
);
assert_eq!(image_edit_dimension_for_area(u64::MAX), 2880);
}
#[test]
fn image_edit_dimension_rounds_down_to_sixteen_pixels() {
let max_squared = IMAGE_EDIT_MAX_DIMENSION_PX * IMAGE_EDIT_MAX_DIMENSION_PX;
let area_just_below_1536 = IMAGE_EDIT_AREA_LIMIT_PX * 1536 * 1536 / max_squared;
let area_at_1536 = (IMAGE_EDIT_AREA_LIMIT_PX * 1536 * 1536).div_ceil(max_squared);
assert_eq!(image_edit_dimension_for_area(area_just_below_1536), 1520);
assert_eq!(image_edit_dimension_for_area(area_at_1536), 1536);
}
}
@@ -1,164 +0,0 @@
use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config};
use crate::ui_editor::commands::separation::image_preprocess;
use crate::ui_editor::commands::separation::prompt::gen_binding_prompt;
use crate::ui_editor::commands::separation::{
validate_binding_response, BindingResp, SeparationNode,
};
use crate::ui_editor::commands::utils::{
parse_limited_llm_tool_arguments, request_ui_editor_llm, run_with_repair_history,
strict_json_schema,
};
use platform_llm::{
LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice,
};
use std::path::PathBuf;
use std::time::Instant;
pub(super) async fn visual_binding(
source_url: String,
processed_url: String,
sidecar: PathBuf,
nodes: &[&SeparationNode],
processed_dimensions: (u32, u32),
) -> Result<BindingResp, String> {
let started = Instant::now();
let result = visual_binding_inner(
source_url,
processed_url,
sidecar,
nodes,
processed_dimensions,
)
.await;
app_log!(
"ui_separation.visual_binding.timing outcome={} elapsed_ms={} nodes={}",
if result.is_ok() { "ok" } else { "error" },
started.elapsed().as_millis(),
nodes.len()
);
result
}
async fn visual_binding_inner(
source_url: String,
processed_url: String,
sidecar: PathBuf,
nodes: &[&SeparationNode],
processed_dimensions: (u32, u32),
) -> Result<BindingResp, String> {
app_log!(
"ui_separation.visual_binding.start nodes={} source_url_chars={} processed_url_chars={}",
nodes.len(),
source_url.chars().count(),
processed_url.chars().count()
);
let preprocess_started = Instant::now();
let binding_processed_url =
match image_preprocess::preprocess_for_visual_binding(processed_url, sidecar).await {
Ok(value) => {
app_log!(
"ui_separation.visual_binding.preprocess.timing outcome=ok elapsed_ms={}",
preprocess_started.elapsed().as_millis()
);
value
}
Err(error) => {
app_log!(
"ui_separation.visual_binding.preprocess.timing outcome=error elapsed_ms={}",
preprocess_started.elapsed().as_millis()
);
app_log!("ui_separation.error stage=visual_binding_preprocess error={error}");
return Err(error);
}
};
let llm_config = load_game_creator_app_config()
.map_err(|e| {
app_log!("ui_separation.error stage=visual_binding reason=load_config error={e}");
e.to_string()
})?
.llm;
let client =
build_game_creator_llm_client_from_llm_config(&llm_config, "llm").map_err(|e| {
app_log!("ui_separation.error stage=visual_binding reason=build_client error={e}");
e.to_string()
})?;
let schema = strict_json_schema::<BindingResp>().map_err(|error| {
app_log!("ui_separation.error stage=visual_binding reason=build_schema error={error}");
error
})?;
let tool = LlmFunctionTool::new(
"bind_ui_elements",
"确认处理图中的区域对应哪些 UI 节点",
schema,
)
.with_strict(true);
let initial_history = vec![
LlmMessage::system(gen_binding_prompt(nodes)),
LlmMessage::user_multimodal(vec![
LlmMessageContentPart::InputText {
text: "processed image:".to_string(),
},
LlmMessageContentPart::InputImage {
image_url: binding_processed_url,
},
LlmMessageContentPart::InputText {
text: "src image:".to_string(),
},
LlmMessageContentPart::InputImage {
image_url: source_url.clone(),
},
]),
];
// 严格工具 schema 由 provider 负责约束正常模型输出;这里的解析失败只代表极小概率
// 的传输/响应损坏,因此沿用有限重试,不再为理论上的坏载荷扩展业务修复协议。
let result = run_with_repair_history(
2,
initial_history,
|history| {
let tool = tool.clone();
let client = client.clone();
let llm_config = llm_config.clone();
async move {
let request = LlmRunRequest::new(history)
.with_function_tools(vec![tool.clone()])
.with_tool_choice(LlmToolChoice::Required);
let request_started = Instant::now();
let response = request_ui_editor_llm(&client, &llm_config, request).await;
app_log!(
"ui_separation.llm.timing outcome={} elapsed_ms={}",
if response.is_ok() { "ok" } else { "error" },
request_started.elapsed().as_millis()
);
response
.map_err(|e| e.to_string())
.and_then(|response| {
response
.tool_calls
.into_iter()
.find(|call| call.name == "bind_ui_elements")
.map(|call| call.arguments)
.ok_or_else(|| "视觉绑定模型未返回工具调用".to_string())
})
.and_then(|arguments| parse_limited_llm_tool_arguments(&arguments))
.and_then(|args| {
serde_json::from_value::<BindingResp>(args)
.map_err(|e| format!("视觉绑定结果无效:{e}"))
})
}
},
|value: &BindingResp| validate_binding_response(value, nodes, processed_dimensions),
)
.await;
match &result {
Ok(value) => app_log!(
"ui_separation.visual_binding.completed nodes={} decisions={}",
nodes.len(),
value.decisions.len()
),
Err(error) => app_log!(
"ui_separation.error stage=visual_binding reason=failed nodes={} error={error}",
nodes.len()
),
}
result
}
@@ -1,132 +0,0 @@
use crate::ui_editor::commands::separation::area::normalize_binding_area;
use crate::ui_editor::commands::separation::model::BindingArea;
use image::ImageFormat;
use std::path::{Path, PathBuf};
use std::time::Instant;
pub(super) async fn cut_processed_image(
source: PathBuf,
area: BindingArea,
target: PathBuf,
) -> Result<(), String> {
let started = Instant::now();
let result = cut_processed_image_inner(source, area, target).await;
app_log!(
"ui_separation.cut_image.timing outcome={} elapsed_ms={}",
if result.is_ok() { "ok" } else { "error" },
started.elapsed().as_millis()
);
result
}
async fn cut_processed_image_inner(
source: PathBuf,
area: BindingArea,
target: PathBuf,
) -> Result<(), String> {
app_log!(
"ui_separation.cut_image.start source_file={} target_file={} area=({}, {}, {}, {})",
source
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>"),
target
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>"),
area.global_pos_x_px,
area.global_pos_y_px,
area.width_px,
area.height_px
);
tokio::task::spawn_blocking(move || cut_processed_image_blocking(&source, &area, &target))
.await
.map_err(|error| format!("裁切处理图任务失败:{error}"))?
}
fn cut_processed_image_blocking(
source: &Path,
area: &BindingArea,
target: &Path,
) -> Result<(), String> {
let image = image::open(source)
.map_err(|e| format!("读取处理图失败:{e}"))?
.to_rgba8();
let normalized = normalize_binding_area(&image, *area)?;
if normalized.transparent {
return Err("分离区域没有可见像素".to_string());
}
let original_area = *area;
let normalized_area = normalized.area;
app_log!(
"ui_separation.cut_image.normalized changed={} clamped={} transparent={} original_area=({}, {}, {}, {}) normalized_area=({}, {}, {}, {})",
normalized.changed,
normalized.clamped,
normalized.transparent,
original_area.global_pos_x_px,
original_area.global_pos_y_px,
original_area.width_px,
original_area.height_px,
normalized_area.global_pos_x_px,
normalized_area.global_pos_y_px,
normalized_area.width_px,
normalized_area.height_px
);
let cropped = image::imageops::crop_imm(
&image,
normalized_area.global_pos_x_px,
normalized_area.global_pos_y_px,
normalized_area.width_px,
normalized_area.height_px,
)
.to_image();
cropped
.save_with_format(target, ImageFormat::Png)
.map_err(|e| format!("写入 cut 图片失败:{e}"))?;
app_log!(
"ui_separation.cut_image.completed target_file={} width={} height={}",
target
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>"),
normalized_area.width_px,
normalized_area.height_px
);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use image::{Rgba, RgbaImage};
#[test]
fn cuts_using_small_processed_image_dimensions() {
let directory = tempfile::tempdir().expect("创建临时目录失败");
let source = directory.path().join("processed.png");
let target = directory.path().join("cut.png");
let mut image = RgbaImage::from_pixel(816, 816, Rgba([0, 0, 0, 0]));
for y in 120..152 {
for x in 700..800 {
image.put_pixel(x, y, Rgba([255, 255, 255, 255]));
}
}
image.save(&source).expect("写入处理图失败");
cut_processed_image_blocking(
&source,
&BindingArea {
global_pos_x_px: 700,
global_pos_y_px: 120,
width_px: 100,
height_px: 32,
},
&target,
)
.expect("裁切处理图失败");
let cropped = image::open(target).expect("读取 cut 图片失败");
assert_eq!(cropped.width(), 100);
assert_eq!(cropped.height(), 32);
}
}
@@ -1,228 +0,0 @@
use crate::platform_session::PlatformSessionSnapshot;
use base64::Engine as _;
use serde::Deserialize;
use std::path::PathBuf;
use std::time::Instant;
use std::{fs, io::Cursor};
#[derive(Deserialize)]
struct RawEditResponse {
data: Vec<RawEditItem>,
}
#[derive(Deserialize)]
struct RawEditItem {
b64_json: String,
}
pub(super) async fn raw_extract(
session: &PlatformSessionSnapshot,
image_data_url: &str,
prompt: &str,
width: u32,
height: u32,
) -> Result<String, String> {
let started = Instant::now();
let result = raw_extract_inner(session, image_data_url, prompt, width, height).await;
app_log!(
"ui_separation.image_edit.timing outcome={} elapsed_ms={} width={} height={}",
if result.is_ok() { "ok" } else { "error" },
started.elapsed().as_millis(),
width,
height
);
result
}
async fn raw_extract_inner(
session: &PlatformSessionSnapshot,
image_data_url: &str,
prompt: &str,
width: u32,
height: u32,
) -> Result<String, String> {
app_log!(
"ui_separation.image_edit.start width={} height={} prompt_chars={}",
width,
height,
prompt.chars().count()
);
let (mime, data) = image_data_url
.split_once(',')
.ok_or_else(|| "界面图 data URL 无效".to_string())?;
let mime = mime
.strip_prefix("data:")
.and_then(|value| value.strip_suffix(";base64"))
.unwrap_or("image/png");
let is_png = mime.eq_ignore_ascii_case("image/png");
let image_bytes = base64::engine::general_purpose::STANDARD
.decode(data.trim())
.map_err(|error| format!("解码源图失败:{error}"))?;
if image_bytes.is_empty() {
return Err("源图不能为空".to_string());
}
let image_bytes = if is_png {
image_bytes
} else {
tokio::task::spawn_blocking(move || normalize_source_image_to_png(image_bytes))
.await
.map_err(|error| format!("转换源图任务失败:{error}"))??
};
let client = crate::http_client::agc_main_site_client_builder()
.timeout(std::time::Duration::from_secs(120))
.build()
.map_err(|error| format!("创建图片编辑客户端失败:{error}"))?;
let url = format!(
"{}/api/raw/v1/images/edit",
session.api_base_url.trim_end_matches('/')
);
let image_part = reqwest::multipart::Part::bytes(image_bytes)
.file_name("image.png")
.mime_str("image/png")
.map_err(|error| format!("构造图片编辑文件部件失败:{error}"))?;
let body = reqwest::multipart::Form::new()
.part("image", image_part)
.text("prompt", prompt.to_string())
.text("width", width.to_string())
.text("height", height.to_string())
.text("output_format", "png")
.text("background", "transparent");
let response = crate::http_client::with_agc_main_site_marker(
client
.post(url)
.bearer_auth(&session.access_token)
.multipart(body),
)
.send()
.await
.map_err(|error| {
app_log!("ui_separation.error stage=image_edit reason=send error={error}");
format!("图片分离请求失败:{error}")
})?;
if !response.status().is_success() {
app_log!(
"ui_separation.error stage=image_edit reason=http_status status={}",
response.status()
);
return Err(format!("图片分离请求失败(HTTP {}", response.status()));
}
let payload = response.json::<RawEditResponse>().await.map_err(|error| {
app_log!("ui_separation.error stage=image_edit reason=parse_response error={error}");
format!("解析图片分离响应失败:{error}")
})?;
let result = payload
.data
.into_iter()
.next()
.map(|item| format!("data:image/png;base64,{}", item.b64_json))
.ok_or_else(|| "图片分离响应没有图像".to_string());
match &result {
Ok(value) => app_log!(
"ui_separation.image_edit.completed data_url_chars={}",
value.chars().count()
),
Err(error) => {
app_log!("ui_separation.error stage=image_edit reason=empty_result error={error}")
}
}
result
}
fn normalize_source_image_to_png(image_bytes: Vec<u8>) -> Result<Vec<u8>, String> {
let image = image::load_from_memory(&image_bytes)
.map_err(|error| format!("解码非 PNG 源图失败:{error}"))?;
let mut png_bytes = Vec::new();
image
.write_to(&mut Cursor::new(&mut png_bytes), image::ImageFormat::Png)
.map_err(|error| format!("将源图转换为 PNG 失败:{error}"))?;
Ok(png_bytes)
}
#[cfg(test)]
mod tests {
use super::normalize_source_image_to_png;
use image::{DynamicImage, ImageFormat, Rgb, RgbImage};
use std::io::Cursor;
#[test]
fn converts_jpeg_source_to_png() {
let image = DynamicImage::ImageRgb8(RgbImage::from_pixel(2, 2, Rgb([255, 0, 0])));
let mut jpeg = Vec::new();
image
.write_to(&mut Cursor::new(&mut jpeg), ImageFormat::Jpeg)
.expect("encode jpeg");
let png = normalize_source_image_to_png(jpeg).expect("convert jpeg");
assert_eq!(&png[..8], b"\x89PNG\r\n\x1a\n");
}
#[test]
fn converts_webp_source_to_png() {
let image = DynamicImage::ImageRgb8(RgbImage::from_pixel(2, 2, Rgb([0, 128, 255])));
let mut webp = Vec::new();
image
.write_to(&mut Cursor::new(&mut webp), ImageFormat::WebP)
.expect("encode webp");
let png = normalize_source_image_to_png(webp).expect("convert webp");
assert_eq!(&png[..8], b"\x89PNG\r\n\x1a\n");
}
}
pub(super) async fn write_processed_image(
processed_url: String,
target: PathBuf,
) -> Result<(u32, u32), String> {
let started = Instant::now();
let result = write_processed_image_inner(processed_url, target).await;
app_log!(
"ui_separation.processed_image.write.timing outcome={} elapsed_ms={}",
if result.is_ok() { "ok" } else { "error" },
started.elapsed().as_millis()
);
result
}
async fn write_processed_image_inner(
processed_url: String,
target: PathBuf,
) -> Result<(u32, u32), String> {
app_log!(
"ui_separation.processed_image.write.start target_file={} data_url_chars={}",
target
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>"),
processed_url.chars().count()
);
tokio::task::spawn_blocking(move || {
let encoded = processed_url
.split_once(',')
.map(|(_, data)| data)
.ok_or_else(|| "处理图 data URL 无效".to_string())?;
let processed_bytes = base64::engine::general_purpose::STANDARD
.decode(encoded)
.map_err(|error| format!("解析处理图失败:{error}"))?;
let dimensions = image::ImageReader::new(Cursor::new(processed_bytes.as_slice()))
.with_guessed_format()
.map_err(|error| format!("识别处理图格式失败:{error}"))?
.into_dimensions()
.map_err(|error| format!("读取处理图尺寸失败:{error}"))?;
let byte_len = processed_bytes.len();
fs::write(&target, processed_bytes)
.map_err(|error| format!("写入处理图失败:{}: {error}", target.display()))
.map(|_| {
app_log!(
"ui_separation.processed_image.write.completed target_file={} bytes={}",
target
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("<unknown>"),
byte_len
);
dimensions
})
})
.await
.map_err(|error| format!("写入处理图任务失败:{error}"))?
}

Some files were not shown because too many files have changed in this diff Show More