Compare commits
53 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 11d4d3d930 | |||
| 418d5732de | |||
| 592b6b4ee7 | |||
| 99c19e3ed9 | |||
| 15e30acb0f | |||
| 56fff8325b | |||
| 884c5ec5ee | |||
| 207b0820dd | |||
| 19f3d62d78 | |||
| 3c1e7bd599 | |||
| dda6b3946a | |||
| aff40f794d | |||
| 187faa607d | |||
| 109cc81740 | |||
| f796aef7e2 | |||
| 4cc7f4dd64 | |||
| b2d4690f94 | |||
| 0400103a19 | |||
| c049d68b8e | |||
| acfcf0c158 | |||
| 5bbedd9a5c | |||
| 40316d82c4 | |||
| 6c77de4a36 | |||
| b3693765ef | |||
| f7edc9d0ee | |||
| f60771dd00 | |||
| f7cccd8592 | |||
| 5ee2d940c1 | |||
| c75a7725ce | |||
| 3a95715990 | |||
| ed86963de2 | |||
| 65e73eee02 | |||
| 6068eeda83 | |||
| 0e244c64ea | |||
| ad7c059603 | |||
| 55d2a393d1 | |||
| e7f66ce950 | |||
| 35dcfea3ef | |||
| cea8bf77fa | |||
| 0136bfe564 | |||
| 22785b4d7b | |||
| a86a2ee4d2 | |||
| b22eae2726 | |||
| 35d0db6377 | |||
| f087ba3e2b | |||
| 7fc1b841be | |||
| a576fdcc37 | |||
| 4dcf38715b | |||
| d1f9f4e1ef | |||
| 5836aaec26 | |||
| 18d60feb04 | |||
| 4e1316e272 | |||
| 302b4addae |
@@ -328,6 +328,15 @@ async fn recognize_ui(
|
||||
ui_editor::commands::recognize_ui_impl(project_path, state).await
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
async fn separate_ui(
|
||||
project_path: String,
|
||||
asset_id: String,
|
||||
state: ui_editor::state::State,
|
||||
) -> Result<ui_editor::commands::SeparationDTO, String> {
|
||||
ui_editor::commands::separate_ui_impl(project_path, asset_id, state).await
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
async fn merge_ui(state: ui_editor::state::State) -> Result<ui_editor::commands::MergeDTO, String> {
|
||||
ui_editor::commands::merge_ui_impl(state).await
|
||||
@@ -2559,6 +2568,7 @@ fn main() {
|
||||
check_ui_editor_font_glyph_coverage,
|
||||
suggest_ui_design_semantic,
|
||||
recognize_ui,
|
||||
separate_ui,
|
||||
merge_ui,
|
||||
bind_components,
|
||||
load_ui_design_state,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
pub mod binding;
|
||||
pub mod merge;
|
||||
pub mod recognition;
|
||||
pub mod separation;
|
||||
pub mod ui_design_suggestion;
|
||||
pub mod utils;
|
||||
|
||||
@@ -10,5 +11,7 @@ pub use merge::MergeDTO;
|
||||
pub(crate) use merge::{merge_ui_impl, merge_ui_impl_with_provider};
|
||||
pub use recognition::RecognitionDTO;
|
||||
pub(crate) use recognition::{recognize_ui_impl, recognize_ui_impl_with_provider};
|
||||
pub(crate) use separation::separate_ui_impl;
|
||||
pub use separation::SeparationDTO;
|
||||
pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl;
|
||||
pub use ui_design_suggestion::UIDesignSuggestionTreeNode;
|
||||
|
||||
@@ -4,6 +4,7 @@ use crate::ui_editor::commands::utils::{
|
||||
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm,
|
||||
strict_json_schema,
|
||||
};
|
||||
use crate::ui_editor::component::Component;
|
||||
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
|
||||
use crate::ui_editor::layout::control_layout::ControlLayout;
|
||||
use crate::ui_editor::layout::dimension::UIRect;
|
||||
@@ -27,16 +28,13 @@ const MAX_RECOGNITION_TREE_NODES: usize = 512;
|
||||
const MAX_RECOGNITION_TREE_DEPTH: usize = 32;
|
||||
|
||||
const SYSTEM_PROMPT: &str = r#"
|
||||
角色:
|
||||
你是游戏 UI 多图结构识别器。
|
||||
|
||||
任务:
|
||||
同时分析同一 UI 系统的全部参考图,建立UI树
|
||||
|
||||
用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构
|
||||
|
||||
识别规则:
|
||||
* 只识别 UI,不识别场景人物、地形、建筑、光影和背景装饰。
|
||||
* 只识别 UI元素. 要区分动态内容, 不要白费力气识别应该由程序生成/绘制的内容.(此类内容应该用一个整体节点+自然语言描述) 除此之外必须完整包含所有元素,结构.
|
||||
* 无法确定类型、层级、关系时,在 UnSure 中写明原因。
|
||||
* 返回的 trees 必须与输入图片一一对应,每张输入图片只能有一棵树,不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id。
|
||||
* 每棵树必须使用自己的输入图片原始像素坐标系(0,0 as left top)输出
|
||||
@@ -45,7 +43,13 @@ const SYSTEM_PROMPT: &str = r#"
|
||||
* 面向用户的字段如名称描述等请用中文
|
||||
* 由于每个截图未必是完整的, 可能是局部的, 每棵树描述清楚每个截图上UI的层次结构即可
|
||||
* 不同树的共用框架/层次/...请使用使用相同的名称描述. 不同状态/变体名称使用相同的前缀, 用后缀区别
|
||||
* 粒度要求: 尽可能细致, 最小单元举例: 进度条的底槽、填充和外框; slider的底槽, dragger等
|
||||
* 粒度要求: 尽可能细致, 以可交互,方便程序化控制的最小单位为准. 包括不限于: icon, 进度条的底槽、填充和外框; slider的底槽, dragger等.
|
||||
* 为每个节点直接返回完整 components.
|
||||
无背景的逻辑容器返回空数组.
|
||||
有背景的容器推荐使用Simple+不锁定宽高比的Image component.
|
||||
目前我们只做识别, 不要求图片字体参数.
|
||||
每个节点当前最多返回一个 Image component 和一个 Text component。
|
||||
文字组件要求: 艺术字等作为图片组件, 其余正常文字要作为单独的节点识别.
|
||||
|
||||
"#;
|
||||
|
||||
@@ -109,6 +113,7 @@ struct RecognitionNode {
|
||||
description: String,
|
||||
children: Vec<RecognitionNode>,
|
||||
confidence: Confidence,
|
||||
components: Vec<Component>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
@@ -312,10 +317,7 @@ fn convert_node(
|
||||
allow_llm_edit_component: true,
|
||||
source: NodeSource::Llm,
|
||||
},
|
||||
// V1 有意把识别结果限定为“结构草稿”:组件绑定属于后续独立阶段。
|
||||
// 因此空组件不是丢失数据,而是等待 visual-binding 阶段补齐 Image/Text。
|
||||
// 约定见 docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md。
|
||||
components: Vec::new(),
|
||||
components: source.components.clone(),
|
||||
children_display_mode: ChildrenDisplayMode::Stack,
|
||||
children,
|
||||
})
|
||||
@@ -323,6 +325,41 @@ fn convert_node(
|
||||
|
||||
fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> {
|
||||
for node in nodes {
|
||||
let image_count = node
|
||||
.components
|
||||
.iter()
|
||||
.filter(|component| matches!(component, Component::Image(_)))
|
||||
.count();
|
||||
let text_count = node
|
||||
.components
|
||||
.iter()
|
||||
.filter(|component| matches!(component, Component::Text(_)))
|
||||
.count();
|
||||
if image_count > 1 || text_count > 1 {
|
||||
return Err("单个节点当前最多包含一个 Image 和一个 Text component".to_string());
|
||||
}
|
||||
if node.components.iter().any(|component| {
|
||||
matches!(
|
||||
component,
|
||||
Component::Image(crate::ui_editor::component::image::ImageComponent {
|
||||
target_graphic: Some(_),
|
||||
..
|
||||
})
|
||||
)
|
||||
}) {
|
||||
return Err("识别阶段不能返回已绑定的 SpriteAssetId".to_string());
|
||||
}
|
||||
if node.components.iter().any(|component| {
|
||||
matches!(
|
||||
component,
|
||||
Component::Text(crate::ui_editor::component::text::TextComponent {
|
||||
font: crate::ui_editor::component::text::FontSource::Bound(_),
|
||||
..
|
||||
})
|
||||
)
|
||||
}) {
|
||||
return Err("识别阶段不能返回已绑定的字体素材".to_string());
|
||||
}
|
||||
if let Confidence::UnSure(reason) = &node.confidence {
|
||||
if reason.trim().is_empty() {
|
||||
return Err("UnSure 必须包含审阅原因".to_string());
|
||||
@@ -387,6 +424,7 @@ mod tests {
|
||||
description: String::new(),
|
||||
children: Vec::new(),
|
||||
confidence: Confidence::Confident,
|
||||
components: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -490,6 +528,31 @@ mod tests {
|
||||
description: String::new(),
|
||||
children: Vec::new(),
|
||||
confidence: Confidence::UnSure(String::new()),
|
||||
components: Vec::new(),
|
||||
};
|
||||
assert!(validate_confidence(&[node]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognition_rejects_bound_font_references() {
|
||||
let mut text = crate::ui_editor::component::text::TextComponent::default();
|
||||
text.font = crate::ui_editor::component::text::FontSource::Bound(
|
||||
crate::ui_editor::utils::FontAssetId::new("font").expect("valid font id"),
|
||||
);
|
||||
let node = RecognitionNode {
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
local_anchor: Anchor::Preset(PresetAnchor {
|
||||
horizontal: HorizontalAnchor::Left,
|
||||
vertical: VerticalAnchor::Top,
|
||||
}),
|
||||
name: "文本".to_string(),
|
||||
description: String::new(),
|
||||
children: Vec::new(),
|
||||
confidence: Confidence::Confident,
|
||||
components: vec![Component::Text(text)],
|
||||
};
|
||||
assert!(validate_confidence(&[node]).is_err());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
mod model;
|
||||
mod persistence;
|
||||
mod prompt;
|
||||
mod tree;
|
||||
mod workflow;
|
||||
|
||||
pub use model::*;
|
||||
pub use persistence::*;
|
||||
pub use tree::*;
|
||||
pub use workflow::apply_batch_patch;
|
||||
pub(crate) use workflow::separate_ui_impl;
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ui_editor::component::Component;
|
||||
use crate::ui_editor::layout::node::Node;
|
||||
use crate::ui_editor::state::{State, UITree};
|
||||
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
|
||||
use std::path::Path;
|
||||
use crate::ui_editor::component::image::{ImageComponent, ImageType};
|
||||
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
|
||||
use crate::ui_editor::layout::control_layout::ControlLayout;
|
||||
use crate::ui_editor::layout::node::{NodeMetadata, NodeSource, StageStatus};
|
||||
use crate::ui_editor::resource::ui_design_image::UIDesignImage;
|
||||
use nalgebra::Vector2;
|
||||
use std::collections::HashMap;
|
||||
use typed_floats::tf32::StrictlyPositiveFinite;
|
||||
|
||||
fn node(id: &str, components: Vec<Component>, children: Vec<Node>) -> Node {
|
||||
Node {
|
||||
id: NodeId::new(id).unwrap(),
|
||||
layout: ControlLayout::default(),
|
||||
metadata: NodeMetadata {
|
||||
name: id.to_string(),
|
||||
description: String::new(),
|
||||
layout_status: StageStatus::NoProblem,
|
||||
components_status: StageStatus::NoProblem,
|
||||
allow_llm_edit_layout: true,
|
||||
allow_llm_edit_component: true,
|
||||
source: NodeSource::Llm,
|
||||
},
|
||||
components,
|
||||
children_display_mode: ChildrenDisplayMode::Stack,
|
||||
children,
|
||||
}
|
||||
}
|
||||
fn state(root: Node) -> State {
|
||||
let image_id = UIDesignImageId::new("page").unwrap();
|
||||
State {
|
||||
ui_trees: vec![UITree {
|
||||
src_ui_design: image_id.clone(),
|
||||
root,
|
||||
}],
|
||||
ui_design_images: HashMap::from([(
|
||||
image_id,
|
||||
UIDesignImage {
|
||||
metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata {
|
||||
name: "page".to_string(),
|
||||
description: String::new(),
|
||||
role: None,
|
||||
slave_to: None,
|
||||
},
|
||||
path: "page.png".to_string(),
|
||||
pixel_size: Vector2::new(100.0, 100.0),
|
||||
pixels_per_unit: StrictlyPositiveFinite::new(1.0).unwrap(),
|
||||
},
|
||||
)]),
|
||||
sprite_assets: HashMap::new(),
|
||||
font_assets: HashMap::new(),
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn construction_filters_pure_nodes_and_passes_children_through() {
|
||||
let image = Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
image_type: ImageType::Simple {
|
||||
preserve_aspect: false,
|
||||
},
|
||||
});
|
||||
let root = node(
|
||||
"root",
|
||||
vec![],
|
||||
vec![node(
|
||||
"container",
|
||||
vec![],
|
||||
vec![node("image", vec![image], vec![])],
|
||||
)],
|
||||
);
|
||||
let result = construct_separation_state(&state(root));
|
||||
assert_eq!(
|
||||
result.unprocessed_trees[0].root.children[0].id.as_str(),
|
||||
"image"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn construction_uses_distinct_root_id_for_root_image() {
|
||||
let image = Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
image_type: ImageType::Simple {
|
||||
preserve_aspect: false,
|
||||
},
|
||||
});
|
||||
let root = node("root-image", vec![image], vec![]);
|
||||
let result = construct_separation_state(&state(root));
|
||||
let tree = &result.unprocessed_trees[0];
|
||||
assert_ne!(tree.root.id, tree.root.children[0].id);
|
||||
assert_eq!(tree.root.children[0].id.as_str(), "root-image");
|
||||
}
|
||||
#[test]
|
||||
fn binding_validation_requires_exact_batch_coverage() {
|
||||
let node = SeparationNode {
|
||||
id: NodeId::new("image").unwrap(),
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
note: SeparationNote {
|
||||
description: "image".to_string(),
|
||||
text_note: String::new(),
|
||||
},
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
};
|
||||
assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err());
|
||||
}
|
||||
#[test]
|
||||
fn sidecar_name_uses_asset_id_digest() {
|
||||
let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap();
|
||||
assert!(dir.to_string_lossy().contains("ui_1-"));
|
||||
assert!(dir.to_string_lossy().ends_with("-separation"));
|
||||
}
|
||||
#[test]
|
||||
fn patch_collects_bound_and_removes_leaf() {
|
||||
let image = Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
image_type: ImageType::Simple {
|
||||
preserve_aspect: false,
|
||||
},
|
||||
});
|
||||
let mut state = construct_separation_state(&state(node(
|
||||
"root",
|
||||
vec![],
|
||||
vec![node("image", vec![image], vec![])],
|
||||
)));
|
||||
let id = NodeId::new("image").unwrap();
|
||||
let decisions = vec![BindingDecision::Ok {
|
||||
to_node: id.clone(),
|
||||
separated_image_area: BindingArea {
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
},
|
||||
}];
|
||||
let paths = HashMap::from([(id.clone(), "ui/.sidecar/cut.png".to_string())]);
|
||||
apply_batch_patch(&mut state, 0, &decisions, &paths).unwrap();
|
||||
assert_eq!(state.bound[0].node_id, id);
|
||||
assert!(state.unprocessed_trees[0].root.children.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use ts_rs::TS;
|
||||
|
||||
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1";
|
||||
pub const MAX_REWORK_COUNT: u32 = 3;
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationNote {
|
||||
pub description: String,
|
||||
pub text_note: String,
|
||||
}
|
||||
impl SeparationNote {
|
||||
pub fn as_prompt(&self) -> String {
|
||||
format!("desc: {} {}", self.description, self.text_note)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationNode {
|
||||
pub id: NodeId,
|
||||
pub global_pos_x_px: u32,
|
||||
pub global_pos_y_px: u32,
|
||||
pub width_px: u32,
|
||||
pub height_px: u32,
|
||||
pub note: SeparationNote,
|
||||
pub children: Vec<SeparationNode>,
|
||||
pub rework_count: u32,
|
||||
}
|
||||
impl SeparationNode {
|
||||
pub fn as_prompt(&self) -> String {
|
||||
format!(
|
||||
"node_id={} area=({}, {}, {}, {}) {}",
|
||||
self.id.as_str(),
|
||||
self.global_pos_x_px,
|
||||
self.global_pos_y_px,
|
||||
self.width_px,
|
||||
self.height_px,
|
||||
self.note.as_prompt()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationTree {
|
||||
pub src_ui_design: UIDesignImageId,
|
||||
pub root: SeparationNode,
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct BoundNode {
|
||||
pub node_id: NodeId,
|
||||
pub cut_image_path: String,
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct ProblematicNode {
|
||||
pub node_id: NodeId,
|
||||
pub problem_description: String,
|
||||
pub rework_count: u32,
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationState {
|
||||
pub schema_version: String,
|
||||
pub unprocessed_trees: Vec<SeparationTree>,
|
||||
pub bound: Vec<BoundNode>,
|
||||
pub problematic_nodes: Vec<ProblematicNode>,
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationDTO {
|
||||
pub bound_nodes: Vec<BoundNode>,
|
||||
pub problematic_nodes: Vec<ProblematicNode>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub struct BindingArea {
|
||||
pub global_pos_x_px: u32,
|
||||
pub global_pos_y_px: u32,
|
||||
pub width_px: u32,
|
||||
pub height_px: u32,
|
||||
}
|
||||
impl BindingArea {
|
||||
pub fn validate_in(&self, w: u32, h: u32) -> Result<(), String> {
|
||||
if self.width_px == 0 || self.height_px == 0 {
|
||||
return Err("BindingArea 宽度和高度必须大于 0".into());
|
||||
}
|
||||
if self
|
||||
.global_pos_x_px
|
||||
.checked_add(self.width_px)
|
||||
.is_none_or(|v| v > w)
|
||||
|| self
|
||||
.global_pos_y_px
|
||||
.checked_add(self.height_px)
|
||||
.is_none_or(|v| v > h)
|
||||
{
|
||||
return Err("BindingArea 超出处理图边界".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub enum BindingDecision {
|
||||
Ok {
|
||||
separated_image_area: BindingArea,
|
||||
to_node: NodeId,
|
||||
},
|
||||
NeedRework {
|
||||
problem_description: String,
|
||||
to_node: NodeId,
|
||||
},
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct BindingResp {
|
||||
pub decisions: Vec<BindingDecision>,
|
||||
}
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
use super::model::*;
|
||||
use crate::ui_editor::commands::separation::*;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
pub fn separation_sidecar_dir(root: &Path, asset_id: &str) -> Result<PathBuf, String> {
|
||||
if asset_id.trim().is_empty() || asset_id.trim() != asset_id {
|
||||
app_log!("ui_separation.error stage=sidecar_dir reason=invalid_asset_id");
|
||||
return Err("UI 资源 ID 无效".to_string());
|
||||
}
|
||||
let dir = root.join("ui").join(format!(
|
||||
".{}-separation",
|
||||
crate::ui_editor::persistence::generated_file_stem(asset_id)
|
||||
));
|
||||
if !dir.starts_with(root) {
|
||||
app_log!("ui_separation.error stage=sidecar_dir reason=path_escape");
|
||||
return Err("separation sidecar 路径越界".to_string());
|
||||
}
|
||||
app_log!(
|
||||
"ui_separation.sidecar_resolved asset_id={} directory={}",
|
||||
asset_id,
|
||||
dir.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("<unknown>")
|
||||
);
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
pub fn write_separation_state(path: &Path, state: &SeparationState) -> Result<(), String> {
|
||||
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
|
||||
app_log!("ui_separation.error stage=state_write reason=schema_mismatch");
|
||||
return Err("不支持的 separation state schema".to_string());
|
||||
}
|
||||
app_log!(
|
||||
"ui_separation.state_write.start file={} trees={} bound={} problematic={}",
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("<unknown>"),
|
||||
state.unprocessed_trees.len(),
|
||||
state.bound.len(),
|
||||
state.problematic_nodes.len()
|
||||
);
|
||||
let bytes = serde_json::to_vec_pretty(state).map_err(|error| {
|
||||
app_log!("ui_separation.error stage=state_write reason=serialize error={error}");
|
||||
format!("序列化 separation state 失败:{error}")
|
||||
})?;
|
||||
let parent = path.parent().ok_or_else(|| {
|
||||
app_log!("ui_separation.error stage=state_write reason=missing_parent");
|
||||
"separation state 路径缺少父目录".to_string()
|
||||
})?;
|
||||
fs::create_dir_all(parent).map_err(|error| {
|
||||
app_log!("ui_separation.error stage=state_write reason=create_parent error={error}");
|
||||
format!("创建 separation sidecar 失败:{error}")
|
||||
})?;
|
||||
let temporary = path.with_extension("json.tmp");
|
||||
fs::write(&temporary, bytes).map_err(|error| {
|
||||
app_log!("ui_separation.error stage=state_write reason=write_temp error={error}");
|
||||
format!("写入 separation state 失败:{error}")
|
||||
})?;
|
||||
fs::rename(&temporary, path).map_err(|error| {
|
||||
app_log!("ui_separation.error stage=state_write reason=install error={error}");
|
||||
format!("安装 separation state 失败:{error}")
|
||||
})?;
|
||||
app_log!(
|
||||
"ui_separation.state_write.completed file={} bytes={} trees={} bound={} problematic={}",
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("<unknown>"),
|
||||
fs::metadata(path)
|
||||
.map(|metadata| metadata.len())
|
||||
.unwrap_or(0),
|
||||
state.unprocessed_trees.len(),
|
||||
state.bound.len(),
|
||||
state.problematic_nodes.len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read_separation_state(path: &Path) -> Result<SeparationState, String> {
|
||||
app_log!(
|
||||
"ui_separation.state_read.start file={}",
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("<unknown>")
|
||||
);
|
||||
let bytes = fs::read(path).map_err(|error| {
|
||||
app_log!("ui_separation.error stage=state_read reason=read error={error}");
|
||||
format!("读取 separation state 失败:{error}")
|
||||
})?;
|
||||
let state: SeparationState = serde_json::from_slice(&bytes).map_err(|error| {
|
||||
app_log!("ui_separation.error stage=state_read reason=parse error={error}");
|
||||
format!("解析 separation state 失败:{error}")
|
||||
})?;
|
||||
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
|
||||
app_log!("ui_separation.error stage=state_read reason=schema_mismatch");
|
||||
return Err("不支持的 separation state schema".to_string());
|
||||
}
|
||||
app_log!(
|
||||
"ui_separation.state_read.completed bytes={} trees={} bound={} problematic={}",
|
||||
bytes.len(),
|
||||
state.unprocessed_trees.len(),
|
||||
state.bound.len(),
|
||||
state.problematic_nodes.len()
|
||||
);
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
pub fn separation_dto(state: &SeparationState) -> SeparationDTO {
|
||||
app_log!(
|
||||
"ui_separation.dto bound_nodes={} problematic_nodes={} remaining_trees={}",
|
||||
state.bound.len(),
|
||||
state.problematic_nodes.len(),
|
||||
state.unprocessed_trees.len()
|
||||
);
|
||||
SeparationDTO {
|
||||
bound_nodes: state.bound.clone(),
|
||||
problematic_nodes: state.problematic_nodes.clone(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
use crate::ui_editor::commands::separation::{SeparationNode, SeparationNote};
|
||||
|
||||
const SHARED_SEPARATION_REQ: &str = r#"
|
||||
|
||||
MUST hard edges; preserve no glow/blur beyond the exact visible shape.
|
||||
NEVER keep its parent's background with it.
|
||||
|
||||
UI elements that needs to extract has been marked with GREEN line frames (only for mark purpose, NEVER wrap a frame in your extraction).
|
||||
On some UI elements, there is some PURPLE filled area, they were removed UI elements, reconstruct the background under where they were.
|
||||
"#;
|
||||
pub(super) fn gen_extract_prompt(separation_notes: Vec<SeparationNote>) -> String {
|
||||
let extract_system_prompt = format!(
|
||||
r#"
|
||||
This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction.
|
||||
Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element.
|
||||
|
||||
MUST keep each element at its original position on a transparent canvas.
|
||||
{SHARED_SEPARATION_REQ}
|
||||
here are UI elements to extract:
|
||||
|
||||
"#
|
||||
);
|
||||
let mut result = extract_system_prompt;
|
||||
result.reserve(512);
|
||||
for elem in separation_notes {
|
||||
result.push_str(&elem.as_prompt());
|
||||
result.push('\n');
|
||||
}
|
||||
result
|
||||
}
|
||||
pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String {
|
||||
let binding_system_prompt = format!(
|
||||
r#"
|
||||
You are working under a UI elements separation workflow.
|
||||
You will be given a src UI design image and a processed image, where some ui elements are separated.
|
||||
Here were the separation requirements:
|
||||
```
|
||||
{SHARED_SEPARATION_REQ}
|
||||
```
|
||||
You need to recognize and review the separation:
|
||||
|
||||
these node need handle:
|
||||
"#
|
||||
);
|
||||
let mut result = binding_system_prompt;
|
||||
result.reserve(512);
|
||||
for elem in nodes {
|
||||
result.push_str(&elem.as_prompt());
|
||||
result.push('\n');
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
use super::model::*;
|
||||
use crate::ui_editor::component::{image::ImageComponent, Component};
|
||||
use crate::ui_editor::layout::node::Node;
|
||||
use crate::ui_editor::state::{State, UITree};
|
||||
use crate::ui_editor::utils::NodeId;
|
||||
fn is_unbound_image(node: &Node) -> bool {
|
||||
node.components.iter().any(|component| {
|
||||
matches!(
|
||||
component,
|
||||
Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
..
|
||||
})
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn node_pixel_rect(
|
||||
node: &Node,
|
||||
parent: &crate::ui_editor::layout::dimension::UIRect,
|
||||
ppu: f32,
|
||||
) -> (u32, u32, u32, u32) {
|
||||
let rect = node.layout.transform.resolve(parent);
|
||||
let x = (rect.min.x * ppu).max(0.0).round() as u32;
|
||||
let y = (rect.min.y * ppu).max(0.0).round() as u32;
|
||||
let w = (rect.size.x * ppu).max(0.0).round() as u32;
|
||||
let h = (rect.size.y * ppu).max(0.0).round() as u32;
|
||||
(x, y, w, h)
|
||||
}
|
||||
|
||||
fn node_description(node: &Node) -> String {
|
||||
let name = node.metadata.name.trim();
|
||||
let description = node.metadata.description.trim();
|
||||
match (name.is_empty(), description.is_empty()) {
|
||||
(true, true) => "未命名 UI 图片元素".to_string(),
|
||||
(false, true) => name.to_string(),
|
||||
(true, false) => description.to_string(),
|
||||
(false, false) => format!("{name}:{description}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_todo_nodes(
|
||||
node: &Node,
|
||||
parent: &crate::ui_editor::layout::dimension::UIRect,
|
||||
ppu: f32,
|
||||
output: &mut Vec<SeparationNode>,
|
||||
) {
|
||||
let mut children = Vec::new();
|
||||
let rect = node.layout.transform.resolve(parent);
|
||||
for child in &node.children {
|
||||
collect_todo_nodes(child, &rect, ppu, &mut children);
|
||||
}
|
||||
if is_unbound_image(node) {
|
||||
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
|
||||
output.push(SeparationNode {
|
||||
id: node.id.clone(),
|
||||
global_pos_x_px: x,
|
||||
global_pos_y_px: y,
|
||||
width_px: w,
|
||||
height_px: h,
|
||||
note: SeparationNote {
|
||||
description: node_description(node),
|
||||
text_note: String::new(),
|
||||
},
|
||||
children,
|
||||
rework_count: 0,
|
||||
});
|
||||
} else {
|
||||
output.extend(children);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn construct_separation_state(state: &State) -> SeparationState {
|
||||
app_log!(
|
||||
"ui_separation.tree_construct.start ui_trees={} ui_images={}",
|
||||
state.ui_trees.len(),
|
||||
state.ui_design_images.len()
|
||||
);
|
||||
let unprocessed_trees = state
|
||||
.ui_trees
|
||||
.iter()
|
||||
.filter_map(|tree| {
|
||||
let Some(image) = state.ui_design_images.get(&tree.src_ui_design) else {
|
||||
app_log!(
|
||||
"ui_separation.error stage=tree_construct reason=missing_ui_image image_id={}",
|
||||
tree.src_ui_design.as_str()
|
||||
);
|
||||
return None;
|
||||
};
|
||||
let ppu = image.pixels_per_unit.get();
|
||||
let size = image.pixel_size / ppu;
|
||||
let root_rect =
|
||||
crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size);
|
||||
let mut children = Vec::new();
|
||||
collect_todo_nodes(&tree.root, &root_rect, ppu, &mut children);
|
||||
app_log!(
|
||||
"ui_separation.tree_construct.tree image_id={} todo_nodes={} pixel_width={} pixel_height={}",
|
||||
tree.src_ui_design.as_str(),
|
||||
count_nodes(&children),
|
||||
image.pixel_size.x.round() as u32,
|
||||
image.pixel_size.y.round() as u32
|
||||
);
|
||||
(!children.is_empty()).then(|| SeparationTree {
|
||||
src_ui_design: tree.src_ui_design.clone(),
|
||||
root: SeparationNode {
|
||||
// The synthetic root must never share an ID with a real
|
||||
// UI node. A single-image design may use the original
|
||||
// tree root as an eligible separation leaf.
|
||||
id: NodeId::new(format!("separation-root-{}", uuid::Uuid::new_v4().simple()))
|
||||
.expect("synthetic separation root id is valid"),
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: image.pixel_size.x.max(0.0).round() as u32,
|
||||
height_px: image.pixel_size.y.max(0.0).round() as u32,
|
||||
note: SeparationNote::default(),
|
||||
children,
|
||||
rework_count: 0,
|
||||
},
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let result = SeparationState {
|
||||
schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(),
|
||||
unprocessed_trees,
|
||||
bound: Vec::new(),
|
||||
problematic_nodes: Vec::new(),
|
||||
};
|
||||
app_log!(
|
||||
"ui_separation.tree_construct.completed trees={}",
|
||||
result.unprocessed_trees.len()
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
fn count_nodes(nodes: &[SeparationNode]) -> usize {
|
||||
nodes
|
||||
.iter()
|
||||
.map(|node| 1 + count_nodes(&node.children))
|
||||
.sum()
|
||||
}
|
||||
|
||||
pub fn next_leaf_batch(tree: &SeparationTree) -> Vec<&SeparationNode> {
|
||||
fn leaves<'a>(node: &'a SeparationNode, output: &mut Vec<&'a SeparationNode>) {
|
||||
if node.children.is_empty() {
|
||||
output.push(node);
|
||||
} else {
|
||||
for child in &node.children {
|
||||
leaves(child, output);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
leaves(&tree.root, &mut output);
|
||||
app_log!(
|
||||
"ui_separation.batch_selected image_id={} leaf_nodes={}",
|
||||
tree.src_ui_design.as_str(),
|
||||
output.len()
|
||||
);
|
||||
output
|
||||
}
|
||||
|
||||
pub fn validate_binding_response(
|
||||
response: &BindingResp,
|
||||
batch: &[&SeparationNode],
|
||||
) -> Result<(), String> {
|
||||
app_log!(
|
||||
"ui_separation.binding_validate.start expected_nodes={} decisions={}",
|
||||
batch.len(),
|
||||
response.decisions.len()
|
||||
);
|
||||
let expected = batch
|
||||
.iter()
|
||||
.map(|node| node.id.clone())
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for decision in &response.decisions {
|
||||
let node_id = match decision {
|
||||
BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => {
|
||||
to_node
|
||||
}
|
||||
};
|
||||
if !expected.contains(node_id) {
|
||||
app_log!(
|
||||
"ui_separation.error stage=binding_validate reason=unknown_node node_id={}",
|
||||
node_id.as_str()
|
||||
);
|
||||
return Err(format!("视觉绑定返回了未知节点:{}", node_id.as_str()));
|
||||
}
|
||||
if !seen.insert(node_id.clone()) {
|
||||
app_log!(
|
||||
"ui_separation.error stage=binding_validate reason=duplicate_node node_id={}",
|
||||
node_id.as_str()
|
||||
);
|
||||
return Err(format!("视觉绑定重复返回节点:{}", node_id.as_str()));
|
||||
}
|
||||
if let BindingDecision::NeedRework {
|
||||
problem_description,
|
||||
..
|
||||
} = decision
|
||||
{
|
||||
if problem_description.trim().is_empty() {
|
||||
app_log!(
|
||||
"ui_separation.error stage=binding_validate reason=empty_problem_description node_id={}",
|
||||
node_id.as_str()
|
||||
);
|
||||
return Err("NeedRework 必须包含问题描述".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
if seen.len() != expected.len() {
|
||||
app_log!(
|
||||
"ui_separation.error stage=binding_validate reason=incomplete_coverage expected={} seen={}",
|
||||
expected.len(),
|
||||
seen.len()
|
||||
);
|
||||
return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string());
|
||||
}
|
||||
app_log!(
|
||||
"ui_separation.binding_validate.completed covered_nodes={}",
|
||||
seen.len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,7 @@ use base64::Engine as _;
|
||||
use platform_llm::{LlmClient, LlmError, LlmRunRequest, LlmRunResponse};
|
||||
use schemars::JsonSchema;
|
||||
use std::fs::File;
|
||||
use std::future::Future;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
@@ -25,6 +26,33 @@ pub(crate) async fn request_ui_editor_llm(
|
||||
request_game_creator_llm_text(client, llm, request).await
|
||||
}
|
||||
|
||||
/// 结构化 LLM 请求的小型 repair harness:第一次请求或校验失败后,
|
||||
/// 将错误反馈给模型并只额外重试一次。网络/模型调用本身的错误也会
|
||||
/// 进入第二次请求的反馈文本;调用方负责在第二次失败后决定业务状态。
|
||||
pub(crate) async fn request_with_feedback<T, Request, Fut, Validate>(
|
||||
more_turn: usize,
|
||||
request: Request,
|
||||
validate: Validate,
|
||||
) -> Result<T, String>
|
||||
where
|
||||
Request: Fn(Option<String>) -> Fut,
|
||||
Fut: Future<Output = Result<T, String>>,
|
||||
Validate: Fn(&T) -> Result<(), String>,
|
||||
{
|
||||
let mut feedback = None;
|
||||
for attempt in 0..=more_turn {
|
||||
let result = request(feedback.clone())
|
||||
.await
|
||||
.and_then(|value| validate(&value).map(|_| value));
|
||||
match result {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(error) if attempt < more_turn => feedback = Some(error),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
unreachable!("repair harness always returns within requested turns")
|
||||
}
|
||||
|
||||
pub(crate) fn parse_limited_llm_tool_arguments(
|
||||
arguments: &str,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
@@ -144,6 +172,27 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feedback_harness_zero_more_turn_calls_once_without_feedback() {
|
||||
let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let seen = calls.clone();
|
||||
let result = request_with_feedback(
|
||||
0,
|
||||
move |feedback| {
|
||||
let seen = seen.clone();
|
||||
async move {
|
||||
seen.lock().unwrap().push(feedback);
|
||||
Ok::<_, String>(serde_json::json!({"ok": true}))
|
||||
}
|
||||
},
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("single turn should succeed");
|
||||
assert_eq!(result, serde_json::json!({"ok": true}));
|
||||
assert_eq!(calls.lock().unwrap().as_slice(), &[None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_image_rejects_file_over_five_mib_before_reading() {
|
||||
let directory = tempfile::tempdir().expect("reference image fixture");
|
||||
|
||||
@@ -196,7 +196,7 @@ pub(crate) fn generate_ui_design_code_at(
|
||||
})
|
||||
}
|
||||
|
||||
fn generated_file_stem(asset_id: &str) -> String {
|
||||
pub(crate) fn generated_file_stem(asset_id: &str) -> String {
|
||||
let mut stem = String::new();
|
||||
for character in asset_id.chars() {
|
||||
if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') {
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# Raw GPT Image 2 图片编辑代理
|
||||
|
||||
更新时间:`2026-09-08`
|
||||
|
||||
## 目标
|
||||
|
||||
提供一个由主站客户端调用的独立同步图片编辑代理:
|
||||
|
||||
```text
|
||||
POST /api/raw/v1/images/edit
|
||||
```
|
||||
|
||||
该入口使用登录态 Bearer access token,不进入 External v1 / MCP OpenAPI,不读取或写入画布、项目资源、素材库、OSS 结果或 `external_generation_job`。
|
||||
|
||||
## 请求合同
|
||||
|
||||
请求使用 `multipart/form-data`,不再接受 JSON/base64 入站格式。图片直接作为文件字段上传,避免 base64 膨胀和入站解码;服务端仍在扣费前完成 PNG 完整解码与资源限制校验。
|
||||
|
||||
```text
|
||||
image: <PNG 文件,必填>
|
||||
mask: <PNG 文件,可选>
|
||||
prompt: 修改图片
|
||||
quality: auto
|
||||
background: auto
|
||||
output_format: png
|
||||
width: 1536
|
||||
height: 1024
|
||||
```
|
||||
|
||||
`image` 和 `mask` 必须是 `image/png` 文件字段;服务端不信任客户端文件名,转发时使用固定文件名。空文件、非 PNG 字节、MIME 不匹配或 mask 与 image 尺寸不一致均在扣费前返回 400。`prompt` 必填,UTF-8 原始字节长度不得超过 `4 KiB`;超限在扣费前返回 400。`quality`、`background` 和 `output_format` 采用 GPT Image 模型支持的值。字段不能重复,未知字段拒绝;缺失的必填字段拒绝。
|
||||
|
||||
`width`、`height` 使用严格输出尺寸规则,均在扣费前校验:
|
||||
|
||||
1. 单边最大值为 `3840px`;
|
||||
2. 宽、高均为 `16px` 的倍数;
|
||||
3. 长边 / 短边不超过 `3:1`;
|
||||
4. 总像素范围为 `655360` 至 `8294400`(含边界)。
|
||||
|
||||
校验通过后按整数尺寸发送给 provider,不静默 clamp 或改写调用者尺寸。
|
||||
|
||||
Raw 路由的 multipart body limit 为 `64 MiB`,覆盖图片和文本字段;文件字段由 multipart 解析器直接收集为字节,随后在阻塞线程中完成 PNG 解码。PNG 解码使用与输出合同一致的资源上限:宽高各不超过 `3840`,解码分配不超过 `8294400 × 4` 字节;不再执行 base64 入站解码。
|
||||
|
||||
服务端发送给 `platform-image` 时固定注入:
|
||||
|
||||
```text
|
||||
model = gpt-image-2
|
||||
n = 1
|
||||
```
|
||||
|
||||
请求不暴露 `model`、`n`、`response_format`、`style`、`user` 或 `output_compression`。
|
||||
|
||||
## 成功响应合同
|
||||
|
||||
响应始终为 JSON,响应只保留 `data` 字段,图片内容只以 base64 返回:
|
||||
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"b64_json": "<base64>"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`data` 保持数组形状,即使服务端固定 `n=1`。响应不重复返回请求参数,不返回 URL、资源 ID、任务 ID、provider 原始 JSON 或 editor 字段。
|
||||
|
||||
## 预检查与计费事务
|
||||
|
||||
所有 multipart 字段、图片结构和 provider 参数检查必须在扣费前完成。预检查失败直接返回 4xx,不产生钱包流水,也不调用 provider。
|
||||
|
||||
检查通过后,api-server 进入现有资产操作计费边界,通过 SpacetimeDB 钱包事务 procedure 原子完成:
|
||||
|
||||
1. 按现有图片编辑算法解析价格:GPT Image 2 长边不超过 1536 使用 1K 价格,否则使用 2K 价格;当前默认价格为 3 / 5 泥点;
|
||||
2. 以认证后的用户、`raw-image-edit` 命名空间和请求 ID 组成幂等扣费流水 ID;
|
||||
3. 原子扣除用户泥点并写入 `asset_operation_consume` 流水。
|
||||
|
||||
provider 调用在 SpacetimeDB 事务之外执行。失败时由现有计费边界把幂等退款事实写入 SpacetimeDB refund outbox,再由 worker 完成退款。
|
||||
|
||||
TODO:新增 raw 操作持久化状态,将“创建 raw 操作事实 + 扣费”收入同一事务,并由恢复 worker 对“已扣费但未收口”状态自动退款,填补进程在扣费后、写入 refund outbox 前崩溃的窗口。
|
||||
|
||||
raw 操作使用独立的 operation / ledger 命名空间,例如 `raw-image-edit`,不能复用编辑器资源 ID、编辑器任务 ID 或 `external_generation_job`。
|
||||
|
||||
## Provider 边界
|
||||
|
||||
`platform-image` 保留 VectorEngine 协议细节。raw handler 只负责:认证、multipart 字段解析、PNG 预检查、计费编排和响应映射。provider 请求仍由 `platform-image` 统一构造,并携带 `model`、`n`、`quality`、`background`、`output_format`、尺寸及图片参考字节。
|
||||
|
||||
provider 响应只提取并透传 `data[].b64_json` 字符串,不在服务端解码图片 base64,也不读取或回传 provider 的 `output_format`(该字段只是请求参数回显)。发送、响应读取、上游状态、响应解析和缺图失败必须生成 `PlatformImageFailureAudit`,由 api-server 写入现有外部 API 失败审计链;成功结果同时写入统一的 `external_generation_run` 追踪事件。raw handler 只将上游 `b64_json` 原样写入 `data[].b64_json`。
|
||||
|
||||
## 代码拆分
|
||||
|
||||
- `server-rs/crates/api-server/src/raw_image.rs`:独立路由 handler、multipart 字段解析、请求/响应 DTO、PNG 输入校验、预检查和 raw billing 编排。
|
||||
- `server-rs/crates/platform-image/src/vector_engine/raw_edit.rs`:raw 编辑选项、严格尺寸校验、独立 provider 请求映射和 `b64_json` 响应透传;不复用现有 editor 图片编辑 client 或其 multipart transport。
|
||||
- `server-rs/crates/api-server/src/modules/raw.rs`:只注册 `/api/raw/v1/images/edit` 并挂载 Bearer middleware。
|
||||
|
||||
不修改 External v1 OpenAPI;不在 `external_editor_api.rs`、编辑器项目模块或外部生成 worker 中增加 raw 分支。
|
||||
|
||||
## 验收
|
||||
|
||||
- 未认证请求被 Bearer middleware 拒绝。
|
||||
- 预检查失败时钱包无扣费、provider 无请求。
|
||||
- 成功响应严格只包含 `data[].b64_json`。
|
||||
- provider 失败时 raw 操作失败事务产生可恢复退款事实。
|
||||
- raw 请求不创建 `external_generation_job`,不写 editor project/resource/asset/OSS。
|
||||
- 运行 api-server 与 platform-image 定向测试、`npm run check:encoding` 和 `git diff --check`。
|
||||
@@ -0,0 +1,81 @@
|
||||
# UI 编辑器自动分离工作流
|
||||
|
||||
更新时间:`2026-09-08`
|
||||
|
||||
## 目标
|
||||
|
||||
将 UI 编辑器现有“用户先提供独立图片/图标,再执行组件绑定”的入口替换为自动分离:结构识别阶段直接返回可渲染组件草稿,分离阶段按整页叶节点批次调用图片编辑模型,再由视觉模型确认处理图中的区域与目标节点。
|
||||
|
||||
## 识别结果
|
||||
|
||||
- `recognize` 返回完整 `Node.components` 草稿,不再要求用户先导入独立素材。
|
||||
- `components` 为空表示纯节点。
|
||||
- `ImageComponent.target_graphic = None` 表示图片组件等待分离结果回填;它不是“明确没有图片”。
|
||||
- 当前约束:需要分离的节点最多包含一个 `ImageComponent`,回填暂使用该节点的第一个图片组件。
|
||||
- 组件容器“一种组件类型最多一个”的正式重构列为 TODO;当前 `Vec<Component>` 仅按上述约束使用。
|
||||
- 组件草稿直接保存在正式 UI Node 中;临时 separation tree 不复制组件。
|
||||
|
||||
## Separation tree
|
||||
|
||||
- recognition 完成后由 UI tree 构造临时 separation tree。
|
||||
- 纯节点、纯 Text 节点和不需要切图的节点在构造时过滤;被过滤节点的可处理 children 向上透传。
|
||||
- separation tree 只保留真实待处理节点。
|
||||
- 一个 batch 是整页当前所有互不重叠叶节点。
|
||||
- 一个 batch 的最小处理单元是:一次 image-edit + 一次 visual binding。
|
||||
- batch 成功后从 pending tree 移除对应叶节点,并把结果放入 bound 容器;失败节点移入 problematic 容器,流程继续消费剩余树。
|
||||
- 不额外维护节点状态枚举;节点是否仍在 pending tree、`rework_count` 和 problematic 容器共同表达状态。
|
||||
|
||||
## 图片编辑与视觉绑定
|
||||
|
||||
- image-edit 使用源 UI design 图片及由 Rust 生成的绿色标记/紫色重建输入。
|
||||
- 请求尺寸始终使用源 UI design 尺寸;Raw GPT Image 2 API 保证返回相同尺寸,客户端不额外做尺寸拒绝检查。
|
||||
- 标记图构建、处理图解码/写入和 cut 裁切属于本地 CPU/文件操作,放入独立的
|
||||
`spawn_blocking` 任务;image-edit 与 visual binding 网络请求仍运行在 async future 中。
|
||||
- 视觉 binding 输入源图与处理图,必须为当前 batch 每个节点恰好返回一次 `Ok` 或 `NeedRework`。
|
||||
- `Ok` 返回 `NodeId + BindingArea`;Rust 仅校验 NodeId、区域边界和非零尺寸,不检查与原节点框的偏差,也不要求区域不重叠。
|
||||
- `NeedRework` 携带短问题描述。结构化工具调用失败时使用可复用 repair harness,把错误反馈给模型并额外请求一次;image-edit 不使用该 harness。
|
||||
- 达到模块级重做常量后,节点移入 problematic;不中断整条工作流,最终统一通知用户。
|
||||
- 父节点背景重建由 image-edit 模型完成,不由 Rust 硬编码重建算法完成。
|
||||
|
||||
## 临时 sidecar
|
||||
|
||||
- separation 状态不写入 UI JSON,也不进入 manifest。
|
||||
- sidecar 目录按 UI manifest `asset_id` 生成,复用 `generated_file_stem(asset_id)` 的安全字符替换和 SHA-256 摘要规则,位于项目 `ui/` 下。
|
||||
- 目录只保存一份当前 separation state,而不是每 batch 一个状态文件。
|
||||
- state 文件只保留 `schema_version`、pending tree、bound 结果和 problematic 节点,不重复保存 `projectId / assetId / uiStateRevision`。
|
||||
- sidecar 只在 separation 未完成期间存在;完成后删除 state JSON。
|
||||
- 当前只持久化已经完成的 batch;正在执行 batch 的恢复语义列 TODO。
|
||||
- 临时图片可跨重启保留。raw image-edit 返回图、绿色/紫色标记图、处理图和 cut 图片当前都保留用于 debug;理论上只应在内存中,清理/归档策略列 TODO。
|
||||
- 并发边界:当前由前端 `isSeparating` 与 `runWithStateLocked` 保证同一 UI 编辑会话
|
||||
同时只有一次 separation。sidecar 是临时恢复状态,不是正式 UI 资产真相,不参与
|
||||
manifest 或项目 revision,因此当前不额外持有项目写锁;若未来支持多窗口/多进程并发,
|
||||
再增加按 UI asset 的 sidecar 进程级锁。
|
||||
|
||||
## bound 与 problematic
|
||||
|
||||
- bound 结果仅保存 `NodeId + cut_image_path`,不保存 `BindingArea` 或 component kind。
|
||||
- problematic 记录原始 NodeId、问题描述和 `rework_count`;原始 UI Node 保留不变。
|
||||
- `SeparationDTO` 不返回计数字段,只返回 `bound_nodes` 与 `problematic_nodes`。
|
||||
- separation Rust 流程不自动登记项目级 SpriteAsset。
|
||||
- 前端调用方消费 `SeparationDTO.bound_nodes`,复制/登记 cut 图片为项目级 SpriteAsset,再回填对应 Node 的第一个 Image component。
|
||||
- 每次重做产生新的 SpriteAssetId,不假设 NodeId 到 SpriteAssetId 的稳定映射。
|
||||
- sidecar 中的图片保留,正式 SpriteAsset 的最终清理策略列 TODO。
|
||||
|
||||
## 重启与 Raw GPT Image 2
|
||||
|
||||
- 已保存的 separation state 是跨重启继续工作的最小单位;重启后从上一个已保存 batch 的状态继续。
|
||||
- 当前执行中的 batch 是否持久化、以及如何避免 image-edit 成功后在 patch 前崩溃导致重复调用,列为 TODO。
|
||||
- Raw endpoint 每次 HTTP 调用都是一次新操作;客户端不保存或复用 raw operation ID,不实现第二套本地幂等账本。
|
||||
- 后端 raw operation 的持久状态与扣费后崩溃恢复窗口,遵循 Raw GPT Image 2 方案中的独立 TODO。
|
||||
|
||||
## TODO
|
||||
|
||||
- `Vec<Component>` 重构为一种组件类型最多一个的容器。
|
||||
- 当前第一个 Image component 回填规则的正式替代方案。
|
||||
- 正在执行 batch 的持久化和恢复。
|
||||
- 前端复制、登记 SpriteAsset、回填 State 的精确 IPC/提交合同。
|
||||
- 临时图片清理/归档策略。
|
||||
- 手动抠图能力。
|
||||
- problematic 对更高层 workflow 完成门禁的最终定义。
|
||||
- separation workflow 与 manifest/stage 的接入。
|
||||
- Raw GPT Image 2 后端 raw operation 持久状态及恢复 worker。
|
||||
Generated
+25
@@ -470,6 +470,7 @@ dependencies = [
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"multer",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"serde_core",
|
||||
@@ -2924,6 +2925,23 @@ dependencies = [
|
||||
"pxfm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multer"
|
||||
version = "3.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-util",
|
||||
"http",
|
||||
"httparse",
|
||||
"memchr",
|
||||
"mime",
|
||||
"spin",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "naga"
|
||||
version = "27.0.3"
|
||||
@@ -4074,6 +4092,7 @@ dependencies = [
|
||||
"image",
|
||||
"platform-oss",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -5628,6 +5647,12 @@ dependencies = [
|
||||
"tokio-tungstenite 0.27.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
|
||||
|
||||
[[package]]
|
||||
name = "sse-stream"
|
||||
version = "0.2.5"
|
||||
|
||||
@@ -7,7 +7,7 @@ license.workspace = true
|
||||
[dependencies]
|
||||
aes = { workspace = true }
|
||||
async-stream = { workspace = true }
|
||||
axum = { workspace = true, features = ["ws"] }
|
||||
axum = { workspace = true, features = ["ws", "multipart"] }
|
||||
base64 = { workspace = true }
|
||||
cbc = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
|
||||
@@ -50,6 +50,7 @@ pub fn build_router(state: AppState) -> Router {
|
||||
.merge(modules::platform::router(state.clone()))
|
||||
.merge(modules::external_generation::router(state.clone()))
|
||||
.merge(modules::platform_support::router(state.clone()))
|
||||
.merge(modules::raw::router(state.clone()))
|
||||
.merge(crate::error_reports::router(state.clone()))
|
||||
.route(
|
||||
"/api/profile/recharge/wechat/notify",
|
||||
|
||||
@@ -67,6 +67,7 @@ mod profile_identity;
|
||||
mod profile_recharge_expiration_listener;
|
||||
mod profile_recharge_refund_reconciliation;
|
||||
mod prompt;
|
||||
mod raw_image;
|
||||
mod refresh_session;
|
||||
mod registration_reward;
|
||||
mod request_context;
|
||||
|
||||
@@ -10,3 +10,4 @@ pub mod internal;
|
||||
pub mod platform;
|
||||
pub mod platform_support;
|
||||
pub mod profile;
|
||||
pub mod raw;
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
use axum::{Router, extract::DefaultBodyLimit, middleware, routing::post};
|
||||
|
||||
use crate::{auth::require_bearer_auth, raw_image::edit_raw_image, state::AppState};
|
||||
|
||||
const RAW_IMAGE_EDIT_BODY_LIMIT_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
pub fn router(state: AppState) -> Router<AppState> {
|
||||
Router::new().route(
|
||||
"/api/raw/v1/images/edit",
|
||||
post(edit_raw_image)
|
||||
.route_layer(middleware::from_fn_with_state(state, require_bearer_auth))
|
||||
.layer(DefaultBodyLimit::max(RAW_IMAGE_EDIT_BODY_LIMIT_BYTES)),
|
||||
)
|
||||
}
|
||||
@@ -414,7 +414,7 @@ impl OpenAiImageSettings {
|
||||
self
|
||||
}
|
||||
|
||||
fn provider_settings(&self) -> VectorEngineImageSettings {
|
||||
pub(crate) fn provider_settings(&self) -> VectorEngineImageSettings {
|
||||
VectorEngineImageSettings {
|
||||
base_url: self.base_url.clone(),
|
||||
api_key: self.api_key.clone(),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,7 @@ base64 = { workspace = true }
|
||||
curl = { workspace = true }
|
||||
image = { workspace = true, features = ["jpeg", "png", "webp"] }
|
||||
reqwest = { workspace = true, features = ["json", "multipart", "rustls-tls"] }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "time"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -10,14 +10,16 @@ pub use pixel_art_snapper::{
|
||||
};
|
||||
pub use vector_engine::{
|
||||
DownloadedImage, GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, GeneratedImages, NANOBANANA_2_MODEL,
|
||||
PlatformImageError, PlatformImageFailureAudit, PlatformImageStatusHint, ReferenceImage,
|
||||
PlatformImageError, PlatformImageFailureAudit, PlatformImageStatusHint,
|
||||
RAW_IMAGE_DIMENSION_ALIGNMENT, RAW_IMAGE_MAX_EDGE, RAW_IMAGE_MAX_PIXELS, RAW_IMAGE_MIN_PIXELS,
|
||||
RawImageEditDimensionError, RawImageEditOptions, RawImageEditResult, ReferenceImage,
|
||||
VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER, VectorEngineImageSettings,
|
||||
build_vector_engine_image_http_client, build_vector_engine_image_request_body,
|
||||
build_vector_engine_nanobanana_generate_content_request_body, create_vector_engine_image_edit,
|
||||
create_vector_engine_image_edit_with_references,
|
||||
create_vector_engine_image_edit_with_references_and_model,
|
||||
create_vector_engine_image_generation, create_vector_engine_image_generation_with_model,
|
||||
create_vector_engine_nanobanana_generate_content, download_remote_image,
|
||||
vector_engine_images_edit_url, vector_engine_images_generation_url,
|
||||
vector_engine_nanobanana_generate_content_url,
|
||||
create_vector_engine_nanobanana_generate_content, create_vector_engine_raw_image_edit,
|
||||
download_remote_image, validate_raw_image_edit_dimensions, vector_engine_images_edit_url,
|
||||
vector_engine_images_generation_url, vector_engine_nanobanana_generate_content_url,
|
||||
};
|
||||
|
||||
@@ -5,3 +5,8 @@ pub const VECTOR_ENGINE_GPT_IMAGE_2_MODEL: &str = GPT_IMAGE_2_MODEL;
|
||||
pub const VECTOR_ENGINE_PROVIDER: &str = "vector-engine";
|
||||
pub const VECTOR_ENGINE_IMAGE_EDIT_MAX_REFERENCE_IMAGES: usize = 5;
|
||||
pub const VECTOR_ENGINE_NANOBANANA_MAX_REFERENCE_IMAGES: usize = 14;
|
||||
|
||||
pub(crate) const GPT_IMAGE_2_MIN_PIXELS: u64 = 655_360;
|
||||
pub(crate) const GPT_IMAGE_2_MAX_PIXELS: u64 = 8_294_400;
|
||||
pub(crate) const GPT_IMAGE_2_MAX_EDGE: u32 = 3_840;
|
||||
pub(crate) const GPT_IMAGE_2_DIMENSION_ALIGNMENT: u32 = 16;
|
||||
|
||||
@@ -6,6 +6,7 @@ mod curl_transport;
|
||||
mod error;
|
||||
mod image_source;
|
||||
mod payload;
|
||||
mod raw_edit;
|
||||
mod request;
|
||||
mod response;
|
||||
mod transport;
|
||||
@@ -25,6 +26,11 @@ pub use constants::{
|
||||
};
|
||||
pub use error::{PlatformImageError, PlatformImageStatusHint};
|
||||
pub use image_source::download_remote_image;
|
||||
pub use raw_edit::{
|
||||
RAW_IMAGE_DIMENSION_ALIGNMENT, RAW_IMAGE_MAX_EDGE, RAW_IMAGE_MAX_PIXELS, RAW_IMAGE_MIN_PIXELS,
|
||||
RawImageEditDimensionError, RawImageEditOptions, create_vector_engine_raw_image_edit,
|
||||
validate_raw_image_edit_dimensions,
|
||||
};
|
||||
pub use request::{
|
||||
build_vector_engine_image_request_body, build_vector_engine_image_request_body_with_model,
|
||||
build_vector_engine_nanobanana_generate_content_request_body, normalize_image_size_for_model,
|
||||
@@ -32,4 +38,6 @@ pub use request::{
|
||||
vector_engine_nanobanana_generate_content_url,
|
||||
};
|
||||
pub use transport::build_vector_engine_image_http_client;
|
||||
pub use types::{DownloadedImage, GeneratedImages, ReferenceImage, VectorEngineImageSettings};
|
||||
pub use types::{
|
||||
DownloadedImage, GeneratedImages, RawImageEditResult, ReferenceImage, VectorEngineImageSettings,
|
||||
};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user