From 55469a2198927eda6eb98fa72e24509ab0194bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=89=A9?= <253518756@qq.com> Date: Mon, 22 Jun 2026 19:13:37 +0800 Subject: [PATCH] UI extraction selection + local FFmpeg dev env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Require an explicit red-box selection before extracting UI-design assets: add rectangular/ellipse/brush selection UI, render red-outline marks into a marked Data URL on the client, and update the extraction prompt to "仅提取被红色框框选的素材并整理成spritesheet". Add frontend overlay/raster models and tests to support the selection flow and adjust numerous image-editor tests and views to wire up the selection lifecycle. On the dev ops side, enhance scripts/dev to detect a local %LOCALAPPDATA%/Genarrative/ffmpeg/bin installation on Windows, prepend it to PATH and inject CHARACTER_ANIMATION_FFMPEG_PATH/FFPROBE_PATH (with tests), and export the helper. Server config now prefers GENARRATIVE_*-prefixed FFmpeg env vars and includes a Rust test to validate reading the prefixed variables. Update docs to describe the new UI extraction behavior and local FFmpeg handling. --- .../shared-memory/decision-log.md | 2 +- docs/project-memory/shared-memory/pitfalls.md | 8 + ...架构】图片画布编辑器MVP接入方案-2026-06-11.md | 4 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 + ...】生成类面板Lovart统一改造方案-2026-06-17.md | 6 +- ...辑器】画板UI设计图生成入口设计-2026-06-17.md | 9 +- scripts/dev.mjs | 119 +++++- scripts/dev.test.ts | 56 ++- server-rs/crates/api-server/src/config.rs | 50 ++- .../crates/api-server/src/editor_project.rs | 4 +- ...CanvasEditorGenerationIntegration.test.tsx | 87 ++++- .../ImageCanvasEditorShellView.test.tsx | 56 ++- .../image-editor/ImageCanvasEditorView.tsx | 28 ++ .../ImageCanvasGenerationModel.ts | 2 +- .../image-editor/ImageCanvasStageView.tsx | 65 +++- .../ImageCanvasUiAssetExtractionModel.ts | 131 +++++++ ...anvasUiAssetExtractionOverlayView.test.tsx | 125 +++++++ ...mageCanvasUiAssetExtractionOverlayView.tsx | 287 +++++++++++++++ ...ImageCanvasUiAssetExtractionRasterModel.ts | 141 +++++++ ...anvasGenerationSubmissionWorkflow.test.tsx | 174 ++++++++- ...ImageCanvasGenerationSubmissionWorkflow.ts | 45 ++- .../useImageCanvasGenerationWorkflow.test.tsx | 123 +++++++ .../useImageCanvasGenerationWorkflow.ts | 346 +++++++++++++++++- src/index.css | 99 +++++ .../image-editor/editorProjectClient.test.ts | 4 +- 25 files changed, 1924 insertions(+), 49 deletions(-) create mode 100644 src/components/image-editor/ImageCanvasUiAssetExtractionModel.ts create mode 100644 src/components/image-editor/ImageCanvasUiAssetExtractionOverlayView.test.tsx create mode 100644 src/components/image-editor/ImageCanvasUiAssetExtractionOverlayView.tsx create mode 100644 src/components/image-editor/ImageCanvasUiAssetExtractionRasterModel.ts diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index dd22306ac..7107f3bd3 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -59,7 +59,7 @@ ## 2026-06-18 图片画布 UI 设计图提取素材保留图集 - 背景：UI 设计图需要从成图中继续抽取可复用独立素材；原图标素材生成只把拆分后的图标放入画布，spritesheet 原图没有保留，后续追溯和二次切图不方便。 -- 决策：`assetKind="ui-design"` 图层浮动工具栏新增 `提取素材`，固定调用 `/api/editor/ui-designs/assets/extractions`，后端固定 `gpt-image-2` 和提示词 `提取画面中的所有独立并整理成spritesheet`，返回结构复用图标 spritesheet 响应。图标生成与 UI 提取都必须把 spritesheet 图集作为 `assetKind="icon-spritesheet"` 图层放到画布，再放拆分后的 `assetKind="icon"` 素材。 +- 决策：`assetKind="ui-design"` 图层浮动工具栏新增 `提取素材`，点击后先进入红框素材框选编辑态，默认矩形框选，并支持椭圆框选和画笔自由框选。至少存在一个框选区域后才能提交；前端把红色轮廓绘入原 UI 设计图并将合成图作为 `/api/editor/ui-designs/assets/extractions` 的参考图。后端固定 `gpt-image-2` 和提示词 `仅提取被红色框框选的素材并整理成spritesheet`，返回结构复用图标 spritesheet 响应。图标生成与 UI 提取都必须把 spritesheet 图集作为 `assetKind="icon-spritesheet"` 图层放到画布，再放拆分后的 `assetKind="icon"` 素材。 - 影响范围：图片画布浮动工具栏、编辑器图片生成 BFF、`platform-image` 图集连通域拆分、画布图层类型和编辑器文档。 - 验证方式：运行图片画布工具栏 / 图集落层 / 生成提交相关前端测试，`cargo test -p platform-image generated_asset_sheets --manifest-path server-rs/Cargo.toml`，以及 `cargo test -p api-server editor_ui_design_asset_extraction_prompt_is_fixed --manifest-path server-rs/Cargo.toml`。 - 关联文档：`docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md`、`docs/【编辑器】画板图标素材生成入口设计-2026-06-15.md`。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 627dedb8c..fa1b8f645 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -95,6 +95,14 @@ - 验证：`cargo test -p api-server editor_character_animation --manifest-path server-rs/Cargo.toml`，其中 `editor_character_animation_extracts_final_sample_from_short_video` 应覆盖本机 FFmpeg 8 的 0 帧回归。 - 关联：`server-rs/crates/api-server/src/character_animation_assets.rs`、`docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md`。 +## Windows 本地角色动画抽帧找不到 ffmpeg 先查 dev 子进程环境 + +- 现象：画板角色动画抽帧报 `抽取动作视频帧失败：无法启动进程 ffmpeg：program not found(requestId:...)`，但新开的 PowerShell 里 `ffmpeg -version` 正常。 +- 原因：长期运行的 `api-server` 可能是在安装 FFmpeg 或更新用户 Path 之前启动的，子进程不会自动继承后续写入的用户环境变量。 +- 处理：Windows 本地默认把 FFmpeg 安装到 `%LOCALAPPDATA%\Genarrative\ffmpeg\bin`，并确保用户 Path 包含该目录；`npm run dev` / `npm run dev:api-server` 会在启动 `api-server` 时自动注入该目录和 `CHARACTER_ANIMATION_FFMPEG_PATH` / `CHARACTER_ANIMATION_FFPROBE_PATH` 绝对路径。修复后需要重启 `api-server`，不能只刷新浏览器。 +- 验证：`where ffmpeg`、`where ffprobe` 能找到本地安装；`npm run test -- scripts/dev.test.ts -t "FFmpeg"`；重启 `npm run dev:api-server` 后访问 `/healthz`。 +- 关联：`scripts/dev.mjs`、`server-rs/crates/api-server/src/config.rs`、`server-rs/crates/api-server/src/character_animation_assets.rs`。 + ## 图片编辑器生成长请求完成态必须由后端写入画布 - 现象：画板角色形象等生成请求已经在服务端返回 `200`，OSS 中也已有 `generated-character-drafts/.../image.png`，但用户刷新或页面重载后仍看到旧生成卡片停在“生成中”。 diff --git a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md index d3aa0f829..99142126f 100644 --- a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md +++ b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md @@ -67,7 +67,7 @@ - `DELETE /api/editor/assets/{assetId}`：删除素材。已放入画布的 project resource 不被级联删除，避免旧画布丢图。 - `POST /api/editor/images/generations`：按提示词调用 VectorEngine 生成图片；角色生成可携带 `model`、`aspectRatio`、`imageSize` 和 `referenceImageSrcs`。`nanobanana2` 参考图作为 `inline_data` 进入 `generateContent`，`gpt-image-2` 参考图进入 edits。携带参考图的快速编辑也走该接口，前端必须把参考图源预读成图片 Data URL 后放入 `referenceImageSrcs`；请求可携带 `projectId`、`assetFolderId`、`assetKind`、`generationInputs` 和 `sourceResourceId`，后端生成成功后创建 project resource / 账号素材并在响应中返回 resource / asset 快照。 - `POST /api/editor/icon-spritesheets/generations`：按图标规范图和素材描述数组生成 spritesheet，再由后端切分为独立透明图标。请求支持 `model`、`aspectRatio`、`imageSize`、`priceMudPoints`、`projectId`、`assetFolderId` 和 `generationInputs`；`priceMudPoints` 必须来自编辑器生成计费配置中对应生图模型的尺寸档位（如 `nanobanana2` 的 `0.5K / 1K / 2K` 或 `gpt-image-2` 的 `1K / 2K`），后端用 `editor_generation_config` 校验后才调用上游；`nanobanana2` 走原生 `generateContent` 并写入 `generationConfig.imageConfig.aspectRatio/imageSize`，`0.5K` 传 `"512"`；`gpt-image-2` 走 `/v1/images/edits`。后端把 spritesheet 和拆分后的 icon 都保存为 project resource / 账号素材，并随响应返回对应快照。 -- `POST /api/editor/ui-designs/assets/extractions`：以 UI 设计图 Data URL 作为参考图，固定 `gpt-image-2` 和提示词 `提取画面中的所有独立并整理成spritesheet` 生成素材 spritesheet，再按连通域自动拆分为 `素材 1..N`，返回结构复用图标 spritesheet 响应。请求可携带 `projectId`、`assetFolderId`、`generationInputs` 和 `spritesheetLabel`，后端保存 spritesheet / 拆分素材并返回对应 resource / asset 快照；前端必须把 spritesheet 原图与拆分素材都加入画布。 +- `POST /api/editor/ui-designs/assets/extractions`：以前端已绘入红色框选轮廓的 UI 设计图 Data URL 作为参考图，固定 `gpt-image-2` 和提示词 `仅提取被红色框框选的素材并整理成spritesheet` 生成素材 spritesheet，再按连通域自动拆分为 `素材 1..N`，返回结构复用图标 spritesheet 响应。请求可携带 `projectId`、`assetFolderId`、`generationInputs` 和 `spritesheetLabel`，后端保存 spritesheet / 拆分素材并返回对应 resource / asset 快照；前端必须把 spritesheet 原图与拆分素材都加入画布。 - `POST /api/editor/images/edits`：按提示词和当前图片 Data URL 调用 VectorEngine edits，返回新的生成图片元数据；请求携带 project / asset 上下文时由后端创建新 resource / asset，前端只消费响应快照。 - `POST /api/editor/videos/generations`：按视频描述、模型、比例、时长、分辨率、模式、声音、默认联网搜索标记和泥点价格生成视频。前端可选模型为 `seedance2.0-fast`、`seedance2.0`、`kling3.0`、`kling3.0-omni`，默认 `seedance2.0-fast`；后端必须将 `seedance2.0-fast` 映射到 `doubao-seedance-2-0-fast-260128`，将 `seedance2.0` 映射到 `doubao-seedance-2-0-260128`，两者不得混用。后端允许 6 类比例、4 到 15 秒整数、`480p / 720p / 1080p`，并拒绝 `seedance2.0-fast + 1080p`；`sound=on/off` 映射 Ark `generate_audio=true/false`。后端复用 Ark / VectorEngine content generation task 轮询链路，下载最终视频并持久化到 OSS，返回 `videoSrc`、尺寸、prompt、model、provider、taskId、durationSeconds、resolution 和 `priceMudPoints`。 @@ -97,7 +97,7 @@ - 快速编辑面板底部必须显示当前选择的比例 / 尺寸和模型，参数按钮可展开修改且视觉对齐其它可编辑面板；参考图请求顺序为额外参考图在前、原图作为隐式最后一张提交，信息面板输入快照只展示用户额外选择的参考图。 - 点击生成、生成规范、生成角色形象或生成图标素材后创建的占位图可继续保留；点击画布空白区域让当前图片或占位图失焦时，关闭当前生成面板并移除图片选中样式，但不删除占位图本身。 - 生成资源显示元数据按钮，元数据窗口展示来源、生成输入快照、model、task、Resolution 和 OSS 引用；生成输入快照只包含用户面板输入和参考图行引用，不包含后端拼接 Prompt，不再展示独立 Size 字段，也不渲染参考图 Data URL 缩略图。 -- 图片选中后的浮动工具栏按钮顺序固定为：快速编辑、分割线、裁扩按钮、去除背景按钮、UI设计图专属提取素材、角色图专属生成动画、分割线、重绘、下载按钮。裁扩通过画布边界拖拉完成，不再展示四边数值输入；默认自由比例，选择固定比例后拖拉边界保持对应比例，完成后在原素材旁边新增裁扩结果图层，扩展区域透明填充。去除背景使用前端轻量去背并回写为新的工程资源快照；前端轻量去背必须复用共享 `chromaKey` 的绿幕 / 近白背景模型，不再只按四角颜色扩散，避免纯白角色主体或主体内部浅色区域被扣碎。UI设计图的提取素材固定走 `gpt-image-2` 和提示词 `提取画面中的所有独立并整理成spritesheet`，生成的 spritesheet 原图和拆分后的独立素材都作为画布图层保留。 +- 图片选中后的浮动工具栏按钮顺序固定为：快速编辑、分割线、裁扩按钮、去除背景按钮、UI设计图专属提取素材、角色图专属生成动画、分割线、重绘、下载按钮。裁扩通过画布边界拖拉完成，不再展示四边数值输入；默认自由比例，选择固定比例后拖拉边界保持对应比例，完成后在原素材旁边新增裁扩结果图层，扩展区域透明填充。去除背景使用前端轻量去背并回写为新的工程资源快照；前端轻量去背必须复用共享 `chromaKey` 的绿幕 / 近白背景模型，不再只按四角颜色扩散，避免纯白角色主体或主体内部浅色区域被扣碎。UI设计图的提取素材必须先进入红框素材框选状态，支持矩形、椭圆和画笔自由框选；至少框选一个区域后才可提交，前端把红色轮廓绘入原图后固定走 `gpt-image-2` 和提示词 `仅提取被红色框框选的素材并整理成spritesheet`，生成的 spritesheet 原图和拆分后的独立素材都作为画布图层保留。 - 重绘生成资源后，右侧出现新生成结果图层，并自动 fit 原图 + 新图，且重绘面板保持打开。 - 快速编辑 / 重绘站内 public 示例图、历史 generated 图或 OSS generated 图时，前端先读取成 `data:image/*;base64,...` 再提交，后端不得再收到 `/creation-type-references/*`、`/generated-*` 或 OSS URL 作为 `referenceImageSrcs/sourceImageSrc`。 - 快速编辑的额外参考图同样必须在前端读取成图片 Data URL 后提交；后端 `referenceImageSrcs` 上限为 9 张，承载 8 张额外参考图加 1 张原图。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 9d02a67b0..fc69301be 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -51,6 +51,8 @@ Linux 本机多用户并发开发时，`npm run dev` 和 `npm run dev:*` 单模 Windows 本地 `npm run dev` / `npm run dev:api-server` 会用空的 `RUSTC_WRAPPER` / `CARGO_BUILD_RUSTC_WRAPPER` 覆盖 `server-rs/.cargo/config.toml` 里的 `sccache`，从而直连真实 `rustc`。不要把 wrapper 绕过值写成 `rustc`；Cargo 会按 wrapper 协议调用 `rustc <真实rustc路径> - ...`，最终报 `multiple input filenames provided` 并导致 api-server 无法启动。排查本地启动失败时，先看 dev 日志是否出现该错误，再确认脚本注入的 wrapper 为空。 +Windows 本地如果已在 `%LOCALAPPDATA%\Genarrative\ffmpeg\bin` 安装 FFmpeg，`npm run dev` / `npm run dev:api-server` 会自动把该目录加入本次 `api-server` 子进程 `Path`，并注入 `CHARACTER_ANIMATION_FFMPEG_PATH` / `CHARACTER_ANIMATION_FFPROBE_PATH` 的绝对路径。这样即使外层终端或长期运行的 dev 进程是在安装 FFmpeg 之前启动，角色动画抽帧也不会继续因为 `ffmpeg: program not found` 失败；若手动配置了上述环境变量或 `GENARRATIVE_CHARACTER_ANIMATION_*` 前缀变量，显式配置优先。 + 开发态 `npm run dev` 与 `npm run dev:api-server` 会默认注入 `GENARRATIVE_DEV_PASSWORD_ENTRY_AUTO_REGISTER_ENABLED=true`，因此密码登录在本地开发环境可直接注册未知手机号账号；生产环境仍按 `api-server` 配置默认关闭该开关。 本地 `npm run dev` 和 `npm run dev:api-server` 默认保留 inline 开发体验：未显式设置 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue` 时，外部生成 handler 会同步复用 worker executor，完成后返回 `completed`，便于快速确认 provider、OSS 和 SpacetimeDB 写回链路。inline 不创建 `external_generation_job`，也不能验证 worker lease、队列等待展示或动态扩缩容。 diff --git a/docs/【编辑器】生成类面板Lovart统一改造方案-2026-06-17.md b/docs/【编辑器】生成类面板Lovart统一改造方案-2026-06-17.md index f720b919f..56cbb40c2 100644 --- a/docs/【编辑器】生成类面板Lovart统一改造方案-2026-06-17.md +++ b/docs/【编辑器】生成类面板Lovart统一改造方案-2026-06-17.md @@ -41,6 +41,7 @@ 8. 多输入框面板必须保留每个字段标题和输入框边界，例如生成规范。图标素材生成不再使用多描述列表，改为复用角色形象生成面板同款单文本输入框。 9. 生成规范下的角色规范、图标规范和自定义规范都使用同一生成类 shell：首行参考图区域、中央字段区、底部生成按钮区，不再出现缺首行参考区或单独 footer 样式。 10. 快速编辑最多允许额外绑定 8 张参考图；原图始终自动作为最后一张隐式参考图提交，不占用额外参考图额度，也不在参考图条里固定展示 `图x`。 +11. 快速编辑打开后，画布视口应调整到面板位于原图下方且不遮挡原图；原图右侧显示竖向框选工具，支持矩形、椭圆和画笔自由框选。每完成一次框选，红色圈选框按完成顺序标注 `1 / 2 / 3...`，并在快速编辑提示词中追加一行 `对N号红色圈选框里的内容做以下修改：`。 ## 参数交互 @@ -173,14 +174,15 @@ - 规范面板比图片生成面板更紧凑，字段间距和输入高度更小，但外层 shell、首行参考图和底部按钮区必须继续对齐生成图片 / 生成角色 / 生成视频。 - 生成规范类图片底部展示禁用态参数按钮 `16:9·2K` 和 `gpt-image-2`，视觉对齐可编辑面板的比例 / 尺寸 / 模型按钮；提交参数也固定为这三项，不出现可展开选项。 - 快速编辑底部展示当前选择的比例 / 尺寸和模型，视觉对齐可编辑面板的比例 / 尺寸 / 模型按钮，并允许展开修改；额外参考图最多 8 张，原图作为隐式最后参考图参与提交但不在参考图条里固定显示 `图x`。 +- 快速编辑打开后面板位于原图下方且不遮挡原图，原图右侧出现竖向矩形 / 椭圆 / 画笔自由框选按钮；完成框选后画布红框显示连续序号，输入框同步追加 `对N号红色圈选框里的内容做以下修改：`。 - 快速编辑提交前必须把提示词里对原图的 `原图`、`当前图片`、`当前图` 或 `图1` 引用改写成最后一张原图编号，例如额外 1 张参考图时改为 `图2`。 -- 快速编辑提交给后端的参考图顺序必须是额外参考图在前、原图在最后。 +- 快速编辑提交给后端的参考图顺序必须是额外参考图在前、原图在最后；若用户已完成红色圈选，最后一张隐式原图必须替换为绘制了红框和序号的标注图。 - 生成中的占位图聚焦后可用 `Delete` / `Backspace` 删除；删除后异步结果不再落回画布，也不显示额外删除 UI。 - 快速编辑生成中的占位图同样只支持键盘 `Delete` / `Backspace` 删除，不新增 UI 删除按钮；删除后异步结果不得再落回画布。 - 生成视频 / 角色形象 / 角色动作 / 音效 / 背景音乐新建后，画布占位空白样式和右上角标签均与对应生成类型一致，不再统一使用图片占位 icon。 - 新建空白待生成占位的尺寸必须和面板参数一致；图片类修改比例 / 尺寸、视频修改清晰度后，画布空白占位同步变更且保持中心点。 - 点击角色图只选中图层并显示工具栏，不自动弹出重绘、快速编辑或角色动画面板；点击工具栏或右键菜单中的 `生成动画` 才创建角色动作占位和面板。 -- 点击 UI 设计图只选中图层并显示工具栏；工具栏在 `去除背景按钮` 后显示 `提取素材`，点击后固定用 `gpt-image-2` 和提示词 `提取画面中的所有独立并整理成spritesheet` 生成 spritesheet，再复用图标素材拆分流程，把 spritesheet 图集和拆分素材都放到画布。 +- 点击 UI 设计图只选中图层并显示工具栏；工具栏在 `去除背景按钮` 后显示 `提取素材`，点击后进入红框素材框选编辑态，默认矩形框选并支持椭圆和画笔自由框选。用户至少框选一个区域后才能点击 `提取`，前端把红色轮廓绘入原图作为参考图，再固定用 `gpt-image-2` 和提示词 `仅提取被红色框框选的素材并整理成spritesheet` 生成 spritesheet，后续复用图标素材拆分流程，把 spritesheet 图集和拆分素材都放到画布。 - 生成游戏音效面板底部不显示字段标题，左下角只有一个时长参数按钮，选项为 Vidu duration `2-10` 秒；右下角固定模型胶囊显示 `Vidu` 并紧贴生成按钮。 - 生成游戏背景音乐面板右下角固定模型胶囊显示 `Suno` 并紧贴生成按钮；`make_instrumental` 不在 UI 中展示。 - 生成视频结果以视频图层加入画布，画布媒体元素标记为 `画布视频：生成视频 N`。 diff --git a/docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md b/docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md index 0d032d46d..f95bbb146 100644 --- a/docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md +++ b/docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md @@ -47,11 +47,14 @@ ## 提取素材 -- 点击浮动工具栏 `提取素材` 后，前端把当前 UI 设计图读取为图片 Data URL，提交到 `POST /api/editor/ui-designs/assets/extractions`。 +- 点击浮动工具栏 `提取素材` 后，进入 UI 设计图素材框选编辑状态，不立即提交提取接口。 +- 框选状态参考微信截图工具栏布局，默认矩形框选，并支持矩形框选、椭圆框选和画笔自由框选三种工具。 +- 用户可在同一张 UI 设计图上多次框选；未至少框选一个区域时，工具栏 `提取` 按钮不可点击。 +- 点击 `提取` 后，前端把所有框选的红色轮廓绘入原 UI 设计图，生成合成图片 Data URL，再提交到 `POST /api/editor/ui-designs/assets/extractions`。 - 后端固定使用 `gpt-image-2` 图片编辑链路，并固定提示词： ```text -提取画面中的所有独立并整理成spritesheet +仅提取被红色框框选的素材并整理成spritesheet ``` - 后端收到 spritesheet 后复用图标素材的连通域拆分能力；未知素材数量时按从上到下、从左到右自动命名为 `素材 1`、`素材 2`。 @@ -64,4 +67,4 @@ - 从画布选择时只能绑定图标规范图片。 - 请求参数包含 `kind: "ui-design"`、`model: "gpt-image-2"`、比例、大小与可选参考图。 - 生成图层信息面板展示 `用户输入` 与 `图标规范`。 -- 选中 UI 设计图时浮动工具栏显示 `提取素材`；点击后调用固定 `gpt-image-2` 提取接口，画布同时出现 spritesheet 图集和拆分后的独立素材。 +- 选中 UI 设计图时浮动工具栏显示 `提取素材`；点击后进入红框素材框选状态，至少框选一个区域后才能调用固定 `gpt-image-2` 提取接口，画布同时出现 spritesheet 图集和拆分后的独立素材。 diff --git a/scripts/dev.mjs b/scripts/dev.mjs index 1e73b8a5b..74f315aa1 100644 --- a/scripts/dev.mjs +++ b/scripts/dev.mjs @@ -37,6 +37,92 @@ const manifestPath = resolve(serverRsDir, 'Cargo.toml'); const modulePath = resolve(serverRsDir, 'crates/spacetime-module'); const viteCliPath = resolve(repoRoot, 'scripts/vite-cli.mjs'); const adminWebDir = resolve(repoRoot, 'apps/admin-web'); + +function resolveLocalGenarrativeFfmpeg(env = process.env, platform = process.platform) { + if (platform !== 'win32') { + return null; + } + + const localAppData = + String(env.LOCALAPPDATA ?? '').trim() || + (String(env.USERPROFILE ?? '').trim() + ? join(String(env.USERPROFILE).trim(), 'AppData', 'Local') + : ''); + if (!localAppData) { + return null; + } + + const binDir = join(localAppData, 'Genarrative', 'ffmpeg', 'bin'); + const ffmpegPath = join(binDir, 'ffmpeg.exe'); + const ffprobePath = join(binDir, 'ffprobe.exe'); + if (!existsSync(ffmpegPath) || !existsSync(ffprobePath)) { + return null; + } + + return {binDir, ffmpegPath, ffprobePath}; +} + +function resolvePathEnvKey(env, platform = process.platform) { + return ( + Object.keys(env).find((key) => key.toLowerCase() === 'path') || + (platform === 'win32' ? 'Path' : 'PATH') + ); +} + +function prependEnvPathEntry(env, entry, platform = process.platform) { + const pathKey = resolvePathEnvKey(env, platform); + const separator = platform === 'win32' ? ';' : ':'; + const currentEntries = String(env[pathKey] ?? '') + .split(separator) + .map((value) => value.trim()) + .filter(Boolean); + const normalize = (value) => + platform === 'win32' ? value.toLowerCase() : value; + if (currentEntries.some((value) => normalize(value) === normalize(entry))) { + return; + } + + env[pathKey] = [entry, ...currentEntries].join(separator); +} + +function setDefaultEnvValue(env, key, value) { + if (!String(env[key] ?? '').trim()) { + env[key] = value; + } +} + +function applyLocalFfmpegEnv(env, platform = process.platform) { + const localFfmpeg = resolveLocalGenarrativeFfmpeg(env, platform); + if (!localFfmpeg) { + return env; + } + + // 本地 dev 进程常在安装 FFmpeg 前已经启动；这里用绝对路径兜底，避免只依赖旧 PATH。 + prependEnvPathEntry(env, localFfmpeg.binDir, platform); + const ffmpegPath = + String(env.CHARACTER_ANIMATION_FFMPEG_PATH ?? '').trim() || + String(env.GENARRATIVE_CHARACTER_ANIMATION_FFMPEG_PATH ?? '').trim() || + localFfmpeg.ffmpegPath; + const ffprobePath = + String(env.CHARACTER_ANIMATION_FFPROBE_PATH ?? '').trim() || + String(env.GENARRATIVE_CHARACTER_ANIMATION_FFPROBE_PATH ?? '').trim() || + localFfmpeg.ffprobePath; + + setDefaultEnvValue(env, 'CHARACTER_ANIMATION_FFMPEG_PATH', ffmpegPath); + setDefaultEnvValue( + env, + 'GENARRATIVE_CHARACTER_ANIMATION_FFMPEG_PATH', + ffmpegPath, + ); + setDefaultEnvValue(env, 'CHARACTER_ANIMATION_FFPROBE_PATH', ffprobePath); + setDefaultEnvValue( + env, + 'GENARRATIVE_CHARACTER_ANIMATION_FFPROBE_PATH', + ffprobePath, + ); + return env; +} + function resolveLocalDevRustcWrapperBypass() { // Windows 下不能把 rustc 自身当成 Cargo wrapper；空值会覆盖仓库 .cargo/config.toml 中的 sccache。 return process.platform === 'win32' ? '' : '/usr/bin/env'; @@ -2149,23 +2235,32 @@ function isSpacetimePublishPermissionError(error) { ); } -function buildApiServerProcessEnv({baseEnv, options, state}) { - return { - ...baseEnv, - // 本地 dev 允许密码入口直接创建账号，生产默认仍由 api-server 配置保持关闭。 - GENARRATIVE_DEV_PASSWORD_ENTRY_AUTO_REGISTER_ENABLED: 'true', - GENARRATIVE_API_HOST: options.apiHost, - GENARRATIVE_API_PORT: String(options.apiPort), - GENARRATIVE_API_LOG: options.apiLog, - GENARRATIVE_SPACETIME_SERVER_URL: state.spacetimeServer, - GENARRATIVE_SPACETIME_DATABASE: options.database, - GENARRATIVE_SPACETIME_TOKEN: baseEnv.GENARRATIVE_SPACETIME_TOKEN || '', - }; +function buildApiServerProcessEnv({ + baseEnv, + options, + state, + platform = process.platform, +}) { + return applyLocalFfmpegEnv( + { + ...baseEnv, + // 本地 dev 允许密码入口直接创建账号，生产默认仍由 api-server 配置保持关闭。 + GENARRATIVE_DEV_PASSWORD_ENTRY_AUTO_REGISTER_ENABLED: 'true', + GENARRATIVE_API_HOST: options.apiHost, + GENARRATIVE_API_PORT: String(options.apiPort), + GENARRATIVE_API_LOG: options.apiLog, + GENARRATIVE_SPACETIME_SERVER_URL: state.spacetimeServer, + GENARRATIVE_SPACETIME_DATABASE: options.database, + GENARRATIVE_SPACETIME_TOKEN: baseEnv.GENARRATIVE_SPACETIME_TOKEN || '', + }, + platform, + ); } export { assertReusableSpacetimeProcessVersionMatchesWorkspace, assertSpacetimeToolVersionMatchesWorkspace, + applyLocalFfmpegEnv, buildApiServerProcessEnv, buildDevStackSnapshot, buildLocalRustProcessEnv, diff --git a/scripts/dev.test.ts b/scripts/dev.test.ts index 105113c67..d80c7d9ec 100644 --- a/scripts/dev.test.ts +++ b/scripts/dev.test.ts @@ -1,4 +1,4 @@ -import {mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; @@ -184,6 +184,60 @@ describe('dev scheduler api-server env', () => { expect(env.GENARRATIVE_API_PORT).toBe('9091'); expect(env.GENARRATIVE_SPACETIME_SERVER_URL).toBe('http://127.0.0.1:3199'); }); + + test('Windows 本地 dev 自动注入已安装的 FFmpeg 路径', () => { + const tempDir = mkdtempSync(join(tmpdir(), 'genarrative-ffmpeg-')); + try { + const binDir = join(tempDir, 'Genarrative', 'ffmpeg', 'bin'); + const ffmpegPath = join(binDir, 'ffmpeg.exe'); + const ffprobePath = join(binDir, 'ffprobe.exe'); + mkdirSync(binDir, {recursive: true}); + writeFileSync(ffmpegPath, '', 'utf8'); + writeFileSync(ffprobePath, '', 'utf8'); + + const {options} = parseArgs(['api-server'], {}); + const env = buildApiServerProcessEnv({ + baseEnv: {LOCALAPPDATA: tempDir, Path: 'C:\\Windows\\System32'}, + options, + state: {spacetimeServer: 'http://127.0.0.1:3199'}, + platform: 'win32', + }); + + expect(env.CHARACTER_ANIMATION_FFMPEG_PATH).toBe(ffmpegPath); + expect(env.CHARACTER_ANIMATION_FFPROBE_PATH).toBe(ffprobePath); + expect(env.GENARRATIVE_CHARACTER_ANIMATION_FFMPEG_PATH).toBe(ffmpegPath); + expect(env.Path?.split(';')[0]).toBe(binDir); + } finally { + rmSync(tempDir, {recursive: true, force: true}); + } + }); + + test('Windows 本地 dev 保留显式 FFmpeg 配置', () => { + const tempDir = mkdtempSync(join(tmpdir(), 'genarrative-ffmpeg-')); + try { + const binDir = join(tempDir, 'Genarrative', 'ffmpeg', 'bin'); + mkdirSync(binDir, {recursive: true}); + writeFileSync(join(binDir, 'ffmpeg.exe'), '', 'utf8'); + writeFileSync(join(binDir, 'ffprobe.exe'), '', 'utf8'); + + const {options} = parseArgs(['api-server'], {}); + const env = buildApiServerProcessEnv({ + baseEnv: { + LOCALAPPDATA: tempDir, + CHARACTER_ANIMATION_FFMPEG_PATH: 'D:\\tools\\ffmpeg.exe', + CHARACTER_ANIMATION_FFPROBE_PATH: 'D:\\tools\\ffprobe.exe', + }, + options, + state: {spacetimeServer: 'http://127.0.0.1:3199'}, + platform: 'win32', + }); + + expect(env.CHARACTER_ANIMATION_FFMPEG_PATH).toBe('D:\\tools\\ffmpeg.exe'); + expect(env.CHARACTER_ANIMATION_FFPROBE_PATH).toBe('D:\\tools\\ffprobe.exe'); + } finally { + rmSync(tempDir, {recursive: true, force: true}); + } + }); }); describe('dev scheduler Rust build env', () => { diff --git a/server-rs/crates/api-server/src/config.rs b/server-rs/crates/api-server/src/config.rs index ab58de933..44793ef52 100644 --- a/server-rs/crates/api-server/src/config.rs +++ b/server-rs/crates/api-server/src/config.rs @@ -1114,13 +1114,19 @@ impl AppConfig { } if let Some(character_animation_ffmpeg_path) = - read_first_non_empty_env(&["CHARACTER_ANIMATION_FFMPEG_PATH"]) + read_first_non_empty_env(&[ + "GENARRATIVE_CHARACTER_ANIMATION_FFMPEG_PATH", + "CHARACTER_ANIMATION_FFMPEG_PATH", + ]) { config.character_animation_ffmpeg_path = character_animation_ffmpeg_path; } if let Some(character_animation_ffprobe_path) = - read_first_non_empty_env(&["CHARACTER_ANIMATION_FFPROBE_PATH"]) + read_first_non_empty_env(&[ + "GENARRATIVE_CHARACTER_ANIMATION_FFPROBE_PATH", + "CHARACTER_ANIMATION_FFPROBE_PATH", + ]) { config.character_animation_ffprobe_path = character_animation_ffprobe_path; } @@ -2076,4 +2082,44 @@ mod tests { std::env::remove_var("GENARRATIVE_EDITOR_GENERATION_PRICING_OVERRIDE_PATH"); } } + + #[test] + fn from_env_reads_prefixed_character_animation_ffmpeg_paths() { + let _guard = ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock should not poison"); + + unsafe { + std::env::remove_var("GENARRATIVE_CHARACTER_ANIMATION_FFMPEG_PATH"); + std::env::remove_var("GENARRATIVE_CHARACTER_ANIMATION_FFPROBE_PATH"); + std::env::remove_var("CHARACTER_ANIMATION_FFMPEG_PATH"); + std::env::remove_var("CHARACTER_ANIMATION_FFPROBE_PATH"); + std::env::set_var( + "GENARRATIVE_CHARACTER_ANIMATION_FFMPEG_PATH", + "C:\\Genarrative\\ffmpeg\\bin\\ffmpeg.exe", + ); + std::env::set_var( + "GENARRATIVE_CHARACTER_ANIMATION_FFPROBE_PATH", + "C:\\Genarrative\\ffmpeg\\bin\\ffprobe.exe", + ); + } + + let config = AppConfig::from_env(); + assert_eq!( + config.character_animation_ffmpeg_path, + "C:\\Genarrative\\ffmpeg\\bin\\ffmpeg.exe" + ); + assert_eq!( + config.character_animation_ffprobe_path, + "C:\\Genarrative\\ffmpeg\\bin\\ffprobe.exe" + ); + + unsafe { + std::env::remove_var("GENARRATIVE_CHARACTER_ANIMATION_FFMPEG_PATH"); + std::env::remove_var("GENARRATIVE_CHARACTER_ANIMATION_FFPROBE_PATH"); + std::env::remove_var("CHARACTER_ANIMATION_FFMPEG_PATH"); + std::env::remove_var("CHARACTER_ANIMATION_FFPROBE_PATH"); + } + } } diff --git a/server-rs/crates/api-server/src/editor_project.rs b/server-rs/crates/api-server/src/editor_project.rs index be00ce083..89e48e315 100644 --- a/server-rs/crates/api-server/src/editor_project.rs +++ b/server-rs/crates/api-server/src/editor_project.rs @@ -68,7 +68,7 @@ const EDITOR_IMAGE_MODEL_NANOBANANA2: &str = "gemini-3.1-flash-image-preview"; const EDITOR_IMAGE_MODEL_NANOBANANA2_DISPLAY_ALIAS: &str = "nanobanana2"; const EDITOR_IMAGE_MODEL_NANOBANANA_LEGACY_ALIAS: &str = "nano-banana"; const EDITOR_ICON_DESCRIPTION_LIMIT: usize = 100; -const EDITOR_UI_DESIGN_ASSET_EXTRACTION_PROMPT: &str = "提取画面中的所有独立并整理成spritesheet"; +const EDITOR_UI_DESIGN_ASSET_EXTRACTION_PROMPT: &str = "仅提取被红色框框选的素材并整理成spritesheet"; const EDITOR_CHARACTER_IMAGE_ASSET_KIND: &str = "editor_character_image"; const EDITOR_CHARACTER_IMAGE_ENTITY_KIND: &str = "editor_project"; const EDITOR_CHARACTER_IMAGE_SLOT: &str = "character"; @@ -3175,7 +3175,7 @@ mod tests { fn editor_ui_design_asset_extraction_prompt_is_fixed() { assert_eq!( EDITOR_UI_DESIGN_ASSET_EXTRACTION_PROMPT, - "提取画面中的所有独立并整理成spritesheet" + "仅提取被红色框框选的素材并整理成spritesheet" ); } diff --git a/src/components/image-editor/ImageCanvasEditorGenerationIntegration.test.tsx b/src/components/image-editor/ImageCanvasEditorGenerationIntegration.test.tsx index d8fb9152b..a2b56f4a4 100644 --- a/src/components/image-editor/ImageCanvasEditorGenerationIntegration.test.tsx +++ b/src/components/image-editor/ImageCanvasEditorGenerationIntegration.test.tsx @@ -9,7 +9,7 @@ import { within, } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; -import { describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { ApiClientError, @@ -23,6 +23,7 @@ import { const generateEditorImageMock = vi.hoisted(() => vi.fn()); const generateEditorIconSpritesheetMock = vi.hoisted(() => vi.fn()); const extractEditorUiDesignAssetsMock = vi.hoisted(() => vi.fn()); +const renderUiDesignAssetExtractionMarkedImageMock = vi.hoisted(() => vi.fn()); const generateEditorCharacterAnimationMock = vi.hoisted(() => vi.fn()); const editEditorImageMock = vi.hoisted(() => vi.fn()); const createEditorAssetMock = vi.hoisted(() => vi.fn()); @@ -64,6 +65,11 @@ vi.mock('../../services/image-editor/editorProjectClient', async () => { }; }); +vi.mock('./ImageCanvasUiAssetExtractionRasterModel', () => ({ + renderUiDesignAssetExtractionMarkedImage: + renderUiDesignAssetExtractionMarkedImageMock, +})); + describe('ImageCanvasEditorView generation integration', () => { setupImageCanvasEditorViewTestLifecycle({ generateEditorImageMock, @@ -85,6 +91,14 @@ describe('ImageCanvasEditorView generation integration', () => { saveEditorProjectLayoutMock, }); + beforeEach(() => { + window.localStorage.clear(); + renderUiDesignAssetExtractionMarkedImageMock.mockReset(); + renderUiDesignAssetExtractionMarkedImageMock.mockResolvedValue( + 'data:image/png;base64,marked-ui-design', + ); + }); + const openGenerationDimensionsPanel = ( panel: HTMLElement, labelPrefix = '生成图片', @@ -2737,8 +2751,8 @@ describe('ImageCanvasEditorView generation integration', () => { height: 96, }, ], - prompt: '提取画面中的所有独立并整理成spritesheet', - actualPrompt: '提取画面中的所有独立并整理成spritesheet', + prompt: '仅提取被红色框框选的素材并整理成spritesheet', + actualPrompt: '仅提取被红色框框选的素材并整理成spritesheet', model: 'gpt-image-2', provider: 'VectorEngine', taskId: 'ui-extract-task-1', @@ -2762,10 +2776,71 @@ describe('ImageCanvasEditorView generation integration', () => { }); fireEvent.click(screen.getByRole('button', { name: '提取素材' })); + const extractionToolbar = await screen.findByRole('toolbar', { + name: 'UI素材框选工具', + }); + expect( + within(extractionToolbar).getByRole('button', { + name: '提取', + }).disabled, + ).toBe(true); + expect(extractEditorUiDesignAssetsMock).not.toHaveBeenCalled(); + + const extractionCanvas = screen.getByRole('application', { + name: 'UI素材框选画布', + }); + vi.spyOn(extractionCanvas, 'getBoundingClientRect').mockReturnValue({ + x: 120, + y: 140, + left: 120, + top: 140, + right: 680, + bottom: 455, + width: 560, + height: 315, + toJSON: () => ({}), + } as DOMRect); + dispatchPointerEvent(extractionCanvas, 'pointerdown', { + pointerId: 201, + clientX: 140, + clientY: 160, + }); + dispatchPointerEvent(extractionCanvas, 'pointermove', { + pointerId: 201, + clientX: 300, + clientY: 260, + }); + dispatchPointerEvent(extractionCanvas, 'pointerup', { + pointerId: 201, + clientX: 300, + clientY: 260, + }); + + expect( + within(extractionToolbar).getByRole('button', { + name: '提取', + }).disabled, + ).toBe(false); + fireEvent.click( + within(extractionToolbar).getByRole('button', { name: '提取' }), + ); + + await waitFor(() => { + expect(renderUiDesignAssetExtractionMarkedImageMock).toHaveBeenCalledWith( + expect.objectContaining({ + source: 'data:image/png;base64,dWktZGVzaWdu', + marks: expect.arrayContaining([ + expect.objectContaining({ + tool: 'rect', + }), + ]), + }), + ); + }); await waitFor(() => { expect(extractEditorUiDesignAssetsMock).toHaveBeenCalledWith( expect.objectContaining({ - sourceImageSrc: 'data:image/png;base64,dWktZGVzaWdu', + sourceImageSrc: 'data:image/png;base64,marked-ui-design', projectId: 'editor-project-ui-extract', assetFolderId: 'project', spritesheetLabel: '战斗UI设计图 素材图集', @@ -2773,7 +2848,7 @@ describe('ImageCanvasEditorView generation integration', () => { fields: expect.arrayContaining([ expect.objectContaining({ title: '提取提示词', - value: '提取画面中的所有独立并整理成spritesheet', + value: '仅提取被红色框框选的素材并整理成spritesheet', }), ]), references: expect.arrayContaining([ @@ -2801,7 +2876,7 @@ describe('ImageCanvasEditorView generation integration', () => { fields: expect.arrayContaining([ expect.objectContaining({ title: '提取提示词', - value: '提取画面中的所有独立并整理成spritesheet', + value: '仅提取被红色框框选的素材并整理成spritesheet', }), ]), }), diff --git a/src/components/image-editor/ImageCanvasEditorShellView.test.tsx b/src/components/image-editor/ImageCanvasEditorShellView.test.tsx index 18ad713ed..16f6b4ce1 100644 --- a/src/components/image-editor/ImageCanvasEditorShellView.test.tsx +++ b/src/components/image-editor/ImageCanvasEditorShellView.test.tsx @@ -1,6 +1,6 @@ /* @vitest-environment jsdom */ -import { fireEvent, render, screen } from '@testing-library/react'; +import { fireEvent, render, screen, within } from '@testing-library/react'; import { createRef } from 'react'; import { describe, expect, it, vi } from 'vitest'; @@ -130,6 +130,10 @@ function createStageProps(): ImageCanvasStageViewProps { generateDialog: null, cropExpandPanel: null, cropExpandSourceLayer: null, + uiAssetExtractionState: null, + uiAssetExtractionSourceLayer: null, + quickEditSelectionState: null, + quickEditSelectionSourceLayer: null, generationComposerStyle: null, selectedToolbarStyle: null, uploadDropTarget: null, @@ -169,6 +173,16 @@ function createStageProps(): ImageCanvasStageViewProps { onOpenCropExpandPanel: vi.fn(), onRemoveBackground: vi.fn(), onExtractUiDesignAssets: vi.fn(), + onUiAssetExtractionToolChange: vi.fn(), + onUiAssetExtractionPointerStart: vi.fn(), + onUiAssetExtractionPointerMove: vi.fn(), + onUiAssetExtractionPointerEnd: vi.fn(), + onCancelUiAssetExtraction: vi.fn(), + onSubmitUiAssetExtraction: vi.fn(), + onQuickEditSelectionToolChange: vi.fn(), + onQuickEditSelectionPointerStart: vi.fn(), + onQuickEditSelectionPointerMove: vi.fn(), + onQuickEditSelectionPointerEnd: vi.fn(), onOpenCharacterAnimationPanel: vi.fn(), onDownloadLayer: vi.fn(), onPasteCanvasClipboard: vi.fn(), @@ -264,4 +278,44 @@ describe('ImageCanvasEditorShellView', () => { expect((preview as HTMLElement).style.left).toBe('0px'); expect((preview as HTMLElement).style.top).toBe('48px'); }); + + it('renders quick edit selection tools through the stage props', () => { + const layer = createLayer(); + const handleQuickEditToolChange = vi.fn(); + const stageProps = { + ...createStageProps(), + layers: [layer], + quickEditSelectionSourceLayer: layer, + quickEditSelectionState: { + sourceLayerId: layer.id, + tool: 'rect' as const, + marks: [], + draftMark: null, + status: 'idle' as const, + }, + onQuickEditSelectionToolChange: handleQuickEditToolChange, + }; + + render( + ()} + uploadInputRef={createRef()} + onUploadInputChange={vi.fn()} + assetDragPreview={null} + sidebarProps={createSidebarProps()} + topbarProps={createTopbarProps()} + stageProps={stageProps} + metadataProps={createMetadataProps()} + />, + ); + + const toolbar = screen.getByRole('toolbar', { + name: '快速编辑框选工具', + }); + fireEvent.click( + within(toolbar).getByRole('button', { name: '椭圆框选' }), + ); + + expect(handleQuickEditToolChange).toHaveBeenCalledWith('ellipse'); + }); }); diff --git a/src/components/image-editor/ImageCanvasEditorView.tsx b/src/components/image-editor/ImageCanvasEditorView.tsx index e1c88c5a4..f365276be 100644 --- a/src/components/image-editor/ImageCanvasEditorView.tsx +++ b/src/components/image-editor/ImageCanvasEditorView.tsx @@ -677,6 +677,20 @@ export function ImageCanvasEditorView() { pickUiDesignSpecFromLayer, hideGeneratedLayerPanelAfterBlur, clearDeletedLayerGenerationState, + uiAssetExtractionState, + uiAssetExtractionSourceLayer, + quickEditSelectionState, + quickEditSelectionSourceLayer, + changeUiAssetExtractionTool, + changeQuickEditSelectionTool, + startUiAssetExtractionPointer, + startQuickEditSelectionPointer, + moveUiAssetExtractionPointer, + moveQuickEditSelectionPointer, + endUiAssetExtractionPointer, + endQuickEditSelectionPointer, + cancelUiAssetExtraction, + submitUiAssetExtraction, generationComposerStyle, generationComposerNode, switchGenerationTool, @@ -1069,6 +1083,10 @@ export function ImageCanvasEditorView() { generateDialog, cropExpandPanel: generationSurface.cropExpandPanel, cropExpandSourceLayer: generationSurface.cropExpandSourceLayer, + uiAssetExtractionState, + uiAssetExtractionSourceLayer, + quickEditSelectionState, + quickEditSelectionSourceLayer, generationComposerStyle, selectedToolbarStyle, uploadDropTarget, @@ -1115,6 +1133,16 @@ export function ImageCanvasEditorView() { onOpenCropExpandPanel: openCropExpandPanel, onRemoveBackground: removeSelectedLayerBackground, onExtractUiDesignAssets: extractUiDesignAssets, + onUiAssetExtractionToolChange: changeUiAssetExtractionTool, + onUiAssetExtractionPointerStart: startUiAssetExtractionPointer, + onUiAssetExtractionPointerMove: moveUiAssetExtractionPointer, + onUiAssetExtractionPointerEnd: endUiAssetExtractionPointer, + onCancelUiAssetExtraction: cancelUiAssetExtraction, + onSubmitUiAssetExtraction: () => void submitUiAssetExtraction(), + onQuickEditSelectionToolChange: changeQuickEditSelectionTool, + onQuickEditSelectionPointerStart: startQuickEditSelectionPointer, + onQuickEditSelectionPointerMove: moveQuickEditSelectionPointer, + onQuickEditSelectionPointerEnd: endQuickEditSelectionPointer, onOpenCharacterAnimationPanel: openCharacterAnimationPanel, onDownloadLayer: exportLayerImage, onPasteCanvasClipboard: pasteCanvasClipboard, diff --git a/src/components/image-editor/ImageCanvasGenerationModel.ts b/src/components/image-editor/ImageCanvasGenerationModel.ts index c595b7041..270939bed 100644 --- a/src/components/image-editor/ImageCanvasGenerationModel.ts +++ b/src/components/image-editor/ImageCanvasGenerationModel.ts @@ -96,7 +96,7 @@ export const DEFAULT_ICON_DESCRIPTIONS = [ '冻结按钮', ]; export const UI_DESIGN_ASSET_EXTRACTION_PROMPT = - '提取画面中的所有独立并整理成spritesheet'; + '仅提取被红色框框选的素材并整理成spritesheet'; export const QUICK_EDIT_SIZE_PRESETS = [ '1024x1024', '1536x1024', diff --git a/src/components/image-editor/ImageCanvasStageView.tsx b/src/components/image-editor/ImageCanvasStageView.tsx index 74f7ef067..eb276b870 100644 --- a/src/components/image-editor/ImageCanvasStageView.tsx +++ b/src/components/image-editor/ImageCanvasStageView.tsx @@ -10,8 +10,8 @@ import type { import { ImageCanvasBottomToolbarView } from './ImageCanvasBottomToolbarView'; import { ImageCanvasContextMenusView } from './ImageCanvasContextMenusView'; import type { - CanvasClipboard, CanvasAssetKind, + CanvasClipboard, CanvasContextMenuState, CanvasGenerationDialogState, CanvasLayer, @@ -28,6 +28,11 @@ import type { import type { StageMinimapModel } from './ImageCanvasInteractionModel'; import { ImageCanvasPanelDockView } from './ImageCanvasPanelDockView'; import { ImageCanvasSelectedLayerToolbarView } from './ImageCanvasSelectedLayerToolbarView'; +import type { + UiAssetExtractionState, + UiAssetExtractionTool, +} from './ImageCanvasUiAssetExtractionModel'; +import { ImageCanvasUiAssetExtractionOverlayView } from './ImageCanvasUiAssetExtractionOverlayView'; import { ImageCanvasWorldView } from './ImageCanvasWorldView'; export type ImageCanvasStageViewProps = { @@ -50,6 +55,10 @@ export type ImageCanvasStageViewProps = { generateDialog: GenerateDialogState | null; cropExpandPanel: CropExpandPanelState | null; cropExpandSourceLayer: CanvasLayer | null; + uiAssetExtractionState: UiAssetExtractionState | null; + uiAssetExtractionSourceLayer: CanvasLayer | null; + quickEditSelectionState: UiAssetExtractionState | null; + quickEditSelectionSourceLayer: CanvasLayer | null; generationComposerStyle: CSSProperties | null; selectedToolbarStyle: CSSProperties | null; uploadDropTarget: 'canvas' | 'assets' | null; @@ -108,6 +117,16 @@ export type ImageCanvasStageViewProps = { onOpenCropExpandPanel: (layer: CanvasLayer) => void; onRemoveBackground: (layer: CanvasLayer) => void; onExtractUiDesignAssets: (layer: CanvasLayer) => void; + onUiAssetExtractionToolChange: (tool: UiAssetExtractionTool) => void; + onUiAssetExtractionPointerStart: (point: { x: number; y: number }) => void; + onUiAssetExtractionPointerMove: (point: { x: number; y: number }) => void; + onUiAssetExtractionPointerEnd: () => void; + onCancelUiAssetExtraction: () => void; + onSubmitUiAssetExtraction: () => void; + onQuickEditSelectionToolChange: (tool: UiAssetExtractionTool) => void; + onQuickEditSelectionPointerStart: (point: { x: number; y: number }) => void; + onQuickEditSelectionPointerMove: (point: { x: number; y: number }) => void; + onQuickEditSelectionPointerEnd: () => void; onOpenCharacterAnimationPanel: (layer: CanvasLayer) => void; onDownloadLayer: (layer: CanvasLayer) => void; onPasteCanvasClipboard: (canvasPoint?: { x: number; y: number }) => void; @@ -165,6 +184,10 @@ export function ImageCanvasStageView({ generateDialog, cropExpandPanel, cropExpandSourceLayer, + uiAssetExtractionState, + uiAssetExtractionSourceLayer, + quickEditSelectionState, + quickEditSelectionSourceLayer, generationComposerStyle, selectedToolbarStyle, uploadDropTarget, @@ -205,6 +228,16 @@ export function ImageCanvasStageView({ onOpenCropExpandPanel, onRemoveBackground, onExtractUiDesignAssets, + onUiAssetExtractionToolChange, + onUiAssetExtractionPointerStart, + onUiAssetExtractionPointerMove, + onUiAssetExtractionPointerEnd, + onCancelUiAssetExtraction, + onSubmitUiAssetExtraction, + onQuickEditSelectionToolChange, + onQuickEditSelectionPointerStart, + onQuickEditSelectionPointerMove, + onQuickEditSelectionPointerEnd, onOpenCharacterAnimationPanel, onDownloadLayer, onPasteCanvasClipboard, @@ -287,7 +320,11 @@ export function ImageCanvasStageView({ /> + + + + Number.isFinite(point.x) && Number.isFinite(point.y), + ); + return points.length >= 2 ? { ...mark, points } : null; + } + if ( + !Number.isFinite(mark.x) || + !Number.isFinite(mark.y) || + !Number.isFinite(mark.width) || + !Number.isFinite(mark.height) + ) { + return null; + } + const left = Math.min(mark.x, mark.x + mark.width); + const top = Math.min(mark.y, mark.y + mark.height); + const width = Math.abs(mark.width); + const height = Math.abs(mark.height); + if (width < 4 || height < 4) { + return null; + } + return { + ...mark, + x: left, + y: top, + width, + height, + }; +} + +export function buildUiAssetExtractionBrushPath( + mark: UiAssetExtractionBrushMark, +) { + const [firstPoint, ...restPoints] = mark.points; + if (!firstPoint) { + return ''; + } + return [ + `M ${firstPoint.x} ${firstPoint.y}`, + ...restPoints.map((point) => `L ${point.x} ${point.y}`), + ].join(' '); +} diff --git a/src/components/image-editor/ImageCanvasUiAssetExtractionOverlayView.test.tsx b/src/components/image-editor/ImageCanvasUiAssetExtractionOverlayView.test.tsx new file mode 100644 index 000000000..e6217093b --- /dev/null +++ b/src/components/image-editor/ImageCanvasUiAssetExtractionOverlayView.test.tsx @@ -0,0 +1,125 @@ +/* @vitest-environment jsdom */ + +import { fireEvent, render, screen, within } from '@testing-library/react'; +import { describe, expect, it, vi } from 'vitest'; + +import type { CanvasLayer } from './ImageCanvasEditorTypes'; +import type { UiAssetExtractionState } from './ImageCanvasUiAssetExtractionModel'; +import { ImageCanvasUiAssetExtractionOverlayView } from './ImageCanvasUiAssetExtractionOverlayView'; + +function createLayer(overrides: Partial = {}): CanvasLayer { + return { + id: 'layer-source', + resourceId: 'resource-source', + title: '源图', + src: 'data:image/png;base64,source', + x: 100, + y: 120, + width: 320, + height: 240, + originalWidth: 640, + originalHeight: 480, + zIndex: 4, + sourceType: 'uploaded', + ...overrides, + }; +} + +function createState( + overrides: Partial = {}, +): UiAssetExtractionState { + return { + sourceLayerId: 'layer-source', + tool: 'rect', + marks: [], + draftMark: null, + status: 'idle', + ...overrides, + }; +} + +describe('ImageCanvasUiAssetExtractionOverlayView', () => { + it('renders quick edit tools vertically on the right without extraction actions', () => { + const handleToolChange = vi.fn(); + + render( + , + ); + + const toolbar = screen.getByRole('toolbar', { + name: '快速编辑框选工具', + }); + + expect(toolbar.className).toContain( + 'image-canvas-editor__ui-extraction-toolbar--right', + ); + expect(within(toolbar).queryByText('框选素材')).toBeNull(); + expect(within(toolbar).queryByText('0')).toBeNull(); + expect( + within(toolbar).queryByRole('button', { name: '提取' }), + ).toBeNull(); + expect( + within(toolbar).getByRole('button', { name: '矩形框选' }), + ).toBeTruthy(); + expect( + within(toolbar).getByRole('button', { name: '椭圆框选' }), + ).toBeTruthy(); + fireEvent.click( + within(toolbar).getByRole('button', { name: '画笔自由框选' }), + ); + + expect(handleToolChange).toHaveBeenCalledWith('brush'); + }); + + it('shows numbered labels for quick edit red selections', () => { + const { container } = render( + , + ); + + const labels = Array.from( + container.querySelectorAll( + '.image-canvas-editor__ui-extraction-mark-label', + ), + ).map((node) => node.textContent); + + expect(labels).toEqual(['1', '2']); + }); +}); diff --git a/src/components/image-editor/ImageCanvasUiAssetExtractionOverlayView.tsx b/src/components/image-editor/ImageCanvasUiAssetExtractionOverlayView.tsx new file mode 100644 index 000000000..884a4324e --- /dev/null +++ b/src/components/image-editor/ImageCanvasUiAssetExtractionOverlayView.tsx @@ -0,0 +1,287 @@ +import { Circle, MousePointer2, Paintbrush, Square, X } from 'lucide-react'; +import type { ComponentType, SVGProps } from 'react'; +import type { PointerEvent as ReactPointerEvent } from 'react'; + +import { PlatformActionButton } from '../common/PlatformActionButton'; +import { PlatformIconButton } from '../common/PlatformIconButton'; +import type { CanvasLayer, CanvasViewport } from './ImageCanvasEditorTypes'; +import type { + UiAssetExtractionMark, + UiAssetExtractionState, + UiAssetExtractionTool, +} from './ImageCanvasUiAssetExtractionModel'; +import { buildUiAssetExtractionBrushPath } from './ImageCanvasUiAssetExtractionModel'; + +type ImageCanvasUiAssetExtractionOverlayViewProps = { + variant?: 'ui-extraction' | 'quick-edit'; + showIndexLabels?: boolean; + sourceLayer: CanvasLayer | null; + viewport: CanvasViewport; + state: UiAssetExtractionState | null; + onToolChange: (tool: UiAssetExtractionTool) => void; + onPointerStart: (point: { x: number; y: number }) => void; + onPointerMove: (point: { x: number; y: number }) => void; + onPointerEnd: () => void; + onCancel?: () => void; + onSubmit?: () => void; +}; + +const TOOL_OPTIONS: Array<{ + value: UiAssetExtractionTool; + label: string; + icon: ComponentType>; +}> = [ + { value: 'rect', label: '矩形框选', icon: Square }, + { value: 'ellipse', label: '椭圆框选', icon: Circle }, + { value: 'brush', label: '画笔自由框选', icon: Paintbrush }, +]; + +function resolvePointerImagePoint( + event: ReactPointerEvent, + layer: CanvasLayer, +) { + const rect = event.currentTarget.getBoundingClientRect(); + const fallbackWidth = Math.max(1, layer.width); + const fallbackHeight = Math.max(1, layer.height); + const ratioX = + (event.clientX - rect.left) / (rect.width > 0 ? rect.width : fallbackWidth); + const ratioY = + (event.clientY - rect.top) / + (rect.height > 0 ? rect.height : fallbackHeight); + return { + x: Math.min( + Math.max(ratioX * Math.max(1, layer.originalWidth), 0), + Math.max(1, layer.originalWidth), + ), + y: Math.min( + Math.max(ratioY * Math.max(1, layer.originalHeight), 0), + Math.max(1, layer.originalHeight), + ), + }; +} + +function resolveMarkLabelPoint(mark: UiAssetExtractionMark) { + if (mark.tool === 'brush') { + return mark.points[0] ?? { x: 0, y: 0 }; + } + return { + x: Math.min(mark.x, mark.x + mark.width), + y: Math.min(mark.y, mark.y + mark.height), + }; +} + +function renderMark( + mark: UiAssetExtractionMark, + index: number, + showIndexLabels: boolean, +) { + const markNode = + mark.tool === 'brush' ? ( + + ) : mark.tool === 'ellipse' ? ( + + ) : ( + + ); + + if (!showIndexLabels) { + return {markNode}; + } + const labelPoint = resolveMarkLabelPoint(mark); + // 中文注释：序号直接落在红色圈选框左上角，生成模型可通过“1号红色圈选框”稳定对应用户修改说明。 + return ( + + {markNode} + + + + {index + 1} + + + + ); +} + +export function ImageCanvasUiAssetExtractionOverlayView({ + variant = 'ui-extraction', + showIndexLabels = false, + sourceLayer, + viewport, + state, + onToolChange, + onPointerStart, + onPointerMove, + onPointerEnd, + onCancel, + onSubmit, +}: ImageCanvasUiAssetExtractionOverlayViewProps) { + if (!sourceLayer || !state) { + return null; + } + const isQuickEdit = variant === 'quick-edit'; + const marks = [...state.marks, ...(state.draftMark ? [state.draftMark] : [])]; + const canSubmit = state.marks.length > 0 && state.status !== 'extracting'; + const screenFrame = { + left: viewport.x + sourceLayer.x * viewport.scale, + top: viewport.y + sourceLayer.y * viewport.scale, + width: sourceLayer.width * viewport.scale, + height: sourceLayer.height * viewport.scale, + }; + + return ( + <> +
event.stopPropagation()} + > + {isQuickEdit ? null : ( + <> + + +
+ { + event.preventDefault(); + event.stopPropagation(); + event.currentTarget.setPointerCapture?.(event.pointerId); + onPointerStart(resolvePointerImagePoint(event, sourceLayer)); + }} + onPointerMove={(event) => { + event.preventDefault(); + event.stopPropagation(); + onPointerMove(resolvePointerImagePoint(event, sourceLayer)); + }} + onPointerUp={(event) => { + event.preventDefault(); + event.stopPropagation(); + onPointerEnd(); + }} + onPointerCancel={(event) => { + event.preventDefault(); + event.stopPropagation(); + onPointerEnd(); + }} + > + + {marks.map((mark, index) => + renderMark(mark, index, showIndexLabels && index < state.marks.length), + )} + + {state.errorMessage ? ( +
+ {state.errorMessage} +
+ ) : null} + + ); +} diff --git a/src/components/image-editor/ImageCanvasUiAssetExtractionRasterModel.ts b/src/components/image-editor/ImageCanvasUiAssetExtractionRasterModel.ts new file mode 100644 index 000000000..008cde50e --- /dev/null +++ b/src/components/image-editor/ImageCanvasUiAssetExtractionRasterModel.ts @@ -0,0 +1,141 @@ +import type { UiAssetExtractionMark } from './ImageCanvasUiAssetExtractionModel'; + +function assertCanvasSupport() { + if (typeof document === 'undefined') { + throw new Error('当前环境不支持图片编辑'); + } +} + +function loadImageElement(source: string) { + return new Promise((resolve, reject) => { + const image = new Image(); + image.crossOrigin = 'anonymous'; + image.onload = () => resolve(image); + image.onerror = () => reject(new Error('图片读取失败')); + image.src = source; + }); +} + +function createRasterCanvas(width: number, height: number) { + assertCanvasSupport(); + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext('2d'); + if (!context) { + throw new Error('当前浏览器不支持图片编辑'); + } + return { canvas, context }; +} + +function resolveImageSize(image: HTMLImageElement) { + return { + width: Math.max(1, image.naturalWidth || image.width || 1), + height: Math.max(1, image.naturalHeight || image.height || 1), + }; +} + +function prepareRedSelectionStroke(context: CanvasRenderingContext2D) { + context.strokeStyle = '#ff2b2b'; + context.lineWidth = 6; + context.lineCap = 'round'; + context.lineJoin = 'round'; +} + +function resolveMarkLabelPoint(mark: UiAssetExtractionMark) { + if (mark.tool === 'brush') { + return mark.points[0] ?? { x: 0, y: 0 }; + } + return { + x: Math.min(mark.x, mark.x + mark.width), + y: Math.min(mark.y, mark.y + mark.height), + }; +} + +function drawRedSelectionIndexLabel( + context: CanvasRenderingContext2D, + mark: UiAssetExtractionMark, + index: number, +) { + const point = resolveMarkLabelPoint(mark); + const radius = 18; + const x = Math.max(radius + 2, Math.min(point.x, context.canvas.width - radius - 2)); + const y = Math.max(radius + 2, Math.min(point.y, context.canvas.height - radius - 2)); + + context.save(); + context.fillStyle = '#ff2b2b'; + context.strokeStyle = '#ffffff'; + context.lineWidth = 4; + context.beginPath(); + context.arc(x, y, radius, 0, Math.PI * 2); + context.fill(); + context.stroke(); + context.fillStyle = '#ffffff'; + context.font = '900 24px sans-serif'; + context.textAlign = 'center'; + context.textBaseline = 'middle'; + context.fillText(String(index + 1), x, y + 1); + context.restore(); +} + +export async function renderUiDesignAssetExtractionMarkedImage({ + source, + marks, + showIndexLabels = false, +}: { + source: string; + marks: UiAssetExtractionMark[]; + showIndexLabels?: boolean; +}) { + if (!marks.length) { + throw new Error('请至少框选一个素材'); + } + const image = await loadImageElement(source); + const { width, height } = resolveImageSize(image); + const { canvas, context } = createRasterCanvas(width, height); + context.drawImage(image, 0, 0, width, height); + prepareRedSelectionStroke(context); + + for (const mark of marks) { + context.beginPath(); + if (mark.tool === 'rect') { + context.rect( + Math.min(mark.x, mark.x + mark.width), + Math.min(mark.y, mark.y + mark.height), + Math.abs(mark.width), + Math.abs(mark.height), + ); + } else if (mark.tool === 'ellipse') { + const width = Math.abs(mark.width); + const height = Math.abs(mark.height); + context.ellipse( + Math.min(mark.x, mark.x + mark.width) + width / 2, + Math.min(mark.y, mark.y + mark.height) + height / 2, + width / 2, + height / 2, + 0, + 0, + Math.PI * 2, + ); + } else if (mark.tool === 'brush') { + const [firstPoint, ...restPoints] = mark.points; + if (!firstPoint) { + continue; + } + context.moveTo(firstPoint.x, firstPoint.y); + restPoints.forEach((point) => { + context.lineTo(point.x, point.y); + }); + context.stroke(); + continue; + } + context.stroke(); + } + if (showIndexLabels) { + marks.forEach((mark, index) => + drawRedSelectionIndexLabel(context, mark, index), + ); + } + + return canvas.toDataURL('image/png'); +} diff --git a/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx b/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx index 3eb162143..cf4717bde 100644 --- a/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx +++ b/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx @@ -20,6 +20,7 @@ import type { SidebarPanel, } from './ImageCanvasEditorTypes'; import { createQuickEditPanelDraft } from './ImageCanvasGenerationDialogModel'; +import type { UiAssetExtractionState } from './ImageCanvasUiAssetExtractionModel'; import { useCanvasGenerationDialogs } from './useCanvasGenerationDialogs'; import { useImageCanvasGenerationSubmissionWorkflow } from './useImageCanvasGenerationSubmissionWorkflow'; @@ -29,6 +30,7 @@ const resolveEditorImageReferenceDataUrlForGenerationMock = vi.hoisted(() => ); const editEditorImageMock = vi.hoisted(() => vi.fn()); const extractEditorUiDesignAssetsMock = vi.hoisted(() => vi.fn()); +const renderUiDesignAssetExtractionMarkedImageMock = vi.hoisted(() => vi.fn()); const generateEditorCharacterAnimationMock = vi.hoisted(() => vi.fn()); const generateEditorIconSpritesheetMock = vi.hoisted(() => vi.fn()); const generateEditorImageMock = vi.hoisted(() => vi.fn()); @@ -55,6 +57,11 @@ vi.mock('../../services/image-editor/editorProjectClient', async () => { }; }); +vi.mock('./ImageCanvasUiAssetExtractionRasterModel', () => ({ + renderUiDesignAssetExtractionMarkedImage: + renderUiDesignAssetExtractionMarkedImageMock, +})); + function createLayer(overrides: Partial = {}): CanvasLayer { return { id: 'layer-source', @@ -116,6 +123,7 @@ function createIconResult(name: string, imageSrc: string) { function SubmissionWorkflowHarness({ initialDialog = null, initialQuickEditPanel = null, + initialQuickEditSelectionState = null, initialCharacterAnimationPanel = null, initialLayers = [createLayer()], projectId = undefined, @@ -123,6 +131,7 @@ function SubmissionWorkflowHarness({ }: { initialDialog?: GenerateDialogState | null; initialQuickEditPanel?: QuickEditPanelState | null; + initialQuickEditSelectionState?: UiAssetExtractionState | null; initialCharacterAnimationPanel?: CharacterAnimationPanelState | null; initialLayers?: CanvasLayer[]; projectId?: string | null; @@ -169,6 +178,7 @@ function SubmissionWorkflowHarness({ layerCounterRef, quickEditPanel, quickEditSourceLayer, + quickEditSelectionState: initialQuickEditSelectionState, setQuickEditPanel, characterAnimationPanel, characterAnimationDialog: activeCanvasDialog, @@ -290,6 +300,27 @@ function SubmissionWorkflowHarness({ > 提取UI素材 + + + + @@ -504,6 +539,32 @@ function GenerationWorkflowHarness({ > 清理源图状态 + + +