From 8bdc728fd3f940ed0011354a9d8ad5abbfa407ec Mon Sep 17 00:00:00 2001 From: Linghong Date: Tue, 21 Jul 2026 16:11:22 +0800 Subject: [PATCH 1/7] Fix/padding (#96) Reviewed-on: http://genarrative-station/git/GenarrativeAI/Genarrative/pulls/96 Co-authored-by: Linghong Co-committed-by: Linghong --- .../shared-memory/decision-log.md | 12 + ...架构】图片画布编辑器MVP接入方案-2026-06-11.md | 5 + ...】server-rs与SpacetimeDB数据契约-2026-05-15.md | 3 +- ...辑器】画板角色形象生成入口设计-2026-06-15.md | 4 + .../src/character_animation_assets.rs | 264 ++++++++++++++++-- 5 files changed, 266 insertions(+), 22 deletions(-) diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 72b4e9aee..bf492c40e 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -16,6 +16,18 @@ --- +## 2026-07-20 角色动作抠图前禁止透明 padding + +- 背景：图片画布角色动作此前在 BgFilter 前复用最终帧 finalizer，把 FFmpeg 抽帧先转成目标尺寸 RGBA 画布并用透明黑像素补边；透明区域进入 BgFilter、阿里云和本地键色共同读取的 OSS 源帧后，会干扰主体边缘判断并降低抠图质量。 +- 决策：仅图片画布角色动作链路在抠图前把 FFmpeg 帧转为 RGB8，按最终帧宽高的 contain 比例使用 `Triangle` 缩放到内容尺寸，不创建最终目标画布、不引入 Alpha、不插入 padding；该 RGB8 PNG owned 上传 OSS 后由三段抠图链共享。抠图返回后继续复用原最终帧 finalizer，转为 RGBA8、居中放入最终目标尺寸，并以 `RGBA(0,0,0,0)` 补边。`560×752 → 323×480` 的固定验收结果为 `323×434 RGB8` 抠图输入和上下各 `23px` 透明补边的 `323×480 RGBA8` 最终帧。 +- 补充（2026-07-21 实现收口）：转 RGB8 时若解码帧携带 Alpha 通道（共享 FFmpeg 抽帧命令不固定 `-pix_fmt`，源视频为 alpha 格式时 PNG 可能是 RGBA），必须先把像素按白底合成为不透明再转 RGB8（`flatten_alpha_onto_white_rgb`），禁止直接丢弃 Alpha——全透明像素下未定义的 RGB 值会以杂色进入抠图输入，重新引入本决策要消除的杂色边缘。该白底合成职责只属于图片画布角色动作的 BgFilter 输入准备阶段，不得为此在共享抽帧命令里固定像素格式。 +- 边界：不改变最终帧的 RGBA/padding 语义与透明帧格式、BgFilter 请求、OSS 上传与签名、抽帧数量和采样时间，也不改变旧 `/api/assets/character-animation/*` 动作发布链路；因为降级链复用同一个 object key，阿里云和本地键色同样读取新的无补边 RGB8 源帧。 +- 影响范围：`server-rs/crates/api-server/src/character_animation_assets.rs`、后端融合架构、角色动作专题和图片画布当前接入方案；不涉及 DTO、前端接口、SpacetimeDB schema 或运维配置。 +- 验证方式：像素测试断言 `560×752 RGB8 → 323×434 RGB8` 且无 Alpha/补边，并断言抠图结果最终成为上下各 `23px` 透明补边的 `323×480 RGBA8`；运行 `cargo test -p api-server character_animation --manifest-path server-rs/Cargo.toml`、`cargo check -p api-server --manifest-path server-rs/Cargo.toml`、`npm run check:encoding` 和 `git diff --check`。 +- 关联文档：`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`、`docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md`、`docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md`。 + +--- + ## 2026-07-20 角色动画帧 OSS 请求使用专用连接池、并发保护与结构化重试 - 背景：角色动作逐帧流水线会同时发起源帧 PUT、透明帧 PUT 和最终帧 HEAD；原路径每次请求新建 `reqwest::Client`，且 OSS 请求错误丢失 HTTP 状态和 timeout/connect/transport 分类，多个动画任务叠加时无法在进程级限制 OSS 在途请求，也无法安全区分 PUT 与 HEAD 的失败。 diff --git a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md index 043e13bf9..59b7c2141 100644 --- a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md +++ b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md @@ -31,6 +31,11 @@ - 画布底部工具栏 / 面板 Dock 提供“画布 Agent”入口。点击后打开右侧独立 Agent 对话面板；桌面端为右侧窄面板，移动端占满可用宽度。该面板只与右上角任务侧栏互斥；素材 / 图层侧栏允许与 Agent 同时展开，切换左侧栏不得关闭 Agent。Agent 面板不得在当前画布内容下方追加内联内容，也不默认展示大段功能说明文案。 - 所有会新建画布生成占位的入口必须先创建 draft，再统一经过 `ImageCanvasGenerationPlacementModel` 计算落点，禁止各入口自行使用当前视口中心裸坐标或原图右侧固定偏移。当前覆盖入口包括 `生成图片`、`生成规范`、`生成角色形象`、`生成图标素材`、`生成视频`、`生成UI设计图` 和 `生成角色动作`。placement 模型的避让对象为所有未隐藏画布图层，以及当前 active / inactive generation dialogs 中仍存在的 placeholder；每个避让矩形按 32px 画布世界坐标间距外扩。候选落点以当前视口世界中心为距离目标，优先选择离视口中心最近且不重叠的占位位置；若中心被占用，会按上下左右和环形候选继续寻找。打开生成面板时必须把避让后的 placeholder 写入 `openCanvasGenerationDialog(...)`，并立即调用 `centerViewportOnPlacement(...)` 居中到新占位中心，保持原 viewport scale 不变；图片快速编辑不属于新建占位入口，提交后覆盖源图。 +### 角色动作帧抠图像素边界 + +- 图片画布角色动作的 FFmpeg 抽帧在上传 OSS 前转为 RGB8，并按最终帧宽高 contain 到内容尺寸；抠图前不创建最终尺寸画布、不引入 Alpha 通道、不增加 padding。同一个无补边 object key 供 `BgFilter → 阿里云通用抠图 → 本地键色` 三段链路使用。抠图完成后才转为最终目标尺寸 RGBA8，并以 `RGBA(0,0,0,0)` 居中补边。`560×752 → 323×480` 的验收样例中，抠图输入为 `323×434 RGB8 PNG`，最终输出为上下各 `23px` 透明补边的 `323×480 RGBA8 PNG`。该规则只作用于图片画布角色动作输入准备，不改变旧动作发布、采样、BgFilter 请求或 OSS 流程。 +- 转 RGB8 时若解码帧携带 Alpha 通道，必须先按白底合成为不透明再转 RGB8，禁止直接丢弃 Alpha：全透明像素下未定义的 RGB 值会以杂色进入抠图输入，重新引入杂色边缘。共享 FFmpeg 抽帧命令保持不固定 `-pix_fmt`，白底合成只发生在 BgFilter 输入准备阶段。 + ## 交互规则 - `适合视图` 的正式语义为“显示画布所有可见元素”，不再回到固定 `x/y/scale`。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 467512157..d45b627c0 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -2,7 +2,7 @@ > 2026-07-18 状态更新：旧创作入口、全部模板业务 API/worker/运行态及 SpacetimeDB 业务逻辑已退役。本文逐玩法路由、流程和 DTO 章节仅作为历史设计记录；相关持久化表仍按原结构作为最小 schema 数据壳编译，当前编译与运行边界以 `server-rs/Cargo.toml`、`server-rs/crates/api-server/src/app.rs` 和 `docs/technical/【架构下线】旧创作模板业务退役方案-2026-07-17.md` 为准。 -更新时间：`2026-07-18` +更新时间：`2026-07-21` ## 后端主线 @@ -241,6 +241,7 @@ npm run check:server-rs-ddd - LLM：通用 LLM 门面继续使用 `GENARRATIVE_LLM_*`；创意 Agent `gpt-5.4-mini` Chat Completions 文本链路已于 2026-06 从 APIMart 迁移到 VectorEngine，使用 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` 构造 OpenAI-compatible client，`api-server` 会把未带 `/v1` 的 VectorEngine base URL 规范化到 `/v1` 后请求 `/chat/completions`。通用 `/api/llm/chat/completions` 代理使用 `GENARRATIVE_LLM_PROVIDER=openai-compatible`、`GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1`、`GENARRATIVE_LLM_MODEL=gpt-5.4-mini`；未单独配置 `GENARRATIVE_LLM_API_KEY` 时可复用 `VECTOR_ENGINE_API_KEY`。`APIMART_BASE_URL` / `APIMART_API_KEY` 只作为历史残留，不再作为创意 Agent gpt-5.4-mini 客户端来源；后续排障时优先确认 VectorEngine `/v1/models`、`/v1/chat/completions` 和 `/v1/responses` 可用性。 - 图片生成：VectorEngine `gpt-image-2` 图片 provider 归属 `platform-image`，密钥只在后端环境变量中；`api-server` 内的 `openai_image_generation.rs` 只是兼容调用面和外部失败审计桥接，不再承载 provider 协议实现。实际外部生成运行记录统一落 `tracking_event`，`event_key = external_generation_run`，metadata 记录开始 / 结束时间、耗时、状态、成功标记、失败原因、provider task id 和结果摘要，不再写回过时的 `ai_task`。DashScope 只按仍在使用的历史能力单独处理，不作为 GPT-image-2 兜底。VectorEngine `/v1/images/generations` 和 `/v1/images/edits` 上游 POST 使用 `libcurl` 发送；`reqwest` 只保留给参考图 URL 下载和响应中图片 URL 下载。`/v1/images/edits` 的 multipart 参考图必须作为 libcurl 文件上传 part 发送，字段名为 `image`，实现上使用 `Form::buffer(file_name, bytes)` 并设置 `Content-Type`；不能只用 `contents(...).filename(...)`，否则上游会把请求转码为缺少图片并返回 `image is required`。`request_send` 阶段的 curl timeout / connect error 按可重试传输错误处理，最多尝试 5 次，并使用指数退避加短抖动；排障时优先看 `attempt`、`max_attempts`、`retry_delay_ms`、`reference_image_bytes_total` 和 `request_params`，不要把 `SendRequest` 当成上游业务错误。 - 抠图输入以私有 OSS 作为内存生命周期边界：生成原图和角色动作抽取帧上传时消费图片字节所有权，上传完成后不保留原图缓冲；手动去背景直接解析并校验已有 OSS object key，不下载原图。BgFilter 必须为 object key 签发 600 秒 GET URL 并通过 multipart `image_url` 提交，不用 `file` 重传；flat 链路进入阿里云 fallback 时由 `platform-matting` URL 接口单独下载并上传 `AuthorizeFileUpload` 临时对象，在推理前释放下载缓冲，继续 fallback 到本地键色时再单独下载一次原图，本地产出后释放本次原图下载缓冲。签名 URL 不得写入日志、审计或持久化。 +- 角色动作抠图输入像素边界：仅图片画布角色动作链路在 FFmpeg 抽帧后、源帧上传 OSS 前，把帧解码为 RGB8，并按最终 `frameWidth × frameHeight` 的 contain 比例使用 `Triangle` 只缩放到内容尺寸；该阶段不得创建最终目标尺寸画布、不得引入 Alpha 通道，也不得插入任何 padding。BgFilter、阿里云通用抠图和本地键色降级共享这个无补边源帧 object key。抠图返回后才统一转为 RGBA8，按相同比例居中放入最终目标尺寸画布，并用 `RGBA(0,0,0,0)` 补齐透明 padding。以 `560×752 → 323×480` 为例，抠图输入固定为无 Alpha、无补边的 `323×434 RGB8 PNG`，最终输出为上下各 `23px` 透明补边的 `323×480 RGBA8 PNG`。旧 `/api/assets/character-animation/*` 动作发布链路继续保留原有帧 finalizer，不适用该输入规则。抽帧解码后若携带 Alpha 通道，必须先把像素按白底合成为不透明再转 RGB8，禁止直接丢弃 Alpha——全透明像素下未定义的 RGB 值会以杂色进入抠图输入，重新引入杂色边缘；共享 FFmpeg 抽帧命令保持不固定 `-pix_fmt`，白底合成只属于该链路的 BgFilter 输入准备阶段。 - 阿里云通用抠图的非上海地域输入不得使用 `viapiutils/GetOssStsToken`、固定 `viapi-customer-temp` 或 OSS V1 PUT。`platform-matting` 必须按官方新版 SDK Advance 协议调用 `AuthorizeFileUpload`，使用动态返回的单对象 Policy 执行 multipart POST，再把临时上海 OSS URL 交给 `SegmentCommonImage`；输入归一化、结果下载与原尺寸 Alpha 回贴继续留在同一适配器内。该协议仍上传图片字节，不等同于阿里云服务端直接抓取任意公网 URL，也不改变上层 BgFilter → 阿里云 → 本地降级顺序。 - 编辑器抠图服务：手动 `POST /api/editor/images/background-removals` 与角色形象生成、图标 spritesheet 生成、UI 设计图素材提取、角色动作抽帧后的透明化统一走 BgFilter，配置为 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL`、`GENARRATIVE_EDITOR_BGFILTER_TOKEN` 和 `GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS`，默认 base URL 为 `http://58.87.105.82/bgfilter`，默认请求超时为 `180000ms`（BgFilter 当前为 CPU 推理，单次抠图较慢，必须留足超时）；旧 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN` 只作为 BgFilter token 的兼容回退别名，原手动去背景专用 base URL / timeout 配置已经删除。手动去背景固定传 `image_url`、`background_mode=complex`、`seg_model=birefnet`、`cross_check=off`，不传 `file` 或 `screen_color`；该 API 接收 `objectKey`、`resourceId` 或 `assetId` 候选引用；BFF 入队前统一拒绝 `data:` / `blob:`；worker 不重复入口校验，只调用 `resolve_editor_reference_object_key_for_owner`，底层 resolver 在解析引用前拒绝内联媒体，并在签名前完成登记状态和 owner 校验；直接签发 OSS URL，不下载原图。标准纯色背景四条链路固定传 `background_mode=flat`，并显式传 `image_url`、`screen_color=`、`seg_model=` 和 `cross_check=`，其中角色形象生成和角色动作逐帧去背传 `cross_check=on`，图标 spritesheet 生成和 UI 设计图素材提取传 `cross_check=off`。前端用户路径不展示抠图模型、模式或 cross-check，固定提交默认 `birefnet`，后端仍识别内部保留的 `anime-seg`；这些参数只属于后端内部供应商策略，不进入前端或外部 OpenAPI。标准纯色背景 BgFilter 调用失败，或连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD`（默认 `3`）并在 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS`（默认 `300`）内打开熔断时，继续复用“阿里云通用抠图 → 本地 `editor_green_screen` 键色扣除”兜底链，熔断期不得直接退化到本地兜底。角色动作视频生成的背景色已与生图链路统一：`screenColor=auto` 时由视觉 LLM（`gpt-5-mini`，Responses 协议、low 推理档）读源角色图自动决策，并经硬过滤器剔除与前景 / 皮肤撞色的候选，手动 hex 则尊重用户选择；透明源角色图在提交 Ark 图生视频前先合成到选定背景色实色，使视频背景等于抠图键色；抽帧后每帧先上传私有 OSS 并释放原帧缓冲，再以该 object key 的签名 URL 固定使用 `seg_model=birefnet`、`cross_check=on` 进入上述三段式链路。阿里云通用抠图配置为 `GENARRATIVE_ALIYUN_MATTING_ENABLED`、`GENARRATIVE_ALIYUN_MATTING_ENDPOINT`、`GENARRATIVE_ALIYUN_MATTING_ACCESS_KEY_ID`、`GENARRATIVE_ALIYUN_MATTING_ACCESS_KEY_SECRET` 和 `GENARRATIVE_ALIYUN_MATTING_REQUEST_TIMEOUT_MS`；未配置专用 AK/SK 时可复用 `ALIBABA_CLOUD_ACCESS_KEY_ID` / `ALIBABA_CLOUD_ACCESS_KEY_SECRET`，默认 endpoint 为 `imageseg.cn-shanghai.aliyuncs.com`。标准纯色背景链路中，BgFilter 调用失败和阿里云抠图链路已开始后的失败（包括源 OSS GET 成功后的解码、尺寸校验和归一化失败）都写入 `external_api_call_failure` 审计；真正开始外部调用前的本地预检不写该审计，并在 `failureStage` 中保留 `source_decode`、`source_validate` 等阶段。 - BgFilter 连接复用、重试与动作帧流水线：api-server 必须在 `AppState` 复用同一个 BgFilter HTTP Client 及 keep-alive 连接池。`GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS` 是所有路径的基准请求超时；角色动作逐帧 BgFilter 的每一次 HTTP attempt 使用“基准超时 + `2000ms × 本次实际帧数`”，默认 `32 / 40 / 48` 帧分别为 `244000 / 260000 / 276000ms`，角色形象单图、图标、UI 和手动去背景仍使用基准值。api-server 只在共享 Client 的单次 RequestBuilder 上覆盖该值；它覆盖从请求发起到响应体读取完成，是单次 attempt 的总 deadline，不是整批帧或 worker job 超时，重试会重新签发 600 秒 OSS URL 并获得同样的 request deadline，整项任务仍受 worker long-job 预算约束。flat 与 complex 请求首次失败后都立即重试 `1` 次；标准纯色背景 flat 请求第二次仍失败才进入“阿里云通用抠图 → 本地键色”降级链，手动 complex 请求第二次仍失败则返回最终错误，不接入依赖纯色键值的降级链，也不改变 flat 路径的熔断状态。角色动作全部 `32 / 40 / 48` 帧按“单帧绿幕源图 owned 上传 OSS 并释放原帧 → 以签名 URL 调 BgFilter/按 object key 降级 → 透明帧落 OSS”独立流水化，使用覆盖本次全部帧的无序在途集合连续发射；不限制 BgFilter、阿里云或本地处理，但角色动画源帧 PUT、透明帧 PUT 和最终帧 HEAD 统一复用 `AppState` 内初始化一次的 OSS HTTP Client（连接池参数为 connect 30 秒、request 60 秒、idle 300 秒、每 host 8 个 idle 连接、TCP keepalive 60 秒），并受进程级 8 路 OSS semaphore 限制。每个 OSS 网络 attempt 单独获取 permit，退避期间释放；PUT/HEAD 动画帧请求最多 3 次（250ms、500ms 退避），只重试无 HTTP 响应的传输错误、timeout、OSS PutObject 的 `400 + RequestTimeout`、PUT `400` 错误体读取失败（未解析出 `Code`，按 timeout/transport 归类）、408、429 和 5xx。动作帧 PUT 只在 400 响应中有界读取最多 16 KiB OSS 错误 XML，并保留 `Code` 与响应头优先的 `x-oss-request-id`；错误体读取超时/断流时保留已读字节，已解析出的 `Code` 优先生效，未解析出 `Code` 则按 timeout/transport 归类重试；除 `RequestTimeout` 与该错误体读取失败情形外的其他 400、401/403/404、配置、URL/签名和空请求体错误不重试。返回结果携带原始帧序并在收口时排序。任一帧最终失败时必须先排空全部已启动 Future，再让整个动作任务失败退款，不能发布缺帧动画。 diff --git a/docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md b/docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md index f2371fc67..edd32af5b 100644 --- a/docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md +++ b/docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md @@ -2,6 +2,8 @@ 日期：`2026-06-15` +更新时间：`2026-07-21` + ## 背景 图片画布编辑器已有普通图片生成与“生成规范”能力。本次新增“生成角色形象”入口，用于在同一画布内生成标注为“角色”的单张角色形象图片，并支持绑定角色规范与常规参考图。 @@ -162,8 +164,10 @@ - 视频生成完成后，后端先把带纯色背景的预览视频登记为 OSS 私有对象、`asset_object`、项目资源和账号素材，再按面板选择抽取对应帧数：`32`、`40` 或 `48`。未传 `assetFolderId` 时进入默认“项目”素材文件夹；后续抽帧或抠图失败不能抹掉这份已经生成成功的可恢复视频。 - 抽帧采样必须按目标帧数预留视频尾部安全步长，例如 `32帧·4秒` 最后一帧采 `3.875s`，避免 FFmpeg 在尾点附近返回成功但输出 `0` 帧。 +- 图片画布角色动作的 FFmpeg 原始帧在上传 OSS 前必须转为 RGB8，并按最终帧宽高的 contain 比例使用 `Triangle` 只缩放到内容尺寸；不得提前创建最终目标尺寸 RGBA 画布，不得引入 Alpha 通道或透明 padding。以 `560×752` 原始帧、`323×480` 最终目标为例，上传给抠图链路的源帧必须是 `323×434 RGB8 PNG`，没有上下补边。抽帧解码后若携带 Alpha 通道，必须先把像素按白底合成为不透明再转 RGB8，禁止直接丢弃 Alpha——全透明像素下未定义的 RGB 值会以杂色进入抠图输入，重新引入杂色边缘；共享 FFmpeg 抽帧命令保持不固定 `-pix_fmt`，白底合成只属于该链路的 BgFilter 输入准备阶段。 - 后端先计算整批精确采样时刻，再用单个 FFmpeg filter graph 统一解码预览视频并输出 `32 / 40 / 48` 张源帧；不得为每帧重新启动 FFmpeg、重复解码同一视频，也不得用会改变现有尾帧安全时刻的粗粒度 `fps` 抽帧替代。批量命令成功后必须逐一确认全部目标帧文件存在，缺少任一帧都按整批失败处理并保留缺帧编号、目标时刻和输出路径诊断。 - 每帧绿幕源图字节由上传 owned 消费（`frame.bytes` 移入 put，上传完成后释放原帧缓冲，不克隆保留）；后续只持 object key。每次 BgFilter attempt 重新签发 600 秒 GET URL，multipart 仅传 `image_url`（加 `background_mode=flat`、`seg_model=birefnet`、`cross_check=on` 与同一次生成已选定的 `screenColor`），不传 `file`。BgFilter 主路径不重新下载原帧；失败后走 `阿里云通用抠图（按签名 URL 单独下载）→ 本地 editor_green_screen（再按 object key 独立下载一次并在产出后释放）`。BgFilter 每一次 HTTP attempt 的 timeout 使用“`GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS` 基准值 + `2000ms × 本次实际帧数`”，默认 `32 / 40 / 48` 帧分别为 `244000 / 260000 / 276000ms`；首次失败后重试 `1` 次。 +- BgFilter、阿里云或本地键色返回透明结果后，后端继续通过现有最终帧 finalizer 转为 RGBA8，按宽高比居中放入最终目标尺寸，并使用 `RGBA(0,0,0,0)` 补边。上述样例最终输出必须为 `323×480 RGBA8 PNG`，顶部和底部各 `23px` 透明 padding，内容区域完整保留抠图结果。 - 全部 `32 / 40 / 48` 帧以覆盖本次所有帧的无序在途集合连续发射，允许乱序完成并最终按 `frameIndex` 排序；任一帧最终失败时先排空全部已启动 Future，再让整项任务失败退款，不发布缺帧动画。 - 抽帧结果写入 OSS，并返回帧路径、帧尺寸、帧数、fps、预览视频路径、模型、价格和实际 prompt。 - 画板前端回填角色动作结果时，必须以 `frames[0].imageSrc` 创建 `mediaType: "image-sequence"`、`assetKind: "character-animation"` 图层，并把完整 `frames` 保存为图层 `imageSequenceFrames`；`previewVideoPath` 只保留为上游预览视频来源，不作为画布主媒体。 diff --git a/server-rs/crates/api-server/src/character_animation_assets.rs b/server-rs/crates/api-server/src/character_animation_assets.rs index 03d37f0fc..43aa0111b 100644 --- a/server-rs/crates/api-server/src/character_animation_assets.rs +++ b/server-rs/crates/api-server/src/character_animation_assets.rs @@ -2320,6 +2320,7 @@ async fn extract_and_persist_editor_character_animation_frames( let plan = AnimationFrameExtractionPlan { frame_count: request.frame_count, apply_chroma_key: false, + prepare_for_bgfilter_input: true, sample_start_ratio: 0.0, sample_end_ratio: 1.0, }; @@ -4077,6 +4078,7 @@ fn normalize_animation_frame_extraction_plan( AnimationFrameExtractionPlan { frame_count, apply_chroma_key, + prepare_for_bgfilter_input: false, sample_start_ratio, sample_end_ratio, } @@ -4155,13 +4157,23 @@ async fn extract_animation_frames_from_preview_video( "message": format!("读取动作抽帧结果失败：{error}"), })) })?; - finalized_frames.push(finalize_animation_frame_payload( - frame_bytes.as_slice(), - "image/png", - frame_width, - frame_height, - plan.apply_chroma_key, - )?); + let finalized_frame = if plan.prepare_for_bgfilter_input { + prepare_editor_character_animation_bgfilter_input( + frame_bytes.as_slice(), + "image/png", + frame_width, + frame_height, + )? + } else { + finalize_animation_frame_payload( + frame_bytes.as_slice(), + "image/png", + frame_width, + frame_height, + plan.apply_chroma_key, + )? + }; + finalized_frames.push(finalized_frame); } Ok::<_, AppError>(finalized_frames) @@ -4457,6 +4469,74 @@ fn run_process_with_timeout( } } +/// BgFilter 只接受不透明 RGB 输入；直接丢弃 alpha 会让全透明像素下未定义的 +/// RGB 值以杂色进入抠图，这里先按白底合成再转 RGB。 +fn flatten_alpha_onto_white_rgb(image: image::DynamicImage) -> image::RgbImage { + if !image.color().has_alpha() { + return image.to_rgb8(); + } + let rgba = image.to_rgba8(); + let mut flattened = image::RgbImage::new(rgba.width(), rgba.height()); + for (source, target) in rgba.pixels().zip(flattened.pixels_mut()) { + let [red, green, blue, alpha] = source.0; + let opacity = f32::from(alpha) / 255.0; + let blend = |channel: u8| -> u8 { + (f32::from(channel) * opacity + 255.0 * (1.0 - opacity)).round() as u8 + }; + target.0 = [blend(red), blend(green), blend(blue)]; + } + flattened +} + +fn prepare_editor_character_animation_bgfilter_input( + source: &[u8], + mime_type: &str, + target_width: u32, + target_height: u32, +) -> Result { + let image = match image_format_from_mime(mime_type) { + Some(format) => image::load_from_memory_with_format(source, format), + None => image::load_from_memory(source), + } + .map_err(|error| { + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "character-animation", + "message": format!("解析 BgFilter 输入动作帧图片失败：{error}"), + })) + })?; + let image = flatten_alpha_onto_white_rgb(image); + + let (draw_width, draw_height) = + compute_contain_dimensions(image.width(), image.height(), target_width, target_height); + let resized = if (draw_width, draw_height) == (image.width(), image.height()) { + image + } else { + image::imageops::resize(&image, draw_width, draw_height, FilterType::Triangle) + }; + + let mut encoded = Vec::new(); + let encoder = PngEncoder::new(&mut encoded); + encoder + .write_image( + resized.as_raw(), + resized.width(), + resized.height(), + ColorType::Rgb8.into(), + ) + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "provider": "character-animation", + "message": format!("编码 BgFilter 输入动作帧 PNG 失败：{error}"), + })) + })?; + + Ok(FinalizedAnimationFrame { + bytes: encoded, + mime_type: "image/png".to_string(), + extension: "png".to_string(), + }) +} + fn finalize_animation_frame_payload( source: &[u8], mime_type: &str, @@ -4464,13 +4544,7 @@ fn finalize_animation_frame_payload( frame_height: u32, apply_chroma_key: bool, ) -> Result { - let image_format = match mime_type { - "image/png" => Some(ImageFormat::Png), - "image/jpeg" | "image/jpg" => Some(ImageFormat::Jpeg), - "image/webp" => Some(ImageFormat::WebP), - _ => None, - }; - let mut image = match image_format { + let mut image = match image_format_from_mime(mime_type) { Some(format) => image::load_from_memory_with_format(source, format), None => image::load_from_memory(source), } @@ -4517,8 +4591,39 @@ fn finalize_animation_frame_payload( fn contain_rgba_image(source: &RgbaImage, target_width: u32, target_height: u32) -> RgbaImage { let mut canvas = RgbaImage::from_pixel(target_width, target_height, Rgba([0, 0, 0, 0])); - let source_width = source.width().max(1); - let source_height = source.height().max(1); + let (draw_width, draw_height) = + compute_contain_dimensions(source.width(), source.height(), target_width, target_height); + let offset_x = ((target_width - draw_width) / 2) as i64; + let offset_y = ((target_height - draw_height) / 2) as i64; + if (draw_width, draw_height) == (source.width(), source.height()) { + image::imageops::overlay(&mut canvas, source, offset_x, offset_y); + } else { + let resized = + image::imageops::resize(source, draw_width, draw_height, FilterType::Triangle); + image::imageops::overlay(&mut canvas, &resized, offset_x, offset_y); + } + canvas +} + +fn image_format_from_mime(mime_type: &str) -> Option { + match mime_type { + "image/png" => Some(ImageFormat::Png), + "image/jpeg" | "image/jpg" => Some(ImageFormat::Jpeg), + "image/webp" => Some(ImageFormat::WebP), + _ => None, + } +} + +fn compute_contain_dimensions( + source_width: u32, + source_height: u32, + target_width: u32, + target_height: u32, +) -> (u32, u32) { + let target_width = target_width.max(1); + let target_height = target_height.max(1); + let source_width = source_width.max(1); + let source_height = source_height.max(1); let scale = (target_width as f32 / source_width as f32) .min(target_height as f32 / source_height as f32); let draw_width = ((source_width as f32 * scale).round() as u32) @@ -4527,11 +4632,7 @@ fn contain_rgba_image(source: &RgbaImage, target_width: u32, target_height: u32) let draw_height = ((source_height as f32 * scale).round() as u32) .max(1) .min(target_height); - let resized = image::imageops::resize(source, draw_width, draw_height, FilterType::Triangle); - let offset_x = ((target_width - draw_width) / 2) as i64; - let offset_y = ((target_height - draw_height) / 2) as i64; - image::imageops::overlay(&mut canvas, &resized, offset_x, offset_y); - canvas + (draw_width, draw_height) } async fn load_media_source_payload( @@ -5848,6 +5949,7 @@ struct BackendFrameExtractionSettings { struct AnimationFrameExtractionPlan { frame_count: u32, apply_chroma_key: bool, + prepare_for_bgfilter_input: bool, sample_start_ratio: f32, sample_end_ratio: f32, } @@ -6356,6 +6458,109 @@ mod tests { ); } + #[test] + fn editor_character_animation_bgfilter_input_is_rgb_without_padding() { + let source = image::RgbImage::from_pixel(560, 752, image::Rgb([17, 99, 201])); + let mut source_png = Vec::new(); + PngEncoder::new(&mut source_png) + .write_image( + source.as_raw(), + source.width(), + source.height(), + ColorType::Rgb8.into(), + ) + .expect("source RGB frame should encode"); + + let prepared = prepare_editor_character_animation_bgfilter_input( + source_png.as_slice(), + "image/png", + 323, + 480, + ) + .expect("BgFilter input should be prepared"); + let decoded = + image::load_from_memory_with_format(prepared.bytes.as_slice(), ImageFormat::Png) + .expect("prepared BgFilter input should decode"); + + assert_eq!(prepared.mime_type, "image/png"); + assert_eq!(prepared.extension, "png"); + assert_eq!(decoded.width(), 323); + assert_eq!(decoded.height(), 434); + assert_eq!(decoded.color(), ColorType::Rgb8); + assert!( + decoded + .to_rgb8() + .pixels() + .all(|pixel| pixel.0 == [17, 99, 201]) + ); + } + + #[test] + fn editor_character_animation_bgfilter_input_flattens_alpha_onto_white() { + let mut source = RgbaImage::from_pixel(8, 8, Rgba([17, 99, 201, 255])); + source.put_pixel(0, 0, Rgba([255, 0, 0, 0])); + source.put_pixel(1, 0, Rgba([0, 0, 0, 127])); + let mut source_png = Vec::new(); + PngEncoder::new(&mut source_png) + .write_image( + source.as_raw(), + source.width(), + source.height(), + ColorType::Rgba8.into(), + ) + .expect("source RGBA frame should encode"); + + let prepared = prepare_editor_character_animation_bgfilter_input( + source_png.as_slice(), + "image/png", + 8, + 8, + ) + .expect("BgFilter input should be prepared"); + let decoded = + image::load_from_memory_with_format(prepared.bytes.as_slice(), ImageFormat::Png) + .expect("prepared BgFilter input should decode") + .to_rgb8(); + + assert_eq!(decoded.get_pixel(0, 0).0, [255, 255, 255]); + assert_eq!(decoded.get_pixel(1, 0).0, [128, 128, 128]); + assert_eq!(decoded.get_pixel(2, 0).0, [17, 99, 201]); + } + + #[test] + fn editor_character_animation_final_frame_adds_transparent_vertical_padding() { + let source = RgbaImage::from_pixel(323, 434, Rgba([31, 127, 223, 191])); + let mut source_png = Vec::new(); + PngEncoder::new(&mut source_png) + .write_image( + source.as_raw(), + source.width(), + source.height(), + ColorType::Rgba8.into(), + ) + .expect("source RGBA frame should encode"); + + let finalized = + finalize_animation_frame_payload(source_png.as_slice(), "image/png", 323, 480, false) + .expect("final transparent frame should be finalized"); + let decoded = + image::load_from_memory_with_format(finalized.bytes.as_slice(), ImageFormat::Png) + .expect("final transparent frame should decode"); + + assert_eq!(decoded.width(), 323); + assert_eq!(decoded.height(), 480); + assert_eq!(decoded.color(), ColorType::Rgba8); + let output = decoded.to_rgba8(); + for (x, y, pixel) in output.enumerate_pixels() { + let expected = if (23..457).contains(&y) { + [31, 127, 223, 191] + } else { + [0, 0, 0, 0] + }; + assert_eq!(pixel.0, expected, "unexpected pixel at ({x}, {y})"); + } + } + #[test] fn editor_character_animation_frames_use_three_stage_matting_fallback() { let source = include_str!("character_animation_assets.rs"); @@ -6365,6 +6570,7 @@ mod tests { "async fn publish_animation_set", &[ "apply_chroma_key: false", + "prepare_for_bgfilter_input: true", "editor_character_animation_bgfilter_request_timeout_ms", "state.config.editor_bgfilter_request_timeout_ms", "process_and_persist_editor_character_animation_frame", @@ -6375,6 +6581,22 @@ mod tests { "frame_errors.sort_by_key", ], ); + assert_function_contains( + source, + "fn normalize_animation_frame_extraction_plan", + "fn normalize_sample_ratio", + &["apply_chroma_key", "prepare_for_bgfilter_input: false"], + ); + assert_function_contains_in_order( + source, + "async fn extract_animation_frames_from_preview_video", + "fn create_animation_temp_dir", + &[ + "plan.prepare_for_bgfilter_input", + "prepare_editor_character_animation_bgfilter_input", + "finalize_animation_frame_payload", + ], + ); assert_function_contains_in_order( source, "async fn process_and_persist_editor_character_animation_frame", From 48c9ee2faeea7d7f0abdb34f3c6c2dd62d687c3d Mon Sep 17 00:00:00 2001 From: kdletters Date: Tue, 21 Jul 2026 16:05:52 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BE=8E=E6=9C=AF=20Agen?= =?UTF-8?q?t=20=E9=95=BF=E7=AD=89=E5=BE=85=E6=8F=90=E7=A4=BA=E4=B8=8E?= =?UTF-8?q?=E8=B6=85=E6=97=B6=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 120 秒硬超时改为请求存活时的耐心等待提示。 收口 provider 安全上限、有限重试和明确失败错误。 补齐等待互斥、计时清理、前后端测试及契约文档。 --- docs/project-memory/shared-memory/pitfalls.md | 8 + ...】server-rs与SpacetimeDB数据契约-2026-05-15.md | 3 +- .../【编辑器】画布Agent对话面板-2026-07-03.md | 6 +- server-rs/crates/api-server/src/state.rs | 12 +- .../platform-editor-agent/src/agent/agent.rs | 6 +- .../src/framework/error.rs | 39 +++- .../EditorAgentConversationPanelView.test.tsx | 41 ++-- .../EditorAgentConversationPanelView.tsx | 5 +- .../MessageBubble.test.tsx | 14 +- .../EditorAgentConversation/MessageBubble.tsx | 20 +- .../useEditorAgentConversation.test.tsx | 189 +++++++++++++++++- .../useEditorAgentConversation.ts | 169 ++++++++++++---- 12 files changed, 430 insertions(+), 82 deletions(-) diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index dcdcd0fe2..8a3749ec8 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -3234,6 +3234,14 @@ - 验证：runner 回归测试必须同时覆盖“待确认工具只调用一次 LLM 并成功结束”和“普通连续工具仍会触发 max-turn 门禁”。 - 关联：`server-rs/crates/platform-editor-agent/src/framework/run.rs`、`server-rs/crates/platform-editor-agent/src/framework/tool.rs`、`server-rs/crates/platform-editor-agent/src/agent/tools/`。 +## 画布 Agent 的规划请求不能关闭瞬时失败重试 + +- 现象：美术 Agent 对话返回红色错误气泡 `completion error: LLM 请求超时，累计尝试 1 次`；HTTP 本身仍返回 200，前端 20 分钟 transport timeout 没有触发。 +- 原因：规划请求虽然有 Agent 专用单次 timeout，但 `editor_agent_llm_client` 把 `max_retries` 硬编码为 0；VectorEngine `gpt-5.4-mini` 的偶发长尾、连接超时或可重试上游状态会在第一次失败后直接持久化成 system error。framework 的英文 `completion error` 前缀也被原样暴露给用户。 +- 处理：120 秒改为前端软提示阈值：POST 仍 pending 时显示不入库的“仍在处理中，请耐心等待”；provider 明确断开/失败才写正式错误。专用 provider 单 attempt 使用 8 分钟 hard timeout，请求发起阶段读取 `GENARRATIVE_LLM_MAX_RETRIES`，但画布 Agent 最多重试 1 次且重试退避最多 60 秒，保证理论上限小于前端 20 分钟 transport timeout；响应头后的体读取/解析错误按明确失败收口。规划错误对用户统一为中文。重试发生在任何生成工具执行前，不会重复提交生成任务或扣费，不要通过提高前端 timeout 或 runner `max_turns` 掩盖 provider 重试缺失。 +- 验证：`platform-editor-agent` 测试锁定 8 分钟 hard timeout 与中文错误；前端 fake timer 用例锁定 120 秒前只显示思考动画、到点后显示耐心等待、成功/失败后移除；`api-server` AppState 测试锁定专用 client 透传 retry 次数。运行态排障按同一 request id 对齐 `platform_llm` failure stage 与 `/messages` 总耗时，并确认仍 pending 的请求不再在 120 秒形成错误气泡。 +- 关联：`server-rs/crates/platform-editor-agent/src/agent/agent.rs`、`server-rs/crates/platform-editor-agent/src/framework/error.rs`、`server-rs/crates/api-server/src/state.rs`、`src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts`、`src/components/image-editor/EditorAgentConversation/MessageBubble.tsx`、`src/services/image-editor/editorAgentClient.ts`。 + ## 前端退役目录不能只靠扫描和 ignore 隔离 - 现象：Tailwind `@source`、TypeScript 根 `include`、ESLint ignore 和 Vitest include 都排除了旧创作目录，但干净打开新版页面时，Vite 仍转换 `services/rpg-entry/index.ts`，构建产物也包含旧作品库和旧 profile 逻辑。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index d45b627c0..5bdd6e6ea 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -75,7 +75,8 @@ npm run check:server-rs-ddd - `/api/editor/projects/{projectId}/agent-conversations` 负责当前工程会话列表和新建；`/api/editor/agent-conversations/{conversationId}` 负责详情读取、终态工具消息懒回填和软删；`POST /api/editor/agent-conversations/{conversationId}/messages` 负责发送消息并返回普通 JSON `EditorAgentMessageResponse`，画布 Agent 不提供 `/messages/stream` SSE 路由。消息请求必须携带最长 128 字符的 `clientMessageId`；前端对该 POST 显式启用 1 次瞬时 transport 重试，并复用同一个序列化 body、`clientMessageId` 和 `x-request-id`。同一会话在锁内按该键幂等，重复键同内容返回已有回合或从已保存用户消息继续，异内容返回 `409`。数字 `EditorAgentMessage.id` 仍只作为工具确认 / 取消的后端消息定位符，不能复用为客户端幂等键。 - `module-editor-agent` 只承载纯领域校验：标题派生、附件上限、消息输入规则和会话软删访问规则；不直接依赖 Axum、SpacetimeDB、OSS、LLM 或 Tokio。 - `spacetime-module` 的 `editor_agent_conversation` 只保存元数据；创建、列表、读取、更新时间和软删通过 `create_editor_agent_conversation_and_return`、`list_editor_agent_conversations_and_return`、`get_editor_agent_conversation_and_return`、`touch_editor_agent_conversation_and_return`、`delete_editor_agent_conversation_and_return` procedure 完成，`api-server` 只能经 `spacetime-client` facade 访问。 -- 完整消息文档存 OSS `editor-agent/{conversationId}.json`，由 `api-server` 负责 2 MiB 上限、会话内串行锁、读改写、消息与工具结果持久化和 `touch` 元数据更新时间；该 JSON 不进入 `editor_canvas.layers_json`，也不作为画布布局真相。LLM / 规划失败必须写入 `role=system`、正文以 `ERROR ` 开头的消息，并通过 `deltaMessages` 返回，`errorMessage` 保持为空；前端隐藏前缀并显示红色错误气泡，后端仍把该 system 消息注入后续 LLM memory，使 Agent 能读取失败上下文。工具失败同样必须形成可回读记录，不能只返回瞬时错误。 +- 完整消息文档存 OSS `editor-agent/{conversationId}.json`，由 `api-server` 负责 2 MiB 上限、会话内串行锁、读改写、消息与工具结果持久化和 `touch` 元数据更新时间；该 JSON 不进入 `editor_canvas.layers_json`，也不作为画布布局真相。LLM 未配置、连接已经断开、请求明确失败、达到最终安全上限或规划不可解析时，必须写入 `role=system`、正文以 `ERROR ` 开头的消息，并通过 `deltaMessages` 返回，`errorMessage` 保持为空；前端隐藏前缀并显示红色错误气泡，面向用户的错误正文使用中文语义，不暴露 `completion error` 等 framework 内部前缀，后端仍把该 system 消息注入后续 LLM memory，使 Agent 能读取失败上下文。普通 JSON POST 尚未结束不形成持久化消息；工具失败同样必须形成可回读记录，不能只返回瞬时错误。 +- 画布 Agent 的 `gpt-5.4-mini` Chat Completions 规划使用 1024 `max_tokens`。前端在 POST pending 120 秒后显示不入库的耐心等待提示；provider request future 明确返回 connect/timeout/HTTP/transport 错误时立即进入正式失败，尚未返回则继续等待。专用 provider 单 attempt hard timeout 为 8 分钟；请求发起阶段的 timeout、连接失败、`408`、`429` 与 `5xx` 读取 `GENARRATIVE_LLM_MAX_RETRIES`，但画布 Agent 最多重试 1 次，显式配置 0 仍可关闭，专用重试退避最多 60 秒，使两次 attempt 的理论上限仍早于前端消息 POST 的 20 分钟 timeout；已收到成功响应头后的响应体读取或解析失败直接按明确失败收口。重试只包围 LLM 规划请求并发生在任何待确认工具执行之前，因此不会重复提交生成任务或扣费。 - 对话附件只允许引用当前工程 `editor_project_resource` 或当前账号 `editor_asset` 的图片；前端可提交展示用 `imageSrc` / `thumbnailSrc`，后端必须按 `resourceId` / `assetId` 重新归一、校验 owner / project 和 `objectKey`，再给 LLM 或生成工具使用。 - 画布 Agent 工具复用既有编辑器图片生成 / 修改 / 图标 spritesheet BFF，并继续使用后端模型定价和 `execute_billable_asset_operation_with_cost`；前端不提交 `priceMudPoints`。 - `/messages/{messageId}/confirm` 与 `/messages/{messageId}/cancel` 只返回成功确认；前端成功后立即重新读取整个会话，以会话详情中的权威消息状态和 `externalJobId` 驱动气泡展示与任务轮询。 diff --git a/docs/【编辑器】画布Agent对话面板-2026-07-03.md b/docs/【编辑器】画布Agent对话面板-2026-07-03.md index 6cf3cc2b6..f2dfc9cd1 100644 --- a/docs/【编辑器】画布Agent对话面板-2026-07-03.md +++ b/docs/【编辑器】画布Agent对话面板-2026-07-03.md @@ -95,12 +95,12 @@ ## LLM 与计费 - 编排复用 `creative_agent_gpt5_client` 的 LLM 接入配置（同 provider/env，独立用途标识），画布 Agent 规划请求固定使用 VectorEngine `gpt-5.4-mini` Chat Completions；function-calling 注册八类工具。 -- 每个用户回合必须由 LLM 返回结构化计划；LLM 未配置、请求失败或返回格式不可解析时，后端写入正文为 `ERROR <错误内容>` 的 system 消息，不使用本地关键词或“收到：...”回显兜底。该错误消息与其它 system 消息一样进入后续 LLM memory，使 Agent 能看到上一轮失败上下文。 +- 每个用户回合必须由 LLM 返回结构化计划；LLM 未配置、连接已经断开、请求明确失败、达到最终安全上限或返回格式不可解析时，后端写入正文为 `ERROR <错误内容>` 的 system 消息，不使用本地关键词或“收到：...”回显兜底。面向用户的规划错误使用中文语义，不暴露 `completion error` 等 framework 内部前缀。该错误消息与其它 system 消息一样进入后续 LLM memory，使 Agent 能看到上一轮失败上下文。普通 JSON POST 尚未结束只表示 provider request future 仍在等待，不能伪装成已持久化失败。 - 规划 prompt 必须自动带入上一条已完成生成结果的 `latestGeneratedImage` 引用，内容只包含上一轮 generation 的 `toolName` / `resourceId` / `objectKey` / `assetObjectId` 等轻量元数据，不把私有签名 URL 或大图内容塞进 prompt。 - 工具参数中的图片 ID 是由真实 object key 或图片地址计算的稳定 SHA-256 标识；真实 data key 仅存于 api-server 的工具上下文映射，所有图片工具在执行时查表恢复，不能把 object key 或图片地址作为 LLM 可见的工具 ID。 - 用户使用「这张」「刚才那个」「上一张」「把衣服换成……」等方式指代或编辑上一张结果图时，LLM 默认选择 `edit_image` 并引用 `latestGeneratedImage` 作为源图；除非用户明确要求全新生成，否则不能因为本轮没有重新上传附件而降级为 `generate_image`。 - 规划 prompt 必须显式区分“规范展板”和“实际素材产出”：规范图、视觉规范图、风格规范图、素材规范展板、角色规范图等规范展板请求走 `generate_image`，并补齐统一视角、线条粗细、色卡、材质、阴影、圆角、状态层级、尺寸标注等要求；实际角色立绘才走 `generate_character`，多个图标素材 / 图集才走 `generate_icon_spritesheet`。 -- 画布 Agent 规划请求使用 Chat Completions、1024 `max_tokens` 和 60 秒 Agent 专用请求超时；生成图片/编辑图片仍走对应生成工具和模型计费。 +- 画布 Agent 规划请求使用 Chat Completions 和 1024 `max_tokens`。发送后 120 秒是前端软提示阈值，不是 provider 失败 deadline：若普通 JSON POST 仍 pending，消息流临时显示“仍在处理中，请耐心等待”并继续等待，提示不写入 OSS 消息历史；连接或请求明确失败则立即按正式错误收口。provider 单 attempt 保留 8 分钟 hard timeout；请求发起阶段的 timeout、连接失败、`408`、`429` 与 `5xx` 读取 `GENARRATIVE_LLM_MAX_RETRIES`，但画布 Agent 最多重试 1 次，专用重试退避最多 60 秒，使两次 attempt 的理论最坏等待仍早于前端 20 分钟 transport timeout；已收到成功响应头后的响应体读取或解析失败直接按明确失败收口。规划重试发生在任何生成工具执行之前，不会重复提交生成任务或扣费；生成图片/编辑图片仍走对应生成工具和模型计费。 - function-calling runner 必须把“等待用户确认”作为显式工具语义：当本批所有工具都校验成功并进入待确认状态时，立即以成功结果结束当前规划回合并持久化助手文本与待确认卡，不得继续依赖 LLM 自行停止；未知工具、参数错误、普通连续工具和不可解析响应仍受 `max_turns` 保护。 - **对话回合免费**（聊天、分析回复不扣泥点），仅 Agent 实际触发生成工具时按对应模型定价扣泥点。 - 工具调用前后端校验泥点余额；不足时该次生成失败并在对话中以明确错误气泡告知，对话本身可继续。 @@ -115,7 +115,7 @@ 4. 消息内生成结果缩略图（纯预览，不显示名称，不点击聚焦图层）； 5. 生成中的进行中动画； 6. 错误气泡（失败/余额不足，带原因）； -7. 普通消息请求等待期间禁用发送按钮，不提供客户端停止操作；前端持续等待后端响应，避免后端已持久化消息但前端中断请求后产生会话状态错位。 +7. 普通消息请求等待期间禁用发送按钮，不提供客户端停止操作；前端持续等待后端响应，超过 120 秒但 POST 仍 pending 时在思考气泡中显示“仍在处理中，请耐心等待”，最终成功或失败后自动移除，避免后端已持久化消息但前端中断请求后产生会话状态错位。 不做（明确排除，防止后人补齐）： diff --git a/server-rs/crates/api-server/src/state.rs b/server-rs/crates/api-server/src/state.rs index 6d0b18d9a..4e263c3f3 100644 --- a/server-rs/crates/api-server/src/state.rs +++ b/server-rs/crates/api-server/src/state.rs @@ -48,6 +48,8 @@ use crate::work_author::{ }; const ADMIN_ROLE: &str = "admin"; +const EDITOR_AGENT_LLM_MAX_RETRIES: u32 = 1; +const EDITOR_AGENT_LLM_MAX_RETRY_BACKOFF_MS: u64 = 60_000; pub(crate) const CHARACTER_ANIMATION_OSS_MAX_CONCURRENCY: usize = 8; pub type HttpRequestPermitPool = Semaphore; @@ -2084,8 +2086,10 @@ fn build_editor_agent_llm_client( api_key.to_string(), platform_llm::EDITOR_AGENT_GPT5_MODEL.to_string(), config.llm_request_timeout_ms, - 0, - config.llm_retry_backoff_ms, + config.llm_max_retries.min(EDITOR_AGENT_LLM_MAX_RETRIES), + config + .llm_retry_backoff_ms + .min(EDITOR_AGENT_LLM_MAX_RETRY_BACKOFF_MS), )?; Ok(Some(LlmClient::new(llm_config)?)) @@ -2354,6 +2358,8 @@ mod tests { fn app_state_builds_editor_agent_llm_client_from_vector_engine_settings() { let mut config = AppConfig::default(); config.llm_api_key = None; + config.llm_max_retries = 2; + config.llm_retry_backoff_ms = 120_000; config.vector_engine_base_url = "https://api.vectorengine.test".to_string(); config.vector_engine_api_key = Some("ve-key".to_string()); @@ -2371,6 +2377,8 @@ mod tests { "https://api.vectorengine.test/v1/chat/completions" ); assert!(!client.config().official_fallback()); + assert_eq!(client.config().max_retries(), 1); + assert_eq!(client.config().retry_backoff_ms(), 60_000); } fn test_feature_gate(gate_key: &str) -> module_runtime::FeatureGateConfigSnapshot { diff --git a/server-rs/crates/platform-editor-agent/src/agent/agent.rs b/server-rs/crates/platform-editor-agent/src/agent/agent.rs index 380f4f2d8..226f8ef61 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/agent.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/agent.rs @@ -8,7 +8,7 @@ use platform_llm::{EDITOR_AGENT_GPT5_MODEL, LlmClient, LlmMessage, LlmTextReques use serde_json::Value; const EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS: u32 = 1024; -const EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS: u64 = 60_000; +const EDITOR_AGENT_LLM_HARD_REQUEST_TIMEOUT_MS: u64 = 480_000; pub struct LlmCompletionModel { client: LlmClient, @@ -41,7 +41,7 @@ fn build_editor_agent_llm_request(messages: Vec) -> LlmTextRequest { LlmTextRequest::new(messages) .with_model(EDITOR_AGENT_GPT5_MODEL) .with_max_tokens(EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS) - .with_request_timeout_ms(EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS) + .with_request_timeout_ms(EDITOR_AGENT_LLM_HARD_REQUEST_TIMEOUT_MS) } pub struct LlmChatAgentBuilder { @@ -188,7 +188,7 @@ mod tests { assert_eq!(request.model.as_deref(), Some(EDITOR_AGENT_GPT5_MODEL)); assert_eq!(request.max_tokens, Some(1024)); - assert_eq!(request.request_timeout_ms, Some(60_000)); + assert_eq!(request.request_timeout_ms, Some(480_000)); assert_eq!(request.messages.len(), 2); } } diff --git a/server-rs/crates/platform-editor-agent/src/framework/error.rs b/server-rs/crates/platform-editor-agent/src/framework/error.rs index bc830dbac..5d2e92581 100644 --- a/server-rs/crates/platform-editor-agent/src/framework/error.rs +++ b/server-rs/crates/platform-editor-agent/src/framework/error.rs @@ -9,14 +9,45 @@ pub enum PromptError { impl std::fmt::Display for PromptError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::CompletionError(msg) => write!(f, "completion error: {msg}"), - Self::ToolError(msg) => write!(f, "tool error: {msg}"), - Self::InternalError(msg) => write!(f, "internal error: {msg}"), + Self::CompletionError(msg) => write!(f, "美术 Agent 规划失败：{msg}"), + Self::ToolError(msg) => write!(f, "美术 Agent 工具执行失败：{msg}"), + Self::InternalError(msg) => write!(f, "美术 Agent 内部错误：{msg}"), Self::MaxTurnsReached { max_turns } => { - write!(f, "max turns reached: {max_turns}") + write!(f, "美术 Agent 规划轮数已达上限：{max_turns}") } } } } impl std::error::Error for PromptError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn completion_error_uses_user_facing_chinese_copy() { + let error = PromptError::CompletionError("LLM 请求超时，累计尝试 2 次".to_string()); + + assert_eq!( + error.to_string(), + "美术 Agent 规划失败：LLM 请求超时，累计尝试 2 次" + ); + } + + #[test] + fn other_errors_do_not_expose_framework_prefixes() { + assert_eq!( + PromptError::ToolError("参数无效".to_string()).to_string(), + "美术 Agent 工具执行失败：参数无效" + ); + assert_eq!( + PromptError::InternalError("序列化失败".to_string()).to_string(), + "美术 Agent 内部错误：序列化失败" + ); + assert_eq!( + PromptError::MaxTurnsReached { max_turns: 3 }.to_string(), + "美术 Agent 规划轮数已达上限：3" + ); + } +} diff --git a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx index 8556f4876..3072fa16c 100644 --- a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx +++ b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx @@ -8,13 +8,14 @@ import { waitFor, within, } from '@testing-library/react'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { EditorAgentMessage, EditorAgentMessageResponse, } from '@/packages/shared/src/contracts'; import type { EditorAgentConversationClient } from '@/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts'; +import { EDITOR_AGENT_PATIENCE_NOTICE_DELAY_MS } from '@/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts'; import { useImageCanvasContextStore } from '@/src/components/image-editor/useImageCanvasContextStore.ts'; import { EditorAgentConversationPanelView } from './EditorAgentConversationPanelView.tsx'; @@ -114,6 +115,10 @@ function createClient(): EditorAgentConversationClient { }; } +afterEach(() => { + vi.useRealTimers(); +}); + function createPendingToolCallMessage(): EditorAgentMessage { return { id: 2, @@ -331,18 +336,26 @@ describe('EditorAgentConversationPanelView', () => { await waitFor(() => { expect(screen.getByText('已经看到画布内容')).toBeTruthy(); }); + vi.useFakeTimers(); fireEvent.change(screen.getByLabelText('发送给画布 Agent'), { target: { value: '继续规划' }, }); fireEvent.click(screen.getByRole('button', { name: '发送' })); - await waitFor(() => { - expect( - screen.getByRole('button', { name: '发送' }).hasAttribute('disabled'), - ).toBe(true); + await act(async () => { + await Promise.resolve(); }); + expect( + screen.getByRole('button', { name: '发送' }).hasAttribute('disabled'), + ).toBe(true); expect(screen.queryByRole('button', { name: '停止' })).toBeNull(); + expect(screen.queryByText('仍在处理中，请耐心等待')).toBeNull(); + + act(() => { + vi.advanceTimersByTime(EDITOR_AGENT_PATIENCE_NOTICE_DELAY_MS); + }); + expect(screen.getByText('仍在处理中，请耐心等待')).toBeTruthy(); await act(async () => { resolveSend({ @@ -355,7 +368,9 @@ describe('EditorAgentConversationPanelView', () => { deltaMessages: [], errorMessage: null, }); + await Promise.resolve(); }); + expect(screen.queryByText('仍在处理中，请耐心等待')).toBeNull(); }); it('uploads pasted images as canvas attachments before sending', async () => { @@ -431,9 +446,7 @@ describe('EditorAgentConversationPanelView', () => { ); }); expect(screen.getByRole('option', { name: '角色参考' })).toBeTruthy(); - expect( - screen.queryByRole('option', { name: 'conversation-1' }), - ).toBeNull(); + expect(screen.queryByRole('option', { name: 'conversation-1' })).toBeNull(); }); it('sends selected attachments even when the text input is empty', async () => { @@ -555,9 +568,9 @@ describe('EditorAgentConversationPanelView', () => { ).toBe('失败后恢复这条草稿'); expect(screen.getByText('角色图层')).toBeTruthy(); expect( - within(screen.getByRole('log', { name: '画布 Agent 消息流' })).queryByText( - '失败后恢复这条草稿', - ), + within( + screen.getByRole('log', { name: '画布 Agent 消息流' }), + ).queryByText('失败后恢复这条草稿'), ).toBeNull(); }); @@ -692,9 +705,9 @@ describe('EditorAgentConversationPanelView', () => { }); expect(screen.getByRole('button', { name: '执行中' })).toBeTruthy(); expect(screen.queryByRole('button', { name: '确认' })).toBeNull(); - expect(screen.getByRole('button', { name: '取消' }).hasAttribute('disabled')).toBe( - true, - ); + expect( + screen.getByRole('button', { name: '取消' }).hasAttribute('disabled'), + ).toBe(true); await act(async () => { resolveConfirmation(); diff --git a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx index 5b055e7c4..073da2c3b 100644 --- a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx +++ b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx @@ -258,6 +258,7 @@ export function EditorAgentConversationPanelView({ isCreatingConversation, isDeletingConversation, isWaiting, + isPatienceNoticeVisible, toolCallAction, isToolCallActionPending, errorMessage, @@ -614,7 +615,9 @@ export function EditorAgentConversationPanelView({ }} /> ))} - {isWaiting ? : null} + {isWaiting ? ( + + ) : null} ) : (
diff --git a/src/components/image-editor/EditorAgentConversation/MessageBubble.test.tsx b/src/components/image-editor/EditorAgentConversation/MessageBubble.test.tsx index 257892551..b9463d4eb 100644 --- a/src/components/image-editor/EditorAgentConversation/MessageBubble.test.tsx +++ b/src/components/image-editor/EditorAgentConversation/MessageBubble.test.tsx @@ -5,7 +5,7 @@ import { describe, expect, it, vi } from 'vitest'; import type { EditorAgentMessage } from '@/packages/shared/src/contracts'; -import { MessageBubble } from './MessageBubble.tsx'; +import { MessageBubble, ThinkingBubble } from './MessageBubble.tsx'; function renderMessage(message: EditorAgentMessage) { return render( @@ -19,6 +19,18 @@ function renderMessage(message: EditorAgentMessage) { } describe('MessageBubble', () => { + it('shows a patience notice only for an extended pending request', () => { + const { rerender } = render(); + + expect(screen.getByLabelText('Agent思考中')).toBeTruthy(); + expect(screen.queryByText('仍在处理中，请耐心等待')).toBeNull(); + + rerender(); + + expect(screen.getByLabelText('Agent仍在处理中')).toBeTruthy(); + expect(screen.getByText('仍在处理中，请耐心等待')).toBeTruthy(); + }); + it('shows prefixed system errors as red Agent errors without the wire prefix', () => { renderMessage({ id: 2, diff --git a/src/components/image-editor/EditorAgentConversation/MessageBubble.tsx b/src/components/image-editor/EditorAgentConversation/MessageBubble.tsx index 91e43eb52..a6afe3c0c 100644 --- a/src/components/image-editor/EditorAgentConversation/MessageBubble.tsx +++ b/src/components/image-editor/EditorAgentConversation/MessageBubble.tsx @@ -14,9 +14,16 @@ function messageRoleLabel(role: EditorAgentMessage['role']) { return 'Agent'; } -export function ThinkingBubble() { +export function ThinkingBubble({ + showPatienceNotice = false, +}: { + showPatienceNotice?: boolean; +}) { return ( -
+
@@ -33,6 +40,9 @@ export function ThinkingBubble() { style={{ animationDelay: '300ms' }} /> + {showPatienceNotice ? ( + 仍在处理中，请耐心等待 + ) : null}
@@ -61,7 +71,11 @@ export function MessageBubble({ ? message.text.slice(EDITOR_AGENT_ERROR_MESSAGE_PREFIX.length) : null; - if (message.role === 'system' && !message.toolCall && systemErrorText === null) { + if ( + message.role === 'system' && + !message.toolCall && + systemErrorText === null + ) { return null; } if ( diff --git a/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx b/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx index 5dcc6ebf9..3df8d34ea 100644 --- a/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx +++ b/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx @@ -1,7 +1,7 @@ /* @vitest-environment jsdom */ import { act, renderHook, waitFor } from '@testing-library/react'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { EditorAgentConversationDetail, @@ -9,6 +9,7 @@ import type { EditorAgentMessageResponse, } from '../../../../packages/shared/src/contracts/editorAgent.ts'; import { + EDITOR_AGENT_PATIENCE_NOTICE_DELAY_MS, type EditorAgentConversationClient, useEditorAgentConversation, } from './useEditorAgentConversation.ts'; @@ -116,6 +117,10 @@ describe('useEditorAgentConversation', () => { vi.clearAllMocks(); }); + afterEach(() => { + vi.useRealTimers(); + }); + it('loads conversations and applies delta messages', async () => { const client = createClient(); const onCanvasRefreshRequested = vi.fn(); @@ -445,6 +450,87 @@ describe('useEditorAgentConversation', () => { ); }); + it('does not send or apply a stale conversation created after switching projects', async () => { + const client = createClient(); + let resolveCreate!: (detail: EditorAgentConversationDetail) => void; + vi.mocked(client.listConversations) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + conversationId: 'conversation-project-2', + projectId: 'project-2', + title: '项目二会话', + updatedAt: '2026-07-03T00:02:00.000Z', + }, + ]); + vi.mocked(client.createConversation).mockImplementationOnce( + () => + new Promise((resolve) => { + resolveCreate = resolve; + }), + ); + vi.mocked(client.getConversation).mockResolvedValueOnce({ + conversationId: 'conversation-project-2', + projectId: 'project-2', + title: '项目二会话', + messages: [ + { + id: 20, + role: 'assistant', + text: '项目二消息', + attachments: [], + toolCall: null, + createdAt: '2026-07-03T00:02:00.000Z', + }, + ], + createdAt: '2026-07-03T00:02:00.000Z', + updatedAt: '2026-07-03T00:02:00.000Z', + }); + const { result, rerender } = renderHook( + ({ projectId }) => useEditorAgentConversation({ projectId, client }), + { initialProps: { projectId: 'project-1' } }, + ); + + await waitFor(() => { + expect(result.current.isLoadingConversations).toBe(false); + }); + + let sendPromise!: Promise; + act(() => { + sendPromise = result.current.sendMessage('旧项目消息'); + }); + await waitFor(() => { + expect(client.createConversation).toHaveBeenCalledWith('project-1', {}); + }); + + rerender({ projectId: 'project-2' }); + await waitFor(() => { + expect(result.current.activeConversationId).toBe( + 'conversation-project-2', + ); + }); + + await act(async () => { + resolveCreate({ + conversationId: 'conversation-project-1', + projectId: 'project-1', + title: '旧项目新会话', + messages: [], + createdAt: '2026-07-03T00:01:00.000Z', + updatedAt: '2026-07-03T00:01:00.000Z', + }); + await sendPromise; + }); + + expect(client.sendMessage).not.toHaveBeenCalled(); + expect(result.current.activeConversationId).toBe('conversation-project-2'); + expect(result.current.messages.map((message) => message.text)).toEqual([ + '项目二消息', + ]); + expect(result.current.isWaiting).toBe(false); + expect(result.current.isPatienceNoticeVisible).toBe(false); + }); + it('allows sending an attachment-only message', async () => { const client = createClient(); const { result } = renderHook(() => @@ -769,7 +855,9 @@ describe('useEditorAgentConversation', () => { ); await waitFor(() => { - expect(result.current.messages[0]?.toolCall?.status).toBe('not_completed'); + expect(result.current.messages[0]?.toolCall?.status).toBe( + 'not_completed', + ); }); const getConversationCallsBeforeCancel = vi.mocked(client.getConversation) .mock.calls.length; @@ -809,6 +897,78 @@ describe('useEditorAgentConversation', () => { expect(result.current.messages).toHaveLength(0); }); + it('replaces the extended patience notice with the actual request failure', async () => { + const client = createClient(); + let rejectSend!: (error: Error) => void; + vi.mocked(client.sendMessage).mockImplementation( + () => + new Promise((_resolve, reject) => { + rejectSend = reject; + }), + ); + const { result } = renderHook(() => + useEditorAgentConversation({ projectId: 'project-1', client }), + ); + + await waitFor(() => { + expect(result.current.activeConversation?.conversationId).toBe( + 'conversation-1', + ); + }); + + vi.useFakeTimers(); + let sendPromise!: Promise; + act(() => { + sendPromise = result.current.sendMessage('请继续'); + }); + await act(async () => { + await Promise.resolve(); + }); + act(() => { + vi.advanceTimersByTime(EDITOR_AGENT_PATIENCE_NOTICE_DELAY_MS); + }); + expect(result.current.isPatienceNoticeVisible).toBe(true); + + await act(async () => { + rejectSend(new Error('LLM 连接已断开')); + await sendPromise.catch(() => undefined); + }); + + expect(result.current.isPatienceNoticeVisible).toBe(false); + expect(result.current.isWaiting).toBe(false); + expect(result.current.errorMessage).toBe('LLM 连接已断开'); + expect(result.current.messages).toHaveLength(0); + }); + + it('cleans the patience timer when the hook unmounts', async () => { + const client = createClient(); + vi.mocked(client.sendMessage).mockImplementation( + () => new Promise(() => undefined), + ); + const { result, unmount } = renderHook(() => + useEditorAgentConversation({ projectId: 'project-1', client }), + ); + + await waitFor(() => { + expect(result.current.activeConversation?.conversationId).toBe( + 'conversation-1', + ); + }); + + vi.useFakeTimers(); + act(() => { + void result.current.sendMessage('请继续'); + }); + await act(async () => { + await Promise.resolve(); + }); + expect(vi.getTimerCount()).toBe(1); + + unmount(); + + expect(vi.getTimerCount()).toBe(0); + }); + it('keeps the active request pending without exposing a stop action', async () => { const client = createClient(); let capturedSignal: AbortSignal | null = null; @@ -830,16 +990,27 @@ describe('useEditorAgentConversation', () => { ); }); - void act(() => { - void result.current.sendMessage('请继续'); + vi.useFakeTimers(); + let sendPromise!: Promise; + act(() => { + sendPromise = result.current.sendMessage('请继续'); + void result.current.sendMessage('不要重复发送'); }); - await waitFor(() => { - expect(result.current.isWaiting).toBe(true); + await act(async () => { + await Promise.resolve(); }); + expect(result.current.isWaiting).toBe(true); + expect(result.current.isPatienceNoticeVisible).toBe(false); + expect(client.sendMessage).toHaveBeenCalledTimes(1); expect(capturedSignal).toBeNull(); expect('stopCurrentTurn' in result.current).toBe(false); + act(() => { + vi.advanceTimersByTime(EDITOR_AGENT_PATIENCE_NOTICE_DELAY_MS); + }); + expect(result.current.isPatienceNoticeVisible).toBe(true); + await act(async () => { resolveSend({ conversation: { @@ -851,10 +1022,10 @@ describe('useEditorAgentConversation', () => { deltaMessages: [], errorMessage: null, }); + await sendPromise; }); - await waitFor(() => { - expect(result.current.isWaiting).toBe(false); - }); + expect(result.current.isWaiting).toBe(false); + expect(result.current.isPatienceNoticeVisible).toBe(false); }); }); diff --git a/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts b/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts index 5c1d66326..765ba77bf 100644 --- a/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts +++ b/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts @@ -40,14 +40,8 @@ export type EditorAgentConversationClient = { payload: EditorAgentMessageRequest, options: SendEditorAgentMessageOptions, ) => Promise; - confirmToolCall: ( - conversationId: string, - messageId: number, - ) => Promise; - cancelToolCall: ( - conversationId: string, - messageId: number, - ) => Promise; + confirmToolCall: (conversationId: string, messageId: number) => Promise; + cancelToolCall: (conversationId: string, messageId: number) => Promise; }; type UseEditorAgentConversationOptions = { @@ -74,6 +68,8 @@ const defaultEditorAgentConversationClient: EditorAgentConversationClient = { cancelToolCall: cancelEditorAgentToolCall, }; +export const EDITOR_AGENT_PATIENCE_NOTICE_DELAY_MS = 120_000; + function createEditorAgentClientMessageId() { const randomId = typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function' @@ -155,16 +151,51 @@ export function useEditorAgentConversation({ const [isCreatingConversation, setIsCreatingConversation] = useState(false); const [isDeletingConversation, setIsDeletingConversation] = useState(false); const [isWaiting, setIsWaiting] = useState(false); + const [patienceNoticeConversationId, setPatienceNoticeConversationId] = + useState(null); const [toolCallAction, setToolCallAction] = useState(null); const [errorMessage, setErrorMessage] = useState(null); + const normalizedProjectIdRef = useRef(normalizedProjectId); const activeConversationIdRef = useRef(null); const activeToolCallActionRef = useRef(null); const conversationLoadRequestIdRef = useRef(0); + const createConversationRequestIdRef = useRef(0); + const isWaitingRef = useRef(false); + const pendingSendRequestIdRef = useRef(0); + const patienceNoticeTimerRef = useRef | null>( + null, + ); useEffect(() => { activeConversationIdRef.current = activeConversationId; }, [activeConversationId]); + normalizedProjectIdRef.current = normalizedProjectId; + + useEffect(() => { + return () => { + pendingSendRequestIdRef.current += 1; + createConversationRequestIdRef.current += 1; + isWaitingRef.current = false; + if (patienceNoticeTimerRef.current !== null) { + clearTimeout(patienceNoticeTimerRef.current); + patienceNoticeTimerRef.current = null; + } + }; + }, []); + + useEffect(() => { + pendingSendRequestIdRef.current += 1; + createConversationRequestIdRef.current += 1; + isWaitingRef.current = false; + if (patienceNoticeTimerRef.current !== null) { + clearTimeout(patienceNoticeTimerRef.current); + patienceNoticeTimerRef.current = null; + } + setIsWaiting(false); + setIsCreatingConversation(false); + setPatienceNoticeConversationId(null); + }, [normalizedProjectId]); const activeConversation = useMemo( () => @@ -193,10 +224,7 @@ export function useEditorAgentConversation({ ); const loadConversation = useCallback( - async ( - conversationId: string, - options: { showLoading?: boolean } = {}, - ) => { + async (conversationId: string, options: { showLoading?: boolean } = {}) => { const requestId = conversationLoadRequestIdRef.current + 1; conversationLoadRequestIdRef.current = requestId; const showLoading = options.showLoading ?? true; @@ -287,19 +315,37 @@ export function useEditorAgentConversation({ if (!normalizedProjectId) { throw new Error('缺少画布项目 ID'); } + const requestedProjectId = normalizedProjectId; + const requestId = createConversationRequestIdRef.current + 1; + createConversationRequestIdRef.current = requestId; setIsCreatingConversation(true); setErrorMessage(null); try { - const detail = await client.createConversation(normalizedProjectId, {}); - applyConversationDetail(detail); + const detail = await client.createConversation(requestedProjectId, {}); + if ( + createConversationRequestIdRef.current === requestId && + normalizedProjectIdRef.current === requestedProjectId + ) { + applyConversationDetail(detail); + } return detail; } catch (error) { - setErrorMessage( - error instanceof Error ? error.message : '创建画布 Agent 会话失败', - ); + if ( + createConversationRequestIdRef.current === requestId && + normalizedProjectIdRef.current === requestedProjectId + ) { + setErrorMessage( + error instanceof Error ? error.message : '创建画布 Agent 会话失败', + ); + } throw error; } finally { - setIsCreatingConversation(false); + if ( + createConversationRequestIdRef.current === requestId && + normalizedProjectIdRef.current === requestedProjectId + ) { + setIsCreatingConversation(false); + } } }, [applyConversationDetail, client, normalizedProjectId]); @@ -328,9 +374,9 @@ export function useEditorAgentConversation({ const toolCall = message.toolCall; return Boolean( toolCall?.externalJobId && - (toolCall.images.length > 0 || - (toolCall.videos?.length ?? 0) > 0 || - (toolCall.audios?.length ?? 0) > 0), + (toolCall.images.length > 0 || + (toolCall.videos?.length ?? 0) > 0 || + (toolCall.audios?.length ?? 0) > 0), ); }) ) { @@ -363,29 +409,49 @@ export function useEditorAgentConversation({ const text = rawText.trim(); if ( (!text && !attachments.length) || - isWaiting || + isWaitingRef.current || activeToolCallActionRef.current !== null || isLoadingConversations || isLoadingMessages ) { return; } - const conversationId = await ensureConversationForSend(); - const clientMessageId = createEditorAgentClientMessageId(); + isWaitingRef.current = true; + const requestedProjectId = normalizedProjectId; + const requestId = pendingSendRequestIdRef.current + 1; + pendingSendRequestIdRef.current = requestId; setErrorMessage(null); setIsWaiting(true); - const optimisticMessage = createLocalUserMessage({ - id: -1, - clientMessageId, - text, - attachments, - }); - setMessages((currentMessages) => [ - ...currentMessages, - optimisticMessage, - ]); + setPatienceNoticeConversationId(null); + let conversationId: string | null = null; + let optimisticMessage: EditorAgentMessage | null = null; try { + conversationId = await ensureConversationForSend(); + if ( + pendingSendRequestIdRef.current !== requestId || + normalizedProjectIdRef.current !== requestedProjectId + ) { + return; + } + const clientMessageId = createEditorAgentClientMessageId(); + const nextOptimisticMessage = createLocalUserMessage({ + id: -1, + clientMessageId, + text, + attachments, + }); + optimisticMessage = nextOptimisticMessage; + setMessages((currentMessages) => [ + ...currentMessages, + nextOptimisticMessage, + ]); + const pendingConversationId = conversationId; + patienceNoticeTimerRef.current = setTimeout(() => { + if (pendingSendRequestIdRef.current === requestId) { + setPatienceNoticeConversationId(pendingConversationId); + } + }, EDITOR_AGENT_PATIENCE_NOTICE_DELAY_MS); const response = await client.sendMessage( conversationId, { @@ -396,6 +462,9 @@ export function useEditorAgentConversation({ {}, ); + if (pendingSendRequestIdRef.current !== requestId) { + return; + } setConversations((currentConversations) => upsertConversationSummary( currentConversations, @@ -413,15 +482,31 @@ export function useEditorAgentConversation({ } catch (error) { const message = error instanceof Error ? error.message : '发送画布 Agent 消息失败'; - if (activeConversationIdRef.current === conversationId) { + const shouldReportError = + pendingSendRequestIdRef.current === requestId && + (!conversationId || + activeConversationIdRef.current === conversationId); + if (shouldReportError) { setErrorMessage(message); - setMessages((currentMessages) => - currentMessages.filter((message) => message !== optimisticMessage), - ); + if (optimisticMessage) { + setMessages((currentMessages) => + currentMessages.filter( + (message) => message !== optimisticMessage, + ), + ); + } + throw error; } - throw error; } finally { - setIsWaiting(false); + if (pendingSendRequestIdRef.current === requestId) { + if (patienceNoticeTimerRef.current !== null) { + clearTimeout(patienceNoticeTimerRef.current); + patienceNoticeTimerRef.current = null; + } + isWaitingRef.current = false; + setIsWaiting(false); + setPatienceNoticeConversationId(null); + } } }, [ @@ -430,7 +515,7 @@ export function useEditorAgentConversation({ applyDeltaMessages, isLoadingConversations, isLoadingMessages, - isWaiting, + normalizedProjectId, ], ); @@ -537,6 +622,8 @@ export function useEditorAgentConversation({ isCreatingConversation, isDeletingConversation, isWaiting, + isPatienceNoticeVisible: + isWaiting && patienceNoticeConversationId === activeConversationId, toolCallAction, isToolCallActionPending: toolCallAction !== null, errorMessage, From 7072e52a253a04a73e6cdcbe5c33d7d78e3d9db1 Mon Sep 17 00:00:00 2001 From: kdletters Date: Tue, 21 Jul 2026 16:44:46 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E6=94=B6=E5=8F=A3=E7=BE=8E=E6=9C=AF=20Agen?= =?UTF-8?q?t=20=E6=95=B4=E4=BD=93=E8=B6=85=E6=97=B6=E9=A2=84=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为消息规划链路增加 18 分钟总 deadline。 修正 LLM 重试响应体错误的 attempt 计数。 统一配置、定价失败中文文案并补齐测试文档。 --- docs/project-memory/shared-memory/pitfalls.md | 4 +- ...】server-rs与SpacetimeDB数据契约-2026-05-15.md | 4 +- .../【编辑器】画布Agent对话面板-2026-07-03.md | 4 +- .../crates/api-server/src/editor_agent/api.rs | 88 ++++++++++++++++-- server-rs/crates/platform-llm/src/lib.rs | 90 +++++++++++++++---- 5 files changed, 164 insertions(+), 26 deletions(-) diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 8a3749ec8..151270cee 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -3238,8 +3238,8 @@ - 现象：美术 Agent 对话返回红色错误气泡 `completion error: LLM 请求超时，累计尝试 1 次`；HTTP 本身仍返回 200，前端 20 分钟 transport timeout 没有触发。 - 原因：规划请求虽然有 Agent 专用单次 timeout，但 `editor_agent_llm_client` 把 `max_retries` 硬编码为 0；VectorEngine `gpt-5.4-mini` 的偶发长尾、连接超时或可重试上游状态会在第一次失败后直接持久化成 system error。framework 的英文 `completion error` 前缀也被原样暴露给用户。 -- 处理：120 秒改为前端软提示阈值：POST 仍 pending 时显示不入库的“仍在处理中，请耐心等待”；provider 明确断开/失败才写正式错误。专用 provider 单 attempt 使用 8 分钟 hard timeout，请求发起阶段读取 `GENARRATIVE_LLM_MAX_RETRIES`，但画布 Agent 最多重试 1 次且重试退避最多 60 秒，保证理论上限小于前端 20 分钟 transport timeout；响应头后的体读取/解析错误按明确失败收口。规划错误对用户统一为中文。重试发生在任何生成工具执行前，不会重复提交生成任务或扣费，不要通过提高前端 timeout 或 runner `max_turns` 掩盖 provider 重试缺失。 -- 验证：`platform-editor-agent` 测试锁定 8 分钟 hard timeout 与中文错误；前端 fake timer 用例锁定 120 秒前只显示思考动画、到点后显示耐心等待、成功/失败后移除；`api-server` AppState 测试锁定专用 client 透传 retry 次数。运行态排障按同一 request id 对齐 `platform_llm` failure stage 与 `/messages` 总耗时，并确认仍 pending 的请求不再在 120 秒形成错误气泡。 +- 处理：120 秒改为前端软提示阈值：POST 仍 pending 时显示不入库的“仍在处理中，请耐心等待”；provider 明确断开/失败才写正式错误。专用 provider 单 attempt 使用 8 分钟 hard timeout，请求发起阶段读取 `GENARRATIVE_LLM_MAX_RETRIES`，但画布 Agent 最多重试 1 次且重试退避最多 60 秒。不要只计算单次 complete 的最坏时间：runner 还可因非法 JSON/工具校验失败进入后续轮次，必须从 handler 入口开始计算 18 分钟总 deadline，进入 `agent.prompt(...)` 时扣除会话锁/上下文准备已用时间，为持久化和前端 20 分钟 timeout 留出余量。响应头后的体读取/解析错误按明确失败收口，必须使用真实 attempt 计数；规划、配置和定价错误对用户统一为中文，原始诊断只记后端日志。重试发生在任何生成工具执行前，不会重复提交生成任务或扣费，不要通过提高前端 timeout 或 runner `max_turns` 掩盖 provider 重试缺失。 +- 验证：`platform-editor-agent` 测试锁定 8 分钟 hard timeout 与中文错误；前端 fake timer 用例锁定 120 秒前只显示思考动画、到点后显示耐心等待、成功/失败后移除；`platform-llm` 回归用例锁定第二次 attempt 成功响应头后的 body timeout 仍报累计 2 次；`api-server` 测试锁定专用 client retry、18 分钟整体 deadline 与中文直达错误。运行态排障按同一 request id 对齐 `platform_llm` failure stage 与 `/messages` 总耗时，并确认仍 pending 的请求不再在 120 秒形成错误气泡。 - 关联：`server-rs/crates/platform-editor-agent/src/agent/agent.rs`、`server-rs/crates/platform-editor-agent/src/framework/error.rs`、`server-rs/crates/api-server/src/state.rs`、`src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts`、`src/components/image-editor/EditorAgentConversation/MessageBubble.tsx`、`src/services/image-editor/editorAgentClient.ts`。 ## 前端退役目录不能只靠扫描和 ignore 隔离 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 5bdd6e6ea..3ef5cad36 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -75,8 +75,8 @@ npm run check:server-rs-ddd - `/api/editor/projects/{projectId}/agent-conversations` 负责当前工程会话列表和新建；`/api/editor/agent-conversations/{conversationId}` 负责详情读取、终态工具消息懒回填和软删；`POST /api/editor/agent-conversations/{conversationId}/messages` 负责发送消息并返回普通 JSON `EditorAgentMessageResponse`，画布 Agent 不提供 `/messages/stream` SSE 路由。消息请求必须携带最长 128 字符的 `clientMessageId`；前端对该 POST 显式启用 1 次瞬时 transport 重试，并复用同一个序列化 body、`clientMessageId` 和 `x-request-id`。同一会话在锁内按该键幂等，重复键同内容返回已有回合或从已保存用户消息继续，异内容返回 `409`。数字 `EditorAgentMessage.id` 仍只作为工具确认 / 取消的后端消息定位符，不能复用为客户端幂等键。 - `module-editor-agent` 只承载纯领域校验：标题派生、附件上限、消息输入规则和会话软删访问规则；不直接依赖 Axum、SpacetimeDB、OSS、LLM 或 Tokio。 - `spacetime-module` 的 `editor_agent_conversation` 只保存元数据；创建、列表、读取、更新时间和软删通过 `create_editor_agent_conversation_and_return`、`list_editor_agent_conversations_and_return`、`get_editor_agent_conversation_and_return`、`touch_editor_agent_conversation_and_return`、`delete_editor_agent_conversation_and_return` procedure 完成，`api-server` 只能经 `spacetime-client` facade 访问。 -- 完整消息文档存 OSS `editor-agent/{conversationId}.json`，由 `api-server` 负责 2 MiB 上限、会话内串行锁、读改写、消息与工具结果持久化和 `touch` 元数据更新时间；该 JSON 不进入 `editor_canvas.layers_json`，也不作为画布布局真相。LLM 未配置、连接已经断开、请求明确失败、达到最终安全上限或规划不可解析时，必须写入 `role=system`、正文以 `ERROR ` 开头的消息，并通过 `deltaMessages` 返回，`errorMessage` 保持为空；前端隐藏前缀并显示红色错误气泡，面向用户的错误正文使用中文语义，不暴露 `completion error` 等 framework 内部前缀，后端仍把该 system 消息注入后续 LLM memory，使 Agent 能读取失败上下文。普通 JSON POST 尚未结束不形成持久化消息；工具失败同样必须形成可回读记录，不能只返回瞬时错误。 -- 画布 Agent 的 `gpt-5.4-mini` Chat Completions 规划使用 1024 `max_tokens`。前端在 POST pending 120 秒后显示不入库的耐心等待提示；provider request future 明确返回 connect/timeout/HTTP/transport 错误时立即进入正式失败，尚未返回则继续等待。专用 provider 单 attempt hard timeout 为 8 分钟；请求发起阶段的 timeout、连接失败、`408`、`429` 与 `5xx` 读取 `GENARRATIVE_LLM_MAX_RETRIES`，但画布 Agent 最多重试 1 次，显式配置 0 仍可关闭，专用重试退避最多 60 秒，使两次 attempt 的理论上限仍早于前端消息 POST 的 20 分钟 timeout；已收到成功响应头后的响应体读取或解析失败直接按明确失败收口。重试只包围 LLM 规划请求并发生在任何待确认工具执行之前，因此不会重复提交生成任务或扣费。 +- 完整消息文档存 OSS `editor-agent/{conversationId}.json`，由 `api-server` 负责 2 MiB 上限、会话内串行锁、读改写、消息与工具结果持久化和 `touch` 元数据更新时间；该 JSON 不进入 `editor_canvas.layers_json`，也不作为画布布局真相。LLM 未配置、连接已经断开、请求明确失败、达到最终安全上限或规划不可解析时，必须写入 `role=system`、正文以 `ERROR ` 开头的消息，并通过 `deltaMessages` 返回，`errorMessage` 保持为空；前端隐藏前缀并显示红色错误气泡，面向用户的错误正文使用中文语义，不暴露 `completion error` 等 framework 内部前缀或原始配置/定价错误；原始诊断只写后端结构化日志。后端仍把该 system 消息注入后续 LLM memory，使 Agent 能读取失败上下文。普通 JSON POST 尚未结束不形成持久化消息；工具失败同样必须形成可回读记录，不能只返回瞬时错误。 +- 画布 Agent 的 `gpt-5.4-mini` Chat Completions 规划使用 1024 `max_tokens`。前端在 POST pending 120 秒后显示不入库的耐心等待提示；provider request future 明确返回 connect/timeout/HTTP/transport 错误时立即进入正式失败，尚未返回则继续等待。专用 provider 单 attempt hard timeout 为 8 分钟；请求发起阶段的 timeout、连接失败、`408`、`429` 与 `5xx` 读取 `GENARRATIVE_LLM_MAX_RETRIES`，但画布 Agent 最多重试 1 次，显式配置 0 仍可关闭，专用重试退避最多 60 秒。消息规划生命周期从 handler 入口开始计入 18 分钟总 deadline，进入 `agent.prompt(...)` 时只使用剩余预算；该 deadline 覆盖会话锁/上下文准备与最多 3 轮规划，并为错误持久化/HTTP 返回预留约 2 分钟，不允许多轮规划绕过前端 20 分钟 timeout。已收到成功响应头后的响应体读取或解析失败直接按明确失败收口，并使用该成功响应所属的真实 attempt 记录错误。重试只包围 LLM 规划请求并发生在任何待确认工具执行之前，因此不会重复提交生成任务或扣费。 - 对话附件只允许引用当前工程 `editor_project_resource` 或当前账号 `editor_asset` 的图片；前端可提交展示用 `imageSrc` / `thumbnailSrc`，后端必须按 `resourceId` / `assetId` 重新归一、校验 owner / project 和 `objectKey`，再给 LLM 或生成工具使用。 - 画布 Agent 工具复用既有编辑器图片生成 / 修改 / 图标 spritesheet BFF，并继续使用后端模型定价和 `execute_billable_asset_operation_with_cost`；前端不提交 `priceMudPoints`。 - `/messages/{messageId}/confirm` 与 `/messages/{messageId}/cancel` 只返回成功确认；前端成功后立即重新读取整个会话，以会话详情中的权威消息状态和 `externalJobId` 驱动气泡展示与任务轮询。 diff --git a/docs/【编辑器】画布Agent对话面板-2026-07-03.md b/docs/【编辑器】画布Agent对话面板-2026-07-03.md index f2dfc9cd1..63749082d 100644 --- a/docs/【编辑器】画布Agent对话面板-2026-07-03.md +++ b/docs/【编辑器】画布Agent对话面板-2026-07-03.md @@ -95,12 +95,12 @@ ## LLM 与计费 - 编排复用 `creative_agent_gpt5_client` 的 LLM 接入配置（同 provider/env，独立用途标识），画布 Agent 规划请求固定使用 VectorEngine `gpt-5.4-mini` Chat Completions；function-calling 注册八类工具。 -- 每个用户回合必须由 LLM 返回结构化计划；LLM 未配置、连接已经断开、请求明确失败、达到最终安全上限或返回格式不可解析时，后端写入正文为 `ERROR <错误内容>` 的 system 消息，不使用本地关键词或“收到：...”回显兜底。面向用户的规划错误使用中文语义，不暴露 `completion error` 等 framework 内部前缀。该错误消息与其它 system 消息一样进入后续 LLM memory，使 Agent 能看到上一轮失败上下文。普通 JSON POST 尚未结束只表示 provider request future 仍在等待，不能伪装成已持久化失败。 +- 每个用户回合必须由 LLM 返回结构化计划；LLM 未配置、连接已经断开、请求明确失败、达到最终安全上限或返回格式不可解析时，后端写入正文为 `ERROR <错误内容>` 的 system 消息，不使用本地关键词或“收到：...”回显兜底。面向用户的规划错误使用中文语义，不暴露 `completion error` 等 framework 内部前缀或原始配置/定价诊断；原始错误只记录在后端日志。该错误消息与其它 system 消息一样进入后续 LLM memory，使 Agent 能看到上一轮失败上下文。普通 JSON POST 尚未结束只表示 provider request future 仍在等待，不能伪装成已持久化失败。 - 规划 prompt 必须自动带入上一条已完成生成结果的 `latestGeneratedImage` 引用，内容只包含上一轮 generation 的 `toolName` / `resourceId` / `objectKey` / `assetObjectId` 等轻量元数据，不把私有签名 URL 或大图内容塞进 prompt。 - 工具参数中的图片 ID 是由真实 object key 或图片地址计算的稳定 SHA-256 标识；真实 data key 仅存于 api-server 的工具上下文映射，所有图片工具在执行时查表恢复，不能把 object key 或图片地址作为 LLM 可见的工具 ID。 - 用户使用「这张」「刚才那个」「上一张」「把衣服换成……」等方式指代或编辑上一张结果图时，LLM 默认选择 `edit_image` 并引用 `latestGeneratedImage` 作为源图；除非用户明确要求全新生成，否则不能因为本轮没有重新上传附件而降级为 `generate_image`。 - 规划 prompt 必须显式区分“规范展板”和“实际素材产出”：规范图、视觉规范图、风格规范图、素材规范展板、角色规范图等规范展板请求走 `generate_image`，并补齐统一视角、线条粗细、色卡、材质、阴影、圆角、状态层级、尺寸标注等要求；实际角色立绘才走 `generate_character`，多个图标素材 / 图集才走 `generate_icon_spritesheet`。 -- 画布 Agent 规划请求使用 Chat Completions 和 1024 `max_tokens`。发送后 120 秒是前端软提示阈值，不是 provider 失败 deadline：若普通 JSON POST 仍 pending，消息流临时显示“仍在处理中，请耐心等待”并继续等待，提示不写入 OSS 消息历史；连接或请求明确失败则立即按正式错误收口。provider 单 attempt 保留 8 分钟 hard timeout；请求发起阶段的 timeout、连接失败、`408`、`429` 与 `5xx` 读取 `GENARRATIVE_LLM_MAX_RETRIES`，但画布 Agent 最多重试 1 次，专用重试退避最多 60 秒，使两次 attempt 的理论最坏等待仍早于前端 20 分钟 transport timeout；已收到成功响应头后的响应体读取或解析失败直接按明确失败收口。规划重试发生在任何生成工具执行之前，不会重复提交生成任务或扣费；生成图片/编辑图片仍走对应生成工具和模型计费。 +- 画布 Agent 规划请求使用 Chat Completions 和 1024 `max_tokens`。发送后 120 秒是前端软提示阈值，不是 provider 失败 deadline：若普通 JSON POST 仍 pending，消息流临时显示“仍在处理中，请耐心等待”并继续等待，提示不写入 OSS 消息历史；连接或请求明确失败则立即按正式错误收口。provider 单 attempt 保留 8 分钟 hard timeout；请求发起阶段的 timeout、连接失败、`408`、`429` 与 `5xx` 读取 `GENARRATIVE_LLM_MAX_RETRIES`，但画布 Agent 最多重试 1 次，专用重试退避最多 60 秒。消息规划生命周期从 handler 入口开始计入 18 分钟总 deadline，进入 `agent.prompt(...)` 时使用扣除会话锁和上下文准备后的剩余预算；该 deadline 覆盖非法 JSON/工具校验失败触发的后续规划轮，并为错误持久化和 HTTP 返回保留约 2 分钟，不再让前端 20 分钟 transport timeout 先触发。已收到成功响应头后的响应体读取或解析失败直接按明确失败收口，错误计数/日志使用该响应所属的真实 attempt。规划重试发生在任何生成工具执行之前，不会重复提交生成任务或扣费；生成图片/编辑图片仍走对应生成工具和模型计费。 - function-calling runner 必须把“等待用户确认”作为显式工具语义：当本批所有工具都校验成功并进入待确认状态时，立即以成功结果结束当前规划回合并持久化助手文本与待确认卡，不得继续依赖 LLM 自行停止；未知工具、参数错误、普通连续工具和不可解析响应仍受 `max_turns` 保护。 - **对话回合免费**（聊天、分析回复不扣泥点），仅 Agent 实际触发生成工具时按对应模型定价扣泥点。 - 工具调用前后端校验泥点余额；不足时该次生成失败并在对话中以明确错误气泡告知，对话本身可继续。 diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index 9e5400993..e26bd6b69 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -1,3 +1,6 @@ +use std::future::IntoFuture; +use std::time::Duration; + use axum::extract::{Path, State}; use axum::{Extension, Json}; use module_editor_agent::{ @@ -75,8 +78,13 @@ use platform_editor_agent::agent::tools::generate_video::{ GenerateVideoTool, GenerateVideoToolArgs, }; use shared_kernel::{build_prefixed_uuid_id, normalize_optional_string, normalize_required_string}; +use tokio::time::{Instant, timeout}; const EDITOR_AGENT_CLIENT_MESSAGE_ID_MAX_CHARS: usize = 128; +const EDITOR_AGENT_PROMPT_TIMEOUT_MS: u64 = 18 * 60_000; +const EDITOR_AGENT_PROMPT_TIMEOUT_MESSAGE: &str = "规划总时长已达到 18 分钟安全上限"; +const EDITOR_AGENT_LLM_UNAVAILABLE_MESSAGE: &str = "美术 Agent 服务暂不可用，请稍后重试"; +const EDITOR_AGENT_PRICING_UNAVAILABLE_MESSAGE: &str = "美术 Agent 生成定价暂不可用，请稍后重试"; pub async fn editor_agent_message( State(state): State, @@ -85,6 +93,7 @@ pub async fn editor_agent_message( Extension(authenticated): Extension, Json(payload): Json, ) -> Result, AppError> { + let message_started_at = Instant::now(); let owner_user_id = authenticated.claims().user_id().to_string(); require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?; let client_message_id = validate_editor_agent_message_request(&payload)?; @@ -211,12 +220,16 @@ pub async fn editor_agent_message( // Build and run agent let Some(llm_client) = state.editor_agent_llm_client() else { + tracing::warn!( + conversation_id = %conversation.conversation_id, + "美术 Agent LLM 客户端未配置" + ); return persist_editor_agent_planning_error( &state, &conversation, &mut document, conversation_summary, - "Editor Agent LLM client not configured", + EDITOR_AGENT_LLM_UNAVAILABLE_MESSAGE, ) .await; }; @@ -224,12 +237,17 @@ pub async fn editor_agent_message( let pricing = match state.editor_generation_pricing().await { Ok(pricing) => pricing, Err(error) => { + tracing::warn!( + conversation_id = %conversation.conversation_id, + error = %error, + "读取美术 Agent 生成定价失败" + ); return persist_editor_agent_planning_error( &state, &conversation, &mut document, conversation_summary, - format!("failed to load editor generation pricing: {error}"), + EDITOR_AGENT_PRICING_UNAVAILABLE_MESSAGE, ) .await; } @@ -264,9 +282,13 @@ pub async fn editor_agent_message( .memory(memory) .build(); - let agent_result = agent - .prompt(LlmMessage::user(user_message.text.clone())) - .await; + let remaining_prompt_duration = + remaining_editor_agent_prompt_duration(message_started_at.elapsed()); + let agent_result = run_editor_agent_prompt_with_timeout( + agent.prompt(LlmMessage::user(user_message.text.clone())), + remaining_prompt_duration, + ) + .await; let assistant_now = now_rfc3339(); @@ -303,6 +325,26 @@ pub async fn editor_agent_message( } } +fn remaining_editor_agent_prompt_duration(elapsed: Duration) -> Duration { + Duration::from_millis(EDITOR_AGENT_PROMPT_TIMEOUT_MS).saturating_sub(elapsed) +} + +async fn run_editor_agent_prompt_with_timeout( + future: F, + duration: Duration, +) -> Result, PromptError> +where + F: IntoFuture, PromptError>>, +{ + timeout(duration, future.into_future()) + .await + .unwrap_or_else(|_| { + Err(PromptError::CompletionError( + EDITOR_AGENT_PROMPT_TIMEOUT_MESSAGE.to_string(), + )) + }) +} + fn build_editor_agent_error_message( message_id: usize, error: impl std::fmt::Display, @@ -512,6 +554,42 @@ mod tests { assert_eq!(message.text, "ERROR planning failed"); assert!(message.tool_call.is_none()); } + + #[test] + fn direct_planning_failures_use_user_facing_chinese_copy() { + assert_eq!( + build_editor_agent_error_message(1, EDITOR_AGENT_LLM_UNAVAILABLE_MESSAGE).text, + "ERROR 美术 Agent 服务暂不可用，请稍后重试" + ); + assert_eq!( + build_editor_agent_error_message(2, EDITOR_AGENT_PRICING_UNAVAILABLE_MESSAGE).text, + "ERROR 美术 Agent 生成定价暂不可用，请稍后重试" + ); + } + + #[tokio::test] + async fn prompt_timeout_applies_to_the_whole_agent_run() { + let error = run_editor_agent_prompt_with_timeout( + std::future::pending::, PromptError>>(), + Duration::from_millis(1), + ) + .await + .expect_err("pending agent run should hit the prompt deadline"); + + assert_eq!(EDITOR_AGENT_PROMPT_TIMEOUT_MS, 1_080_000); + assert_eq!( + remaining_editor_agent_prompt_duration(Duration::from_secs(17 * 60)), + Duration::from_secs(60) + ); + assert_eq!( + remaining_editor_agent_prompt_duration(Duration::from_secs(18 * 60)), + Duration::ZERO + ); + assert_eq!( + error.to_string(), + "美术 Agent 规划失败：规划总时长已达到 18 分钟安全上限" + ); + } } fn editor_agent_system_prompt() -> &'static str { r#" diff --git a/server-rs/crates/platform-llm/src/lib.rs b/server-rs/crates/platform-llm/src/lib.rs index fa2ab2dbd..8c5e41aae 100644 --- a/server-rs/crates/platform-llm/src/lib.rs +++ b/server-rs/crates/platform-llm/src/lib.rs @@ -189,6 +189,11 @@ pub struct LlmClient { http_client: Client, } +struct LlmAttemptResponse { + response: reqwest::Response, + attempt: u32, +} + #[derive(Serialize)] #[serde(untagged)] enum LlmRequestBody { @@ -753,14 +758,15 @@ impl LlmClient { pub async fn request_text(&self, request: LlmTextRequest) -> Result { request.validate()?; let resolved_model = request.resolved_model(self.config.model()).to_string(); - let response = self.execute_request(&request, false).await?; + let LlmAttemptResponse { response, attempt } = + self.execute_request(&request, false).await?; let raw_text = response.text().await.map_err(|error| { - let llm_error = map_stream_read_error(error, 1); + let llm_error = map_stream_read_error(error, attempt); log_llm_raw_failure( &self.config, &request, false, - 1, + attempt, "read_response_failed", llm_error.to_string().as_str(), ); @@ -778,7 +784,7 @@ impl LlmClient { &self.config, &request, false, - 1, + attempt, "parse_response_failed", raw_text.as_str(), ); @@ -805,7 +811,10 @@ impl LlmClient { { request.validate()?; let resolved_model = request.resolved_model(self.config.model()).to_string(); - let mut response = self.execute_request(&request, true).await?; + let LlmAttemptResponse { + mut response, + attempt, + } = self.execute_request(&request, true).await?; let response_id = response .headers() .get("x-request-id") @@ -819,12 +828,12 @@ impl LlmClient { loop { let next_chunk = response.chunk().await.map_err(|error| { - let llm_error = map_stream_read_error(error, 1); + let llm_error = map_stream_read_error(error, attempt); log_llm_raw_failure( &self.config, &request, true, - 1, + attempt, "read_stream_failed", parser.raw_text().as_str(), ); @@ -842,7 +851,7 @@ impl LlmClient { &self.config, &request, true, - 1, + attempt, "decode_stream_failed", parser.raw_text().as_str(), ); @@ -857,7 +866,7 @@ impl LlmClient { &self.config, &request, true, - 1, + attempt, "parse_stream_failed", parser.raw_text().as_str(), ); @@ -889,7 +898,7 @@ impl LlmClient { &self.config, &request, true, - 1, + attempt, "decode_stream_failed", parser.raw_text().as_str(), ); @@ -901,7 +910,7 @@ impl LlmClient { &self.config, &request, true, - 1, + attempt, "parse_stream_failed", parser.raw_text().as_str(), ); @@ -932,7 +941,7 @@ impl LlmClient { &self.config, &request, true, - 1, + attempt, "parse_stream_failed", parser.raw_text().as_str(), ); @@ -962,7 +971,7 @@ impl LlmClient { &self.config, &request, true, - 1, + attempt, "empty_stream_response", parser.raw_text().as_str(), ); @@ -999,7 +1008,7 @@ impl LlmClient { &self, request: &LlmTextRequest, stream: bool, - ) -> Result { + ) -> Result { let request_body = build_request_body(request, &self.config, stream); let model = request.resolved_model(self.config.model()); let url = match request.protocol { @@ -1039,7 +1048,7 @@ impl LlmClient { attempt, response.status().as_u16() ); - return Ok(response); + return Ok(LlmAttemptResponse { response, attempt }); } Ok(response) => { let status = response.status(); @@ -1936,6 +1945,57 @@ mod tests { assert_eq!(response.response_id.as_deref(), Some("resp_retry")); } + #[tokio::test] + async fn request_text_reports_the_successful_attempt_when_response_body_times_out() { + let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener should have addr"); + let server_handle = thread::spawn(move || { + let (mut first_stream, _) = listener.accept().expect("first request should connect"); + let _ = read_request(&mut first_stream); + write_response( + &mut first_stream, + MockResponse { + status_line: "500 Internal Server Error", + content_type: "application/json; charset=utf-8", + body: r#"{"error":{"message":"temporary upstream failure"}}"#.to_string(), + extra_headers: Vec::new(), + }, + ); + + let (mut second_stream, _) = listener.accept().expect("second request should connect"); + let _ = read_request(&mut second_stream); + second_stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 128\r\nConnection: close\r\n\r\n", + ) + .expect("response headers should be written"); + second_stream + .flush() + .expect("response headers should be flushed"); + thread::sleep(StdDuration::from_millis(200)); + }); + + let config = LlmConfig::new( + LlmProvider::Ark, + format!("http://{address}"), + "test-key".to_string(), + "test-model".to_string(), + 50, + 1, + 1, + ) + .expect("config should be valid"); + let client = LlmClient::new(config).expect("client should be created"); + + let error = client + .request_single_message_text("系统", "用户") + .await + .expect_err("the second response body should time out"); + + assert_eq!(error, LlmError::Timeout { attempts: 2 }); + server_handle.join().expect("mock server should finish"); + } + #[tokio::test] async fn request_text_uses_request_level_timeout_override() { let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); From 6bed37542c64f6e36ae785683b4982442510f6b1 Mon Sep 17 00:00:00 2001 From: kdletters Date: Tue, 21 Jul 2026 17:06:28 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E8=B0=83=E6=95=B4=20VectorEngine=20?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E6=A8=A1=E5=9E=8B=E5=85=9C=E5=BA=95=E9=A1=BA?= =?UTF-8?q?=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 统一 gpt-image-2 首发并在符合条件的失败后切换 gpt-image-2-c 保留总尝试预算、worker deadline 与首选失败审计 补齐响应图片校验、脚本兜底和回归测试 同步后端架构、开发运维和项目共享记忆文档 --- .codex/skills/gpt-image-2-apimart/SKILL.md | 8 +- .../generate-anthro-cat-illustrations.mjs | 149 +++- .../scripts/generate-template-samples.mjs | 149 +++- .../shared-memory/decision-log.md | 7 + docs/project-memory/shared-memory/pitfalls.md | 2 +- ...】server-rs与SpacetimeDB数据契约-2026-05-15.md | 4 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 6 +- .../api-server/src/openai_image_generation.rs | 26 +- server-rs/crates/platform-image/src/lib.rs | 2 +- .../platform-image/src/vector_engine/audit.rs | 5 +- .../src/vector_engine/budget.rs | 4 + .../src/vector_engine/client.rs | 729 ++++++++++++++---- .../src/vector_engine/constants.rs | 1 + .../src/vector_engine/curl_transport.rs | 3 + .../platform-image/src/vector_engine/error.rs | 39 + .../src/vector_engine/image_source.rs | 18 +- .../platform-image/src/vector_engine/mod.rs | 4 +- .../src/vector_engine/request.rs | 11 +- .../src/vector_engine/response.rs | 39 + .../platform-image/src/vector_engine/types.rs | 3 + .../platform-image/src/vector_engine/util.rs | 7 + .../platform-image/tests/vector_engine.rs | 381 ++++++++- 22 files changed, 1372 insertions(+), 225 deletions(-) diff --git a/.codex/skills/gpt-image-2-apimart/SKILL.md b/.codex/skills/gpt-image-2-apimart/SKILL.md index 99caa76b4..f350d98a5 100644 --- a/.codex/skills/gpt-image-2-apimart/SKILL.md +++ b/.codex/skills/gpt-image-2-apimart/SKILL.md @@ -1,11 +1,11 @@ --- name: gpt-image-2-apimart -description: Generate or inspect project image assets through this repository's VectorEngine gpt-image-2 workflow. Use when Codex needs to create puzzle template sample images, reproduce the server-rs gpt-image-2 request body, dry-run image prompts, batch-generate local project thumbnails, or debug VECTOR_ENGINE_BASE_URL / VECTOR_ENGINE_API_KEY image-generation configuration without exposing secrets. The directory name is historical. +description: Generate or inspect project image assets through this repository's VectorEngine gpt-image-2 workflow with gpt-image-2-c fallback. Use when Codex needs to create puzzle template sample images, reproduce the server-rs image request body, dry-run image prompts, batch-generate local project thumbnails, or debug VECTOR_ENGINE_BASE_URL / VECTOR_ENGINE_API_KEY image-generation configuration without exposing secrets. The directory name is historical. --- # gpt-image-2 VectorEngine -Use this skill for project-local image asset generation that must match the repository's `server-rs` VectorEngine `gpt-image-2` path. The folder still contains `apimart` in its name for compatibility with existing local plugin references. +Use this skill for project-local image asset generation that must match the repository's `server-rs` VectorEngine image path. Keep the product/price model identifier and primary provider request as `gpt-image-2`, then fall back once to `gpt-image-2-c` for eligible provider failures. The folder still contains `apimart` in its name for compatibility with existing local plugin references. ## Workflow @@ -65,9 +65,9 @@ size=1024x1024 image=@reference.png ``` -In this repository, calls with no reference images use `POST /v1/images/generations`; calls with any reference image use `POST /v1/images/edits` and pass references as one or more `image` form parts. Match3D container UI generation embeds `public/match3d-background-references/pot-fused-reference.png` into the edit request as an `image` part. +In this repository, calls with no reference images use `POST /v1/images/generations`; calls with any reference image use `POST /v1/images/edits` and pass references as one or more `image` form parts. Both paths prefer `gpt-image-2`; on an eligible upstream/model failure they retry with `gpt-image-2-c`. Do not fall back for authentication, local validation, request-budget exhaustion, uncertain send/connection failure, content-safety rejection, or a generated image URL download failure. Match3D container UI generation embeds `public/match3d-background-references/pot-fused-reference.png` into the edit request as an `image` part. -Accept image output from `data[].url`, `data[].b64_json`, or direct nested `url` fields. VectorEngine GPT-image-2 currently returns synchronously; do not poll APIMart task endpoints. +Accept image output from `data[].url`, `data[].b64_json`, or direct nested `url` fields. VectorEngine image generation currently returns synchronously; do not poll APIMart task endpoints. ## Environment diff --git a/.codex/skills/gpt-image-2-apimart/scripts/generate-anthro-cat-illustrations.mjs b/.codex/skills/gpt-image-2-apimart/scripts/generate-anthro-cat-illustrations.mjs index e3b1f99d7..071301778 100644 --- a/.codex/skills/gpt-image-2-apimart/scripts/generate-anthro-cat-illustrations.mjs +++ b/.codex/skills/gpt-image-2-apimart/scripts/generate-anthro-cat-illustrations.mjs @@ -9,6 +9,8 @@ const skillRoot = path.resolve(__dirname, '..'); const repoRoot = path.resolve(skillRoot, '..', '..', '..'); const defaultOutDir = path.join(repoRoot, 'public', 'anthro-cat-illustrations'); const defaultTimeoutMs = 1000000; +const preferredImageModel = 'gpt-image-2'; +const fallbackImageModel = 'gpt-image-2-c'; const prompts = [ { @@ -165,6 +167,25 @@ function extractBase64Images(payload) { return values; } +function decodeStrictBase64Image(raw) { + const normalized = String(raw || '').trim(); + if ( + !normalized || + normalized.length % 4 !== 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test( + normalized, + ) + ) { + return null; + } + const bytes = Buffer.from(normalized, 'base64'); + return bytes.length > 0 && + bytes.toString('base64') === normalized && + inferExtensionFromBytes(bytes) + ? bytes + : null; +} + function inferExtensionFromContentType(contentType) { const normalized = contentType.split(';')[0]?.trim().toLowerCase(); if (normalized === 'image/png') { @@ -192,7 +213,13 @@ function inferExtensionFromBytes(bytes) { ) { return 'webp'; } - return 'png'; + if ( + bytes.subarray(0, 6).toString('ascii') === 'GIF87a' || + bytes.subarray(0, 6).toString('ascii') === 'GIF89a' + ) { + return 'gif'; + } + return null; } async function fetchJson(url, options, timeoutMs) { @@ -205,9 +232,20 @@ async function fetchJson(url, options, timeoutMs) { }); const text = await response.text(); if (!response.ok) { - throw new Error(`VectorEngine ${response.status}: ${text.slice(0, 600)}`); + const error = new Error( + `VectorEngine ${response.status}: ${text.slice(0, 600)}`, + ); + error.vectorEngineStatus = response.status; + error.vectorEngineBody = text; + throw error; + } + try { + return JSON.parse(text); + } catch (error) { + error.vectorEngineResponseParse = true; + error.vectorEngineBody = text; + throw error; } - return JSON.parse(text); } catch (error) { if (error?.name === 'AbortError') { throw new Error(`VectorEngine request timed out after ${timeoutMs}ms`); @@ -218,6 +256,83 @@ async function fetchJson(url, options, timeoutMs) { } } +function shouldFallbackImageModel(error) { + const raw = `${error?.message || ''}\n${error?.vectorEngineBody || ''}`.toLowerCase(); + if (error?.vectorEngineResponseParse) { + return !containsContentRejection(raw); + } + const status = Number(error?.vectorEngineStatus || 0); + if (status === 408 || status >= 500) { + return true; + } + if (status === 429) { + return !containsContentRejection(raw); + } + const mentionsImageModel = + raw.includes('model') || + raw.includes('模型') || + raw.includes(preferredImageModel) || + raw.includes(fallbackImageModel); + return ( + [400, 404, 422].includes(status) && + mentionsImageModel && + /(not found|not supported|unsupported|unavailable|does not exist|invalid model|unknown model|不存在|不支持|不可用|未开通)/u.test( + raw, + ) + ); +} + +function containsContentRejection(raw) { + return /(invalid_prompt|safety|content[_ ]policy|moderation|prompt rejected|content rejected|prompt refusal|content refusal|rejected by safety|rejected by moderation|敏感|违规|安全策略|内容审核|提示词拒绝|内容拒绝)/u.test( + raw, + ); +} + +async function requestImagePayload(env, entry) { + for (const model of [preferredImageModel, fallbackImageModel]) { + const requestBody = { + model, + prompt: buildPrompt(entry), + n: 1, + size: '1024x1024', + }; + try { + const payload = await fetchJson( + buildVectorEngineImagesGenerationUrl(env.baseUrl), + { + method: 'POST', + headers: { + Authorization: `Bearer ${env.apiKey}`, + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + }, + env.timeoutMs, + ); + const base64Image = decodeStrictBase64Image(extractBase64Images(payload)[0]); + if ( + extractImageUrls(payload)[0] || + base64Image + ) { + return payload; + } + const error = new Error(`VectorEngine returned no image for ${entry.id}`); + error.vectorEngineResponseParse = true; + error.vectorEngineBody = JSON.stringify(payload).slice(0, 600); + throw error; + } catch (error) { + if (model !== preferredImageModel || !shouldFallbackImageModel(error)) { + throw error; + } + console.warn( + `VectorEngine ${preferredImageModel} failed, retrying with ${fallbackImageModel}: ${error.message}`, + ); + } + } + throw new Error(`VectorEngine returned no image for ${entry.id}`); +} + async function downloadUrl(url, timeoutMs) { const abortController = new AbortController(); const timer = setTimeout(() => abortController.abort(), timeoutMs); @@ -244,25 +359,7 @@ async function downloadUrl(url, timeoutMs) { } async function generateOne(env, entry, outDir) { - const requestBody = { - model: 'gpt-image-2', - prompt: buildPrompt(entry), - n: 1, - size: '1024x1024', - }; - const payload = await fetchJson( - buildVectorEngineImagesGenerationUrl(env.baseUrl), - { - method: 'POST', - headers: { - Authorization: `Bearer ${env.apiKey}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }, - env.timeoutMs, - ); + const payload = await requestImagePayload(env, entry); const urls = extractImageUrls(payload); const b64Images = extractBase64Images(payload); @@ -271,7 +368,10 @@ async function generateOne(env, entry, outDir) { if (urls[0]) { image = await downloadUrl(urls[0], env.timeoutMs); } else if (b64Images[0]) { - const bytes = Buffer.from(b64Images[0], 'base64'); + const bytes = decodeStrictBase64Image(b64Images[0]); + if (!bytes) { + throw new Error(`VectorEngine returned invalid base64 image for ${entry.id}`); + } image = { bytes, extension: inferExtensionFromBytes(bytes), @@ -304,8 +404,9 @@ if (dryRun) { requests: selectedPrompts.map((entry) => ({ id: entry.id, title: entry.title, + fallbackModel: fallbackImageModel, body: { - model: 'gpt-image-2', + model: preferredImageModel, prompt: buildPrompt(entry), n: 1, size: '1024x1024', diff --git a/.codex/skills/gpt-image-2-apimart/scripts/generate-template-samples.mjs b/.codex/skills/gpt-image-2-apimart/scripts/generate-template-samples.mjs index 165dac017..4f1c37e42 100644 --- a/.codex/skills/gpt-image-2-apimart/scripts/generate-template-samples.mjs +++ b/.codex/skills/gpt-image-2-apimart/scripts/generate-template-samples.mjs @@ -14,6 +14,8 @@ const promptsPath = path.join( ); const defaultOutDir = path.join(repoRoot, 'public', 'puzzle-creation-templates'); const defaultTimeoutMs = 1000000; +const preferredImageModel = 'gpt-image-2'; +const fallbackImageModel = 'gpt-image-2-c'; const args = new Map(); for (let index = 2; index < process.argv.length; index += 1) { @@ -131,6 +133,25 @@ function extractBase64Images(payload) { return values; } +function decodeStrictBase64Image(raw) { + const normalized = String(raw || '').trim(); + if ( + !normalized || + normalized.length % 4 !== 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test( + normalized, + ) + ) { + return null; + } + const bytes = Buffer.from(normalized, 'base64'); + return bytes.length > 0 && + bytes.toString('base64') === normalized && + inferExtensionFromBytes(bytes) + ? bytes + : null; +} + function inferExtensionFromContentType(contentType) { const normalized = contentType.split(';')[0]?.trim().toLowerCase(); if (normalized === 'image/png') { @@ -158,7 +179,13 @@ function inferExtensionFromBytes(bytes) { ) { return 'webp'; } - return 'png'; + if ( + bytes.subarray(0, 6).toString('ascii') === 'GIF87a' || + bytes.subarray(0, 6).toString('ascii') === 'GIF89a' + ) { + return 'gif'; + } + return null; } async function fetchJson(url, options, timeoutMs) { @@ -171,9 +198,20 @@ async function fetchJson(url, options, timeoutMs) { }); const text = await response.text(); if (!response.ok) { - throw new Error(`VectorEngine ${response.status}: ${text.slice(0, 600)}`); + const error = new Error( + `VectorEngine ${response.status}: ${text.slice(0, 600)}`, + ); + error.vectorEngineStatus = response.status; + error.vectorEngineBody = text; + throw error; + } + try { + return JSON.parse(text); + } catch (error) { + error.vectorEngineResponseParse = true; + error.vectorEngineBody = text; + throw error; } - return JSON.parse(text); } catch (error) { if (error?.name === 'AbortError') { throw new Error(`VectorEngine request timed out after ${timeoutMs}ms`); @@ -184,6 +222,83 @@ async function fetchJson(url, options, timeoutMs) { } } +function shouldFallbackImageModel(error) { + const raw = `${error?.message || ''}\n${error?.vectorEngineBody || ''}`.toLowerCase(); + if (error?.vectorEngineResponseParse) { + return !containsContentRejection(raw); + } + const status = Number(error?.vectorEngineStatus || 0); + if (status === 408 || status >= 500) { + return true; + } + if (status === 429) { + return !containsContentRejection(raw); + } + const mentionsImageModel = + raw.includes('model') || + raw.includes('模型') || + raw.includes(preferredImageModel) || + raw.includes(fallbackImageModel); + return ( + [400, 404, 422].includes(status) && + mentionsImageModel && + /(not found|not supported|unsupported|unavailable|does not exist|invalid model|unknown model|不存在|不支持|不可用|未开通)/u.test( + raw, + ) + ); +} + +function containsContentRejection(raw) { + return /(invalid_prompt|safety|content[_ ]policy|moderation|prompt rejected|content rejected|prompt refusal|content refusal|rejected by safety|rejected by moderation|敏感|违规|安全策略|内容审核|提示词拒绝|内容拒绝)/u.test( + raw, + ); +} + +async function requestImagePayload(env, template) { + for (const model of [preferredImageModel, fallbackImageModel]) { + const requestBody = { + model, + prompt: buildPrompt(template), + n: 1, + size: '1024x1024', + }; + try { + const payload = await fetchJson( + buildVectorEngineImagesGenerationUrl(env.baseUrl), + { + method: 'POST', + headers: { + Authorization: `Bearer ${env.apiKey}`, + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + }, + env.timeoutMs, + ); + const base64Image = decodeStrictBase64Image(extractBase64Images(payload)[0]); + if ( + extractImageUrls(payload)[0] || + base64Image + ) { + return payload; + } + const error = new Error(`VectorEngine returned no image for ${template.id}`); + error.vectorEngineResponseParse = true; + error.vectorEngineBody = JSON.stringify(payload).slice(0, 600); + throw error; + } catch (error) { + if (model !== preferredImageModel || !shouldFallbackImageModel(error)) { + throw error; + } + console.warn( + `VectorEngine ${preferredImageModel} failed, retrying with ${fallbackImageModel}: ${error.message}`, + ); + } + } + throw new Error(`VectorEngine returned no image for ${template.id}`); +} + async function downloadUrl(url, timeoutMs) { const abortController = new AbortController(); const timer = setTimeout(() => abortController.abort(), timeoutMs); @@ -210,25 +325,7 @@ async function downloadUrl(url, timeoutMs) { } async function generateOne(env, template, outDir) { - const requestBody = { - model: 'gpt-image-2', - prompt: buildPrompt(template), - n: 1, - size: '1024x1024', - }; - const payload = await fetchJson( - buildVectorEngineImagesGenerationUrl(env.baseUrl), - { - method: 'POST', - headers: { - Authorization: `Bearer ${env.apiKey}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }, - env.timeoutMs, - ); + const payload = await requestImagePayload(env, template); const urls = extractImageUrls(payload); const b64Images = extractBase64Images(payload); @@ -237,7 +334,10 @@ async function generateOne(env, template, outDir) { if (urls[0]) { image = await downloadUrl(urls[0], env.timeoutMs); } else if (b64Images[0]) { - const bytes = Buffer.from(b64Images[0], 'base64'); + const bytes = decodeStrictBase64Image(b64Images[0]); + if (!bytes) { + throw new Error(`VectorEngine returned invalid base64 image for ${template.id}`); + } image = { bytes, extension: inferExtensionFromBytes(bytes), @@ -274,8 +374,9 @@ if (dryRun) { requests: selectedTemplates.map((template) => ({ id: template.id, title: template.title, + fallbackModel: fallbackImageModel, body: { - model: 'gpt-image-2', + model: preferredImageModel, prompt: buildPrompt(template), n: 1, size: '1024x1024', diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index bf492c40e..9d3707cb2 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -4366,3 +4366,10 @@ ## 2026-07-20 VectorEngine 图片任务预算收口到 worker deadline - 决策：`editor_image_generation`、`editor_image_edit`、`editor_icon_spritesheet_generation` 和 `editor_ui_design_asset_extraction` 使用默认 `1800s` long job 预算。worker 从同一起点计算绝对 job deadline，并向 provider 提前保留 `min(60s, job 预算 / 2)` 作为审计、OSS 和终态写回窗口。deadline 只经进程内 `RequestContext` 传递；VectorEngine 单 attempt 取配置 timeout 与剩余预算的较小值，退避加下一次 attempt 无法落在同一 deadline 内时停止重试，参考图和响应图片下载也受同一 deadline 限制。普通 HTTP / `inline` 保持无 deadline 行为；`VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS` 默认仍为 `1000000`，配置加载层允许显式值更低。lease 续租 / fencing、迟到写回仲裁、attempt 耗尽和原子退款语义不变。 + +## 2026-07-21 VectorEngine 图片首选 gpt-image-2 并以 gpt-image-2-c 兜底 + +- 决策：前端、DTO、计费配置、持久化和 `platform-image` 的 `/v1/images/generations` / `/v1/images/edits` provider 首选请求统一使用 `gpt-image-2`；符合条件时才回退到兜底模型 `gpt-image-2-c`。不在业务 handler、前端或价格表中新增平行模型。 +- 回退边界：明确模型不存在 / 不支持、408、非内容拒绝类 429、5xx、响应解析失败或非拒绝类缺图可以切模型；401 / 403、普通参数 / 内容安全拒绝、本地配置与参考图错误、发送 / 连接错误、request budget 耗尽和已生成图片下载失败不切模型。一次业务请求总发送上限仍为 5 次，两个模型共享同一 worker provider deadline 和 attempt 预算。 +- 观测边界：审计 `image_model` 记录实际 provider attempt；首选 `gpt-image-2` 失败但兜底 `gpt-image-2-c` 恢复成功时，首选失败仍写入 `external_api_call_failure`，最终成功运行摘要记录 `recoveredFailureCount`。日志用 `fallback_from_model` / `fallback_to_model` 标识切换，不改变业务模型、扣费、素材 metadata 或终态语义。 +- 脚本边界：仓库 `gpt-image-2-apimart` skill 的现役生成脚本采用同一首选 / 回退顺序；认证、请求发送不确定错误和下载失败不重新生图，避免重复上游成本。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 151270cee..cf3d0996f 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -1510,7 +1510,7 @@ - 现象：配置了 `APIMART_BASE_URL` / `APIMART_API_KEY` 后，RPG、拼图或方洞的 GPT-image-2 生图仍返回缺配置，或请求体里还出现 `official_fallback` / `image_urls`。 - 原因：2026-05-21 后 GPT-image-2 图片生成按 VectorEngine 创建/编辑接口分流；2026-07-05 后创意 Agent 文本链路也改为 VectorEngine Chat Completions `gpt-5.4-mini`，APIMart 不再作为当前创意 Agent 来源。 -- 处理：为图片生成配置 `VECTOR_ENGINE_BASE_URL=https://api.vectorengine.ai`、`VECTOR_ENGINE_API_KEY`、`VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS`；排查请求体时确认无参考图路径为 `/v1/images/generations`、有参考图路径为 `/v1/images/edits`，模型为 `gpt-image-2`。 +- 处理：为图片生成配置 `VECTOR_ENGINE_BASE_URL=https://api.vectorengine.ai`、`VECTOR_ENGINE_API_KEY`、`VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS`；排查请求体时确认无参考图路径为 `/v1/images/generations`、有参考图路径为 `/v1/images/edits`，业务 / 计费与 provider 首发模型均为 `gpt-image-2`，仅在符合条件的 provider 失败后切到兜底模型 `gpt-image-2-c`。 - 验证：运行 `cargo test -p api-server openai_image --manifest-path server-rs/Cargo.toml` 和相关玩法图片生成测试；真实联调只在本地私密环境放置 VectorEngine key。 - 关联：`docs/technical/VECTOR_ENGINE_GPT_IMAGE_2_GENERATION_2026-05-09.md`、`server-rs/crates/api-server/src/openai_image_generation.rs`。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 3ef5cad36..c631b90c2 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -158,7 +158,7 @@ npm run check:server-rs-ddd 2. Adapter 输入应显式包含 provider、prompt、reference images、OSS prefix/path/file name、asset kind、entity kind/id、slot、owner/profile/source job、metadata 和可选透明背景后处理。 3. Adapter 输出应保留 legacy public path、object key、asset object id、MIME、extension、task id 和实际 prompt。 4. Adapter 不负责扣费、退款或钱包读取；计费仍由调用方显式包裹。 -5. 图片 provider 协议不再放在玩法模块里实现。VectorEngine `gpt-image-2` 创建 / 编辑协议、URL / base64 图片解析、远端图片下载、请求超时 / 上游状态 / 响应解析 / 缺图 / 下载失败的结构化日志统一在 `server-rs/crates/platform-image/src/vector_engine/`；其中 `client.rs` 只保留 provider 调用编排，`transport.rs` 负责 HTTP client 与 reqwest 错误归一，`request.rs` 负责请求体和路径，`payload.rs` 负责响应 JSON 字段提取，`response.rs` 负责响应状态分流和图片结果归一。`api-server` 只负责配置校验、玩法 prompt 编排、OSS / asset object / binding 持久化、计费和外部 API 失败审计落库。 +5. 图片 provider 协议不再放在玩法模块里实现。产品、计费、DTO、持久化和 VectorEngine 创建 / 编辑首选请求统一使用 `gpt-image-2`；只有符合回退条件时，provider 边界才切到兜底模型 `gpt-image-2-c`。URL / base64 图片解析、远端图片下载、请求超时 / 上游状态 / 响应解析 / 缺图 / 下载失败的结构化日志统一在 `server-rs/crates/platform-image/src/vector_engine/`；其中 `client.rs` 只保留 provider 调用编排，`transport.rs` 负责 HTTP client 与 reqwest 错误归一，`request.rs` 负责请求体和路径，`payload.rs` 负责响应 JSON 字段提取，`response.rs` 负责响应状态分流和图片结果归一。`api-server` 只负责配置校验、玩法 prompt 编排、OSS / asset object / binding 持久化、计费和外部 API 失败审计落库。 6. OSS 平台适配日志统一在 `server-rs/crates/platform-oss` 输出，覆盖 `sign_post_object`、`sign_get_object_url`、`head_object` 和 `put_object`。日志字段固定使用 `provider`、`operation`、`bucket`、`endpoint`、`object_key` / `key_prefix`、`access`、`content_type`、`content_length`、`status`、`status_class`、`error_kind` 和 `elapsed_ms`，只记录对象定位和排障信息；不得输出 AccessKey、policy、signature、Authorization header 或完整 signed URL。generated 私有对象上传时必须由 OSS 对象头承载浏览器 / CDN 缓存策略，默认写入 `Cache-Control: public, max-age=31536000, immutable`，不得改成 api-server 本地磁盘静态资源兜底。 7. Puzzle、Match3D、音频、GLB、视频等复杂媒体可以复用 OSS + asset object + binding 的底层持久化能力，但玩法专属处理规则留在各自编排层，不塞进公共接口。 8. 拼图入口页与结果页新增关卡的本地参考图不走浏览器直传 OSS，前端读取为 Data URL 后随创作 action 提交，并在读取前限制 6MB、显示“图片≤6MB”。`api-server` 必须对 Data URL 实际字节数再次校验；历史图片才提交 `referenceImageAssetObjectId(s)`，后端校验 `asset_object` 的 bucket、kind、图片 MIME、大小和 owner 后签发只读 URL 给 VectorEngine 读取。 @@ -240,7 +240,7 @@ npm run check:server-rs-ddd ## 外部服务与资产 - LLM：通用 LLM 门面继续使用 `GENARRATIVE_LLM_*`；创意 Agent `gpt-5.4-mini` Chat Completions 文本链路已于 2026-06 从 APIMart 迁移到 VectorEngine，使用 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` 构造 OpenAI-compatible client，`api-server` 会把未带 `/v1` 的 VectorEngine base URL 规范化到 `/v1` 后请求 `/chat/completions`。通用 `/api/llm/chat/completions` 代理使用 `GENARRATIVE_LLM_PROVIDER=openai-compatible`、`GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1`、`GENARRATIVE_LLM_MODEL=gpt-5.4-mini`；未单独配置 `GENARRATIVE_LLM_API_KEY` 时可复用 `VECTOR_ENGINE_API_KEY`。`APIMART_BASE_URL` / `APIMART_API_KEY` 只作为历史残留，不再作为创意 Agent gpt-5.4-mini 客户端来源；后续排障时优先确认 VectorEngine `/v1/models`、`/v1/chat/completions` 和 `/v1/responses` 可用性。 -- 图片生成：VectorEngine `gpt-image-2` 图片 provider 归属 `platform-image`，密钥只在后端环境变量中；`api-server` 内的 `openai_image_generation.rs` 只是兼容调用面和外部失败审计桥接，不再承载 provider 协议实现。实际外部生成运行记录统一落 `tracking_event`，`event_key = external_generation_run`，metadata 记录开始 / 结束时间、耗时、状态、成功标记、失败原因、provider task id 和结果摘要，不再写回过时的 `ai_task`。DashScope 只按仍在使用的历史能力单独处理，不作为 GPT-image-2 兜底。VectorEngine `/v1/images/generations` 和 `/v1/images/edits` 上游 POST 使用 `libcurl` 发送；`reqwest` 只保留给参考图 URL 下载和响应中图片 URL 下载。`/v1/images/edits` 的 multipart 参考图必须作为 libcurl 文件上传 part 发送，字段名为 `image`，实现上使用 `Form::buffer(file_name, bytes)` 并设置 `Content-Type`；不能只用 `contents(...).filename(...)`，否则上游会把请求转码为缺少图片并返回 `image is required`。`request_send` 阶段的 curl timeout / connect error 按可重试传输错误处理，最多尝试 5 次，并使用指数退避加短抖动；排障时优先看 `attempt`、`max_attempts`、`retry_delay_ms`、`reference_image_bytes_total` 和 `request_params`，不要把 `SendRequest` 当成上游业务错误。 +- 图片生成：VectorEngine 图片 provider 归属 `platform-image`，密钥只在后端环境变量中；逻辑 SKU 与 provider 首选模型均固定为 `gpt-image-2`，只在明确模型不可用、408 / 非拒绝类 429 / 5xx、响应解析失败或非拒绝类缺图时切换兜底模型 `gpt-image-2-c`。401 / 403、普通参数或安全拒绝、本地配置 / 参考图错误、无法确认上游是否已受理的发送错误、request budget 耗尽和生成成功后的图片下载失败不得切模型。一次业务请求总发送上限仍为 5 次；切换兜底模型会消耗后续 attempt，不允许两个模型各重试 5 次。`api-server` 内的 `openai_image_generation.rs` 只是兼容调用面和外部失败审计桥接，不再承载 provider 协议实现。实际外部生成运行记录统一落 `tracking_event`，`event_key = external_generation_run`，metadata 记录开始 / 结束时间、耗时、状态、成功标记、失败原因、provider task id、结果摘要和 recovered failure 数量；首选模型失败但兜底模型成功时，首选失败仍落 `external_api_call_failure`。DashScope 只按仍在使用的历史能力单独处理，不作为 GPT-image-2 兜底。VectorEngine `/v1/images/generations` 和 `/v1/images/edits` 上游 POST 使用 `libcurl` 发送；`reqwest` 只保留给参考图 URL 下载和响应中图片 URL 下载。`/v1/images/edits` 的 multipart 参考图必须作为 libcurl 文件上传 part 发送，字段名为 `image`，实现上使用 `Form::buffer(file_name, bytes)` 并设置 `Content-Type`；不能只用 `contents(...).filename(...)`，否则上游会把请求转码为缺少图片并返回 `image is required`。`request_send` 阶段的 curl timeout / connect error 按可重试传输错误处理，最多尝试 5 次，并使用指数退避加短抖动；排障时优先看 `attempt`、`max_attempts`、`retry_delay_ms`、`fallback_from_model`、`fallback_to_model`、`reference_image_bytes_total` 和 `request_params`，不要把 `SendRequest` 当成上游业务错误。 - 抠图输入以私有 OSS 作为内存生命周期边界：生成原图和角色动作抽取帧上传时消费图片字节所有权，上传完成后不保留原图缓冲；手动去背景直接解析并校验已有 OSS object key，不下载原图。BgFilter 必须为 object key 签发 600 秒 GET URL 并通过 multipart `image_url` 提交，不用 `file` 重传；flat 链路进入阿里云 fallback 时由 `platform-matting` URL 接口单独下载并上传 `AuthorizeFileUpload` 临时对象，在推理前释放下载缓冲，继续 fallback 到本地键色时再单独下载一次原图，本地产出后释放本次原图下载缓冲。签名 URL 不得写入日志、审计或持久化。 - 角色动作抠图输入像素边界：仅图片画布角色动作链路在 FFmpeg 抽帧后、源帧上传 OSS 前，把帧解码为 RGB8，并按最终 `frameWidth × frameHeight` 的 contain 比例使用 `Triangle` 只缩放到内容尺寸；该阶段不得创建最终目标尺寸画布、不得引入 Alpha 通道，也不得插入任何 padding。BgFilter、阿里云通用抠图和本地键色降级共享这个无补边源帧 object key。抠图返回后才统一转为 RGBA8，按相同比例居中放入最终目标尺寸画布，并用 `RGBA(0,0,0,0)` 补齐透明 padding。以 `560×752 → 323×480` 为例，抠图输入固定为无 Alpha、无补边的 `323×434 RGB8 PNG`，最终输出为上下各 `23px` 透明补边的 `323×480 RGBA8 PNG`。旧 `/api/assets/character-animation/*` 动作发布链路继续保留原有帧 finalizer，不适用该输入规则。抽帧解码后若携带 Alpha 通道，必须先把像素按白底合成为不透明再转 RGB8，禁止直接丢弃 Alpha——全透明像素下未定义的 RGB 值会以杂色进入抠图输入，重新引入杂色边缘；共享 FFmpeg 抽帧命令保持不固定 `-pix_fmt`，白底合成只属于该链路的 BgFilter 输入准备阶段。 - 阿里云通用抠图的非上海地域输入不得使用 `viapiutils/GetOssStsToken`、固定 `viapi-customer-temp` 或 OSS V1 PUT。`platform-matting` 必须按官方新版 SDK Advance 协议调用 `AuthorizeFileUpload`，使用动态返回的单对象 Policy 执行 multipart POST，再把临时上海 OSS URL 交给 `SegmentCommonImage`；输入归一化、结果下载与原尺寸 Alpha 回贴继续留在同一适配器内。该协议仍上传图片字节，不等同于阿里云服务端直接抓取任意公网 URL，也不改变上层 BgFilter → 阿里云 → 本地降级顺序。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 635934ce3..24cd48b95 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -153,9 +153,9 @@ spacetime sql "SELECT * FROM runtime_setting LIMIT 1" --server http:/ 本地 `spacetime` CLI / standalone 版本必须和 `server-rs/Cargo.toml` 里锁定的 `spacetimedb` 版本一致；当前统一版本为 `2.6.1`。若版本错配，procedure 返回值可能在宿主侧触发 `Failed to BSATN deserialize procedure return value`，api-server 最终表现为现役 settings、editor project 或 profile procedure 超时。排障时先运行 `spacetime --version`，再对照 `server-rs/Cargo.toml` 的 `spacetimedb = "..."`；遇到版本不匹配时直接执行 `spacetime version install && spacetime version use `，或在目标就是最新版本时执行 `spacetime version upgrade`，升级后重启 `npm run dev:spacetime` 再重试。当前 `scripts/dev.mjs` 会在启动和复用本地 SpacetimeDB 前写入并校验 `dev-spacetime-tool-version`。2.6.1 修复了 procedure context 中调用者 `Identity` / `ConnectionId` 始终为空的回归，依赖 `ctx.sender` 鉴权时必须同时确认宿主已升级。 -本地 `.env`、`.env.local` 或 `.env.secrets.local` 修改后必须重启 `api-server` 才会生效；若已经通过 `npm run dev` 启动完整联调，可在该终端输入 `rs api-server`。排查图片编辑器 VectorEngine 生成链路时，确认 `VECTOR_ENGINE_BASE_URL`、`VECTOR_ENGINE_API_KEY` 和 `VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS` 只在本地或服务器密钥文件中配置，不能写入 Git。`VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS` 是单次 attempt 的配置上限，默认 `1000000`；配置加载层允许显式值低于该默认值，不再在读取环境变量时强制抬高。VectorEngine `gpt-image-2` 图片协议、URL / base64 响应解析、远端图片下载和 provider 侧结构化日志在 `server-rs/crates/platform-image`；`api-server` 只做编辑器请求编排、OSS / asset 持久化、计费和失败审计落库。`platform-image` 会在 JSON 生成和 multipart 编辑请求发送前归一显式像素尺寸；若请求发送失败，先按同一 `request_id` 查看 provider 日志与 `external_api_call_failure.metadata_json.errorSource`，当前 multipart `/v1/images/edits` 单独强制 HTTP/1.1。 +本地 `.env`、`.env.local` 或 `.env.secrets.local` 修改后必须重启 `api-server` 才会生效；若已经通过 `npm run dev` 启动完整联调，可在该终端输入 `rs api-server`。排查图片编辑器 VectorEngine 生成链路时，确认 `VECTOR_ENGINE_BASE_URL`、`VECTOR_ENGINE_API_KEY` 和 `VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS` 只在本地或服务器密钥文件中配置，不能写入 Git。`VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS` 是单次 attempt 的配置上限，默认 `1000000`；配置加载层允许显式值低于该默认值，不再在读取环境变量时强制抬高。业务模型和 VectorEngine provider 首选请求都使用 `gpt-image-2`，符合条件时才回退到兜底模型 `gpt-image-2-c`；图片协议、URL / base64 响应解析、远端图片下载和 provider 侧结构化日志在 `server-rs/crates/platform-image`，`api-server` 只做编辑器请求编排、OSS / asset 持久化、计费和失败审计落库。`platform-image` 会在 JSON 生成和 multipart 编辑请求发送前按同一 GPT-image-2 family 规则归一显式像素尺寸；若请求发送失败，先按同一 `request_id` 查看 provider 日志与 `external_api_call_failure.metadata_json.errorSource`，当前 multipart `/v1/images/edits` 单独强制 HTTP/1.1。 -VectorEngine 图片生成 / 编辑在 `request_send` 阶段出现 `timeout`、`connect`、libcurl 35 SSL connect reset、libcurl 56 receive error / `unexpected eof while reading`、recv failure 等临时传输错误，或在 `upstream_status` 阶段收到 408 / 429 / 5xx（例如 Nginx HTML `502 Bad Gateway`）时，`platform-image` 会对同一请求最多发送 5 次；multipart 图片编辑每次重试都会重新构造 form，避免复用已消费的 body。worker 从 job 开始的同一时钟起点计算绝对 deadline，常规保留最后 `60` 秒给审计、OSS 和终态写回；job 预算小于 `120` 秒时保留一半。VectorEngine 单次 attempt timeout 取配置值和剩余 provider 预算的较小值；退避后已没有下一次 attempt 的预算时立即停止重试。该 deadline 覆盖参考图、provider 请求 / 响应和响应图片下载的整次 provider future，但只在 worker 进程内通过 `RequestContext` 传递；普通 HTTP / `inline` 没有该 deadline，继续保持原有 timeout 和重试行为。日志中 `VectorEngine 图片请求发送失败，准备重试` 或 `VectorEngine 图片上游状态可重试，准备重试` 表示本次失败确有预算进入下一次尝试；预算耗尽或最终仍失败时才会写入 `external_api_call_failure` 并返回 504 / 502。排查生产失败时应同时统计 retry 前的尝试日志和最终 audit，避免把一次用户请求内的多次发送误判成多个用户请求。这项收口不修改 lease 续租 / fencing、迟到写回仲裁、attempt 耗尽与原子退款语义。 +VectorEngine 图片生成 / 编辑在 `request_send` 阶段出现 `timeout`、`connect`、libcurl 35 SSL connect reset、libcurl 56 receive error / `unexpected eof while reading`、recv failure 等临时传输错误，或在 `upstream_status` 阶段收到 408 / 429 / 5xx（例如 Nginx HTML `502 Bad Gateway`）时，`platform-image` 会在一次业务请求总上限 5 次内处理；multipart 图片编辑每次重试都会重新构造 form，避免复用已消费的 body。首个 provider attempt 使用 `gpt-image-2`；明确模型不可用、408 / 非拒绝类 429 / 5xx、响应解析失败或非拒绝类缺图时，下一 attempt 直接切兜底模型 `gpt-image-2-c`，之后只在剩余次数内重试兜底模型。发送 / 连接错误无法确认上游是否已受理，只重试同一首选模型，不切模型；认证、普通参数、安全拒绝、图片下载和 budget 错误同样不切。worker 从 job 开始的同一时钟起点计算绝对 deadline，常规保留最后 `60` 秒给审计、OSS 和终态写回；job 预算小于 `120` 秒时保留一半。VectorEngine 单次 attempt timeout 取配置值和剩余 provider 预算的较小值；退避或模型切换后已没有下一次 attempt 的预算时立即停止。该 deadline 覆盖参考图、provider 请求 / 响应和响应图片下载的整次 provider future，但只在 worker 进程内通过 `RequestContext` 传递；普通 HTTP / `inline` 没有该 deadline，继续保持原有 timeout 和重试行为。日志中 `VectorEngine 首选图片模型失败，切换兼容模型` 会携带 `fallback_from_model` / `fallback_to_model`；即使回退成功，首选模型错误仍写入 `external_api_call_failure`，成功运行摘要的 `recoveredFailureCount` 同时递增。排查生产失败时应同时统计 fallback / retry 日志和最终 audit，避免把一次用户请求内的多次发送误判成多个用户请求。这项收口不修改 lease 续租 / fencing、迟到写回仲裁、attempt 耗尽与原子退款语义。 图片编辑器生成属于持久队列长任务：提交接口返回 job 后，前端通过 `/api/runtime/external-generation/jobs/{jobId}` 与编辑器项目资源状态收敛。生产排查小程序或 WebView `Failed to fetch` 时，若 Nginx access log 为 `499`、`upstream_status=-`，先按提交请求的 `request_id`、job id、worker 日志和 `external_api_call_failure` 对齐真实任务，不把客户端断开直接判定为 provider 失败。 @@ -603,7 +603,7 @@ OpenTelemetry 现阶段默认开启 OTLP traces / metrics / logs，但本地日 - debug exporter / Rider 转发都会同时接收 traces、metrics 和 logs。 - api-server 会随 metrics 发送进程级指标：`process.memory.usage`、`process.memory.virtual`、`process.cpu.time`、`genarrative.process.cpu.usage_percent`、`process.thread.count`、`genarrative.process.memory.private`；Windows 额外发送 `process.windows.handle.count`，Linux 额外发送 `process.unix.file_descriptor.count`。这些指标只描述当前进程，不携带请求、用户或作品 label。 - HTTP 运行态补充发送 `genarrative.http.server.response_bodies.in_flight` 与 `genarrative.http.server.request_permits.available`，后者带低基数 `pool=default|gallery|detail|admin` label，用于区分业务 handler / 背压 permit 是否仍被占用；拼图广场热点缓存补充发送 `genarrative.puzzle_gallery.cache.*` 指标，记录 fresh hit、stale hit、未命中、后台刷新开始 / 失败、重建耗时和预序列化 data JSON 字节数。 -- 外部 API 失败统一发送 OTLP 并落库。当前 VectorEngine `gpt-image-2` 图片生成 / 编辑失败由 `platform-image` provider 输出结构化日志字段，字段包括 provider、endpoint、failure_stage、status、source、source_chain、source_chain_depth、timeout、retryable、latency_ms、prompt_chars、reference_image_count、image_model、request_params 和 raw_excerpt；图片编辑请求参数日志还会带 reference_image_bytes_total，并在 request_params.referenceImages 中记录每个 multipart `image` part 的 fileName、mimeType 和 bytes，不记录 API key 或原始图片 bytes；`api-server` 再记录指标 `genarrative.external_api.failures{provider,failure_stage,status_class,retryable}`，并写入 `tracking_event`，`event_key = external_api_call_failure`、`module_key = external-api`、`scope_kind = module`、`scope_id = provider`。调用方能拿到身份上下文时，失败事件还会在行级 `user_id` / `owner_user_id` / `profile_id` 和 `metadata_json.userId` / `metadata_json.profileId` / `metadata_json.requestId` / `metadata_json.errorSource` 中记录触发者、草稿 / 作品作用域、请求标识和传输错误链。排障时先按 provider / failureStage 聚合，再下钻 userId / profileId，最后结合 request 日志、errorSource 和上游响应 excerpt 判断是限流、超时、解析失败还是未返回图片。 +- 外部 API 失败统一发送 OTLP 并落库。当前 VectorEngine 图片生成 / 编辑失败由 `platform-image` provider 输出结构化日志字段，字段包括 provider、endpoint、failure_stage、status、source、source_chain、source_chain_depth、timeout、retryable、latency_ms、prompt_chars、reference_image_count、实际 provider `image_model`、request_params 和 raw_excerpt；发生模型回退时另带 `fallback_from_model` / `fallback_to_model`。图片编辑请求参数日志还会带 reference_image_bytes_total，并在 request_params.referenceImages 中记录每个 multipart `image` part 的 fileName、mimeType 和 bytes，不记录 API key 或原始图片 bytes；`api-server` 再记录指标 `genarrative.external_api.failures{provider,failure_stage,status_class,retryable}`，并写入 `tracking_event`，`event_key = external_api_call_failure`、`module_key = external-api`、`scope_kind = module`、`scope_id = provider`。调用方能拿到身份上下文时，失败事件还会在行级 `user_id` / `owner_user_id` / `profile_id` 和 `metadata_json.userId` / `metadata_json.profileId` / `metadata_json.requestId` / `metadata_json.errorSource` 中记录触发者、草稿 / 作品作用域、请求标识和传输错误链。排障时先按 provider / failureStage / imageModel 聚合，再下钻 userId / profileId，最后结合 request 日志、errorSource 和上游响应 excerpt 判断是模型不可用、限流、超时、解析失败还是未返回图片。 - OSS 平台适配器也输出结构化日志，覆盖 `sign_post_object`、`sign_get_object_url`、`head_object` 和 `put_object`。排查资产签名、上传或确认失败时，先按 `provider=aliyun-oss` 与 `operation` 过滤，再看 `object_key` / `key_prefix`、`status`、`status_class`、`error_kind`、`content_length`、`content_type` 和 `elapsed_ms`；角色动画逐帧额外按 `frame_index`、`operation=source_put|final_put|final_head`、`attempt/max_attempts`、`will_retry`、`oss_code` 和 `oss_request_id` 对齐同一对象的请求尝试。`请求 OSS 失败` 时，`timeout/connect/transport=true` 表示传输类失败，OSS PutObject 的 `status=400, oss_code=RequestTimeout, timeout=true`、`status=429` 或 `500–599` 表示暂时性失败，PUT 的 `status=400`、`oss_code` 为空且 `timeout=true` 或 `transport=true`（message 含「错误响应体读取失败」，即 400 错误体读取超时/断流）也会重试；除这两类例外外，其他 400、401/403/404、配置、URL 和签名错误是确定性失败，不会重试。最终帧 HEAD 失败只会重试 HEAD，不会重复 PUT。日志不得包含 AccessKey、policy、signature、Authorization header、完整 signed URL 或 OSS 错误响应体；`oss_request_id` 只用于关联 OSS 服务端排障。排查 generated 图片重复下载时，先确认前端输入是否为 `/generated-*` legacy path 或可归一化的 `https://*.oss-*.aliyuncs.com/generated-*`；正确链路应先调 `/api/assets/read-url`，再由浏览器请求 signed URL，且同一路径、同一 `refreshKey` 版本和未临近过期的 signed URL 应复用。新上传 generated 私有对象应带 `Cache-Control: public, max-age=31536000, immutable`；旧对象若只有 `ETag` / `Last-Modified`，浏览器会走 304 协商缓存而不是长期强缓存，可通过刷新 OSS 元数据或 CDN 配置补齐。 - SpacetimeDB 观测分为两类：procedure / reducer 调用继续用 `genarrative.spacetime.procedure.*`，订阅本地 cache 读使用 `genarrative.spacetime.read.*`。`read=list_puzzle_gallery` 表示拼图广场当前从 `puzzle_gallery_card_view` 本地 cache 读取，不再每个 HTTP 请求调用 `list_puzzle_gallery` procedure。 - 本地 Windows 直连压测的内存高水位要结合 K6 VU / 连接数解释。250 RPS 下过高 `PREALLOCATED_VUS` 可能让 300 个本地 Established 连接把 `api-server` private memory 瞬时推到 GB 级，且 `/healthz` 小响应也能复现；若压测结束后回落、`response_bodies.in_flight` 和背压 permit 未显示业务积压，应优先按连接 / 发送链路高水位处理，而不是判断为 SpacetimeDB 或 JSON 缓存泄漏。 diff --git a/server-rs/crates/api-server/src/openai_image_generation.rs b/server-rs/crates/api-server/src/openai_image_generation.rs index 55ee30c46..e3e381eb5 100644 --- a/server-rs/crates/api-server/src/openai_image_generation.rs +++ b/server-rs/crates/api-server/src/openai_image_generation.rs @@ -434,6 +434,9 @@ async fn map_platform_image_result( ) -> Result { match result { Ok(value) => { + for audit in &value.recovered_failure_audits { + record_openai_image_failure_audit_if_configured(settings, audit).await; + } if let Some(state) = settings.external_api_audit_state.as_ref() { record_external_generation_run_after_success( state, @@ -448,6 +451,7 @@ async fn map_platform_image_result( Some(json!({ "imageCount": value.images.len(), "actualPromptChars": value.actual_prompt.as_ref().map(|prompt| prompt.chars().count()), + "recoveredFailureCount": value.recovered_failure_audits.len(), })), ) .await; @@ -455,6 +459,9 @@ async fn map_platform_image_result( Ok(value) } Err(error) => { + for audit in error.recovered_failure_audits() { + record_openai_image_failure_audit_if_configured(settings, audit).await; + } if let Some(state) = settings.external_api_audit_state.as_ref() { record_external_generation_run_after_success( state, @@ -478,14 +485,21 @@ async fn map_platform_image_result( pub(crate) async fn record_openai_image_failure_if_configured( settings: &OpenAiImageSettings, error: &PlatformImageError, +) { + let Some(audit) = error.audit() else { + return; + }; + record_openai_image_failure_audit_if_configured(settings, audit).await; +} + +async fn record_openai_image_failure_audit_if_configured( + settings: &OpenAiImageSettings, + audit: &platform_image::PlatformImageFailureAudit, ) { let Some(state) = settings.external_api_audit_state.as_ref() else { return; }; - let Some(draft) = build_openai_image_failure_audit_draft(error) else { - return; - }; - let draft = draft + let draft = build_external_api_failure_draft_from_platform_image_audit(audit) .with_user_id(settings.external_api_audit_user_id.clone()) .with_profile_id(settings.external_api_audit_profile_id.clone()) .with_request_id(settings.external_api_audit_request_id.clone()); @@ -501,6 +515,7 @@ pub(crate) fn build_openai_image_failure_audit_draft( } pub(crate) fn map_platform_image_error(error: PlatformImageError) -> AppError { + let error = error.into_final_error(); let status = match error.status_hint() { PlatformImageStatusHint::BadRequest => StatusCode::BAD_REQUEST, PlatformImageStatusHint::ServiceUnavailable => StatusCode::SERVICE_UNAVAILABLE, @@ -545,6 +560,9 @@ pub(crate) fn map_platform_image_error(error: PlatformImageError) -> AppError { details["rawExcerpt"] = json!(raw_excerpt); } PlatformImageError::MissingImage { .. } => {} + PlatformImageError::FallbackFailed { .. } => { + unreachable!("fallback wrapper should be removed before HTTP error mapping") + } } if let Some(audit) = error.audit() { diff --git a/server-rs/crates/platform-image/src/lib.rs b/server-rs/crates/platform-image/src/lib.rs index 4e7aaa896..95495b218 100644 --- a/server-rs/crates/platform-image/src/lib.rs +++ b/server-rs/crates/platform-image/src/lib.rs @@ -3,7 +3,7 @@ pub mod generated_assets; pub mod vector_engine; pub use vector_engine::{ - DownloadedImage, GPT_IMAGE_2_MODEL, GeneratedImages, PlatformImageError, + DownloadedImage, GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, GeneratedImages, PlatformImageError, PlatformImageFailureAudit, PlatformImageStatusHint, ReferenceImage, VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER, VectorEngineImageSettings, build_vector_engine_image_http_client, build_vector_engine_image_request_body, diff --git a/server-rs/crates/platform-image/src/vector_engine/audit.rs b/server-rs/crates/platform-image/src/vector_engine/audit.rs index c28381d57..41bef9a6d 100644 --- a/server-rs/crates/platform-image/src/vector_engine/audit.rs +++ b/server-rs/crates/platform-image/src/vector_engine/audit.rs @@ -1,4 +1,4 @@ -use super::constants::{VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER}; +use super::constants::VECTOR_ENGINE_PROVIDER; #[derive(Clone, Debug)] pub struct PlatformImageFailureAudit { @@ -33,6 +33,7 @@ pub(crate) fn build_failure_audit( latency_ms: Option, prompt_chars: Option, reference_image_count: Option, + image_model: Option<&'static str>, ) -> PlatformImageFailureAudit { PlatformImageFailureAudit { provider: VECTOR_ENGINE_PROVIDER, @@ -49,7 +50,7 @@ pub(crate) fn build_failure_audit( latency_ms, prompt_chars, reference_image_count, - image_model: Some(VECTOR_ENGINE_GPT_IMAGE_2_MODEL), + image_model, } } diff --git a/server-rs/crates/platform-image/src/vector_engine/budget.rs b/server-rs/crates/platform-image/src/vector_engine/budget.rs index a7c82ec46..02f099344 100644 --- a/server-rs/crates/platform-image/src/vector_engine/budget.rs +++ b/server-rs/crates/platform-image/src/vector_engine/budget.rs @@ -53,6 +53,7 @@ fn retry_delay_fits_request_deadline_at( pub(crate) fn request_budget_exhausted_error( request_url: &str, operation: &str, + image_model: Option<&'static str>, latency_ms: Option, prompt_chars: Option, reference_image_count: Option, @@ -73,6 +74,7 @@ pub(crate) fn request_budget_exhausted_error( latency_ms, prompt_chars, reference_image_count, + image_model, ); tracing::warn!( provider = VECTOR_ENGINE_PROVIDER, @@ -82,6 +84,7 @@ pub(crate) fn request_budget_exhausted_error( elapsed_ms = latency_ms, prompt_chars, reference_image_count, + image_model, operation, "VectorEngine 图片请求执行预算已耗尽" ); @@ -162,6 +165,7 @@ mod tests { let error = request_budget_exhausted_error( "https://vector.example/v1/images/generations", "生成图片失败", + None, Some(900), Some(12), Some(1), diff --git a/server-rs/crates/platform-image/src/vector_engine/client.rs b/server-rs/crates/platform-image/src/vector_engine/client.rs index 54647031f..c71b2c8ab 100644 --- a/server-rs/crates/platform-image/src/vector_engine/client.rs +++ b/server-rs/crates/platform-image/src/vector_engine/client.rs @@ -9,7 +9,7 @@ use super::{ effective_request_timeout_ms, request_budget_exhausted_error, retry_delay_fits_request_deadline, }, - constants::{GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER}, + constants::{GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER}, curl_transport::{ map_curl_error, send_vector_engine_json_request_with_curl, send_vector_engine_multipart_edit_request_with_curl, @@ -19,7 +19,7 @@ use super::{ request::{ build_vector_engine_image_edit_request_log_params, build_vector_engine_image_request_body_with_model, - build_vector_engine_nanobanana_generate_content_request_body, + build_vector_engine_nanobanana_generate_content_request_body, is_gpt_image_2_family_model, normalize_image_size_for_model, normalize_vector_engine_image_model, vector_engine_images_edit_url, vector_engine_images_generation_url, vector_engine_nanobanana_generate_content_url, @@ -65,7 +65,7 @@ pub async fn create_vector_engine_image_generation_with_model( reference_images: &[String], failure_context: &str, ) -> Result { - let model = normalize_vector_engine_image_model(model); + let requested_model = normalize_vector_engine_image_model(model); if !reference_images.is_empty() { let resolved_references = resolve_reference_images( http_client, @@ -77,7 +77,7 @@ pub async fn create_vector_engine_image_generation_with_model( return create_vector_engine_image_edit_with_references_and_model( http_client, settings, - model, + requested_model, prompt, negative_prompt, size, @@ -89,30 +89,36 @@ pub async fn create_vector_engine_image_generation_with_model( } let request_url = vector_engine_images_generation_url(settings); - let normalized_size = normalize_image_size_for_model(model, size); - let request_body = build_vector_engine_image_request_body_with_model( - model, - prompt, - negative_prompt, - normalized_size.as_str(), - candidate_count, - reference_images, - ); + let normalized_size = normalize_image_size_for_model(requested_model, size); let started_at = std::time::Instant::now(); + let mut upstream_model = preferred_vector_engine_upstream_model(requested_model); + let mut recovered_failure_audits = Vec::new(); let mut attempt = 1; - let response = loop { + loop { + let request_body = build_vector_engine_image_request_body_with_model( + upstream_model, + prompt, + negative_prompt, + normalized_size.as_str(), + candidate_count, + reference_images, + ); let Some(attempt_timeout_ms) = effective_request_timeout_ms(settings.request_timeout_ms, settings.request_deadline) else { - return Err(request_budget_exhausted_error( - request_url.as_str(), - failure_context, - Some(started_at.elapsed().as_millis() as u64), - Some(prompt.chars().count()), - Some(reference_images.len()), + return Err(finish_vector_engine_model_fallback_error( + request_budget_exhausted_error( + request_url.as_str(), + failure_context, + auditable_vector_engine_image_model(upstream_model), + Some(started_at.elapsed().as_millis() as u64), + Some(prompt.chars().count()), + Some(reference_images.len()), + ), + &mut recovered_failure_audits, )); }; - match send_vector_engine_json_request_with_curl( + let response = match send_vector_engine_json_request_with_curl( request_url.as_str(), settings.api_key.as_str(), &request_body, @@ -121,7 +127,56 @@ pub async fn create_vector_engine_image_generation_with_model( .await { Ok(response) => { - if should_retry_vector_engine_upstream_status(response.status, attempt) { + if should_retry_vector_engine_upstream_response( + response.status, + response.body.as_str(), + attempt, + ) { + let primary_error = if upstream_model == GPT_IMAGE_2_MODEL { + handle_vector_engine_response( + http_client, + request_url.as_str(), + response.status, + response.body.as_str(), + auditable_vector_engine_image_model(upstream_model), + failure_context, + started_at.elapsed().as_millis() as u64, + Some(prompt.chars().count()), + Some(reference_images.len()), + candidate_count, + "vector-engine", + settings.request_deadline, + ) + .await + .err() + } else { + None + }; + if primary_error.as_ref().is_some_and(|error| { + should_fallback_to_gpt_image_2_c( + requested_model, + upstream_model, + attempt, + error, + settings, + ) + }) { + let error = primary_error.expect("primary error checked above"); + record_vector_engine_model_fallback( + "generation", + request_url.as_str(), + upstream_model, + GPT_IMAGE_2_C_MODEL, + attempt, + &error, + ); + if let Some(audit) = error.audit().cloned() { + recovered_failure_audits.push(audit); + } + upstream_model = GPT_IMAGE_2_C_MODEL; + attempt += 1; + continue; + } if retry_vector_engine_upstream_status_after_delay( "generation", request_url.as_str(), @@ -140,7 +195,7 @@ pub async fn create_vector_engine_image_generation_with_model( continue; } } - break response; + response } Err(error) => { if should_retry_vector_engine_curl_send_error(&error, attempt) { @@ -166,48 +221,91 @@ pub async fn create_vector_engine_image_generation_with_model( continue; } } - return Err(map_curl_error( - format!("{failure_context}：创建图片生成任务失败").as_str(), + return Err(finish_vector_engine_model_fallback_error( + map_curl_error( + format!("{failure_context}：创建图片生成任务失败").as_str(), + request_url.as_str(), + "request_send", + auditable_vector_engine_image_model(upstream_model), + error, + started_at.elapsed().as_millis() as u64, + Some(prompt.chars().count()), + Some(reference_images.len()), + Some(&request_body), + ), + &mut recovered_failure_audits, + )); + } + }; + let response_status = response.status; + tracing::info!( + provider = VECTOR_ENGINE_PROVIDER, + endpoint = %request_url, + status = response_status, + image_model = upstream_model, + requested_image_model = requested_model, + prompt_chars = prompt.chars().count(), + size = %normalized_size, + reference_image_count = reference_images.len(), + attempt, + elapsed_ms = started_at.elapsed().as_millis() as u64, + failure_context, + "VectorEngine 图片生成 HTTP 返回" + ); + let response_text = response.body; + match handle_vector_engine_response( + http_client, + request_url.as_str(), + response_status, + response_text.as_str(), + auditable_vector_engine_image_model(upstream_model), + failure_context, + started_at.elapsed().as_millis() as u64, + Some(prompt.chars().count()), + Some(reference_images.len()), + candidate_count, + "vector-engine", + settings.request_deadline, + ) + .await + { + Ok(mut generated) => { + generated + .recovered_failure_audits + .append(&mut recovered_failure_audits); + return Ok(generated); + } + Err(error) + if should_fallback_to_gpt_image_2_c( + requested_model, + upstream_model, + attempt, + &error, + settings, + ) => + { + record_vector_engine_model_fallback( + "generation", request_url.as_str(), - "request_send", + upstream_model, + GPT_IMAGE_2_C_MODEL, + attempt, + &error, + ); + if let Some(audit) = error.audit().cloned() { + recovered_failure_audits.push(audit); + } + upstream_model = GPT_IMAGE_2_C_MODEL; + attempt += 1; + } + Err(error) => { + return Err(finish_vector_engine_model_fallback_error( error, - started_at.elapsed().as_millis() as u64, - Some(prompt.chars().count()), - Some(reference_images.len()), - Some(&request_body), + &mut recovered_failure_audits, )); } } - }; - let response_status = response.status; - tracing::info!( - provider = VECTOR_ENGINE_PROVIDER, - endpoint = %request_url, - status = response_status, - image_model = model, - prompt_chars = prompt.chars().count(), - size = %normalized_size, - reference_image_count = reference_images.len(), - attempt, - elapsed_ms = started_at.elapsed().as_millis() as u64, - failure_context, - "VectorEngine 图片生成 HTTP 返回" - ); - let response_text = response.body; - handle_vector_engine_response( - http_client, - request_url.as_str(), - response_status, - response_text.as_str(), - failure_context, - started_at.elapsed().as_millis() as u64, - Some(prompt.chars().count()), - Some(reference_images.len()), - candidate_count, - "vector-engine", - settings.request_deadline, - ) - .await + } } #[allow(clippy::too_many_arguments)] @@ -260,6 +358,7 @@ pub async fn create_vector_engine_nanobanana_generate_content( return Err(request_budget_exhausted_error( request_url.as_str(), failure_context, + None, Some(started_at.elapsed().as_millis() as u64), Some(prompt.chars().count()), Some(reference_image_count), @@ -274,7 +373,11 @@ pub async fn create_vector_engine_nanobanana_generate_content( .await { Ok(response) => { - if should_retry_vector_engine_upstream_status(response.status, attempt) { + if should_retry_vector_engine_upstream_response( + response.status, + response.body.as_str(), + attempt, + ) { if retry_vector_engine_upstream_status_after_delay( "nanobanana_generate_content", request_url.as_str(), @@ -323,6 +426,7 @@ pub async fn create_vector_engine_nanobanana_generate_content( format!("{failure_context}：创建 nanobanana2 图片生成任务失败").as_str(), request_url.as_str(), "request_send", + None, error, started_at.elapsed().as_millis() as u64, Some(prompt.chars().count()), @@ -355,6 +459,7 @@ pub async fn create_vector_engine_nanobanana_generate_content( request_url.as_str(), response_status, response_text.as_str(), + None, failure_context, started_at.elapsed().as_millis() as u64, Some(prompt.chars().count()), @@ -424,7 +529,7 @@ pub async fn create_vector_engine_image_edit_with_references_and_model( reference_images: &[ReferenceImage], failure_context: &str, ) -> Result { - let model = normalize_vector_engine_image_model(model); + let requested_model = normalize_vector_engine_image_model(model); if reference_images.is_empty() { return Err(PlatformImageError::InvalidRequest { provider: VECTOR_ENGINE_PROVIDER, @@ -433,15 +538,7 @@ pub async fn create_vector_engine_image_edit_with_references_and_model( } let request_url = vector_engine_images_edit_url(settings); - let normalized_size = normalize_image_size_for_model(model, size); - let request_params = build_vector_engine_image_edit_request_log_params( - model, - prompt, - negative_prompt, - normalized_size.as_str(), - candidate_count, - reference_images, - ); + let normalized_size = normalize_image_size_for_model(requested_model, size); let reference_image_count = reference_images.iter().take(5).count(); let reference_image_bytes_total: usize = reference_images @@ -450,43 +547,59 @@ pub async fn create_vector_engine_image_edit_with_references_and_model( .map(|image| image.bytes.len()) .sum(); let started_at = std::time::Instant::now(); - tracing::info!( - provider = VECTOR_ENGINE_PROVIDER, - endpoint = %request_url, - image_model = model, - size = %normalized_size, - candidate_count = candidate_count.clamp(1, 4), - requested_candidate_count = candidate_count, - prompt_chars = prompt.trim().chars().count(), - negative_prompt_chars = negative_prompt - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::chars) - .map(Iterator::count) - .unwrap_or_default(), - reference_image_count, - reference_image_bytes_total, - request_params = %request_params, - failure_context, - "VectorEngine 图片编辑请求参数" - ); + let mut upstream_model = preferred_vector_engine_upstream_model(requested_model); + let mut recovered_failure_audits = Vec::new(); let mut attempt = 1; - let response = loop { + loop { + let request_params = build_vector_engine_image_edit_request_log_params( + upstream_model, + prompt, + negative_prompt, + normalized_size.as_str(), + candidate_count, + reference_images, + ); + tracing::info!( + provider = VECTOR_ENGINE_PROVIDER, + endpoint = %request_url, + image_model = upstream_model, + requested_image_model = requested_model, + size = %normalized_size, + candidate_count = candidate_count.clamp(1, 4), + requested_candidate_count = candidate_count, + prompt_chars = prompt.trim().chars().count(), + negative_prompt_chars = negative_prompt + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::chars) + .map(Iterator::count) + .unwrap_or_default(), + reference_image_count, + reference_image_bytes_total, + request_params = %request_params, + attempt, + failure_context, + "VectorEngine 图片编辑请求参数" + ); let Some(attempt_timeout_ms) = effective_request_timeout_ms(settings.request_timeout_ms, settings.request_deadline) else { - return Err(request_budget_exhausted_error( - request_url.as_str(), - failure_context, - Some(started_at.elapsed().as_millis() as u64), - Some(prompt.chars().count()), - Some(reference_image_count), + return Err(finish_vector_engine_model_fallback_error( + request_budget_exhausted_error( + request_url.as_str(), + failure_context, + auditable_vector_engine_image_model(upstream_model), + Some(started_at.elapsed().as_millis() as u64), + Some(prompt.chars().count()), + Some(reference_image_count), + ), + &mut recovered_failure_audits, )); }; - match send_vector_engine_multipart_edit_request_with_curl( + let response = match send_vector_engine_multipart_edit_request_with_curl( request_url.as_str(), settings.api_key.as_str(), - model, + upstream_model, prompt, negative_prompt, normalized_size.as_str(), @@ -497,7 +610,56 @@ pub async fn create_vector_engine_image_edit_with_references_and_model( .await { Ok(response) => { - if should_retry_vector_engine_upstream_status(response.status, attempt) { + if should_retry_vector_engine_upstream_response( + response.status, + response.body.as_str(), + attempt, + ) { + let primary_error = if upstream_model == GPT_IMAGE_2_MODEL { + handle_vector_engine_response( + http_client, + request_url.as_str(), + response.status, + response.body.as_str(), + auditable_vector_engine_image_model(upstream_model), + failure_context, + started_at.elapsed().as_millis() as u64, + Some(prompt.chars().count()), + Some(reference_image_count), + candidate_count, + "vector-engine-edit", + settings.request_deadline, + ) + .await + .err() + } else { + None + }; + if primary_error.as_ref().is_some_and(|error| { + should_fallback_to_gpt_image_2_c( + requested_model, + upstream_model, + attempt, + error, + settings, + ) + }) { + let error = primary_error.expect("primary error checked above"); + record_vector_engine_model_fallback( + "edit", + request_url.as_str(), + upstream_model, + GPT_IMAGE_2_C_MODEL, + attempt, + &error, + ); + if let Some(audit) = error.audit().cloned() { + recovered_failure_audits.push(audit); + } + upstream_model = GPT_IMAGE_2_C_MODEL; + attempt += 1; + continue; + } if retry_vector_engine_upstream_status_after_delay( "edit", request_url.as_str(), @@ -516,7 +678,7 @@ pub async fn create_vector_engine_image_edit_with_references_and_model( continue; } } - break response; + response } Err(error) => { if should_retry_vector_engine_curl_send_error(&error, attempt) { @@ -542,49 +704,238 @@ pub async fn create_vector_engine_image_edit_with_references_and_model( continue; } } - return Err(map_curl_error( - format!("{failure_context}：创建图片编辑任务失败").as_str(), + return Err(finish_vector_engine_model_fallback_error( + map_curl_error( + format!("{failure_context}：创建图片编辑任务失败").as_str(), + request_url.as_str(), + "request_send", + auditable_vector_engine_image_model(upstream_model), + error, + started_at.elapsed().as_millis() as u64, + Some(prompt.chars().count()), + Some(reference_image_count), + Some(&request_params), + ), + &mut recovered_failure_audits, + )); + } + }; + let response_status = response.status; + tracing::info!( + provider = VECTOR_ENGINE_PROVIDER, + endpoint = %request_url, + status = response_status, + image_model = upstream_model, + requested_image_model = requested_model, + prompt_chars = prompt.chars().count(), + size = %normalized_size, + reference_image_count, + reference_image_bytes_total, + request_params = %request_params, + attempt, + elapsed_ms = started_at.elapsed().as_millis() as u64, + failure_context, + "VectorEngine 图片编辑 HTTP 返回" + ); + let response_text = response.body; + match handle_vector_engine_response( + http_client, + request_url.as_str(), + response_status, + response_text.as_str(), + auditable_vector_engine_image_model(upstream_model), + failure_context, + started_at.elapsed().as_millis() as u64, + Some(prompt.chars().count()), + Some(reference_image_count), + candidate_count, + "vector-engine-edit", + settings.request_deadline, + ) + .await + { + Ok(mut generated) => { + generated + .recovered_failure_audits + .append(&mut recovered_failure_audits); + return Ok(generated); + } + Err(error) + if should_fallback_to_gpt_image_2_c( + requested_model, + upstream_model, + attempt, + &error, + settings, + ) => + { + record_vector_engine_model_fallback( + "edit", request_url.as_str(), - "request_send", + upstream_model, + GPT_IMAGE_2_C_MODEL, + attempt, + &error, + ); + if let Some(audit) = error.audit().cloned() { + recovered_failure_audits.push(audit); + } + upstream_model = GPT_IMAGE_2_C_MODEL; + attempt += 1; + } + Err(error) => { + return Err(finish_vector_engine_model_fallback_error( error, - started_at.elapsed().as_millis() as u64, - Some(prompt.chars().count()), - Some(reference_image_count), - Some(&request_params), + &mut recovered_failure_audits, )); } } - }; - let response_status = response.status; - tracing::info!( + } +} + +fn preferred_vector_engine_upstream_model(requested_model: &str) -> &str { + if is_gpt_image_2_family_model(requested_model) { + GPT_IMAGE_2_MODEL + } else { + requested_model + } +} + +fn finish_vector_engine_model_fallback_error( + error: PlatformImageError, + recovered_failure_audits: &mut Vec, +) -> PlatformImageError { + error.with_recovered_failure_audits(std::mem::take(recovered_failure_audits)) +} + +fn auditable_vector_engine_image_model(model: &str) -> Option<&'static str> { + match model { + GPT_IMAGE_2_C_MODEL => Some(GPT_IMAGE_2_C_MODEL), + GPT_IMAGE_2_MODEL => Some(GPT_IMAGE_2_MODEL), + _ => None, + } +} + +fn should_fallback_to_gpt_image_2_c( + requested_model: &str, + upstream_model: &str, + attempt: u32, + error: &PlatformImageError, + settings: &VectorEngineImageSettings, +) -> bool { + if !is_gpt_image_2_family_model(requested_model) + || upstream_model != GPT_IMAGE_2_MODEL + || attempt >= VECTOR_ENGINE_SEND_MAX_ATTEMPTS + || effective_request_timeout_ms(settings.request_timeout_ms, settings.request_deadline) + .is_none() + { + return false; + } + + match error { + PlatformImageError::Upstream { + upstream_status, + message, + raw_excerpt, + .. + } => match *upstream_status { + 408 => true, + 429 => !contains_vector_engine_content_rejection(message, raw_excerpt), + status if status >= 500 => true, + 400 | 404 | 422 => contains_vector_engine_model_unavailable(message, raw_excerpt), + _ => false, + }, + PlatformImageError::ResponseParse { + message, + raw_excerpt, + .. + } => !contains_vector_engine_content_rejection(message, raw_excerpt), + PlatformImageError::MissingImage { message, audit, .. } => { + let raw_excerpt = audit + .as_ref() + .and_then(|audit| audit.raw_excerpt.as_deref()) + .unwrap_or_default(); + !contains_vector_engine_content_rejection(message, raw_excerpt) + } + PlatformImageError::InvalidConfig { .. } + | PlatformImageError::InvalidRequest { .. } + | PlatformImageError::Request { .. } + | PlatformImageError::FallbackFailed { .. } => false, + } +} + +fn contains_vector_engine_model_unavailable(message: &str, raw_excerpt: &str) -> bool { + let haystack = format!("{message}\n{raw_excerpt}").to_ascii_lowercase(); + let mentions_model = haystack.contains("model") + || haystack.contains("模型") + || haystack.contains(GPT_IMAGE_2_MODEL) + || haystack.contains(GPT_IMAGE_2_C_MODEL); + let unavailable = [ + "not found", + "not supported", + "unsupported", + "unavailable", + "does not exist", + "invalid model", + "unknown model", + "不存在", + "不支持", + "不可用", + "未开通", + ] + .iter() + .any(|marker| haystack.contains(marker)); + mentions_model && unavailable +} + +fn contains_vector_engine_content_rejection(message: &str, raw_excerpt: &str) -> bool { + let haystack = format!("{message}\n{raw_excerpt}").to_ascii_lowercase(); + [ + "invalid_prompt", + "safety", + "content policy", + "content_policy", + "moderation", + "prompt rejected", + "content rejected", + "prompt refusal", + "content refusal", + "rejected by safety", + "rejected by moderation", + "敏感", + "违规", + "安全策略", + "内容审核", + "提示词拒绝", + "内容拒绝", + ] + .iter() + .any(|marker| haystack.contains(marker)) +} + +fn record_vector_engine_model_fallback( + request_kind: &'static str, + request_url: &str, + from_model: &str, + to_model: &str, + attempt: u32, + error: &PlatformImageError, +) { + let audit = error.audit(); + tracing::warn!( provider = VECTOR_ENGINE_PROVIDER, endpoint = %request_url, - status = response_status, - prompt_chars = prompt.chars().count(), - size = %normalized_size, - reference_image_count, - reference_image_bytes_total, - request_params = %request_params, + request_kind, + fallback_from_model = from_model, + fallback_to_model = to_model, attempt, - elapsed_ms = started_at.elapsed().as_millis() as u64, - failure_context, - "VectorEngine 图片编辑 HTTP 返回" + next_attempt = attempt + 1, + max_attempts = VECTOR_ENGINE_SEND_MAX_ATTEMPTS, + failure_stage = audit.map(|audit| audit.failure_stage).unwrap_or("unknown"), + status = audit.and_then(|audit| audit.status_code).unwrap_or_default(), + error = %error.message(), + "VectorEngine 首选图片模型失败，切换兼容模型" ); - let response_text = response.body; - handle_vector_engine_response( - http_client, - request_url.as_str(), - response_status, - response_text.as_str(), - failure_context, - started_at.elapsed().as_millis() as u64, - Some(prompt.chars().count()), - Some(reference_image_count), - candidate_count, - "vector-engine-edit", - settings.request_deadline, - ) - .await } fn should_retry_vector_engine_curl_send_error( @@ -595,8 +946,11 @@ fn should_retry_vector_engine_curl_send_error( && (error.is_timeout() || error.is_connect() || error.is_transient_transport()) } -fn should_retry_vector_engine_upstream_status(status: u16, attempt: u32) -> bool { - attempt < VECTOR_ENGINE_SEND_MAX_ATTEMPTS && (status == 408 || status == 429 || status >= 500) +fn should_retry_vector_engine_upstream_response(status: u16, raw_body: &str, attempt: u32) -> bool { + attempt < VECTOR_ENGINE_SEND_MAX_ATTEMPTS + && (status == 408 + || status >= 500 + || (status == 429 && !contains_vector_engine_content_rejection("", raw_body))) } async fn retry_vector_engine_send_after_delay( @@ -795,11 +1149,100 @@ mod tests { #[test] fn vector_engine_send_retry_policy_treats_upstream_502_as_retryable() { - assert!(should_retry_vector_engine_upstream_status(502, 1)); - assert!(should_retry_vector_engine_upstream_status(429, 1)); - assert!(should_retry_vector_engine_upstream_status(408, 1)); - assert!(!should_retry_vector_engine_upstream_status(400, 1)); - assert!(!should_retry_vector_engine_upstream_status(502, 5)); + assert!(should_retry_vector_engine_upstream_response(502, "", 1)); + assert!(should_retry_vector_engine_upstream_response(429, "", 1)); + assert!(should_retry_vector_engine_upstream_response( + 429, + "request rejected due to rate limit", + 1, + )); + assert!(should_retry_vector_engine_upstream_response(408, "", 1)); + assert!(!should_retry_vector_engine_upstream_response( + 429, + "内容审核拒绝", + 1, + )); + assert!(!should_retry_vector_engine_upstream_response(400, "", 1)); + assert!(!should_retry_vector_engine_upstream_response(502, "", 5)); + } + + #[test] + fn model_fallback_only_accepts_eligible_provider_failures() { + let settings = VectorEngineImageSettings { + base_url: "https://vector.example/v1".to_string(), + api_key: "test-key".to_string(), + request_timeout_ms: 1_000, + request_deadline: None, + }; + let unsupported_model = PlatformImageError::Upstream { + provider: VECTOR_ENGINE_PROVIDER, + message: "model gpt-image-2 is not supported".to_string(), + upstream_status: 400, + raw_excerpt: "unknown model".to_string(), + audit: None, + }; + let content_rejection = PlatformImageError::Upstream { + provider: VECTOR_ENGINE_PROVIDER, + message: "moderation blocked".to_string(), + upstream_status: 429, + raw_excerpt: "invalid_prompt".to_string(), + audit: None, + }; + let uncertain_send_failure = PlatformImageError::Request { + provider: VECTOR_ENGINE_PROVIDER, + message: "send failed".to_string(), + endpoint: None, + timeout: true, + connect: false, + request: true, + body: false, + status_code: None, + source: None, + audit: None, + }; + + assert!(should_fallback_to_gpt_image_2_c( + GPT_IMAGE_2_MODEL, + GPT_IMAGE_2_MODEL, + 1, + &unsupported_model, + &settings, + )); + assert!(!should_fallback_to_gpt_image_2_c( + GPT_IMAGE_2_MODEL, + GPT_IMAGE_2_MODEL, + 1, + &content_rejection, + &settings, + )); + let rate_limit_rejection = PlatformImageError::Upstream { + provider: VECTOR_ENGINE_PROVIDER, + message: "request rejected due to rate limit".to_string(), + upstream_status: 429, + raw_excerpt: "rate_limit_exceeded".to_string(), + audit: None, + }; + assert!(should_fallback_to_gpt_image_2_c( + GPT_IMAGE_2_MODEL, + GPT_IMAGE_2_MODEL, + 1, + &rate_limit_rejection, + &settings, + )); + assert!(!should_fallback_to_gpt_image_2_c( + GPT_IMAGE_2_MODEL, + GPT_IMAGE_2_MODEL, + 1, + &uncertain_send_failure, + &settings, + )); + assert!(!should_fallback_to_gpt_image_2_c( + GPT_IMAGE_2_MODEL, + GPT_IMAGE_2_MODEL, + VECTOR_ENGINE_SEND_MAX_ATTEMPTS, + &unsupported_model, + &settings, + )); } #[test] diff --git a/server-rs/crates/platform-image/src/vector_engine/constants.rs b/server-rs/crates/platform-image/src/vector_engine/constants.rs index 4cfa0432b..afbe17d03 100644 --- a/server-rs/crates/platform-image/src/vector_engine/constants.rs +++ b/server-rs/crates/platform-image/src/vector_engine/constants.rs @@ -1,3 +1,4 @@ pub const GPT_IMAGE_2_MODEL: &str = "gpt-image-2"; +pub const GPT_IMAGE_2_C_MODEL: &str = "gpt-image-2-c"; pub const VECTOR_ENGINE_GPT_IMAGE_2_MODEL: &str = GPT_IMAGE_2_MODEL; pub const VECTOR_ENGINE_PROVIDER: &str = "vector-engine"; diff --git a/server-rs/crates/platform-image/src/vector_engine/curl_transport.rs b/server-rs/crates/platform-image/src/vector_engine/curl_transport.rs index ce7dd389c..2098718a3 100644 --- a/server-rs/crates/platform-image/src/vector_engine/curl_transport.rs +++ b/server-rs/crates/platform-image/src/vector_engine/curl_transport.rs @@ -148,6 +148,7 @@ pub(crate) fn map_curl_error( context: &str, request_url: &str, failure_stage: &'static str, + image_model: Option<&'static str>, error: VectorEngineCurlError, latency_ms: u64, prompt_chars: Option, @@ -172,6 +173,7 @@ pub(crate) fn map_curl_error( Some(latency_ms), prompt_chars, reference_image_count, + image_model, ); tracing::warn!( provider = VECTOR_ENGINE_PROVIDER, @@ -189,6 +191,7 @@ pub(crate) fn map_curl_error( elapsed_ms = latency_ms, prompt_chars, reference_image_count, + image_model, request_params = %request_params .map(|value| value.to_string()) .unwrap_or_default(), diff --git a/server-rs/crates/platform-image/src/vector_engine/error.rs b/server-rs/crates/platform-image/src/vector_engine/error.rs index c98edf2dc..cb820e08a 100644 --- a/server-rs/crates/platform-image/src/vector_engine/error.rs +++ b/server-rs/crates/platform-image/src/vector_engine/error.rs @@ -42,6 +42,10 @@ pub enum PlatformImageError { message: String, audit: Option, }, + FallbackFailed { + final_error: Box, + recovered_failure_audits: Vec, + }, } impl PlatformImageError { @@ -53,6 +57,7 @@ impl PlatformImageError { | Self::Upstream { provider, .. } | Self::ResponseParse { provider, .. } | Self::MissingImage { provider, .. } => provider, + Self::FallbackFailed { final_error, .. } => final_error.provider(), } } @@ -64,6 +69,7 @@ impl PlatformImageError { | Self::Upstream { message, .. } | Self::ResponseParse { message, .. } | Self::MissingImage { message, .. } => message, + Self::FallbackFailed { final_error, .. } => final_error.message(), } } @@ -73,10 +79,42 @@ impl PlatformImageError { | Self::Upstream { audit, .. } | Self::ResponseParse { audit, .. } | Self::MissingImage { audit, .. } => audit.as_ref(), + Self::FallbackFailed { final_error, .. } => final_error.audit(), Self::InvalidConfig { .. } | Self::InvalidRequest { .. } => None, } } + pub fn recovered_failure_audits(&self) -> &[PlatformImageFailureAudit] { + match self { + Self::FallbackFailed { + recovered_failure_audits, + .. + } => recovered_failure_audits.as_slice(), + _ => &[], + } + } + + pub(crate) fn with_recovered_failure_audits( + self, + recovered_failure_audits: Vec, + ) -> Self { + if recovered_failure_audits.is_empty() { + self + } else { + Self::FallbackFailed { + final_error: Box::new(self), + recovered_failure_audits, + } + } + } + + pub fn into_final_error(self) -> Self { + match self { + Self::FallbackFailed { final_error, .. } => final_error.into_final_error(), + error => error, + } + } + pub fn status_hint(&self) -> PlatformImageStatusHint { match self { Self::InvalidConfig { .. } => PlatformImageStatusHint::ServiceUnavailable, @@ -93,6 +131,7 @@ impl PlatformImageError { | Self::Upstream { .. } | Self::ResponseParse { .. } | Self::MissingImage { .. } => PlatformImageStatusHint::BadGateway, + Self::FallbackFailed { final_error, .. } => final_error.status_hint(), } } } diff --git a/server-rs/crates/platform-image/src/vector_engine/image_source.rs b/server-rs/crates/platform-image/src/vector_engine/image_source.rs index e2a59706c..b9c65e853 100644 --- a/server-rs/crates/platform-image/src/vector_engine/image_source.rs +++ b/server-rs/crates/platform-image/src/vector_engine/image_source.rs @@ -69,6 +69,7 @@ async fn download_remote_image_with_deadline( return Err(request_budget_exhausted_error( image_url, operation, + None, Some(0), None, None, @@ -83,6 +84,7 @@ async fn download_remote_image_with_deadline( request_budget_exhausted_error( image_url, operation, + None, Some(started_at.elapsed().as_millis() as u64), None, None, @@ -117,6 +119,7 @@ pub(crate) async fn download_images_from_urls( task_id, actual_prompt: None, images, + recovered_failure_audits: Vec::new(), }) } @@ -222,12 +225,13 @@ pub(crate) fn images_from_base64( task_id, actual_prompt: None, images, + recovered_failure_audits: Vec::new(), } } pub(crate) fn decode_generated_image_base64(raw: &str) -> Option { let bytes = BASE64_STANDARD.decode(raw.trim()).ok()?; - let mime_type = infer_image_mime_type(bytes.as_slice()); + let mime_type = infer_image_mime_type(bytes.as_slice())?; Some(DownloadedImage { extension: mime_to_extension(mime_type.as_str()).to_string(), mime_type, @@ -258,20 +262,20 @@ pub(crate) fn mime_to_extension(mime_type: &str) -> &str { } } -pub(crate) fn infer_image_mime_type(bytes: &[u8]) -> String { +pub(crate) fn infer_image_mime_type(bytes: &[u8]) -> Option { if bytes.starts_with(b"\x89PNG\r\n\x1A\n") { - return "image/png".to_string(); + return Some("image/png".to_string()); } if bytes.starts_with(b"\xFF\xD8\xFF") { - return "image/jpeg".to_string(); + return Some("image/jpeg".to_string()); } if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP") { - return "image/webp".to_string(); + return Some("image/webp".to_string()); } if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { - return "image/gif".to_string(); + return Some("image/gif".to_string()); } - "image/png".to_string() + None } fn map_simple_request_error(message: String, endpoint: Option) -> PlatformImageError { diff --git a/server-rs/crates/platform-image/src/vector_engine/mod.rs b/server-rs/crates/platform-image/src/vector_engine/mod.rs index 8f771c54a..99511b9ac 100644 --- a/server-rs/crates/platform-image/src/vector_engine/mod.rs +++ b/server-rs/crates/platform-image/src/vector_engine/mod.rs @@ -19,7 +19,9 @@ pub use client::{ create_vector_engine_image_generation, create_vector_engine_image_generation_with_model, create_vector_engine_nanobanana_generate_content, }; -pub use constants::{GPT_IMAGE_2_MODEL, VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER}; +pub use constants::{ + GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER, +}; pub use error::{PlatformImageError, PlatformImageStatusHint}; pub use image_source::download_remote_image; pub use request::{ diff --git a/server-rs/crates/platform-image/src/vector_engine/request.rs b/server-rs/crates/platform-image/src/vector_engine/request.rs index 91d6075c4..a0daa53c4 100644 --- a/server-rs/crates/platform-image/src/vector_engine/request.rs +++ b/server-rs/crates/platform-image/src/vector_engine/request.rs @@ -1,7 +1,7 @@ use serde_json::{Map, Value, json}; use super::{ - constants::GPT_IMAGE_2_MODEL, + constants::{GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL}, types::{ReferenceImage, VectorEngineImageSettings}, }; @@ -90,6 +90,13 @@ pub fn normalize_vector_engine_image_model(model: &str) -> &str { } } +pub(crate) fn is_gpt_image_2_family_model(model: &str) -> bool { + matches!( + normalize_vector_engine_image_model(model), + GPT_IMAGE_2_MODEL | GPT_IMAGE_2_C_MODEL + ) +} + pub fn normalize_image_size_for_model(model: &str, size: &str) -> String { let size = size.trim(); let normalized_size = match size { @@ -102,7 +109,7 @@ pub fn normalize_image_size_for_model(model: &str, size: &str) -> String { _ => "1024x1024".to_string(), }; - if normalize_vector_engine_image_model(model) == GPT_IMAGE_2_MODEL { + if is_gpt_image_2_family_model(model) { clamp_gpt_image_2_pixel_size(normalized_size.as_str()) } else { normalized_size diff --git a/server-rs/crates/platform-image/src/vector_engine/response.rs b/server-rs/crates/platform-image/src/vector_engine/response.rs index 112889930..438b5b3a6 100644 --- a/server-rs/crates/platform-image/src/vector_engine/response.rs +++ b/server-rs/crates/platform-image/src/vector_engine/response.rs @@ -17,6 +17,7 @@ pub(crate) async fn handle_vector_engine_response( request_url: &str, response_status: u16, response_text: &str, + image_model: Option<&'static str>, failure_context: &str, latency_ms: u64, prompt_chars: Option, @@ -42,6 +43,7 @@ pub(crate) async fn handle_vector_engine_response( Some(latency_ms), prompt_chars, reference_image_count, + image_model, ); tracing::warn!( provider = VECTOR_ENGINE_PROVIDER, @@ -79,6 +81,7 @@ pub(crate) async fn handle_vector_engine_response( Some(latency_ms), prompt_chars, reference_image_count, + image_model, ); tracing::warn!( provider = VECTOR_ENGINE_PROVIDER, @@ -136,6 +139,7 @@ pub(crate) async fn handle_vector_engine_response( Some(download_started_at.elapsed().as_millis() as u64), prompt_chars, reference_image_count, + image_model, ); return Err(error.with_audit(audit)); } @@ -154,6 +158,40 @@ pub(crate) async fn handle_vector_engine_response( let b64_images = extract_b64_images(&response_json.payload); if !b64_images.is_empty() { let mut generated = images_from_base64(task_id, b64_images, candidate_count); + if generated.images.is_empty() { + let message = format!("{failure_context}：VectorEngine 返回的 base64 图片无法解码"); + let raw_excerpt = truncate_raw(response_text); + let audit = build_failure_audit( + request_url, + failure_context, + "response_parse", + Some(response_status), + None, + false, + false, + message.as_str(), + None, + Some(raw_excerpt.clone()), + Some(latency_ms), + prompt_chars, + reference_image_count, + image_model, + ); + tracing::warn!( + provider = VECTOR_ENGINE_PROVIDER, + endpoint = %request_url, + status = response_status, + image_model, + raw_excerpt = %raw_excerpt, + "VectorEngine 图片 base64 解码失败" + ); + return Err(PlatformImageError::ResponseParse { + provider: VECTOR_ENGINE_PROVIDER, + message, + raw_excerpt, + audit: Some(audit), + }); + } generated.actual_prompt = actual_prompt; tracing::info!( provider = VECTOR_ENGINE_PROVIDER, @@ -180,6 +218,7 @@ pub(crate) async fn handle_vector_engine_response( Some(latency_ms), prompt_chars, reference_image_count, + image_model, ); tracing::warn!( provider = VECTOR_ENGINE_PROVIDER, diff --git a/server-rs/crates/platform-image/src/vector_engine/types.rs b/server-rs/crates/platform-image/src/vector_engine/types.rs index 328173567..77fbd19f9 100644 --- a/server-rs/crates/platform-image/src/vector_engine/types.rs +++ b/server-rs/crates/platform-image/src/vector_engine/types.rs @@ -1,3 +1,5 @@ +use super::audit::PlatformImageFailureAudit; + #[derive(Clone, Debug)] pub struct VectorEngineImageSettings { pub base_url: String, @@ -11,6 +13,7 @@ pub struct GeneratedImages { pub task_id: String, pub actual_prompt: Option, pub images: Vec, + pub recovered_failure_audits: Vec, } #[derive(Clone, Debug)] diff --git a/server-rs/crates/platform-image/src/vector_engine/util.rs b/server-rs/crates/platform-image/src/vector_engine/util.rs index ed6c487ab..621d57c6a 100644 --- a/server-rs/crates/platform-image/src/vector_engine/util.rs +++ b/server-rs/crates/platform-image/src/vector_engine/util.rs @@ -79,6 +79,13 @@ impl PlatformImageError { message, audit: Some(audit), }, + Self::FallbackFailed { + final_error, + recovered_failure_audits, + } => Self::FallbackFailed { + final_error: Box::new(final_error.with_audit(audit)), + recovered_failure_audits, + }, Self::InvalidConfig { .. } | Self::InvalidRequest { .. } => self, } } diff --git a/server-rs/crates/platform-image/tests/vector_engine.rs b/server-rs/crates/platform-image/tests/vector_engine.rs index 6485e8d6c..c4b1801db 100644 --- a/server-rs/crates/platform-image/tests/vector_engine.rs +++ b/server-rs/crates/platform-image/tests/vector_engine.rs @@ -1,6 +1,6 @@ use platform_image::vector_engine::{ - GPT_IMAGE_2_MODEL, PlatformImageError, ReferenceImage, VECTOR_ENGINE_PROVIDER, - VectorEngineImageSettings, build_vector_engine_image_http_client, + GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, PlatformImageError, ReferenceImage, + VECTOR_ENGINE_PROVIDER, VectorEngineImageSettings, build_vector_engine_image_http_client, build_vector_engine_image_request_body, build_vector_engine_image_request_body_with_model, build_vector_engine_nanobanana_generate_content_request_body, create_vector_engine_image_edit, create_vector_engine_image_generation, create_vector_engine_nanobanana_generate_content, @@ -17,6 +17,7 @@ use std::{ use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, + sync::Mutex, }; #[test] @@ -32,6 +33,7 @@ fn vector_engine_module_exposes_provider_protocol_helpers() { build_vector_engine_image_request_body("雾海神殿", Some("文字，水印"), "16:9", 9, &[]); assert_eq!(GPT_IMAGE_2_MODEL, "gpt-image-2"); + assert_eq!(GPT_IMAGE_2_C_MODEL, "gpt-image-2-c"); assert_eq!(VECTOR_ENGINE_PROVIDER, "vector-engine"); assert_eq!(body["model"], GPT_IMAGE_2_MODEL); assert_eq!(body["size"], "1536x1024"); @@ -109,8 +111,17 @@ fn vector_engine_only_enforces_the_gpt_image_2_pixel_budget_for_that_model() { 1, &[], ); + let fallback_gpt_body = build_vector_engine_image_request_body_with_model( + GPT_IMAGE_2_C_MODEL, + "小尺寸图", + None, + "640x640", + 1, + &[], + ); assert_eq!(gpt_body["size"], "816x816"); + assert_eq!(fallback_gpt_body["size"], "816x816"); assert_eq!(nanobanana_body["size"], "640x640"); assert_eq!(oversized_gpt_body["size"], "2880x2880"); } @@ -207,6 +218,8 @@ async fn vector_engine_image_edit_retries_send_timeout_once_and_succeeds() { .expect("mock server address should be readable"); let request_count = Arc::new(AtomicUsize::new(0)); let request_count_for_server = Arc::clone(&request_count); + let requests = Arc::new(Mutex::new(Vec::new())); + let requests_for_server = Arc::clone(&requests); let server = tokio::spawn(async move { loop { @@ -214,9 +227,13 @@ async fn vector_engine_image_edit_retries_send_timeout_once_and_succeeds() { break; }; let request_index = request_count_for_server.fetch_add(1, Ordering::SeqCst); + let requests_for_connection = Arc::clone(&requests_for_server); tokio::spawn(async move { - let mut buffer = [0_u8; 4096]; - let _ = stream.read(&mut buffer).await; + let request = read_http_request(&mut stream).await; + requests_for_connection + .lock() + .await + .push(String::from_utf8_lossy(request.as_slice()).into_owned()); if request_index == 0 { tokio::time::sleep(Duration::from_millis(120)).await; return; @@ -261,10 +278,55 @@ async fn vector_engine_image_edit_retries_send_timeout_once_and_succeeds() { assert_eq!(generated.images.len(), 1); assert_eq!(generated.images[0].mime_type, "image/png"); + assert!(generated.recovered_failure_audits.is_empty()); assert_eq!(request_count.load(Ordering::SeqCst), 2); + let requests = requests.lock().await; + assert!( + requests + .iter() + .all(|request| request.contains("\r\n\r\ngpt-image-2\r\n")) + ); server.abort(); } +async fn read_http_request(stream: &mut tokio::net::TcpStream) -> Vec { + let mut request = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + let Ok(read) = stream.read(&mut buffer).await else { + return request; + }; + if read == 0 { + return request; + } + request.extend_from_slice(&buffer[..read]); + let Some(header_start) = request.windows(4).position(|window| window == b"\r\n\r\n") else { + continue; + }; + let header_end = header_start + 4; + let headers = String::from_utf8_lossy(&request[..header_end]); + let content_length = headers + .lines() + .find_map(|line| { + line.strip_prefix("Content-Length:") + .or_else(|| line.strip_prefix("content-length:")) + }) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or_default(); + let expected_len = header_end + content_length; + while request.len() < expected_len { + let Ok(read) = stream.read(&mut buffer).await else { + return request; + }; + if read == 0 { + return request; + } + request.extend_from_slice(&buffer[..read]); + } + return request; + } +} + #[tokio::test] async fn vector_engine_deadline_clips_stalled_attempt_and_prevents_retry() { let listener = TcpListener::bind("127.0.0.1:0") @@ -393,7 +455,7 @@ async fn nanobanana_generate_content_posts_native_body_and_reads_inline_data() { } #[tokio::test] -async fn vector_engine_image_generation_retries_upstream_502_once_and_succeeds() { +async fn vector_engine_image_generation_falls_back_after_upstream_502_and_succeeds() { let listener = TcpListener::bind("127.0.0.1:0") .await .expect("mock server should bind"); @@ -402,6 +464,8 @@ async fn vector_engine_image_generation_retries_upstream_502_once_and_succeeds() .expect("mock server address should be readable"); let request_count = Arc::new(AtomicUsize::new(0)); let request_count_for_server = Arc::clone(&request_count); + let requests = Arc::new(Mutex::new(Vec::new())); + let requests_for_server = Arc::clone(&requests); let server = tokio::spawn(async move { loop { @@ -409,9 +473,13 @@ async fn vector_engine_image_generation_retries_upstream_502_once_and_succeeds() break; }; let request_index = request_count_for_server.fetch_add(1, Ordering::SeqCst); + let requests_for_connection = Arc::clone(&requests_for_server); tokio::spawn(async move { - let mut buffer = [0_u8; 4096]; - let _ = stream.read(&mut buffer).await; + let request = read_http_request(&mut stream).await; + requests_for_connection + .lock() + .await + .push(String::from_utf8_lossy(request.as_slice()).into_owned()); if request_index == 0 { let body = "502 Bad Gateway

502 Bad Gateway

nginx
"; let response = format!( @@ -458,6 +526,305 @@ async fn vector_engine_image_generation_retries_upstream_502_once_and_succeeds() assert_eq!(generated.images.len(), 1); assert_eq!(generated.images[0].mime_type, "image/png"); + assert_eq!(generated.recovered_failure_audits.len(), 1); + assert_eq!( + generated.recovered_failure_audits[0].image_model, + Some(GPT_IMAGE_2_MODEL) + ); assert_eq!(request_count.load(Ordering::SeqCst), 2); + let requests = requests.lock().await; + assert!(requests[0].contains("\"model\":\"gpt-image-2\"")); + assert!(requests[1].contains("\"model\":\"gpt-image-2-c\"")); server.abort(); } + +#[tokio::test] +async fn vector_engine_image_generation_uses_gpt_image_2_without_fallback_on_success() { + let (base_url, server, requests) = start_http_response_sequence(vec![MockResponse { + status: "200 OK", + content_type: "application/json", + body: r#"{"data":[{"b64_json":"iVBORw0KGgpyZXN0"}]}"#, + }]) + .await; + let settings = test_vector_engine_settings(base_url); + let http_client = + build_vector_engine_image_http_client(&settings).expect("client should build"); + + let generated = create_vector_engine_image_generation( + &http_client, + &settings, + "测试提示词", + None, + "1024x1024", + 1, + &[], + "测试 VectorEngine 图片生成失败", + ) + .await + .expect("preferred model should generate image"); + + assert_eq!(generated.images.len(), 1); + assert!(generated.recovered_failure_audits.is_empty()); + let requests = requests.lock().await; + assert_eq!(requests.len(), 1); + assert!(requests[0].contains("\"model\":\"gpt-image-2\"")); + server.abort(); +} + +#[tokio::test] +async fn vector_engine_image_edit_falls_back_when_preferred_model_is_unsupported() { + let (base_url, server, requests) = start_http_response_sequence(vec![ + MockResponse { + status: "400 Bad Request", + content_type: "application/json", + body: r#"{"error":{"message":"model gpt-image-2 is not supported"}}"#, + }, + MockResponse { + status: "200 OK", + content_type: "application/json", + body: r#"{"data":[{"b64_json":"iVBORw0KGgpyZXN0"}]}"#, + }, + ]) + .await; + let settings = test_vector_engine_settings(base_url); + let http_client = + build_vector_engine_image_http_client(&settings).expect("client should build"); + let reference = ReferenceImage { + bytes: b"reference".to_vec(), + mime_type: "image/png".to_string(), + file_name: "reference.png".to_string(), + }; + + let generated = create_vector_engine_image_edit( + &http_client, + &settings, + "测试提示词", + None, + "1024x1024", + &reference, + "测试 VectorEngine 图片编辑失败", + ) + .await + .expect("fallback model should recover unsupported preferred model"); + + assert_eq!(generated.images.len(), 1); + assert_eq!(generated.recovered_failure_audits.len(), 1); + assert_eq!( + generated.recovered_failure_audits[0].image_model, + Some(GPT_IMAGE_2_MODEL) + ); + let requests = requests.lock().await; + assert_eq!(requests.len(), 2); + assert!(requests[0].contains("\r\n\r\ngpt-image-2\r\n")); + assert!(requests[1].contains("\r\n\r\ngpt-image-2-c\r\n")); + server.abort(); +} + +#[tokio::test] +async fn vector_engine_image_generation_does_not_fallback_on_auth_failure() { + let (base_url, server, requests) = start_http_response_sequence(vec![MockResponse { + status: "401 Unauthorized", + content_type: "application/json", + body: r#"{"error":{"message":"invalid api key"}}"#, + }]) + .await; + let settings = test_vector_engine_settings(base_url); + let http_client = + build_vector_engine_image_http_client(&settings).expect("client should build"); + + let error = create_vector_engine_image_generation( + &http_client, + &settings, + "测试提示词", + None, + "1024x1024", + 1, + &[], + "测试 VectorEngine 图片生成失败", + ) + .await + .expect_err("authentication failure should remain terminal"); + + assert!(matches!( + error, + PlatformImageError::Upstream { + upstream_status: 401, + .. + } + )); + let requests = requests.lock().await; + assert_eq!(requests.len(), 1); + server.abort(); +} + +#[tokio::test] +async fn vector_engine_image_generation_falls_back_after_non_image_base64_response() { + let (base_url, server, requests) = start_http_response_sequence(vec![ + MockResponse { + status: "200 OK", + content_type: "application/json", + body: r#"{"data":[{"b64_json":"bm90IGFuIGltYWdl"}]}"#, + }, + MockResponse { + status: "200 OK", + content_type: "application/json", + body: r#"{"data":[{"b64_json":"iVBORw0KGgpyZXN0"}]}"#, + }, + ]) + .await; + let settings = test_vector_engine_settings(base_url); + let http_client = + build_vector_engine_image_http_client(&settings).expect("client should build"); + + let generated = create_vector_engine_image_generation( + &http_client, + &settings, + "测试提示词", + None, + "1024x1024", + 1, + &[], + "测试 VectorEngine 图片生成失败", + ) + .await + .expect("fallback model should recover invalid preferred response"); + + assert_eq!(generated.images.len(), 1); + assert_eq!(generated.recovered_failure_audits.len(), 1); + assert_eq!( + generated.recovered_failure_audits[0].failure_stage, + "response_parse" + ); + let requests = requests.lock().await; + assert_eq!(requests.len(), 2); + assert!(requests[0].contains("\"model\":\"gpt-image-2\"")); + assert!(requests[1].contains("\"model\":\"gpt-image-2-c\"")); + server.abort(); +} + +#[tokio::test] +async fn vector_engine_image_generation_preserves_primary_audit_when_fallback_also_fails() { + let (base_url, server, requests) = start_http_response_sequence(vec![ + MockResponse { + status: "502 Bad Gateway", + content_type: "text/html", + body: "

502 Bad Gateway

", + }, + MockResponse { + status: "401 Unauthorized", + content_type: "application/json", + body: r#"{"error":{"message":"invalid api key"}}"#, + }, + ]) + .await; + let settings = test_vector_engine_settings(base_url); + let http_client = + build_vector_engine_image_http_client(&settings).expect("client should build"); + + let error = create_vector_engine_image_generation( + &http_client, + &settings, + "测试提示词", + None, + "1024x1024", + 1, + &[], + "测试 VectorEngine 图片生成失败", + ) + .await + .expect_err("fallback authentication failure should remain terminal"); + + assert_eq!(error.recovered_failure_audits().len(), 1); + assert_eq!( + error.recovered_failure_audits()[0].image_model, + Some(GPT_IMAGE_2_MODEL) + ); + assert_eq!( + error.audit().and_then(|audit| audit.image_model), + Some(GPT_IMAGE_2_C_MODEL) + ); + let requests = requests.lock().await; + assert_eq!(requests.len(), 2); + server.abort(); +} + +#[tokio::test] +async fn vector_engine_image_generation_does_not_fallback_on_safety_parse_failure() { + let (base_url, server, requests) = start_http_response_sequence(vec![MockResponse { + status: "200 OK", + content_type: "application/json", + body: "safety refusal: 内容审核拒绝", + }]) + .await; + let settings = test_vector_engine_settings(base_url); + let http_client = + build_vector_engine_image_http_client(&settings).expect("client should build"); + + let error = create_vector_engine_image_generation( + &http_client, + &settings, + "测试提示词", + None, + "1024x1024", + 1, + &[], + "测试 VectorEngine 图片生成失败", + ) + .await + .expect_err("content rejection should not switch models"); + + assert!(matches!(error, PlatformImageError::ResponseParse { .. })); + let requests = requests.lock().await; + assert_eq!(requests.len(), 1); + server.abort(); +} + +#[derive(Clone, Copy)] +struct MockResponse { + status: &'static str, + content_type: &'static str, + body: &'static str, +} + +async fn start_http_response_sequence( + responses: Vec, +) -> (String, tokio::task::JoinHandle<()>, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("mock server should bind"); + let server_addr = listener + .local_addr() + .expect("mock server address should be readable"); + let requests = Arc::new(Mutex::new(Vec::new())); + let requests_for_server = Arc::clone(&requests); + let server = tokio::spawn(async move { + for response_spec in responses { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let request = read_http_request(&mut stream).await; + requests_for_server + .lock() + .await + .push(String::from_utf8_lossy(request.as_slice()).into_owned()); + let response = format!( + "HTTP/1.1 {}\r\nContent-Type: {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + response_spec.status, + response_spec.content_type, + response_spec.body.len(), + response_spec.body, + ); + let _ = stream.write_all(response.as_bytes()).await; + } + }); + (format!("http://{server_addr}/v1"), server, requests) +} + +fn test_vector_engine_settings(base_url: String) -> VectorEngineImageSettings { + VectorEngineImageSettings { + base_url, + api_key: "test-key".to_string(), + request_timeout_ms: 1_000, + request_deadline: None, + } +} From 7e58fd2195ac385f0016645b6d92077f9c795626 Mon Sep 17 00:00:00 2001 From: kdletters Date: Tue, 21 Jul 2026 17:15:49 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E6=8E=A5=E5=85=A5Gitea=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E6=A3=80=E6=9F=A5=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 master 推送、PR 和手工触发的三路项目检查工作流 收紧 PR 基线、只读权限、action 固定版本与依赖锁漂移检查 补充 Gitea 版本、隔离 runner 和 required context 运维说明 --- .gitea/workflows/project-ci.yml | 280 ++++++++++++++++++ .../shared-memory/development-workflow.md | 9 + ...发运维】本地开发验证与生产运维-2026-05-15.md | 18 +- 3 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 .gitea/workflows/project-ci.yml diff --git a/.gitea/workflows/project-ci.yml b/.gitea/workflows/project-ci.yml new file mode 100644 index 000000000..a6844ad08 --- /dev/null +++ b/.gitea/workflows/project-ci.yml @@ -0,0 +1,280 @@ +name: Project CI + +on: + push: + branches: + - master + pull_request: + workflow_dispatch: + +permissions: + contents: read + +env: + CI: 'true' + CARGO_INCREMENTAL: '0' + CARGO_TERM_COLOR: always + RUSTC_WRAPPER: '' + CARGO_BUILD_RUSTC_WRAPPER: '' + +jobs: + repository-checks: + name: Repository checks + runs-on: ubuntu-latest + steps: + - name: Checkout full history + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install base tools + shell: bash + run: | + set -euo pipefail + command -v apt-get >/dev/null 2>&1 || { + echo 'ubuntu-latest runner must provide an Ubuntu or Debian environment.' >&2 + exit 1 + } + sudo_command='' + if command -v sudo >/dev/null 2>&1; then + sudo_command='sudo' + fi + ${sudo_command} apt-get update + ${sudo_command} env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + curl + + - name: Set up Node.js 22 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + + - name: Resolve comparison base + shell: bash + run: | + set -euo pipefail + base_ref="$(node -e ' + const fs = require("node:fs"); + const event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); + process.stdout.write(event.pull_request?.base?.sha ?? event.before ?? ""); + ')" + if [[ -n "${base_ref}" && ! "${base_ref}" =~ ^0+$ ]]; then + git cat-file -e "${base_ref}^{commit}" 2>/dev/null || { + echo "comparison base commit is unavailable: ${base_ref}" >&2 + exit 1 + } + else + base_ref="$(git merge-base HEAD origin/master 2>/dev/null || git rev-parse HEAD)" + fi + if [[ "${GITHUB_EVENT_NAME:-}" == 'pull_request' ]] \ + && ! git merge-base --is-ancestor "${base_ref}" HEAD; then + echo 'pull request head does not contain the latest base commit; update the branch and rerun CI.' >&2 + exit 1 + fi + echo "SPACETIME_SCHEMA_BASE_REF=${base_ref}" >> "${GITHUB_ENV}" + + - name: Set up repository Rust toolchain + shell: bash + run: | + set -euo pipefail + if ! command -v rustup >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --profile minimal --default-toolchain none + fi + echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" + export PATH="${HOME}/.cargo/bin:${PATH}" + toolchain="$(sed -n 's/^channel = "\([^"]*\)"/\1/p' rust-toolchain.toml)" + test -n "${toolchain}" + rustup toolchain install "${toolchain}" --profile minimal --component rustfmt + rustc --version + cargo --version + rustfmt --version + + - name: Install npm dependencies + run: npm ci + + - name: Run repository lint gates + run: npm run lint + + - name: Run frontend and script tests + run: npm run test + + - name: Build web applications + run: npm run build + + - name: Validate content data + run: npm run check:content + + - name: Check committed whitespace + shell: bash + run: | + set -euo pipefail + base_ref="${SPACETIME_SCHEMA_BASE_REF:-}" + test -n "${base_ref}" + git cat-file -e "${base_ref}^{commit}" + git diff --check "${base_ref}"...HEAD + + backend-checks: + name: Backend checks + runs-on: ubuntu-latest + steps: + - name: Checkout full history + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install backend build dependencies + shell: bash + run: | + set -euo pipefail + command -v apt-get >/dev/null 2>&1 || { + echo 'ubuntu-latest runner must provide an Ubuntu or Debian environment.' >&2 + exit 1 + } + sudo_command='' + if command -v sudo >/dev/null 2>&1; then + sudo_command='sudo' + fi + ${sudo_command} apt-get update + ${sudo_command} env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + curl \ + libclang-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + lld \ + pkg-config + + - name: Set up Node.js 22 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + + - name: Resolve comparison base + shell: bash + run: | + set -euo pipefail + base_ref="$(node -e ' + const fs = require("node:fs"); + const event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); + process.stdout.write(event.pull_request?.base?.sha ?? event.before ?? ""); + ')" + if [[ -n "${base_ref}" && ! "${base_ref}" =~ ^0+$ ]]; then + git cat-file -e "${base_ref}^{commit}" 2>/dev/null || { + echo "comparison base commit is unavailable: ${base_ref}" >&2 + exit 1 + } + else + base_ref="$(git merge-base HEAD origin/master 2>/dev/null || git rev-parse HEAD)" + fi + if [[ "${GITHUB_EVENT_NAME:-}" == 'pull_request' ]] \ + && ! git merge-base --is-ancestor "${base_ref}" HEAD; then + echo 'pull request head does not contain the latest base commit; update the branch and rerun CI.' >&2 + exit 1 + fi + echo "SPACETIME_SCHEMA_BASE_REF=${base_ref}" >> "${GITHUB_ENV}" + + - name: Set up repository Rust toolchain + shell: bash + run: | + set -euo pipefail + if ! command -v rustup >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --profile minimal --default-toolchain none + fi + echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" + export PATH="${HOME}/.cargo/bin:${PATH}" + toolchain="$(sed -n 's/^channel = "\([^"]*\)"/\1/p' rust-toolchain.toml)" + test -n "${toolchain}" + rustup toolchain install "${toolchain}" --profile minimal --component rustfmt + rustc --version + cargo --version + rustfmt --version + + - name: Install npm dependencies + run: npm ci + + - name: Check server-rs boundaries + run: npm run check:server-rs-ddd + + - name: Check api-server targets + run: cargo check --locked -p api-server --all-targets --manifest-path server-rs/Cargo.toml + + - name: Check SpacetimeDB module + run: cargo check --locked -p spacetime-module --manifest-path server-rs/Cargo.toml + + native-shell-checks: + name: Native shell checks + runs-on: ubuntu-latest + steps: + - name: Checkout full history + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install native shell build dependencies + shell: bash + run: | + set -euo pipefail + command -v apt-get >/dev/null 2>&1 || { + echo 'ubuntu-latest runner must provide an Ubuntu or Debian environment.' >&2 + exit 1 + } + sudo_command='' + if command -v sudo >/dev/null 2>&1; then + sudo_command='sudo' + fi + ${sudo_command} apt-get update + ${sudo_command} env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + curl \ + file \ + libayatana-appindicator3-dev \ + libssl-dev \ + libwebkit2gtk-4.1-dev \ + libxdo-dev \ + librsvg2-dev \ + lld \ + patchelf \ + pkg-config \ + wget + + - name: Set up Node.js 22 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + + - name: Set up repository Rust toolchain + shell: bash + run: | + set -euo pipefail + if ! command -v rustup >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --profile minimal --default-toolchain none + fi + echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" + export PATH="${HOME}/.cargo/bin:${PATH}" + toolchain="$(sed -n 's/^channel = "\([^"]*\)"/\1/p' rust-toolchain.toml)" + test -n "${toolchain}" + rustup toolchain install "${toolchain}" --profile minimal --component rustfmt + rustc --version + cargo --version + rustfmt --version + + - name: Install npm dependencies + run: npm ci + + - name: Run native shell gates + run: npm run check:native-shells + + - name: Ensure native lockfile is unchanged + run: git diff --exit-code -- apps/desktop-shell/src-tauri/Cargo.lock diff --git a/docs/project-memory/shared-memory/development-workflow.md b/docs/project-memory/shared-memory/development-workflow.md index 5630daee9..535861061 100644 --- a/docs/project-memory/shared-memory/development-workflow.md +++ b/docs/project-memory/shared-memory/development-workflow.md @@ -262,6 +262,15 @@ DDD 边界检查： npm run check:server-rs-ddd ``` +## Gitea CI 与 PR 检查 + +- 仓库 CI 入口是 `.gitea/workflows/project-ci.yml`，向 `master` 推送和所有 PR 创建、更新时必须运行，也允许手工触发。 +- CI 固定拆分为 `Repository checks`、`Backend checks`、`Native shell checks` 三个 required job；对应 PR context 完整名称是 `Project CI / Repository checks (pull_request)`、`Project CI / Backend checks (pull_request)`、`Project CI / Native shell checks (pull_request)`，首次运行后仍须从 Gitea 最近一周 context 表复核。前两项覆盖仓库/Web 与后端边界/编译，原生壳验收单独运行以便定位重型构建失败。 +- 三个 job 共同覆盖 `npm run check`，并追加 `npm run check:server-rs-ddd`、`cargo check -p api-server --all-targets --manifest-path server-rs/Cargo.toml` 和 `cargo check -p spacetime-module --manifest-path server-rs/Cargo.toml`。 +- checkout 必须使用完整历史。PR 将 base SHA 写入 `SPACETIME_SCHEMA_BASE_REF`，直接推送 `master` 使用 before SHA；事件基线不可解析时直接失败。Gitea 检查的是 PR head 而非预合并 commit，workflow 必须拒绝不包含最新 base commit 的过期 PR，分支保护同时保持“PR 过期禁止合并”。 +- 普通 PR job 不读取业务 secret，不运行真实 API/SpacetimeDB/OSS/支付/生成/live smoke，也不执行会修改外部状态的维护、迁移、发布或备份命令。 +- Gitea 至少升级到 `1.26.4` 后才能注册执行 PR job 的 runner；`ubuntu-latest` 标签只映射到固定 digest 的 Ubuntu 24.04 级 Docker/临时隔离镜像，不使用浮动镜像 tag，不映射 host，不向 job 暴露 Docker socket、业务 secret 或不必要内网。runner 能访问 Gitea、GitHub Actions 与 `actions/node-versions`、nodejs.org、npm、Rust 分发和 crates.io；workflow 的官方 action 固定完整 commit，若内网禁用 GitHub，先在当前 Gitea 镜像对应 commit 并改用绝对 URL。受控镜像优先预装 rustup。Gitea 1.26 的任务超时由 runner 全局配置控制；首次运行成功后，`master` 分支保护必须要求上述三个 job 全部成功。 + ## 后端相关默认验证 后端修改后，按 DDD 文档中的验收命令执行。涉及 API smoke 时： diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 24cd48b95..4b9cf53cd 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -1,6 +1,6 @@ # 本地开发验证与生产运维 -更新时间：`2026-07-17` +更新时间：`2026-07-21` ## 标准开发流程 @@ -198,6 +198,22 @@ npm run check `npm run build` 由 `scripts/build-gate.mjs` 串行构建主站和后台；该门禁会把 Vite warning 当成失败处理。若看到 `Build gate failed because warnings were emitted`，先看 warning 原文，例如 chunk 体积超过 `vite.config.ts` / `apps/admin-web/vite.config.ts` 的 `chunkSizeWarningLimit`，不要先按 Rust 编译失败排查。 +### Gitea Actions PR 门禁 + +仓库级 Gitea Actions 工作流固定为 `.gitea/workflows/project-ci.yml`，在向 `master` 推送、创建或更新 PR，以及手工触发时运行。工作流拆成三个必须通过的 job： + +- `Repository checks`：执行 `npm run lint`、完整 Vitest、主站与后台生产构建、内容数据检查和提交差异空白检查。 +- `Backend checks`：执行 `npm run check:server-rs-ddd`、`api-server --all-targets` 编译和 `spacetime-module` 编译。 +- `Native shell checks`：独立执行 `npm run check:native-shells`，覆盖微信壳、Expo 和 Tauri 的完整验收，并确认 Tauri `Cargo.lock` 没有被构建过程改写，避免把重型原生壳或依赖锁漂移隐藏在基础检查末尾。 + +三个 job 合起来覆盖根 `npm run check`，并补齐根检查没有包含的 server-rs DDD 与现役后端编译门禁。普通 PR CI 不注入业务密钥，不启动真实 API、SpacetimeDB、OSS、支付、图片生成或生产 live smoke；需要现场环境、可变外部状态、Docker 编排或发布凭据的 `check:*` 继续按对应专题和 Jenkins 发布流程执行，不能遍历所有同名前缀脚本冒充 PR 门禁。 + +PR checkout 必须保留完整 Git 历史，并把 PR base SHA 传给 `SPACETIME_SCHEMA_BASE_REF`。`check:spacetime-schema` 依赖该基线识别已有表字段删除、改名、重排和改类型；事件给出的基线缺失或本地不可解析时必须直接失败，不能退化为空差异检查。Gitea 的 PR checkout 是 PR head，不是与目标分支的预合并 commit，因此 workflow 还会验证 PR head 包含事件中的最新 base commit；分支保护必须继续开启“PR 过期禁止合并”，过期分支先更新再重跑。向 `master` 直接推送时使用 push before SHA，手工触发时回退到 `origin/master`。 + +启用或注册执行 PR job 的 runner 前，Gitea 服务端必须至少升级到 `1.26.4`；不得在 `1.26.2` 上执行不受信任 PR 代码。runner 必须提供 `ubuntu-latest` 标签，并将其映射到经验证且固定 digest 的 Ubuntu 24.04 级 Docker/临时隔离镜像；禁止使用浮动镜像 tag，禁止将该标签映射到 host 执行器，禁止向 job 暴露 Docker socket、业务环境变量、业务密钥或不必要的内网。workflow 会安装 Node 22、仓库 `rust-toolchain.toml` 固定的 Rust 1.96.0，以及 clang/lld 和 Tauri Linux 依赖；受控 runner 镜像应预装 rustup，fallback 下载只用于首次引导。runner 仍需能访问 Gitea、GitHub Actions 与 `actions/node-versions`、nodejs.org、npm registry、Rust 分发和 crates.io。workflow 中的 `actions/checkout` / `actions/setup-node` 固定到完整 commit；内网 runner 不允许访问 GitHub 时，先把对应 commit 镜像到当前 Gitea 并把 workflow 改为绝对 action URL。首版不使用 Actions cache，避免未配置 runner cache 网络时把缓存恢复错误变成 PR 失败。Gitea 1.26 不执行 workflow 的 `timeout-minutes`，任务最长运行时间在 runner 全局配置收口，不能只在 YAML 写一个不会生效的超时值。 + +workflow 首次成功运行后，在 Gitea `master` 分支保护中把 `Project CI / Repository checks (pull_request)`、`Project CI / Backend checks (pull_request)`、`Project CI / Native shell checks (pull_request)` 三个完整 context 都设为合并必需检查，并从最近一周已上报 context 表复核名称后再保存。不能只填裸 job 名，否则无法匹配 Gitea 实际上报的 ` / ()`。只提交 workflow 文件不会自动创建 runner，也不会自动修改分支保护；如果 Actions 长时间停留在等待状态，先到仓库或组织的 Actions runner 页面确认存在在线、带 `ubuntu-latest` 标签的 runner。 + 视觉小说负向扫描与验收门禁： ```bash From 31a93c6842f8784fbeaa6b73a184c952d639244e Mon Sep 17 00:00:00 2001 From: kdletters Date: Tue, 21 Jul 2026 18:12:43 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E8=A1=A5=E9=BD=90Gitea=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 拆分前端后端与原生壳测试任务 接入server-rs正式workspace全量测试 修正过期Rust断言和Vitest遗漏路径 将AI游戏创作分支纳入CI触发范围 --- .gitea/workflows/project-ci.yml | 36 +++++++++++++++---- .../shared-memory/development-workflow.md | 8 ++--- ...发运维】本地开发验证与生产运维-2026-05-15.md | 13 +++---- .../src/external_generation_worker.rs | 4 +-- .../crates/api-server/src/runtime_profile.rs | 4 +-- .../tests/generated_asset_sheets.rs | 6 ++-- vitest.config.ts | 2 ++ 7 files changed, 49 insertions(+), 24 deletions(-) diff --git a/.gitea/workflows/project-ci.yml b/.gitea/workflows/project-ci.yml index a6844ad08..d519e74b8 100644 --- a/.gitea/workflows/project-ci.yml +++ b/.gitea/workflows/project-ci.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - codex/ai-game-creator-app pull_request: workflow_dispatch: @@ -97,9 +98,6 @@ jobs: - name: Run repository lint gates run: npm run lint - - name: Run frontend and script tests - run: npm run test - - name: Build web applications run: npm run build @@ -115,8 +113,28 @@ jobs: git cat-file -e "${base_ref}^{commit}" git diff --check "${base_ref}"...HEAD - backend-checks: - name: Backend checks + frontend-tests: + name: Frontend tests + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + + - name: Set up Node.js 22 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + + - name: Install npm dependencies + run: npm ci + + - name: Run frontend and script tests + run: npm run test + + backend-tests: + name: Backend tests runs-on: ubuntu-latest steps: - name: Checkout full history @@ -144,6 +162,7 @@ jobs: clang \ cmake \ curl \ + ffmpeg \ libclang-dev \ libcurl4-openssl-dev \ libssl-dev \ @@ -202,14 +221,17 @@ jobs: - name: Check server-rs boundaries run: npm run check:server-rs-ddd + - name: Run server-rs workspace tests + run: cargo test --locked --workspace --no-fail-fast --manifest-path server-rs/Cargo.toml + - name: Check api-server targets run: cargo check --locked -p api-server --all-targets --manifest-path server-rs/Cargo.toml - name: Check SpacetimeDB module run: cargo check --locked -p spacetime-module --manifest-path server-rs/Cargo.toml - native-shell-checks: - name: Native shell checks + native-shell-tests: + name: Native shell tests runs-on: ubuntu-latest steps: - name: Checkout full history diff --git a/docs/project-memory/shared-memory/development-workflow.md b/docs/project-memory/shared-memory/development-workflow.md index 535861061..dc8ea1061 100644 --- a/docs/project-memory/shared-memory/development-workflow.md +++ b/docs/project-memory/shared-memory/development-workflow.md @@ -264,12 +264,12 @@ npm run check:server-rs-ddd ## Gitea CI 与 PR 检查 -- 仓库 CI 入口是 `.gitea/workflows/project-ci.yml`，向 `master` 推送和所有 PR 创建、更新时必须运行，也允许手工触发。 -- CI 固定拆分为 `Repository checks`、`Backend checks`、`Native shell checks` 三个 required job；对应 PR context 完整名称是 `Project CI / Repository checks (pull_request)`、`Project CI / Backend checks (pull_request)`、`Project CI / Native shell checks (pull_request)`，首次运行后仍须从 Gitea 最近一周 context 表复核。前两项覆盖仓库/Web 与后端边界/编译，原生壳验收单独运行以便定位重型构建失败。 -- 三个 job 共同覆盖 `npm run check`，并追加 `npm run check:server-rs-ddd`、`cargo check -p api-server --all-targets --manifest-path server-rs/Cargo.toml` 和 `cargo check -p spacetime-module --manifest-path server-rs/Cargo.toml`。 +- 仓库 CI 入口是 `.gitea/workflows/project-ci.yml`，向 `master`、`codex/ai-game-creator-app` 推送和所有 PR 创建、更新时必须运行，也允许手工触发。 +- CI 固定拆分为 `Repository checks`、`Frontend tests`、`Backend tests`、`Native shell tests` 四个 required job；对应 PR context 完整名称是 `Project CI / Repository checks (pull_request)`、`Project CI / Frontend tests (pull_request)`、`Project CI / Backend tests (pull_request)`、`Project CI / Native shell tests (pull_request)`，首次运行后仍须从 Gitea 最近一周 context 表复核。测试使用独立 job，不能只藏在综合检查 step 中；原生壳验收单独运行以便定位重型构建失败。 +- 四个 job 共同覆盖 `npm run check`，并追加 `npm run check:server-rs-ddd`、`cargo test --locked --workspace --no-fail-fast --manifest-path server-rs/Cargo.toml`、`cargo check -p api-server --all-targets --manifest-path server-rs/Cargo.toml` 和 `cargo check -p spacetime-module --manifest-path server-rs/Cargo.toml`。后端 runner 安装 `ffmpeg`，避免视频抽帧测试因工具缺失提前返回。`codex/ai-game-creator-app` 分支的原生壳入口还必须覆盖 `npm run ai-game-creator-shell:check` 和 release build smoke。 - checkout 必须使用完整历史。PR 将 base SHA 写入 `SPACETIME_SCHEMA_BASE_REF`，直接推送 `master` 使用 before SHA；事件基线不可解析时直接失败。Gitea 检查的是 PR head 而非预合并 commit，workflow 必须拒绝不包含最新 base commit 的过期 PR，分支保护同时保持“PR 过期禁止合并”。 - 普通 PR job 不读取业务 secret，不运行真实 API/SpacetimeDB/OSS/支付/生成/live smoke，也不执行会修改外部状态的维护、迁移、发布或备份命令。 -- Gitea 至少升级到 `1.26.4` 后才能注册执行 PR job 的 runner；`ubuntu-latest` 标签只映射到固定 digest 的 Ubuntu 24.04 级 Docker/临时隔离镜像，不使用浮动镜像 tag，不映射 host，不向 job 暴露 Docker socket、业务 secret 或不必要内网。runner 能访问 Gitea、GitHub Actions 与 `actions/node-versions`、nodejs.org、npm、Rust 分发和 crates.io；workflow 的官方 action 固定完整 commit，若内网禁用 GitHub，先在当前 Gitea 镜像对应 commit 并改用绝对 URL。受控镜像优先预装 rustup。Gitea 1.26 的任务超时由 runner 全局配置控制；首次运行成功后，`master` 分支保护必须要求上述三个 job 全部成功。 +- Gitea 至少升级到 `1.26.4` 后才能注册执行 PR job 的 runner；`ubuntu-latest` 标签只映射到固定 digest 的 Ubuntu 24.04 级 Docker/临时隔离镜像，不使用浮动镜像 tag，不映射 host，不向 job 暴露 Docker socket、业务 secret 或不必要内网。runner 能访问 Gitea、GitHub Actions 与 `actions/node-versions`、nodejs.org、npm、Rust 分发和 crates.io；workflow 的官方 action 固定完整 commit，若内网禁用 GitHub，先在当前 Gitea 镜像对应 commit 并改用绝对 URL。受控镜像优先预装 rustup。Gitea 1.26 的任务超时由 runner 全局配置控制；首次运行成功后，`master` 分支保护必须要求上述四个 job 全部成功。 ## 后端相关默认验证 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 4b9cf53cd..18c85ddcb 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -200,19 +200,20 @@ npm run check ### Gitea Actions PR 门禁 -仓库级 Gitea Actions 工作流固定为 `.gitea/workflows/project-ci.yml`，在向 `master` 推送、创建或更新 PR，以及手工触发时运行。工作流拆成三个必须通过的 job： +仓库级 Gitea Actions 工作流固定为 `.gitea/workflows/project-ci.yml`，在向 `master` 或 `codex/ai-game-creator-app` 推送、创建或更新 PR，以及手工触发时运行。工作流拆成四个必须通过的 job： -- `Repository checks`：执行 `npm run lint`、完整 Vitest、主站与后台生产构建、内容数据检查和提交差异空白检查。 -- `Backend checks`：执行 `npm run check:server-rs-ddd`、`api-server --all-targets` 编译和 `spacetime-module` 编译。 -- `Native shell checks`：独立执行 `npm run check:native-shells`，覆盖微信壳、Expo 和 Tauri 的完整验收，并确认 Tauri `Cargo.lock` 没有被构建过程改写，避免把重型原生壳或依赖锁漂移隐藏在基础检查末尾。 +- `Repository checks`：执行 `npm run lint`、主站与后台生产构建、内容数据检查和提交差异空白检查。 +- `Frontend tests`：独立执行根 `npm run test`，让 Vitest 文件数和测试数在 Gitea job 列表中明确可见。 +- `Backend tests`：执行 `npm run check:server-rs-ddd`、`cargo test --locked --workspace --no-fail-fast`、`api-server --all-targets` 编译和 `spacetime-module` 编译；runner 安装 `ffmpeg`，避免视频抽帧测试因工具缺失提前返回。依赖真实服务或密钥的测试必须显式 `ignored`，不能让普通 PR job访问现场环境。 +- `Native shell tests`：独立执行 `npm run check:native-shells`，覆盖微信壳、Expo 和 Tauri 的完整验收，并确认 Tauri `Cargo.lock` 没有被构建过程改写，避免把重型原生壳或依赖锁漂移隐藏在基础检查末尾。`codex/ai-game-creator-app` 分支的同名脚本还会执行 `npm run ai-game-creator-shell:check` 和 AI 游戏创作壳 release build smoke。 -三个 job 合起来覆盖根 `npm run check`，并补齐根检查没有包含的 server-rs DDD 与现役后端编译门禁。普通 PR CI 不注入业务密钥，不启动真实 API、SpacetimeDB、OSS、支付、图片生成或生产 live smoke；需要现场环境、可变外部状态、Docker 编排或发布凭据的 `check:*` 继续按对应专题和 Jenkins 发布流程执行，不能遍历所有同名前缀脚本冒充 PR 门禁。 +四个 job 合起来覆盖根 `npm run check`，并补齐根检查没有包含的 server-rs DDD、正式 workspace Rust 测试与现役后端编译门禁。普通 PR CI 不注入业务密钥，不启动真实 API、SpacetimeDB、OSS、支付、图片生成或生产 live smoke；需要现场环境、可变外部状态、Docker 编排或发布凭据的 `check:*` 继续按对应专题和 Jenkins 发布流程执行，不能遍历所有同名前缀脚本冒充 PR 门禁。 PR checkout 必须保留完整 Git 历史，并把 PR base SHA 传给 `SPACETIME_SCHEMA_BASE_REF`。`check:spacetime-schema` 依赖该基线识别已有表字段删除、改名、重排和改类型；事件给出的基线缺失或本地不可解析时必须直接失败，不能退化为空差异检查。Gitea 的 PR checkout 是 PR head，不是与目标分支的预合并 commit，因此 workflow 还会验证 PR head 包含事件中的最新 base commit；分支保护必须继续开启“PR 过期禁止合并”，过期分支先更新再重跑。向 `master` 直接推送时使用 push before SHA，手工触发时回退到 `origin/master`。 启用或注册执行 PR job 的 runner 前，Gitea 服务端必须至少升级到 `1.26.4`；不得在 `1.26.2` 上执行不受信任 PR 代码。runner 必须提供 `ubuntu-latest` 标签，并将其映射到经验证且固定 digest 的 Ubuntu 24.04 级 Docker/临时隔离镜像；禁止使用浮动镜像 tag，禁止将该标签映射到 host 执行器，禁止向 job 暴露 Docker socket、业务环境变量、业务密钥或不必要的内网。workflow 会安装 Node 22、仓库 `rust-toolchain.toml` 固定的 Rust 1.96.0，以及 clang/lld 和 Tauri Linux 依赖；受控 runner 镜像应预装 rustup，fallback 下载只用于首次引导。runner 仍需能访问 Gitea、GitHub Actions 与 `actions/node-versions`、nodejs.org、npm registry、Rust 分发和 crates.io。workflow 中的 `actions/checkout` / `actions/setup-node` 固定到完整 commit；内网 runner 不允许访问 GitHub 时，先把对应 commit 镜像到当前 Gitea 并把 workflow 改为绝对 action URL。首版不使用 Actions cache，避免未配置 runner cache 网络时把缓存恢复错误变成 PR 失败。Gitea 1.26 不执行 workflow 的 `timeout-minutes`，任务最长运行时间在 runner 全局配置收口，不能只在 YAML 写一个不会生效的超时值。 -workflow 首次成功运行后，在 Gitea `master` 分支保护中把 `Project CI / Repository checks (pull_request)`、`Project CI / Backend checks (pull_request)`、`Project CI / Native shell checks (pull_request)` 三个完整 context 都设为合并必需检查，并从最近一周已上报 context 表复核名称后再保存。不能只填裸 job 名，否则无法匹配 Gitea 实际上报的 ` / ()`。只提交 workflow 文件不会自动创建 runner，也不会自动修改分支保护；如果 Actions 长时间停留在等待状态，先到仓库或组织的 Actions runner 页面确认存在在线、带 `ubuntu-latest` 标签的 runner。 +workflow 首次成功运行后，在 Gitea `master` 分支保护中把 `Project CI / Repository checks (pull_request)`、`Project CI / Frontend tests (pull_request)`、`Project CI / Backend tests (pull_request)`、`Project CI / Native shell tests (pull_request)` 四个完整 context 都设为合并必需检查，并从最近一周已上报 context 表复核名称后再保存。不能只填裸 job 名，否则无法匹配 Gitea 实际上报的 ` / ()`。只提交 workflow 文件不会自动创建 runner，也不会自动修改分支保护；如果 Actions 长时间停留在等待状态，先到仓库或组织的 Actions runner 页面确认存在在线、带 `ubuntu-latest` 标签的 runner。 视觉小说负向扫描与验收门禁： diff --git a/server-rs/crates/api-server/src/external_generation_worker.rs b/server-rs/crates/api-server/src/external_generation_worker.rs index ea682e4a1..4280c81ae 100644 --- a/server-rs/crates/api-server/src/external_generation_worker.rs +++ b/server-rs/crates/api-server/src/external_generation_worker.rs @@ -1702,8 +1702,8 @@ mod tests { serde_json::from_str(&editor_generation_result_payload_json(&job, &response)) .expect("worker 结果应是合法 JSON"); - assert_eq!(payload["sourceModule"], json!("puzzle")); - assert_eq!(payload["sourceEntityId"], json!("session-1:puzzle-level-1")); + assert_eq!(payload["sourceModule"], json!("editor")); + assert_eq!(payload["sourceEntityId"], json!("project-1")); assert_eq!( payload["warning"], json!({ diff --git a/server-rs/crates/api-server/src/runtime_profile.rs b/server-rs/crates/api-server/src/runtime_profile.rs index c25ba1f81..58bbdcfe8 100644 --- a/server-rs/crates/api-server/src/runtime_profile.rs +++ b/server-rs/crates/api-server/src/runtime_profile.rs @@ -2682,7 +2682,7 @@ mod tests { } #[tokio::test] - async fn profile_play_stats_requires_authentication() { + async fn retired_profile_play_stats_route_is_not_mounted() { let app = build_router(AppState::new(AppConfig::default()).expect("state should build")); let response = app @@ -2696,7 +2696,7 @@ mod tests { .await .expect("request should succeed"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(response.status(), StatusCode::NOT_FOUND); } #[tokio::test] diff --git a/server-rs/crates/platform-image/tests/generated_asset_sheets.rs b/server-rs/crates/platform-image/tests/generated_asset_sheets.rs index 83f5edc31..d3bc61705 100644 --- a/server-rs/crates/platform-image/tests/generated_asset_sheets.rs +++ b/server-rs/crates/platform-image/tests/generated_asset_sheets.rs @@ -198,7 +198,7 @@ fn generated_asset_sheet_muted_green_alpha_requires_explicit_option() { } #[test] -fn generated_asset_sheet_magenta_key_preserves_green_white_and_disconnected_key_subject() { +fn generated_asset_sheet_magenta_key_preserves_subject_and_removes_internal_hole() { let mut sheet = RgbaImage::from_pixel(28, 28, Rgba([255, 0, 255, 255])); for y in 6..22 { for x in 6..14 { @@ -227,8 +227,8 @@ fn generated_asset_sheet_magenta_key_preserves_green_white_and_disconnected_key_ assert_eq!(cleaned.get_pixel(18, 8).0[3], 255); assert_eq!( cleaned.get_pixel(13, 13).0[3], - 255, - "非边缘连通的 key 色像素不应被当成背景清掉" + 0, + "达到阈值的主体内部 key 色镂空区域应被清理" ); } diff --git a/vitest.config.ts b/vitest.config.ts index e9eeeb49d..b3035b116 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -20,6 +20,7 @@ export default defineConfig({ 'src/services/activeAppTitle.test.ts', 'src/services/authService.test.ts', 'src/services/apiClient.test.ts', + 'src/services/clipboard.test.ts', 'src/services/host-bridge/**/*.test.ts', 'src/services/image-editor/**/*.test.ts', 'src/services/external-generation/**/*.test.ts', @@ -43,6 +44,7 @@ export default defineConfig({ 'src/components/platform-entry/PlatformProfileWalletLedgerModal.test.tsx', 'src/components/platform-entry/platformProfile*.test.ts', 'src/components/platform-entry/usePlatformProfileCenterController.test.tsx', + 'src/hooks/useHostNavigationCanGoBack.test.tsx', 'apps/admin-web/src/**/*.test.ts', 'apps/admin-web/src/**/*.test.tsx', 'miniprogram/**/*.test.js', From 29369e6376203d8a95556ad42ecae817911420a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Tue, 21 Jul 2026 20:50:03 +0800 Subject: [PATCH 7/7] =?UTF-8?q?fix:=20=E7=94=BB=E5=B8=83=E9=BC=A0=E6=A0=87?= =?UTF-8?q?/=E8=A7=A6=E6=91=B8=E6=9D=BF=E5=8F=AF=E4=BB=A5=E4=B8=8A?= =?UTF-8?q?=E4=B8=8B=E6=BB=9A=E5=8A=A8=EF=BC=8C=E6=97=A0=E6=B3=95=E5=B7=A6?= =?UTF-8?q?=E5=8F=B3=E6=BB=9A=E5=8A=A8=20(#100)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-on: http://genarrative-station/git/GenarrativeAI/Genarrative/pulls/100 Co-authored-by: 王德宇 Co-committed-by: 王德宇 --- .../shared-memory/decision-log.md | 6 +++ ...架构】图片画布编辑器MVP接入方案-2026-06-11.md | 4 +- ...构】图片画布编辑器前端拆分计划-2026-06-17.md | 4 +- .../ImageCanvasEditorView.test.tsx | 13 ++++- .../ImageCanvasInteractionModel.test.ts | 10 ++-- .../ImageCanvasInteractionModel.ts | 14 +++-- .../image-editor/ImageCanvasShortcutModel.ts | 3 +- .../useImageCanvasStageInteractions.test.tsx | 16 ++++++ .../useImageCanvasViewportControls.test.tsx | 54 +++++++++++++++---- .../useImageCanvasViewportControls.ts | 12 ++++- 10 files changed, 109 insertions(+), 27 deletions(-) diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 9d3707cb2..01345b3cb 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -4373,3 +4373,9 @@ - 回退边界：明确模型不存在 / 不支持、408、非内容拒绝类 429、5xx、响应解析失败或非拒绝类缺图可以切模型；401 / 403、普通参数 / 内容安全拒绝、本地配置与参考图错误、发送 / 连接错误、request budget 耗尽和已生成图片下载失败不切模型。一次业务请求总发送上限仍为 5 次，两个模型共享同一 worker provider deadline 和 attempt 预算。 - 观测边界：审计 `image_model` 记录实际 provider attempt；首选 `gpt-image-2` 失败但兜底 `gpt-image-2-c` 恢复成功时，首选失败仍写入 `external_api_call_failure`，最终成功运行摘要记录 `recoveredFailureCount`。日志用 `fallback_from_model` / `fallback_to_model` 标识切换，不改变业务模型、扣费、素材 metadata 或终态语义。 - 脚本边界：仓库 `gpt-image-2-apimart` skill 的现役生成脚本采用同一首选 / 回退顺序；认证、请求发送不确定错误和下载失败不重新生图，避免重复上游成本。 + +## 2026-07-21 图片画布滚轮与中键平移统一为二维视口移动 + +- 背景：画布中键拖拽的平移模型已同时计算 X / Y，但普通滚轮分支只消费 `deltaY`，横向滚轮或触控板的 `deltaX` 被丢弃，且缺少中键横向拖动的状态机回归覆盖。 +- 决策：普通滚轮原样消费设备上报的 `deltaX / deltaY` 二维平移 viewport；当按住 Shift 且设备上报 `deltaX = 0` 时，输入适配层把 `deltaY` 映射为横向位移并将纵向位移置零，核心平移模型不感知修饰键。`Ctrl / Cmd + 滚轮` 继续只负责围绕指针缩放；中键和抓手拖拽继续同时更新 X / Y。 +- 验证：交互模型单测覆盖原始 `deltaX / deltaY` 和缩放边界；viewport hook 单测覆盖二维滚轮、Shift 横向适配与 Ctrl 缩放；stage 状态机单测覆盖中键水平、垂直同时移动。 diff --git a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md index 59b7c2141..cca6665d0 100644 --- a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md +++ b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md @@ -11,7 +11,7 @@ - 创作 Tab 顶部提供编辑器入口，入口只负责跳转，不参与玩法创作链路。 - 编辑器顶部栏采用紧凑高度，项目标题和重命名入口贴近返回项目按钮；右侧复用与主站相同的公共泥点资产入口。顶部总余额优先展示画板按扣费、退回、到账和页面恢复链路刷新的 `profileDashboard.walletBalance`，充值中心 `mudPointBalance` 仅用于展开面板的账户明细，不覆盖已刷新的总余额。余额区只展开不限时、每日免费及重置信息，会员周期限时泥点保留在后端 read model 中用于存量兼容和结算但不展示；独立“充值”按钮进入“购买更多泥点”弹窗，“使用详情”进入泥点账单。 - 编辑器左侧为图片素材栏，可展开 / 收起；移动端优先保持素材栏可折叠。 -- 中央画布支持背景拖拽平移、滚轮缩放、缩放百分比菜单、显示所有元素和固定比例缩放。 +- 中央画布支持背景拖拽平移、滚轮二维平移、`Ctrl / Cmd + 滚轮` 缩放、缩放百分比菜单、显示所有元素和固定比例缩放。 - 画布左下角提供 Lovart 式状态控件：背景色圆点、素材 / 图层入口、小地图开关；小地图显示图层缩略分布和当前视口框，点击小地图执行显示所有元素。 - 画布 chrome 的边框、hover / 选中态、吸附 / 框选参考线、生成类按钮和通用 active 控件使用陶泥儿暖色主题（以 `--platform-accent`、陶土橙主按钮和深棕文字为基准），不得回退为黑色或蓝色主题；元素类型自身的识别色可继续保留。 - 画布中的图片可展示、悬浮显示图片 Resolution 尺寸与边框，点击后在图片上方显示浮动工具栏；浮动工具栏只保留当前可执行的编辑动作，不放调整 / 复制 / 删除 / 查看信息占位按钮。图片右上角素材类型标签、图片信息角标和悬浮尺寸标签在画布缩小时必须按 viewport 反向缩放，保持屏幕可读尺寸；无 `assetKind` 的素材右上角显示 `未知` 标签，点击标签弹出独立标签选择菜单并可写回图层 `assetKind`，不能触发图层选择 / 拖拽事件；图片信息角标使用圆形 `i` 图标，不使用中括号或花括号样式。图片不再维护独立展示 `Size` 字段，画布显示宽高统一取 `originalWidth/originalHeight`（图片信息中的 `Resolution`）。 @@ -149,7 +149,7 @@ - 素材面板支持按素材名、文件夹名、生成信息、模型、任务和媒体类型搜索，并支持选择模式框选，一次选中多个素材，并可批量移动或删除上传素材。 - 图层面板支持按图层名、生成信息、模型、任务和媒体类型搜索；支持选择多个图层后创建图层组，组名和 groupId 随画布布局保存。 - 小地图支持拖拽视口框，拖动时画布 viewport 跟随移动；`pointermove` 更新必须通过 `requestAnimationFrame` 合帧，结束拖拽时 flush 最后一帧，避免高频 pointermove 直接压垮 React 渲染和项目持久化链路。 -- 鼠标滚轮默认垂直滚动画布视口；按住 Ctrl / Cmd 滚轮才缩放画布，并阻止浏览器页面缩放。缩放比例显示保持现有换算口径，最低可缩小到 `5%`。 +- 鼠标滚轮默认按设备原始 `deltaX / deltaY` 二维平移画布视口；当按住 Shift 且设备上报 `deltaX = 0` 时，视口输入适配层把 `deltaY` 映射为横向位移并将纵向位移置零，核心平移模型仍只消费归一化后的 X / Y。按住 Ctrl / Cmd 滚轮才缩放画布，并阻止浏览器页面缩放。缩放比例显示保持现有换算口径，最低可缩小到 `5%`。 - 工程刷新后能从后端恢复资源、图层布局和 viewport。 - “我的”页项目入口能进入 `/project`；项目页能列出工程、重命名 / 删除单个工程、批量选择和批量删除；点击工程后进入 `/editor/canvas?projectid=` 并按 query 加载该工程。 diff --git a/docs/technical/【前端架构】图片画布编辑器前端拆分计划-2026-06-17.md b/docs/technical/【前端架构】图片画布编辑器前端拆分计划-2026-06-17.md index 4a633479f..68c306b07 100644 --- a/docs/technical/【前端架构】图片画布编辑器前端拆分计划-2026-06-17.md +++ b/docs/technical/【前端架构】图片画布编辑器前端拆分计划-2026-06-17.md @@ -68,7 +68,7 @@ ## 第六阶段模块 - `ImageCanvasInteractionModel.ts` - - 承载画布交互纯计算：适合视图、中心缩放、普通滚轮纵向滚动、Ctrl / Cmd 滚轮缩放、画布坐标换算、框选命中、平移、生成占位框拖拽、图层拖拽吸附、小地图投影、小地图点击定位和小地图拖拽视图移动。 + - 承载画布交互纯计算：适合视图、中心缩放、普通滚轮按原始 `deltaX / deltaY` 二维平移、Shift 且 `deltaX = 0` 时由输入适配层把 `deltaY` 映射为横向位移、Ctrl / Cmd 滚轮缩放、画布坐标换算、框选命中、平移、生成占位框拖拽、图层拖拽吸附、小地图投影、小地图点击定位和小地图拖拽视图移动。 - 主视图继续负责 React 事件对象、pointer capture、history 快照、生成对象回写、选中态和 `setState`。 - 该模块用独立单测覆盖小地图灵敏度、吸附、多选拖拽和滚轮缩放等之前容易回退的交互规则。 @@ -151,7 +151,7 @@ ## 第十七阶段模块 - `useImageCanvasViewportControls.ts` - - 承载画布视口控制：`viewport`、`canvasSize`、小地图投影、适合视图、中心缩放、普通滚轮纵向滚动、Ctrl / Cmd 滚轮缩放、屏幕点到画布 / 世界坐标换算和小地图点击 / 拖拽移动视图。 + - 承载画布视口控制：`viewport`、`canvasSize`、小地图投影、适合视图、中心缩放、普通滚轮按原始 `deltaX / deltaY` 二维平移、Shift 且 `deltaX = 0` 时的横向位移适配、Ctrl / Cmd 滚轮缩放、屏幕点到画布 / 世界坐标换算和小地图点击 / 拖拽移动视图。 - 主视图继续负责图层拖拽、生成占位框拖拽、框选、多选、历史触发时机、上传 drop 分流和小地图 pointer down 事件；该 hook 只作为视口控制协调器，不接管画布完整 pointer 状态机。 - 该 hook 用独立单测覆盖尺寸同步、适合视图、中心缩放、坐标换算、滚轮语义和小地图移动，为后续抽 `useImageCanvasStageInteractions` 预留更清晰的视口接口。 diff --git a/src/components/image-editor/ImageCanvasEditorView.test.tsx b/src/components/image-editor/ImageCanvasEditorView.test.tsx index 6add43c9a..b4380fdb2 100644 --- a/src/components/image-editor/ImageCanvasEditorView.test.tsx +++ b/src/components/image-editor/ImageCanvasEditorView.test.tsx @@ -2320,15 +2320,24 @@ describe('ImageCanvasEditorView', () => { }).not.toThrow(); }); - it('uses normal wheel for vertical canvas scroll and ctrl wheel for zoom', () => { + it('uses normal wheel for two-axis canvas pan and ctrl wheel for zoom', () => { render(); const viewport = screen.getByLabelText('画布工作区'); + const world = screen.getByTestId('image-canvas-world') as HTMLElement; expect( screen.getByRole('button', { name: '当前缩放比例 100%' }), ).toBeTruthy(); - fireEvent.wheel(viewport, { deltaY: 120, clientX: 400, clientY: 280 }); + fireEvent.wheel(viewport, { + deltaX: 80, + deltaY: 120, + clientX: 400, + clientY: 280, + }); + expect(world.style.transform).toBe( + 'translate(-80px, -120px) scale(0.5)', + ); expect( screen.getByRole('button', { name: '当前缩放比例 100%' }), ).toBeTruthy(); diff --git a/src/components/image-editor/ImageCanvasInteractionModel.test.ts b/src/components/image-editor/ImageCanvasInteractionModel.test.ts index f9579cf51..2428748af 100644 --- a/src/components/image-editor/ImageCanvasInteractionModel.test.ts +++ b/src/components/image-editor/ImageCanvasInteractionModel.test.ts @@ -19,8 +19,8 @@ import { moveViewportFromMinimapDrag, moveViewportFromMinimapPointer, moveViewportFromPan, + panViewportFromWheel, scaleViewportFromScreenPoint, - scrollViewportVertically, selectCanvasObjectsInsideMarquee, selectLayersInsideMarquee, zoomViewportFromWheel, @@ -133,11 +133,13 @@ describe('ImageCanvasInteractionModel', () => { ).toEqual({ x: 100, y: 60 }); }); - it('scrolls vertically and zooms around a screen point', () => { + it('pans from two-axis wheel input and zooms around a screen point', () => { const viewport = { x: 10, y: 20, scale: 1 }; - expect(scrollViewportVertically(viewport, 120)).toEqual({ - x: 10, + expect( + panViewportFromWheel({ viewport, deltaX: 40, deltaY: 120 }), + ).toEqual({ + x: -30, y: -100, scale: 1, }); diff --git a/src/components/image-editor/ImageCanvasInteractionModel.ts b/src/components/image-editor/ImageCanvasInteractionModel.ts index 3780870d8..1140366cd 100644 --- a/src/components/image-editor/ImageCanvasInteractionModel.ts +++ b/src/components/image-editor/ImageCanvasInteractionModel.ts @@ -212,12 +212,18 @@ export function scaleViewportFromScreenPoint({ }; } -export function scrollViewportVertically( - viewport: CanvasViewport, - deltaY: number, -) { +export function panViewportFromWheel({ + viewport, + deltaX, + deltaY, +}: { + viewport: CanvasViewport; + deltaX: number; + deltaY: number; +}) { return { ...viewport, + x: viewport.x - deltaX, y: viewport.y - deltaY, }; } diff --git a/src/components/image-editor/ImageCanvasShortcutModel.ts b/src/components/image-editor/ImageCanvasShortcutModel.ts index 518793a36..7863e599a 100644 --- a/src/components/image-editor/ImageCanvasShortcutModel.ts +++ b/src/components/image-editor/ImageCanvasShortcutModel.ts @@ -29,7 +29,8 @@ export const IMAGE_CANVAS_SHORTCUT_SECTIONS: ImageCanvasShortcutSection[] = [ { title: '视图', items: [ - { action: '纵向滚动画布', keys: ['鼠标滚轮'], status: 'ready' }, + { action: '平移画布', keys: ['鼠标滚轮 / 触控板'], status: 'ready' }, + { action: '横向平移画布', keys: ['Shift', '鼠标滚轮'], status: 'ready' }, { action: '以鼠标位置缩放', keys: ['Ctrl', '鼠标滚轮'], status: 'ready' }, { action: '显示全部元素', keys: ['Ctrl', '0'], status: 'new' }, { action: '缩放至 100%', keys: ['Ctrl', '1'], status: 'new' }, diff --git a/src/components/image-editor/useImageCanvasStageInteractions.test.tsx b/src/components/image-editor/useImageCanvasStageInteractions.test.tsx index 3ec5856a0..7d62ce7f7 100644 --- a/src/components/image-editor/useImageCanvasStageInteractions.test.tsx +++ b/src/components/image-editor/useImageCanvasStageInteractions.test.tsx @@ -1083,6 +1083,22 @@ describe('useImageCanvasStageInteractions', () => { }); expect(screen.getByTestId('panning').textContent).toBe('true'); + act(() => { + fireEvent( + viewport, + new MouseEvent('pointermove', { + bubbles: true, + clientX: 140, + clientY: 70, + button: 1, + buttons: 4, + }), + ); + }); + expect(screen.getByTestId('viewport-state').textContent).toBe( + '40.0,-30.0,1', + ); + act(() => { fireEvent( viewport, diff --git a/src/components/image-editor/useImageCanvasViewportControls.test.tsx b/src/components/image-editor/useImageCanvasViewportControls.test.tsx index c9f3baf14..0f0ab01eb 100644 --- a/src/components/image-editor/useImageCanvasViewportControls.test.tsx +++ b/src/components/image-editor/useImageCanvasViewportControls.test.tsx @@ -158,19 +158,12 @@ describe('useImageCanvasViewportControls', () => { expect(captureCanvasHistory).toHaveBeenCalledTimes(2); }); - it('handles vertical wheel scroll, ctrl wheel zoom and minimap movement', () => { - const rafCallbacks: FrameRequestCallback[] = []; - vi.spyOn(window, 'requestAnimationFrame').mockImplementation( - (callback: FrameRequestCallback) => { - rafCallbacks.push(callback); - return rafCallbacks.length; - }, - ); - vi.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => {}); + it('keeps ordinary mouse wheel vertical and maps Shift wheel to horizontal', () => { const { result, viewportElement } = renderViewportControls(); act(() => { fireEvent.wheel(viewportElement, { + deltaX: 0, deltaY: 120, clientX: 260, clientY: 190, @@ -182,6 +175,47 @@ describe('useImageCanvasViewportControls', () => { scale: 0.5, }); + act(() => { + fireEvent.wheel(viewportElement, { + shiftKey: true, + deltaX: 0, + deltaY: 80, + clientX: 260, + clientY: 190, + }); + }); + expectViewport(result.current.viewport, { + x: -340, + y: -50, + scale: 0.5, + }); + }); + + it('handles native two-axis wheel pan, ctrl wheel zoom and minimap movement', () => { + const rafCallbacks: FrameRequestCallback[] = []; + vi.spyOn(window, 'requestAnimationFrame').mockImplementation( + (callback: FrameRequestCallback) => { + rafCallbacks.push(callback); + return rafCallbacks.length; + }, + ); + vi.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => {}); + const { result, viewportElement } = renderViewportControls(); + + act(() => { + fireEvent.wheel(viewportElement, { + deltaX: 40, + deltaY: 120, + clientX: 260, + clientY: 190, + }); + }); + expectViewport(result.current.viewport, { + x: -300, + y: -50, + scale: 0.5, + }); + act(() => { fireEvent.wheel(viewportElement, { ctrlKey: true, @@ -191,7 +225,7 @@ describe('useImageCanvasViewportControls', () => { }); }); expectViewport(result.current.viewport, { - x: -312, + x: -356, y: -74, scale: 0.55, }); diff --git a/src/components/image-editor/useImageCanvasViewportControls.ts b/src/components/image-editor/useImageCanvasViewportControls.ts index 4e5d0ba1d..2f74c1b8f 100644 --- a/src/components/image-editor/useImageCanvasViewportControls.ts +++ b/src/components/image-editor/useImageCanvasViewportControls.ts @@ -23,8 +23,8 @@ import { getWorldPointFromClient, moveViewportFromMinimapDrag as resolveViewportFromMinimapDrag, moveViewportFromMinimapPointer as resolveViewportFromMinimapPointer, + panViewportFromWheel, scaleViewportFromScreenPoint, - scrollViewportVertically, zoomViewportFromWheel, } from './ImageCanvasInteractionModel'; @@ -260,8 +260,16 @@ export function useImageCanvasViewportControls({ } if (!event.ctrlKey && !event.metaKey) { + const useShiftWheelHorizontalFallback = + event.shiftKey && event.deltaX === 0; setViewport((currentViewport) => - scrollViewportVertically(currentViewport, event.deltaY), + panViewportFromWheel({ + viewport: currentViewport, + deltaX: useShiftWheelHorizontalFallback + ? event.deltaY + : event.deltaX, + deltaY: useShiftWheelHorizontalFallback ? 0 : event.deltaY, + }), ); return; }