From 324f99efdf2335aafa315efe760031f31e3968ba Mon Sep 17 00:00:00 2001 From: kdletters Date: Sat, 11 Jul 2026 13:55:05 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=96=E9=83=A8=E7=94=9F?= =?UTF-8?q?=E6=88=90=E4=BB=BB=E5=8A=A1=E5=A4=A7=E8=BD=BD=E8=8D=B7=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84=E5=86=85=E5=AD=98=E8=86=A8=E8=83=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 阻止图片编辑器将 Data URL 和超限 JSON 写入生成任务，复用 objectKey 等轻量引用 新增外部生成任务摘要投影与分批 payload 压缩回填流程，避免正式列表读取大字段 收紧 SpacetimeDB 发布为永不删数据并移除 Jenkins 普通清库入口，补齐维护窗口门禁 更新生成绑定、运维文档和回归测试，并完成生产数据副本迁移与压缩验证 --- .../shared-memory/decision-log.md | 8 + docs/project-memory/shared-memory/pitfalls.md | 9 + ...架构】图片画布编辑器MVP接入方案-2026-06-11.md | 6 +- ...】server-rs与SpacetimeDB数据契约-2026-05-15.md | 12 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 4 + ...nkinsfile.production-full-build-and-deploy | 17 + .../Jenkinsfile.production-stdb-module-build | 2 +- ...Jenkinsfile.production-stdb-module-publish | 8 +- package.json | 1 + scripts/build-production-release.sh | 3 +- scripts/check-production-ops-guardrails.mjs | 73 + scripts/deploy/production-api-deploy.sh | 7 +- scripts/deploy/production-stdb-publish.sh | 92 +- ...time-maintain-external-generation-jobs.mjs | 162 ++ scripts/spacetime-migration-common.mjs | 54 +- scripts/spacetime-migration-common.test.ts | 35 + .../api-server/src/editor_generation_queue.rs | 133 +- .../api-server/src/external_generation.rs | 125 +- .../src/external_generation.rs | 75 + server-rs/crates/spacetime-client/src/lib.rs | 3 +- .../crates/spacetime-client/src/mapper.rs | 5 +- .../src/mapper/external_generation.rs | 91 + .../spacetime-client/src/module_bindings.rs | 52 + ...tion_job_summaries_and_return_procedure.rs | 62 + ...tion_job_summaries_and_return_procedure.rs | 61 + ...ation_job_payloads_and_return_procedure.rs | 61 + ...ation_job_payload_compaction_input_type.rs | 18 + ...ayload_compaction_procedure_result_type.rs | 26 + ...eration_job_summary_backfill_input_type.rs | 18 + ..._summary_backfill_procedure_result_type.rs | 22 + ...ation_job_summary_procedure_result_type.rs | 24 + ...al_generation_job_summary_snapshot_type.rs | 30 + .../external_generation_job_summary_table.rs | 169 ++ .../external_generation_job_summary_type.rs | 103 ++ ...ration_job_summary_and_return_procedure.rs | 59 + ...tion_job_summaries_and_return_procedure.rs | 62 + .../src/external_generation.rs | 1462 ++++++++++++++++- .../crates/spacetime-module/src/migration.rs | 6 +- ...anvasGenerationSubmissionWorkflow.test.tsx | 220 ++- ...ImageCanvasGenerationSubmissionWorkflow.ts | 408 ++++- .../useImageCanvasGenerationWorkflow.test.tsx | 30 +- .../useImageCanvasGenerationWorkflow.ts | 27 +- 42 files changed, 3528 insertions(+), 317 deletions(-) create mode 100644 scripts/spacetime-maintain-external-generation-jobs.mjs create mode 100644 scripts/spacetime-migration-common.test.ts create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/acknowledge_external_generation_job_summaries_and_return_procedure.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/backfill_external_generation_job_summaries_and_return_procedure.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/compact_external_generation_job_payloads_and_return_procedure.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_payload_compaction_input_type.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_payload_compaction_procedure_result_type.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_backfill_input_type.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_backfill_procedure_result_type.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_procedure_result_type.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_snapshot_type.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_table.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_type.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_job_summary_and_return_procedure.rs create mode 100644 server-rs/crates/spacetime-client/src/module_bindings/list_external_generation_job_summaries_and_return_procedure.rs diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 5a01edd7b..599161124 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -16,6 +16,14 @@ --- +## 2026-07-10 外部生成任务只持久化轻量媒体引用并独立维护摘要投影 + +- 背景：编辑器 worker 化后直接把同步接口 payload 序列化进 `external_generation_job.request_payload_json`；前端又把已有 OSS `objectKey` 下载成 Data URL 再提交，导致单个任务 JSON 膨胀到数 MB，正式任务列表读取 20 条任务时同时搬运约 65 MB payload，并放大为 SpacetimeDB 与 api-server 的瞬时内存峰值。此前“禁止 Data URL 持久化”只覆盖工程、素材、图层和元数据，遗漏了正式生成任务表。 +- 决策：`external_generation_job.request_payload_json` / `result_payload_json` 同样属于正式持久化边界。对于本次事故涉及的 `source_module = editor-canvas` 任务，只允许普通业务参数和 `objectKey` / `resourceId` / `assetId` 等已登记轻量引用；任意层级 `data:` / `blob:` 与超限 JSON 必须由 api-server 和 SpacetimeDB 双重拒绝。编辑器已有媒体直接传正式引用，本地红框标记图先上传 OSS 后再入队，上传目录与文件名使用同一个强唯一 ID。其它玩法现存 Data URL 请求契约不在本次事故修复中被静默禁用，后续必须先完成各自资源化再扩大 DB 门禁。用户任务列表、单任务状态和 acknowledge 只读取不含 request/result payload 的 `external_generation_job_summary` 投影；acknowledge 只更新摘要小表并保留审计事件，不为确认通知加载 / 重写主任务 payload。提示词在入队时提前提取；错误摘要统一去除内联媒体并限制为 2048 字符；列表在单次 owner 扫描中只保留固定大小 top-N，不再收集全量历史后截断。历史终态 payload 仅允许迁移操作员通过默认 dry-run、`editor-canvas + job_id` B-tree cursor 显式分批压缩，pending / running 永不压缩；cursor 选择最多读取 `limit + 1` 行，apply 再逐条主键读取。首次发布默认 fail-closed 暂停在 Stdb 与 API 之间，保持维护模式并停止旧 API/controller/worker，完成压缩和摘要回填后才由指定审批人放行 API。 +- 影响范围：编辑器生成提交 workflow、`external_generation_job`、`external_generation_job_summary`、外部生成 procedure / typed client / BFF、SpacetimeDB bindings、历史数据维护流程和图片画布文档。 +- 验证方式：覆盖编辑器嵌套内联媒体与 payload 上限拒绝、非编辑器既有任务不被本轮门禁误伤、正式任务接口类型不含 payload、终态分批压缩不修改活动任务、已有 objectKey 不转 Data URL、本地标记图先上传再提交；运行外部生成定向 Rust / Vitest、`npm run spacetime:generate`、`npm run check:spacetime-schema`、`npm run typecheck`、`npm run check:encoding` 和 `git diff --check`。 +- 关联文档：`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`、`docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md`、`docs/project-memory/shared-memory/pitfalls.md`。 + ## 2026-07-10 BgFilter segModel 保留内部字段，不进入外部 OpenAPI - 背景：`api-server` 的图片生成、图标 spritesheet 与 UI 素材提取请求仍可反序列化 `segModel`，并识别 `birefnet` / `anime-seg`，以兼容内部调用和既有任务；但 BgFilter 当前受服务进程内存与并发容量约束，不同分割模型的内存占用并非可由外部调用方自由选择的稳定契约。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 13c92451c..0cda9834e 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -14,6 +14,15 @@ - 关联：相关文件、文档、提交或 Issue ``` +## 禁止 Data URL 持久化时不要漏掉异步任务 JSON + +- 现象：工程、素材、图层和元数据都已禁止 Data URL 后，服务器仍在生成高峰出现 SpacetimeDB / api-server 内存急剧膨胀甚至 OOM；读取少量正式生成任务也会造成远大于响应体的瞬时内存增长。 +- 原因：同步接口 worker 化时把原请求整体序列化到 `external_generation_job.request_payload_json`，而前端又把已有 `objectKey` 下载成 Data URL 提交。任务表也是正式持久化边界；列表 procedure 若先收集完整任务行再截断，还会把 request/result 大字段在 SpacetimeDB、SDK mapper 和 BFF 多次持有。 +- 处理：先在事故涉及的编辑器持久任务 JSON 上由 api-server 与 SpacetimeDB 两层递归拒绝 `data:` / `blob:` 并限制字节数；已有媒体传 `objectKey` / `resourceId` / `assetId`，本地派生图先用强唯一 key 上传。其它玩法若仍以 Data URL 作为正式请求契约，必须先资源化，不能直接扩大门禁造成玩法回归。列表、详情和 acknowledge 只走无 payload 的摘要投影，ack 不能为了同步旧字段重写大任务行；摘要错误文本也必须清除内联媒体并设硬上限，列表只能维护有界 top-N，不能先收集 owner 全量历史再截断。历史只通过迁移操作员的 dry-run + B-tree cursor 分批 procedure 压缩 `editor-canvas` 终态任务，cursor 选择读取量必须受 limit 约束，绝不全表扫描、绝不处理 pending / running；dry-run 后 apply 同一批时保持输入 cursor 不变，最后一批即使 `has_more=false` 只要仍有命中也必须 apply，只有 apply 成功后才推进到返回 cursor。SpacetimeDB CLI 2.5 的 `Option` 非空参数必须使用 SATS sum 编码；维护脚本要统一编码 `cursor_job_id`、`owner_user_id` 和 `completed_before_micros`，否则首批空 cursor 可运行，但第二批或带截止时间的调用会在写入前被拒绝。 +- 发布门禁：生产发布入口必须固定 `--delete-data=never` 与 scoped `--yes=migrate,break-clients`，普通 Jenkins 参数不得暴露清库开关；需要删数据的 schema 冲突必须直接阻断并重新检查 artifact/schema，不能靠裸 `--yes` 放行。 +- 验证：构造嵌套 Data URL、Blob URL 和超限 JSON 确认入队失败；检查正式 UI procedure / client record 不含 request/result payload；用 dry-run 和 apply 测试确认活动任务不变、终态普通提示词保留且内联媒体被替换；至少带一次非空 `--cursor-job-id` 与 `--completed-before-micros` 验证 CLI Option 编码，而不是只测首批空 cursor。 +- 关联：`server-rs/crates/api-server/src/editor_generation_queue.rs`、`server-rs/crates/spacetime-module/src/external_generation.rs`、`server-rs/crates/api-server/src/external_generation.rs`、`src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.ts`、`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`。 + ## React 测试因内部状态或实现细节正常重构就碎 - 现象：修改组件结构、按钮排序、图标库 class、提示文案或 hook 内部状态名后，React 测试大量失败，但真实用户流程和对外契约没有变化。 diff --git a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md index 767b59d62..8b45587f6 100644 --- a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md +++ b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md @@ -86,8 +86,8 @@ - `POST /api/editor/images/generations`：按提示词调用 VectorEngine 生成图片；角色生成可携带 `model`、`screenColor`、`segModel`、`aspectRatio`、`imageSize` 和 `referenceImageSrcs`，生成成功后 api-server 先保存带纯色背景源图，再调用 BgFilter 并传入 `screen_color=`、`seg_model=` 生成透明 PNG。宣发素材携带 `kind: "publication-material"` 时固定归一为 `gpt-image-2`，不支持 `nanobanana2`。`nanobanana2` 参考图作为 `inline_data` 进入 `generateContent`，`gpt-image-2` 参考图进入 edits。普通重绘继续走该接口并把当前图层图片作为参考图；图片快速编辑不走该接口。请求可携带 `projectId`、`assetFolderId`、`assetKind`、`generationInputs` 和 `sourceResourceId`，后端生成成功后创建 project resource / 账号素材并在响应中返回 resource / asset 快照。 - `POST /api/editor/images/background-removals`：接收当前图片源，校验登录态后由 api-server 解析为图片文件并转发到 BiRefNet 去背景服务；请求可携带 `projectId`、`targetLayerId`、`assetFolderId`、`assetLabel`、`sourceResourceId` 和 `canvasCompletion`，有 `canvasCompletion` 时完成后按生成占位写入结果图层，否则沿用旧的目标图层替换路径；响应返回 `imageSrc`、`objectKey`、`assetObjectId`、`width`、`height`、`taskId`、`elapsedMs`、`provider` 和可选 `project` 快照。服务地址由 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL` 配置，令牌只在服务端通过 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN` 注入。 - `POST /api/editor/icon-spritesheets/generations`：按图标规范图和素材描述数组生成 spritesheet，生成成功后 api-server 先保存带纯色背景 spritesheet 源图，再调用 BgFilter 生成透明 spritesheet。请求支持 `model`、`screenColor`、`segModel`、`aspectRatio`、`imageSize`、`priceMudPoints`、`projectId`、`assetFolderId` 和 `generationInputs`；`priceMudPoints` 必须来自编辑器生成计费配置中对应生图模型的尺寸档位（如 `nanobanana2` 的 `0.5K / 1K / 2K` 或 `gpt-image-2` 的 `1K / 2K`），后端用 `editor_generation_config` 校验后才调用上游；`nanobanana2` 走原生 `generateContent` 并写入 `generationConfig.imageConfig.aspectRatio/imageSize`，`0.5K` 传 `"512"`；`gpt-image-2` 走 `/v1/images/edits`。后端保存透明 spritesheet project resource / 账号素材，并随响应返回对应快照。 -- `POST /api/editor/ui-designs/assets/extractions`：以前端已绘入红色框选轮廓的 UI 设计图 Data URL 作为参考图，固定 `gpt-image-2` 和自动决策纯色背景素材提取提示词生成素材 spritesheet，生成成功后 api-server 先保存带纯色背景 spritesheet 源图，再调用 BgFilter 生成透明 spritesheet，并按连通域自动拆分为 `素材 1..N`，返回结构复用图标 spritesheet 响应。请求必须携带 `screenColor`、`segModel`、`aspectRatio: "1:1"`、`imageSize: "1K" | "2K"` 和 `priceMudPoints`；框选数量不超过 6 个时前端按 `1:1·1K` 与 gpt-image-2 1K 价格提交，超过 6 个时按 `1:1·2K` 与 2K 价格提交。后端必须在调用上游前校验比例、尺寸和泥点价格，只允许 `1:1 / 1K / 2K`。请求可携带 `projectId`、`assetFolderId`、`generationInputs` 和 `spritesheetLabel`，后端保存 spritesheet / 拆分素材并返回对应 resource / asset 快照；前端必须把 spritesheet 原图与拆分素材都加入画布。 -- `POST /api/editor/images/edits`：按提示词和当前图片 Data URL 调用 VectorEngine edits，返回新的生成图片元数据；接口能力仍可接收明确参考图，但图片快速编辑当前只提交 `sourceImageSrc`，不提交隐藏的 `referenceImageSrcs`。请求携带 project / asset 上下文时由后端创建新 resource / asset，前端只消费响应快照。 +- `POST /api/editor/ui-designs/assets/extractions`：前端把红色框选轮廓绘入本地临时图后，先将该图上传 OSS 并确认 asset object，再以返回的 `objectKey` 作为参考图入队；Data URL / Blob URL 只允许停留在上传前的浏览器临时态。接口固定 `gpt-image-2` 和自动决策纯色背景素材提取提示词生成素材 spritesheet，生成成功后 api-server 先保存带纯色背景 spritesheet 源图，再调用 BgFilter 生成透明 spritesheet，并按连通域自动拆分为 `素材 1..N`，返回结构复用图标 spritesheet 响应。请求必须携带 `screenColor`、`segModel`、`aspectRatio: "1:1"`、`imageSize: "1K" | "2K"` 和 `priceMudPoints`；框选数量不超过 6 个时前端按 `1:1·1K` 与 gpt-image-2 1K 价格提交，超过 6 个时按 `1:1·2K` 与 2K 价格提交。后端必须在调用上游前校验比例、尺寸和泥点价格，只允许 `1:1 / 1K / 2K`。请求可携带 `projectId`、`assetFolderId`、`generationInputs` 和 `spritesheetLabel`，后端保存 spritesheet / 拆分素材并返回对应 resource / asset 快照；前端必须把 spritesheet 原图与拆分素材都加入画布。 +- `POST /api/editor/images/edits`：按提示词和当前图片的已登记 `objectKey` / `resourceId` 调用 VectorEngine edits，返回新的生成图片元数据；图片快速编辑当前只提交 `sourceImageSrc`，不提交隐藏的 `referenceImageSrcs`。本地红框标记图必须先上传再提交 objectKey；请求携带 project / asset 上下文时由后端创建新 resource / asset，前端只消费响应快照。 - `POST /api/editor/videos/generations`：按视频描述、模型、比例、时长、分辨率、模式、声音、默认联网搜索标记和泥点价格生成视频。前端可选模型为 `seedance2.0-fast`、`seedance2.0`、`kling3.0`、`kling3.0-omni`，默认 `seedance2.0-fast`；后端必须将 `seedance2.0-fast` 映射到 `doubao-seedance-2-0-fast-260128`，将 `seedance2.0` 映射到 `doubao-seedance-2-0-260128`，两者不得混用。后端允许 6 类比例、4 到 15 秒整数、`480p / 720p / 1080p`，并拒绝 `seedance2.0-fast + 1080p`；`sound=on/off` 映射 Ark `generate_audio=true/false`。后端复用 Ark / VectorEngine content generation task 轮询链路，下载最终视频并持久化到 OSS；请求携带 `projectId` / `assetFolderId` 时同步创建 project resource / 账号素材并返回 `project` / `asset` 快照，基础响应返回 `videoSrc`、尺寸、prompt、model、provider、taskId、durationSeconds、resolution 和 `priceMudPoints`。 - `POST /api/editor/audios/sound-effects/generations` 与 `POST /api/editor/audios/background-music/generations`：按音效 / 背景音乐参数生成音频并持久化到 OSS；请求携带 `projectId` / `assetFolderId` 时同步创建 project resource / 账号素材并返回 `project` / `resource` / `asset` 快照，基础响应返回 `audioSrc`、prompt、model、provider、taskId、duration、歌词和 `priceMudPoints`。 @@ -128,7 +128,7 @@ - 画布 Agent 会话刷新后能从后端恢复会话标题、消息、附件和生成记录；前端不得根据本地临时状态伪造会话持久化结果。 - 图片选中后的浮动工具栏按钮顺序固定为：快速编辑、分割线、裁扩按钮、去除背景按钮、UI设计图专属提取素材、角色图专属生成动画、分割线、重绘、下载按钮。裁扩通过画布边界拖拉完成，不再展示四边数值输入；默认自由比例，选择固定比例后拖拉边界保持对应比例，完成后在原素材旁边新增裁扩结果图层，扩展区域透明填充。去除背景调用同源 BFF `POST /api/editor/images/background-removals`，由 api-server 代理远端 BiRefNet 服务并持久化结果；有项目上下文时先在画布创建关闭面板的去背景生成占位，完成后由后端通过 `canvasCompletion` 把新 project resource 写入该占位并返回快照，无占位上下文时才用新的 project resource 引用替换当前图层。画布任务侧栏按“排队/生成中”和“已完成”分页，生成中排在排队前，生成中耗时从任务开始时间戳实时计算，排队中不计时；进行中任务只显示阶段文本和已用时，不显示百分比；完成态生成任务副标题显示用户提示词并单行截断；点击任务只聚焦对应画布内容，不激活生成面板或改变任务顺序，聚焦时必须预留图片上方工具栏、底部工具栏和可见生成对话框空间。UI设计图的提取素材必须先进入红框素材框选状态，默认启用矩形框选，右侧框选工具与快速编辑统一且可再次点击取消启用态，当前启用工具按钮必须保持高亮。素材提取面板必须在素材下方，使用与生成新素材一致的面板宽度和底部模型 / 按钮样式，提示语显示 `使用框选工具框选你希望从画面中提取的素材`，并展示按原图坐标准确裁剪的框选区域截图预览、固定模型 `gpt-image-2`、左下角计划规格 `1:1·1K/2K` 和 `提取 · N泥点` 按钮，不显示额外取消按钮；点击素材和面板以外的画布区域即退出 UI 素材提取。至少框选一个区域后才可提交，前端把红色轮廓绘入原图后固定走 `gpt-image-2` 和自动决策纯色背景素材提取提示词；生成的透明 spritesheet 原图和拆分后的独立素材都作为画布图层保留。 - 重绘生成资源后，右侧出现新生成结果图层，并自动 fit 原图 + 新图，且重绘面板保持打开。 -- 快速编辑 / 重绘站内 public 示例图、历史 generated 图或 OSS generated 图时，前端先读取成 `data:image/*;base64,...` 再提交，后端不得再收到 `/creation-type-references/*`、`/generated-*` 或 OSS URL 作为 `referenceImageSrcs/sourceImageSrc`。 +- 快速编辑 / 重绘站内 public 示例图、历史 generated 图或 OSS generated 图时，优先复用当前图层已有 `objectKey` / `resourceId` / `sourceAssetId`；只有尚未登记的浏览器本地图片才先上传并取得 objectKey。前端不得再把正式对象下载成 `data:image/*;base64,...` 后提交，也不得把 Data URL / Blob URL 写入外部生成持久任务 JSON；后端收到引用后统一做 owner 归属校验并签名读取。 - 快速编辑不保留额外参考图入口；点击修改时只把原图或红框序号标注图作为 `/api/editor/images/edits` 的 `sourceImageSrc` 提交给后端。 - 素材文件夹可以新建、折叠、重命名和删除；删除普通文件夹后，其素材移动到“项目素材”。普通上传默认落入“上传素材”文件夹；素材库缺少该文件夹时，前端在首次普通上传前创建一次并复用，拖到指定文件夹或点击指定文件夹上传时仍进入目标文件夹。 - 上传按钮和拖拽上传都支持多文件；底部工具栏的上传入口选择文件后直接进入“上传素材”并在当前画布视口中心创建画布图层，素材栏文件夹内的上传入口只写入对应素材文件夹、不自动入画布；拖到文件夹或该文件夹内素材时进入目标文件夹；拖到画布时进入“上传素材”并在投放点创建画布图层。上传图片必须在创建占位素材、画布图层和账号级素材记录前先读取原图 Resolution，图层宽高、`originalWidth/originalHeight` 和素材库 `width/height` 都使用图片本身尺寸；上传视频同样在创建素材和图层前读取视频 metadata 宽高，保证单层下载或 ZIP 导出的真实视频文件重新导入后仍按文件自身尺寸入画布；仅在无法解析尺寸时才使用对应媒体兜底尺寸。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 36a5d4e0c..200f5a84c 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -221,7 +221,7 @@ npm run check:server-rs-ddd 5. 编辑器图片生成 / 图片修改 / 图标 spritesheet / UI 设计图提取素材 / 视频 / 角色动作 / 音效 / 背景音乐必须在后端计算模型价格后使用 `execute_billable_asset_operation_with_cost` 预扣泥点；预扣失败必须 fail-closed，不得继续提交 VectorEngine、Ark、Suno 或 Vidu 上游任务。 6. 队列任务按 `job_id + claim_attempt` 使用独立 consume/refund ledger。新 attempt 结算旧 attempt 时必须先写 `asset_operation_wallet_settlement`：旧 consume 已存在则原子退款，尚不存在则写取消 intent；迟到 consume 在同一 SpacetimeDB 事务内看到 intent 后必须失败关闭。重复 consume/refund 只有用户、金额、来源和配对 ledger 全部一致时才可视为幂等成功。lease 过期时只有 `attempt < max_attempts` 才能递增并重领；最终 attempt 已耗尽时，claim transaction 必须直接把 job 收口为 `failed`、清理 lease、写失败事件并结算当前 attempt，不能再把任务返回 worker 或调用 provider。 7. 音频生成的编辑器链路虽然任务提交和结果发布分离，仍必须把提交时后端计算出的模型价格写入 `AudioAssetBindingTarget.billing_points_cost`，最终发布落资产时按该价格扣费；创作音频目标未提供该字段时才使用旧的创作音频固定成本。 -8. 编辑器图片生成、图片修改、图标 spritesheet 和 UI 设计图提取素材的参考图可以提交 Data URL 或已登记的 generated objectKey；objectKey 必须归属于当前账号的 `editor_project_resource`、`editor_asset` 或 `asset_object`，后端通过归属校验后才签名读取 OSS。图标素材和 UI 素材提取的额外参考图必须真正传入 provider，不得只写入 `generationInputs` 展示快照；图片快速编辑当前不开放额外参考图，只提交原图或红框序号标注图作为 `sourceImageSrc`。UI 素材提取额外参考图上限为 5 张，普通图片生成上限 5 张，图标素材上限 8 张额外参考图。 +8. 编辑器进入外部生成持久队列的图片生成、图片修改、去背景、图标 spritesheet、UI 设计图提取、角色动作和视频参考图，只允许提交已登记的 generated `objectKey`、`resourceId` 或 `assetId`；任务 `request_payload_json` / `result_payload_json` 任意层级都禁止 `data:` / `blob:`，并受统一字节上限保护。objectKey 必须归属于当前账号的 `editor_project_resource`、`editor_asset` 或 `asset_object`，后端通过归属校验后才签名读取 OSS。本地红框序号标注图必须先上传并确认对象，再把 objectKey 入队；不得把既有 objectKey 下载成 Data URL 后写入任务。图标素材和 UI 素材提取的额外参考图必须真正传入 provider，不得只写入 `generationInputs` 展示快照；图片快速编辑当前不开放额外参考图。UI 素材提取额外参考图上限为 5 张，普通图片生成上限 5 张，图标素材上限 8 张额外参考图。同步且不持久化的历史兼容入口即使仍能解析 Data URL，也不能把该值转存到工程、素材、元数据、审计或任务表。 ## 外部服务与资产 @@ -266,7 +266,15 @@ npm run check:server-rs-ddd - Rust 结构体：`ExternalGenerationJob` - 源码：`server-rs/crates/spacetime-module/src/external_generation.rs` -- 用途：外部生成 worker 的持久任务队列和用户可见生成任务列表；`GENARRATIVE_EXTERNAL_GENERATION_MODE=queue` 时，`api-server` HTTP 角色只入队，`external-generation-worker` 角色通过 claim lease 领取、续租、执行，并用 `lease_token` 栅栏回写完成 / 失败。队列行同时保存 `price_mud_points`、`refund_ledger_id` 和 `notification_acknowledged_at`，BFF 通过 `GET /api/runtime/external-generation/jobs` 返回当前账号的正式生成任务列表、价格、状态和未确认终态数量；前端只能展示该后端事实，完成 / 失败提示展示后后台调用 `POST /api/runtime/external-generation/jobs/acknowledge` 由后端写确认时间，关闭按钮只收起本地弹窗，未确认终态任务会在下次登录后再次集中弹出。拼图 `compile_puzzle_draft` 的前置 `compile_puzzle_agent_draft`、`generate_puzzle_images` 与 `generate_puzzle_ui_background` 的业务写回也在对应 SpacetimeDB transaction 内校验 `job_id + worker_id + lease_token`、job kind、owner 和 source entity，避免过期 worker 写 session / work profile；图片画布编辑器的 `editor_image_generation`、`editor_image_edit`、`editor_background_removal`、`editor_icon_spritesheet_generation`、`editor_ui_design_asset_extraction`、`editor_character_animation_generation`、`editor_video_generation`、`editor_sound_effect_generation` 和 `editor_background_music_generation` 复用同一队列表，worker 成功后经 `api-server` facade 写入 `editor_project_resource` / `editor_asset` / `editor_canvas.layers_json`，前端只通过 BFF job 状态轮询和项目快照读取恢复完成态。`GENARRATIVE_EXTERNAL_GENERATION_MODE=inline` 时不创建该队列行，三个 external generation guard 字段必须同时为空才允许 api-server 受控同步写回，半空 guard 仍会拒绝。worker 成功写回业务事实后才能 complete job；业务失败态写回成功后才能 fail job，失败态未写回时保留租约等待后续重领。 +- 用途：外部生成 worker 的内部持久任务队列；`GENARRATIVE_EXTERNAL_GENERATION_MODE=queue` 时，`api-server` HTTP 角色只入队，`external-generation-worker` 角色通过 claim lease 领取、续租、执行，并用 `lease_token` 栅栏回写完成 / 失败。队列行继续保存 worker 执行、计费与滚动发布兼容所需字段，但用户可见任务列表、价格、状态、未确认终态数量和通知确认时间的正式读取事实源已经迁到 `external_generation_job_summary`；BFF 不得再为列表 / 详情 / acknowledge 读取该大表。拼图 `compile_puzzle_draft` 的前置 `compile_puzzle_agent_draft`、`generate_puzzle_images` 与 `generate_puzzle_ui_background` 的业务写回也在对应 SpacetimeDB transaction 内校验 `job_id + worker_id + lease_token`、job kind、owner 和 source entity，避免过期 worker 写 session / work profile；图片画布编辑器的 `editor_image_generation`、`editor_image_edit`、`editor_background_removal`、`editor_icon_spritesheet_generation`、`editor_ui_design_asset_extraction`、`editor_character_animation_generation`、`editor_video_generation`、`editor_sound_effect_generation` 和 `editor_background_music_generation` 复用同一队列表，worker 成功后经 `api-server` facade 写入 `editor_project_resource` / `editor_asset` / `editor_canvas.layers_json`，前端只通过 BFF job 状态轮询和项目快照读取恢复完成态。`GENARRATIVE_EXTERNAL_GENERATION_MODE=inline` 时不创建该队列行，三个 external generation guard 字段必须同时为空才允许 api-server 受控同步写回，半空 guard 仍会拒绝。worker 成功写回业务事实后才能 complete job；业务失败态写回成功后才能 fail job，失败态未写回时保留租约等待后续重领。 +- 载荷约束：本次先对 `source_module = editor-canvas` 的 `request_payload_json` / `result_payload_json` 实施有限大小合法 JSON、任意层级禁止 `data:` / `blob:` 的双层门禁，只保存 worker 执行必需的普通参数和已登记媒体引用；其它玩法在完成各自参考图资源化之前不由本次门禁静默改变既有请求契约。该主表只供 worker claim / 执行和受控维护读取；正式用户任务列表、单任务状态、队列概览与 acknowledge 不得再返回或解析这两个 payload。 + +### `external_generation_job_summary` + +- Rust 结构体：`ExternalGenerationJobSummary` +- 源码：`server-rs/crates/spacetime-module/src/external_generation.rs` +- 用途：外部生成正式任务列表的轻量投影，按 `job_id` 保存 owner、来源、状态、价格、有界错误摘要、通知确认时间、各阶段时间和入队时提取的 `request_prompt`，不包含 request/result payload、worker lease 或 dedupe 内部字段。错误摘要统一拒绝内联媒体并限制为 2048 字符；列表在单次 owner 扫描中同时计数并只保留请求 limit 的固定大小 top-N，不得先收集全量历史再截断。enqueue、claim、renew、complete、fail 事务同步投影；acknowledge 只更新该轻量表并写审计事件，后续主任务同步必须保留已有确认时间，禁止为了写确认时间加载 / 重写大 payload 行。BFF 的列表、状态和确认只调用 summary procedure。历史终态任务由迁移操作员的游标分批 maintenance procedure 在压缩 payload 时同步回填摘要，正式列表不得为兼容旧数据回扫完整主表。 +- 正式读取 procedure 为 `get_external_generation_job_summary_and_return`、`list_external_generation_job_summaries_and_return` 和 `acknowledge_external_generation_job_summaries_and_return`。历史维护 procedure 为 `compact_external_generation_job_payloads_and_return` 与 `backfill_external_generation_job_summaries_and_return`，仅 migration operator 可调用；运维入口统一使用 `npm run spacetime:external-generation:maintain -- ...`，默认 dry-run、单批最多 25 条。B-tree cursor 选择阶段最多反序列化 `limit + 1` 行，apply 再按主键逐条读取选中行；怀疑存在单行异常巨型 JSON 时必须先使用 `--limit 1`。payload 压缩额外固定使用 `source_module = editor-canvas` 的复合 cursor 索引，不得静默改写其它玩法历史任务。 ### `external_generation_job_event` diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index b9c713dfa..e6d84db8e 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -67,6 +67,8 @@ lease 过期后不代表任务一定再次执行：claim transaction 只有在 ` `我的` 页签或排障面板展示队列等待时，只读取 BFF 队列接口：`GET /api/runtime/external-generation/queue-overview` 查看当前用户可见队列概览，`GET /api/runtime/external-generation/jobs/{jobId}` 查看单 job 状态。生成页 / 进度页不承接队列概览，只展示当前玩法业务进度；队列接口只提供等待 / 运行 / 失败 / 完成状态补充，最终草稿、作品和结果页仍要轮询对应玩法 session/detail 接口收敛到 ready 或 failed；不要直接查询 `external_generation_job` private table，也不要把 worker 内部 payload 暴露到前端。 +外部生成任务摘要投影与历史 payload 维护使用 `npm run spacetime:external-generation:maintain -- ...`，且只能由已授权 migration operator 的 SpacetimeDB CLI 登录态执行。脚本默认 dry-run、每次只处理一批，绝不自动循环全表；`--apply` 才写入。先发布包含 `external_generation_job_summary` 与 cursor 索引的 SpacetimeDB 模块，在维护模式内对事故时间以前的编辑器终态任务执行小批 dry-run，例如 `npm run spacetime:external-generation:maintain -- --database --server-url --limit 5 --completed-before-micros `；核对 `matched_count`、`before_bytes`、`after_bytes` 和 `inline_media_count` 后，保持本批输入 cursor 不变并追加 `--apply` 重跑同一批，即使最后一批 `has_more = false`，只要 dry-run 仍有 `matched_count` / `selected_count` 也必须 apply；只有 apply 成功后才使用它返回的 `next_cursor_job_id` 继续。B-tree cursor 的选择阶段最多反序列化 `limit + 1` 行，apply 会再按主键逐条读取选中行但不会同时保留整批 payload；如怀疑存在单行异常巨型历史 JSON，先用 `--limit 1`。payload 压缩硬限制 `source_module = editor-canvas`；终态压缩完成后，用 `--backfill-summaries` 先 dry-run、再 `--apply` 分批补齐仍缺失的活动任务或无内联媒体历史任务摘要，直到 `has_more = false`，最后再切换使用 summary procedure 的 api-server。Stdb 构建 artifact 和完整 release 包都必须包含 `scripts/spacetime-maintain-external-generation-jobs.mjs` 与 `scripts/spacetime-migration-common.mjs`。首次上线不得让 Full Build 从 Stdb 自动直落 API：`STDB_API_ROLLOUT_MODE` 默认 fail-closed 为 `pause-after-stdb`，必须填写受限的 `STDB_API_ROLLOUT_APPROVERS`；Stdb Publish 通过 `KEEP_MAINTENANCE_MODE` 保持维护文件并停止旧 API/controller/worker，暂停点最多等待 4 小时，完成上述维护并确认无后续批次后才由指定审批人放行 API。定时构建缺少审批人时必须在发布前失败，不能静默退回 `normal`；也可分开运行 Stdb publish、维护、API deploy 三个受控 Job。任一批次都不得处理 pending / running payload；不要用 runtime writer、bootstrap secret 或匿名 identity 代替 migration operator，也不要在未核对 dry-run 时直接 apply。 + 需要验证“更新 API 不停 worker”和“worker 是否持续消费队列”时，优先使用隔离容器 smoke：`npm run container:worker-smoke -- smoke`。该脚本生成 gitignored 的 `deploy/container/worker-smoke/api-server.env`，启动独立 compose project 与独立 SpacetimeDB，发布当前 `spacetime-module` 后写入 `worker_smoke_unsupported` 测试 job；预期 worker claim 后执行 unsupported 失败分支，再执行 API-only recreate 并确认 worker 容器 ID 不变，最后再次入队验证 API 更新后队列仍可消费。`external_generation_job` 是 private table，脚本通过 worker 日志确认 job_id 被消费，不用 CLI SQL 查询私表。该 smoke 不读取 `.env.local`，也不依赖真实 VectorEngine / OSS 密钥；真实生图链路联调再在本地私有 env 中补齐 provider 配置。worker-smoke 默认把本机 `spacetime` CLI 打成轻量 SpacetimeDB 镜像，避免本机首次 smoke 依赖官方大镜像下载。若容器内 Cargo 拉取 crates.io 依赖不稳定，可用 `npm run container:worker-smoke -- smoke --local-binary` 让容器内 Cargo 复用本机 Cargo 缓存构建当前二进制，再打入 Debian bookworm smoke runtime 临时镜像；可用 `GENARRATIVE_WORKER_SMOKE_LOCAL_BASE_IMAGE` 覆盖运行时基础镜像；若隔离端口或库数据需要重建，追加 `--force`。完成 queue 链路验证时，还要用队列概览 BFF 和单 job 状态接口确认 job 从 queued/running 收敛，并用对应玩法 session/detail 接口确认业务状态同步完成。 本地只做账号/UI smoke 且需要短信登录时，`SMS_AUTH_PROVIDER` 应显式设为 `mock`，并把 `SMS_AUTH_MOCK_VERIFY_CODE` 设为固定值（当前常用 `123456`），再重启 `npm run dev` 或 `npm run dev:api-server`。如果 `.env.local` 还保留 `SMS_AUTH_PROVIDER=aliyun`，`POST /api/auth/phone/login` 用 mock 验证码会稳定报“验证码错误”，不是前端表单问题。真实短信联调再切回 `aliyun` 并重启。 @@ -389,6 +391,8 @@ Pingora current release 自审脚本 `scripts/ops/pingora-current-release-audit. `Genarrative-Stdb-Module-Build` 的 Jenkins 归档产物必须包含 `build//spacetime_module.wasm`、`spacetime_module.wasm.sha256`、`release-manifest.json`、`scripts/deploy/production-stdb-publish.sh`、`scripts/deploy/production-runtime-writer-identity-rotate.mjs`、`scripts/deploy/maintenance-on.sh`、`scripts/deploy/maintenance-off.sh`、`scripts/spacetime-migration-common.mjs` 和 `scripts/database-backup-to-oss.mjs`，不得包含 `migration-bootstrap-secret.txt` 或任何原始 bootstrap secret。`Genarrative-Stdb-Module-Build` 只接受 `MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID` 指向的受保护 Jenkins Secret File：构建 shell 从临时文件读取原始值，强制校验为 64 位十六进制，计算 SHA-256，随后只通过 `GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET_SHA256` 注入 Rust 编译；WASM 因而只包含摘要，不包含可下载的原文，Stdb `release-manifest.json` 以 `migration_bootstrap_secret_sha256` 记录该非敏感摘要。`Genarrative-Stdb-Module-Publish` 只通过 `copyArtifacts` 复制上述非敏感产物，不在目标机器 checkout Git，并在发布阶段用同一个凭据 ID 再次挂载 Secret File；publish 必须再次校验 64 位十六进制、重算 SHA-256，并与 manifest 的 `migration_bootstrap_secret_sha256` 强制匹配后才可发布。Full Build 必须保证 Stdb Build / Publish 的 `MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID` 完全相同并把同一个 ID 同时透传，不能从构建 artifact 传 secret；ID 不同、manifest 缺摘要或摘要不匹配都必须在发布前失败。 +生产 Stdb publish 固定传 `--delete-data=never --yes=migrate,break-clients`，普通 Stdb Jenkins Job 不提供 `CLEAR_DATABASE`；任何需要删除数据的迁移都必须失败并重新核对 schema 与 artifact，不能在发布路径内切换清库继续。 + 生产运行时不把 bootstrap secret 明文写进 `/etc/genarrative/*.env`。`api-server.env` 和 worker env 只登记固定 FILE 路径 `GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE=/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt`；若检测到明文 `GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET` 或其他 FILE 路径，Server-Provision / API deploy 必须失败。Full Build 先执行 Stdb publish、后执行 API deploy，因此两段必须透传同一 `API_ENV_FILE` / `WORKER_ENV_FILE`；Stdb Publish 把 Secret File 路径作为 `--migration-bootstrap-secret-file` 传给随包 `production-stdb-publish.sh`。脚本先进入维护模式、按所选模式完成发布前冷备份、校验 checksum 并发布 module；成功后拒绝符号链接目标，把 secret 安装成 `root:genarrative 0440`、目录收紧为 `root:genarrative 0750`，原子补齐 API / worker env 的固定 FILE 配置，再快照并重启发布前为 active 的 API、controller 和 worker。`systemctl is-active` 只有明确返回合法的非 active 状态时才允许跳过，查询错误或 active worker 的 `list-units` 失败都必须保留维护模式并阻断；所有原 active 服务重启后必须重新确认为 `active`。如果 API 原本 active，还必须在 `maintenance-off` 前通过本机 `http://127.0.0.1:8082/healthz` readiness；可用 `--api-health-url` / `GENARRATIVE_STDB_PUBLISH_API_HEALTH_URL` 调整本机 URL，并用 `--api-readiness-timeout-seconds` / `GENARRATIVE_STDB_PUBLISH_API_READINESS_TIMEOUT_SECONDS` 调整超时。这样旧服务器首次 rollout 也不会等到后续 API deploy 才拿到 FILE；只覆盖 secret 文件或只补 env 而不重启都不生效，因为 `AppConfig` 在进程启动时读取 secret。人工执行 `npm run build:production-release -- --component spacetime-module --name ` 且未显式提供 secret / SHA-256 时，原始随机 secret 只写入 gitignored 的 `server-rs/.spacetimedb/build-secrets/.txt`，目录权限 `0700`、文件权限 `0600`，发布包和 `release-manifest.json` 都不收录它；必须把该受保护文件另行交给 publish 阶段。旧 `npm run deploy:rust:remote` Ubuntu 直传入口也使用同一 sidecar 目录，发布包内不含原文；上传模式通过独立 SSH 标准输入把 secret 原子安装到远端发布目录并收紧为 `0600`，`--skip-upload` 时必须单独受保护交付。 本地 dev 的原始值也只能进入 api-server，不得扩散给 Web / Vite，任何控制台、Jenkins 日志、归档或生成 README 都不得输出明文。相关变更至少运行 `bash -n scripts/deploy/production-stdb-publish.sh scripts/deploy/production-api-deploy.sh scripts/deploy-rust-remote.sh scripts/jenkins-server-provision.sh`、`node --check scripts/dev.mjs scripts/check-production-ops-guardrails.mjs`、`npm run check:production-ops`、`npm run check:encoding` 和 `git diff --check`。 生产 runtime writer 不能通过替换 bootstrap secret 或重启服务隐式轮换。migration operator 与 runtime writer 必须身份互斥：operator 不能成为 writer，当前 writer 不能授权为 operator；已有任一 operator 后，bootstrap secret 不得新增或接管 operator。先准备新 api-server identity，并使用当前已授权 migration operator 的 CLI 登录态执行 `node scripts/deploy/production-runtime-writer-identity-rotate.mjs --database --server-url --operator-identity --operator-user-id --next-writer-identity --confirm-next-writer-identity --note `。CLI 会校验当前登录 identity、双录新 identity 和审计原因；模块 procedure 还会拒绝把 writer 设为任一已登记 migration operator。成功后必须核对 `editor_generation_runtime_identity_rotation` 的旧 writer、新 writer、operator identity、操作人、原因和服务端时间，再切换 API token；轮换只改 writer，不改模型价格。 diff --git a/jenkins/Jenkinsfile.production-full-build-and-deploy b/jenkins/Jenkinsfile.production-full-build-and-deploy index 23ea50342..d35d74ba4 100644 --- a/jenkins/Jenkinsfile.production-full-build-and-deploy +++ b/jenkins/Jenkinsfile.production-full-build-and-deploy @@ -40,6 +40,8 @@ pipeline { string(name: 'SPACETIME_RUN_AS_USER', defaultValue: 'spacetimedb', description: 'Stdb 发布使用的本机用户') string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'API 与 Stdb publish 共用的 api-server 环境文件') string(name: 'WORKER_ENV_FILE', defaultValue: '/etc/genarrative/external-generation-worker.env', description: 'API 与 Stdb publish 共用的 worker 环境文件') + choice(name: 'STDB_API_ROLLOUT_MODE', choices: ['pause-after-stdb', 'normal'], description: '默认 fail-closed 暂停在 Stdb 与 API 之间；确认无需维护时才显式选 normal') + string(name: 'STDB_API_ROLLOUT_APPROVERS', defaultValue: '', description: 'pause-after-stdb 必填：允许放行 API 的 Jenkins 用户/组，多个值用逗号分隔') } stages { @@ -104,6 +106,9 @@ pipeline { if (params.DEPLOY_TARGET == 'release' && !params.CONFIRM_RELEASE_DEPLOY_AGENT) { error('release 部署需要先配置独立 release 部署 agent，并勾选 CONFIRM_RELEASE_DEPLOY_AGENT。当前 Linux 开发/构建/开发部署 agent 不能执行 release 部署。') } + if (params.STDB_API_ROLLOUT_MODE == 'pause-after-stdb' && !params.STDB_API_ROLLOUT_APPROVERS?.trim()) { + error('pause-after-stdb 必须填写 STDB_API_ROLLOUT_APPROVERS；定时构建默认在此 fail-closed，不得无人值守发布 Stdb/API。') + } } } } @@ -184,6 +189,7 @@ pipeline { string(name: 'SPACETIME_RUN_AS_USER', value: params.SPACETIME_RUN_AS_USER ?: 'spacetimedb'), string(name: 'API_ENV_FILE', value: params.API_ENV_FILE ?: '/etc/genarrative/api-server.env'), string(name: 'WORKER_ENV_FILE', value: params.WORKER_ENV_FILE ?: '/etc/genarrative/external-generation-worker.env'), + booleanParam(name: 'KEEP_MAINTENANCE_MODE', value: params.STDB_API_ROLLOUT_MODE == 'pause-after-stdb'), string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET), booleanParam(name: 'CONFIRM_RELEASE_DEPLOY_AGENT', value: params.CONFIRM_RELEASE_DEPLOY_AGENT), string(name: 'BUILD_JOB_NAME', value: params.STDB_BUILD_JOB_NAME), @@ -192,6 +198,17 @@ pipeline { } } + stage('Stdb / Api Rollout Gate') { + when { + expression { return params.STDB_API_ROLLOUT_MODE == 'pause-after-stdb' } + } + steps { + timeout(time: 4, unit: 'HOURS') { + input message: 'SpacetimeDB module 已发布，站点和外部生成服务保持维护态。请在目标部署 agent 使用本次 Stdb artifact 中的外部生成任务维护脚本完成 dry-run、分批 apply 与摘要回填；确认无 has_more 后再继续部署 API。', ok: '确认维护完成，继续部署 API', submitter: params.STDB_API_ROLLOUT_APPROVERS.trim(), submitterParameter: 'STDB_API_ROLLOUT_APPROVED_BY' + } + } + } + stage('Deploy Api') { steps { build job: params.API_DEPLOY_JOB_NAME, diff --git a/jenkins/Jenkinsfile.production-stdb-module-build b/jenkins/Jenkinsfile.production-stdb-module-build index a5e7dac66..ab793bc76 100644 --- a/jenkins/Jenkinsfile.production-stdb-module-build +++ b/jenkins/Jenkinsfile.production-stdb-module-build @@ -148,7 +148,7 @@ pipeline { stage('Archive') { steps { - archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/spacetime_module.wasm,build/${env.EFFECTIVE_BUILD_VERSION}/spacetime_module.wasm.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,scripts/deploy/production-stdb-publish.sh,scripts/deploy/production-runtime-writer-identity-rotate.mjs,scripts/deploy/maintenance-on.sh,scripts/deploy/maintenance-off.sh,scripts/spacetime-migration-common.mjs,scripts/database-backup-to-oss.mjs", fingerprint: true + archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/spacetime_module.wasm,build/${env.EFFECTIVE_BUILD_VERSION}/spacetime_module.wasm.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,scripts/deploy/production-stdb-publish.sh,scripts/deploy/production-runtime-writer-identity-rotate.mjs,scripts/deploy/maintenance-on.sh,scripts/deploy/maintenance-off.sh,scripts/spacetime-migration-common.mjs,scripts/spacetime-maintain-external-generation-jobs.mjs,scripts/database-backup-to-oss.mjs", fingerprint: true } } diff --git a/jenkins/Jenkinsfile.production-stdb-module-publish b/jenkins/Jenkinsfile.production-stdb-module-publish index d7eebc84f..146bb77f6 100644 --- a/jenkins/Jenkinsfile.production-stdb-module-publish +++ b/jenkins/Jenkinsfile.production-stdb-module-publish @@ -24,7 +24,7 @@ pipeline { string(name: 'SPACETIME_RUN_AS_USER', defaultValue: 'spacetimedb', description: '执行 spacetime publish 的本机用户，默认使用自托管服务用户') string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: '需补齐 runtime bootstrap secret FILE 的 api-server 环境文件') string(name: 'WORKER_ENV_FILE', defaultValue: '/etc/genarrative/external-generation-worker.env', description: '需补齐 runtime bootstrap secret FILE 的 worker 环境文件；文件不存在时跳过') - booleanParam(name: 'CLEAR_DATABASE', defaultValue: false, description: '是否清空数据库后发布') + booleanParam(name: 'KEEP_MAINTENANCE_MODE', defaultValue: false, description: '发布 module 后保持维护模式并停止旧 API/controller/worker，等待受控维护和后续 API deploy') choice(name: 'DATABASE_BACKUP_MODE', choices: ['async', 'sync', 'skip'], description: '数据库备份策略：async 在 publish 前生成本地冷备份、后台上传 OSS；sync 在 publish 前等待上传完成且失败阻断；skip 跳过') } @@ -93,7 +93,7 @@ pipeline { copyArtifacts( projectName: params.BUILD_JOB_NAME, selector: specific(params.BUILD_NUMBER_TO_DEPLOY), - filter: "build/${params.BUILD_VERSION}/spacetime_module.wasm,build/${params.BUILD_VERSION}/spacetime_module.wasm.sha256,build/${params.BUILD_VERSION}/release-manifest.json,scripts/deploy/production-stdb-publish.sh,scripts/deploy/production-runtime-writer-identity-rotate.mjs,scripts/deploy/maintenance-on.sh,scripts/deploy/maintenance-off.sh,scripts/spacetime-migration-common.mjs,scripts/database-backup-to-oss.mjs", + filter: "build/${params.BUILD_VERSION}/spacetime_module.wasm,build/${params.BUILD_VERSION}/spacetime_module.wasm.sha256,build/${params.BUILD_VERSION}/release-manifest.json,scripts/deploy/production-stdb-publish.sh,scripts/deploy/production-runtime-writer-identity-rotate.mjs,scripts/deploy/maintenance-on.sh,scripts/deploy/maintenance-off.sh,scripts/spacetime-migration-common.mjs,scripts/spacetime-maintain-external-generation-jobs.mjs,scripts/database-backup-to-oss.mjs", target: '.', fingerprintArtifacts: true ) @@ -106,7 +106,7 @@ pipeline { } steps { script { - def clearArg = params.CLEAR_DATABASE ? '--clear-database' : '' + def keepMaintenanceArg = params.KEEP_MAINTENANCE_MODE ? '--keep-maintenance-mode' : '' def backupMode = params.DATABASE_BACKUP_MODE?.trim() ? params.DATABASE_BACKUP_MODE.trim() : 'async' if (!(backupMode in ['async', 'sync', 'skip'])) { error("DATABASE_BACKUP_MODE 只能是 async、sync 或 skip: ${backupMode}") @@ -146,7 +146,7 @@ pipeline { --migration-bootstrap-secret-file "\${MIGRATION_BOOTSTRAP_SECRET_FILE:?MIGRATION_BOOTSTRAP_SECRET_FILE 不能为空}" \\ --api-env-file "${params.API_ENV_FILE}" \\ --worker-env-file "${params.WORKER_ENV_FILE}" \\ - ${clearArg} \\ + ${keepMaintenanceArg} \\ ${backupArg} ' """ diff --git a/package.json b/package.json index 69954509c..c912347e7 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "admin-web:typecheck": "node scripts/admin-web-build.mjs typecheck", "admin-web:preview": "npm --prefix apps/admin-web run preview --", "spacetime:generate": "node scripts/generate-spacetime-bindings.mjs", + "spacetime:external-generation:maintain": "node scripts/spacetime-maintain-external-generation-jobs.mjs", "check:api-server-env": "node scripts/check-api-server-env.mjs", "check:spacetime-runtime-access": "node scripts/check-spacetime-runtime-access.mjs", "deploy:rust:remote": "node scripts/run-bash-script.mjs scripts/deploy-rust-remote.sh", diff --git a/scripts/build-production-release.sh b/scripts/build-production-release.sh index 458b6193f..fb368e3ce 100644 --- a/scripts/build-production-release.sh +++ b/scripts/build-production-release.sh @@ -546,6 +546,7 @@ chmod +x \ copy_required_file "${SCRIPT_DIR}/spacetime-export-migration-json.mjs" "${TARGET_DIR}/scripts/database-export.mjs" "数据库导出脚本" copy_required_file "${SCRIPT_DIR}/spacetime-import-migration-json.mjs" "${TARGET_DIR}/scripts/database-import.mjs" "数据库导入脚本" copy_required_file "${SCRIPT_DIR}/spacetime-migration-common.mjs" "${TARGET_DIR}/scripts/spacetime-migration-common.mjs" "数据库迁移公共脚本" +copy_required_file "${SCRIPT_DIR}/spacetime-maintain-external-generation-jobs.mjs" "${TARGET_DIR}/scripts/spacetime-maintain-external-generation-jobs.mjs" "外部生成任务维护脚本" copy_required_file "${SCRIPT_DIR}/spacetime-authorize-migration-operator.mjs" "${TARGET_DIR}/scripts/spacetime-authorize-migration-operator.mjs" "数据库迁移授权脚本" copy_required_file "${SCRIPT_DIR}/spacetime-revoke-migration-operator.mjs" "${TARGET_DIR}/scripts/spacetime-revoke-migration-operator.mjs" "数据库迁移撤权脚本" copy_required_file "${SCRIPT_DIR}/database-backup-to-oss.mjs" "${TARGET_DIR}/scripts/database-backup-to-oss.mjs" "数据库 OSS 备份脚本" @@ -584,7 +585,7 @@ cat >"${TARGET_DIR}/README.md" < 0)', + reason: '历史维护最后一批即使 has_more=false，只要 dry-run 仍命中数据也必须提示同 cursor apply。', + }, + { + file: 'scripts/spacetime-maintain-external-generation-jobs.mjs', + includes: 'cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null)', + reason: '历史维护续批游标必须编码为 CLI SATS Option，不能把非空字符串直接传给 procedure。', + }, + { + file: 'scripts/spacetime-maintain-external-generation-jobs.mjs', + includes: 'completed_before_micros: encodeSpacetimeCliOption(', + reason: '历史 payload 截止时间必须编码为 CLI SATS Option，确保事故时间过滤可调用。', + }, { file: 'scripts/deploy-rust-remote.sh', excludes: diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index 2b0ced40c..90d1af69b 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -688,6 +688,7 @@ DEPLOY_COMPLETED=0 PINGORA_INCLUDED=0 REQUIRE_PINGORA_GATEWAY=0 MAINTENANCE_ENABLED_BY_DEPLOY=0 +MAINTENANCE_FILE="${GENARRATIVE_MAINTENANCE_FILE:-/var/lib/genarrative/maintenance/enabled}" CURRENT_LINK_SWITCHED=0 RELEASE_DIR="" STAGING_RELEASE_DIR="" @@ -847,8 +848,12 @@ on_exit() { trap on_exit EXIT +if [[ ! -f "${MAINTENANCE_FILE}" ]]; then + MAINTENANCE_ENABLED_BY_DEPLOY=1 +else + echo "[production-api-deploy] 继承已有维护模式；部署失败时不得误退出上游维护窗口: ${MAINTENANCE_FILE}" +fi bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}" -MAINTENANCE_ENABLED_BY_DEPLOY=1 echo "[production-api-deploy] 校验 api-server" ( diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index a1425646e..96817420b 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -6,16 +6,18 @@ umask 077 usage() { cat <<'EOF' 用法: - ./scripts/deploy/production-stdb-publish.sh --source-dir build/ --database --migration-bootstrap-secret-file [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--api-health-url http://127.0.0.1:8082/healthz] [--api-readiness-timeout-seconds 60] [--clear-database] [--backup-mode async|sync|skip] + ./scripts/deploy/production-stdb-publish.sh --source-dir build/ --database --migration-bootstrap-secret-file [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--api-health-url http://127.0.0.1:8082/healthz] [--api-readiness-timeout-seconds 60] [--keep-maintenance-mode] [--backup-mode async|sync|skip] 说明: 进入维护模式，校验 spacetime_module.wasm.sha256，并在生产实例本机执行 spacetime publish。 + publish 固定使用 --delete-data=never 与 scoped --yes=migrate,break-clients；任何需要删除数据的 schema 冲突都会阻断发布。 默认使用 http://127.0.0.1:3101，避免与部署机本机 Git/Web 服务的 3000 端口冲突。 默认使用 /stdb 作为 spacetime CLI root-dir，并以 spacetimedb 用户发布，避免 root CLI 身份污染自托管实例。 发布时固定追加 --no-config，只使用显式参数，避免工作区或用户目录里的 spacetime 配置干扰目标。 async 模式会在 publish 前先做本地冷备份，再在 publish 完成后后台上传 OSS，避免低带宽上传阻塞部署。 如需强制等待备份完成并在失败时阻断 publish，传入 --backup-mode sync。 发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径，再重启并验活重启前 active 的服务。 + --keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker，并在成功后保持维护态，交由后续 API deploy 恢复服务。 migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。 如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。 失败时保留维护模式。 @@ -51,7 +53,7 @@ RUN_AS_USER="spacetimedb" MIGRATION_BOOTSTRAP_SECRET_FILE="" API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-server.env}" WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}" -CLEAR_DATABASE=0 +KEEP_MAINTENANCE_MODE=0 BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}" DEPLOY_COMPLETED=0 PUBLISH_TMP_DIR="" @@ -285,6 +287,54 @@ restart_runtime_services_after_bootstrap_secret_install() { fi } +stop_runtime_services_for_rollout_gate() { + local api_state="" + local controller_state="" + local worker_service="" + local worker_units_output="" + local -a services_to_stop=() + + api_state="$(get_runtime_service_active_state genarrative-api.service)" + controller_state="$(get_runtime_service_active_state genarrative-external-generation-controller.service)" + if ! worker_units_output="$( + run_privileged systemctl list-units \ + --type=service \ + --state=active \ + --no-legend \ + --plain \ + 'genarrative-external-generation-worker@*.service' + )"; then + echo "[production-stdb-publish] 查询 active worker 服务失败，无法建立受控维护窗口。" >&2 + return 1 + fi + + if [[ "${controller_state}" == "active" ]]; then + services_to_stop+=(genarrative-external-generation-controller.service) + fi + while read -r worker_service _; do + if [[ "${worker_service}" =~ ^genarrative-external-generation-worker@[A-Za-z0-9_.@:-]+\.service$ ]]; then + services_to_stop+=("${worker_service}") + fi + done <<<"${worker_units_output}" + if [[ "${api_state}" == "active" ]]; then + services_to_stop+=(genarrative-api.service) + fi + + if [[ "${#services_to_stop[@]}" -eq 0 ]]; then + echo "[production-stdb-publish] 受控维护窗口开始前没有 active API/controller/worker。" + return 0 + fi + + echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}" + run_privileged systemctl stop "${services_to_stop[@]}" + for worker_service in "${services_to_stop[@]}"; do + if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then + echo "[production-stdb-publish] 运行时服务停止后仍为 active: ${worker_service}" >&2 + return 1 + fi + done +} + while [[ $# -gt 0 ]]; do case "$1" in -h|--help) @@ -336,8 +386,8 @@ while [[ $# -gt 0 ]]; do API_READINESS_TIMEOUT_SECONDS="${2:?缺少 --api-readiness-timeout-seconds 的值}" shift 2 ;; - --clear-database) - CLEAR_DATABASE=1 + --keep-maintenance-mode) + KEEP_MAINTENANCE_MODE=1 shift ;; --skip-backup) @@ -455,6 +505,7 @@ on_exit() { trap on_exit EXIT prepare_async_backup() { + local -a restart_service_args=() ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs" if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" @@ -464,6 +515,10 @@ prepare_async_backup() { exit 1 fi + if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then + restart_service_args+=(--restart-service-after genarrative-api.service) + fi + ASYNC_BACKUP_STATUS_FILE="$(mktemp /tmp/genarrative-stdb-backup-status.XXXXXX.json)" echo "[production-stdb-publish] publish 前生成本地冷备份，随后会异步上传 OSS" node -- "${ASYNC_BACKUP_SCRIPT}" \ @@ -471,7 +526,7 @@ prepare_async_backup() { --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ --stop-service spacetimedb.service \ - --restart-service-after genarrative-api.service \ + "${restart_service_args[@]}" \ --defer-upload \ --result-file "${ASYNC_BACKUP_STATUS_FILE}" } @@ -555,12 +610,16 @@ wait_for_api_healthz_ready() { } "${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}" +if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then + stop_runtime_services_for_rollout_gate +fi case "${BACKUP_MODE}" in async) prepare_async_backup ;; sync) + SYNC_BACKUP_RESTART_SERVICE_ARGS=() BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs" if [[ ! -f "${BACKUP_SCRIPT}" ]]; then BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" @@ -569,6 +628,9 @@ case "${BACKUP_MODE}" in echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2 exit 1 fi + if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then + SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service) + fi echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布" node -- "${BACKUP_SCRIPT}" \ @@ -576,7 +638,7 @@ case "${BACKUP_MODE}" in --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ --stop-service spacetimedb.service \ - --restart-service-after genarrative-api.service + "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}" ;; skip) echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份" @@ -596,7 +658,8 @@ PUBLISH_ARGS=( publish "${DATABASE}" --bin-path "${SOURCE_DIR}/spacetime_module.wasm" - --yes + --delete-data=never + --yes=migrate,break-clients --no-config ) @@ -606,10 +669,6 @@ else PUBLISH_ARGS+=(--server "${SERVER_ALIAS}") fi -if [[ "${CLEAR_DATABASE}" -eq 1 ]]; then - PUBLISH_ARGS+=(--clear-database) -fi - if [[ -n "${SERVER_URL}" ]]; then echo "[production-stdb-publish] 发布 SpacetimeDB module: ${DATABASE} -> ${SERVER_URL}, root=${SPACETIME_ROOT_DIR}" else @@ -629,7 +688,8 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then publish "${DATABASE}" --bin-path "${PUBLISH_TMP_DIR}/spacetime_module.wasm" - --yes + --delete-data=never + --yes=migrate,break-clients --no-config ) if [[ -n "${SERVER_URL}" ]]; then @@ -637,9 +697,6 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then else PUBLISH_ARGS+=(--server "${SERVER_ALIAS}") fi - if [[ "${CLEAR_DATABASE}" -eq 1 ]]; then - PUBLISH_ARGS+=(--clear-database) - fi runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}" else spacetime "${PUBLISH_ARGS[@]}" @@ -668,6 +725,11 @@ run_privileged runuser -u genarrative -- test -r "${RUNTIME_SERVICE_BOOTSTRAP_SE echo "[production-stdb-publish] 已安装运行时服务身份引导密钥: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" ensure_runtime_bootstrap_secret_env_file "${API_ENV_FILE}" true ensure_runtime_bootstrap_secret_env_file "${WORKER_ENV_FILE}" false +if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then + echo "[production-stdb-publish] module 发布完成；按参数保持维护模式和旧运行时服务停止状态，等待人工维护与 API deploy。" + DEPLOY_COMPLETED=1 + exit 0 +fi restart_runtime_services_after_bootstrap_secret_install wait_for_api_healthz_ready diff --git a/scripts/spacetime-maintain-external-generation-jobs.mjs b/scripts/spacetime-maintain-external-generation-jobs.mjs new file mode 100644 index 000000000..69ead2f3c --- /dev/null +++ b/scripts/spacetime-maintain-external-generation-jobs.mjs @@ -0,0 +1,162 @@ +#!/usr/bin/env node + +import { + callSpacetimeProcedureViaCli, + encodeSpacetimeCliOption, + ensureProcedureOk, + parsePositiveInteger, +} from './spacetime-migration-common.mjs'; + +const MAX_BATCH_SIZE = 25; + +function usage() { + return `用法： + node scripts/spacetime-maintain-external-generation-jobs.mjs --database [选项] + +默认只 dry-run 一批历史终态任务 payload 压缩，不修改数据库。 + +公共选项： + --database 目标数据库（必填，也可用 GENARRATIVE_SPACETIME_DATABASE） + --server spacetime CLI server 名或 URL + --server-url 显式 server URL + --limit <1-${MAX_BATCH_SIZE}> 单批任务数，默认 10 + --cursor-job-id 从上一批 next_cursor_job_id 继续 + --apply 执行写入；省略时始终 dry-run + --backfill-summaries 改为回填轻量摘要投影 + --owner-user-id 仅摘要回填可选，限定 owner + --completed-before-micros 仅 payload 压缩可选，限定终态完成时间 + --help 显示帮助 + +必须使用已授权 migration operator 的 spacetime CLI 登录态。脚本每次只处理一批； +根据返回的 next_cursor_job_id 与 has_more 手工继续，避免在生产一次长事务扫完整历史。`; +} + +function parseOptions(argv) { + const options = { + apply: false, + backfillSummaries: false, + completedBeforeMicros: null, + cursorJobId: '', + database: process.env.GENARRATIVE_SPACETIME_DATABASE || '', + limit: 10, + ownerUserId: '', + passthrough: [], + server: process.env.GENARRATIVE_SPACETIME_SERVER || '', + serverUrl: process.env.GENARRATIVE_SPACETIME_SERVER_URL || '', + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const readValue = (name) => { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error(`${name} 缺少参数值。`); + } + index += 1; + return value; + }; + + if (arg === '--database') { + options.database = readValue(arg); + } else if (arg === '--server') { + options.server = readValue(arg); + } else if (arg === '--server-url') { + options.serverUrl = readValue(arg); + } else if (arg === '--limit') { + options.limit = parsePositiveInteger(readValue(arg), arg); + } else if (arg === '--cursor-job-id') { + options.cursorJobId = readValue(arg).trim(); + } else if (arg === '--completed-before-micros') { + const value = readValue(arg); + if (!/^-?[0-9]+$/u.test(value)) { + throw new Error(`${arg} 必须是整数。`); + } + const parsed = Number.parseInt(value, 10); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${arg} 超出 JavaScript 安全整数范围。`); + } + options.completedBeforeMicros = parsed; + } else if (arg === '--owner-user-id') { + options.ownerUserId = readValue(arg).trim(); + } else if (arg === '--apply') { + options.apply = true; + } else if (arg === '--backfill-summaries') { + options.backfillSummaries = true; + } else if (arg === '--help' || arg === '-h') { + options.help = true; + } else { + throw new Error(`未知参数: ${arg}`); + } + } + + if (options.limit > MAX_BATCH_SIZE) { + throw new Error(`--limit 不能超过 ${MAX_BATCH_SIZE}。`); + } + if (options.ownerUserId && !options.backfillSummaries) { + throw new Error('--owner-user-id 只能与 --backfill-summaries 一起使用。'); + } + if (options.completedBeforeMicros !== null && options.backfillSummaries) { + throw new Error('--completed-before-micros 不能用于摘要回填。'); + } + return options; +} + +try { + const options = parseOptions(process.argv.slice(2)); + if (options.help) { + console.log(usage()); + process.exit(0); + } + if (!options.database) { + throw new Error( + '必须传入 --database，或设置 GENARRATIVE_SPACETIME_DATABASE。', + ); + } + + const procedureName = options.backfillSummaries + ? 'backfill_external_generation_job_summaries_and_return' + : 'compact_external_generation_job_payloads_and_return'; + const input = options.backfillSummaries + ? { + owner_user_id: encodeSpacetimeCliOption(options.ownerUserId || null), + limit: options.limit, + cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), + dry_run: !options.apply, + } + : { + dry_run: !options.apply, + limit: options.limit, + cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), + completed_before_micros: encodeSpacetimeCliOption( + options.completedBeforeMicros, + ), + }; + const result = await callSpacetimeProcedureViaCli( + options, + procedureName, + input, + ); + ensureProcedureOk(result); + + console.log(JSON.stringify({ procedure: procedureName, ...result }, null, 2)); + const pendingApplyCount = options.backfillSummaries + ? Number(result.selected_count ?? 0) + : Number(result.matched_count ?? 0); + if (result.has_more && options.apply) { + console.log( + `仍有后续批次；下一次追加 --cursor-job-id ${result.next_cursor_job_id ?? ''}。`, + ); + } else if (!options.apply && (result.has_more || pendingApplyCount > 0)) { + const currentCursor = options.cursorJobId + ? `保留 --cursor-job-id ${options.cursorJobId}` + : '仍从首批开始'; + console.log( + `当前仅 dry-run；请${currentCursor}并追加 --apply 重跑同一批。apply 成功后再使用其 next_cursor_job_id 进入下一批。`, + ); + } +} catch (error) { + console.error( + `[spacetime:external-generation:maintenance] ${error instanceof Error ? error.message : String(error)}`, + ); + process.exit(1); +} diff --git a/scripts/spacetime-migration-common.mjs b/scripts/spacetime-migration-common.mjs index 62d7d0037..0dc8d3fd1 100644 --- a/scripts/spacetime-migration-common.mjs +++ b/scripts/spacetime-migration-common.mjs @@ -109,6 +109,10 @@ export function parsePositiveInteger(value, name) { return parsed; } +export function encodeSpacetimeCliOption(value) { + return value === null || value === undefined ? null : [0, value]; +} + function parseOptionalPositiveInteger(value, name) { if (!value) { return 0; @@ -171,7 +175,7 @@ export async function callSpacetimeProcedure(options, procedureName, input) { ); } - return parseProcedureResult(text); + return parseProcedureResult(text, procedureName); } export async function createSpacetimeWebIdentity(options) { @@ -225,7 +229,7 @@ export async function callSpacetimeProcedureAuto(options, procedureName, input) export async function callSpacetimeProcedureViaCli(options, procedureName, input) { const args = buildSpacetimeCallArgs(options, procedureName, input); const output = await runSpacetimeCli(args); - return parseProcedureResult(output); + return parseProcedureResult(output, procedureName); } export function validateSpacetimeDatabaseName(database) { @@ -236,7 +240,7 @@ export function validateSpacetimeDatabaseName(database) { } } -export function parseProcedureResult(output) { +export function parseProcedureResult(output, procedureName = '') { const candidates = []; const trimmed = output.trim(); if (trimmed) { @@ -252,7 +256,7 @@ export function parseProcedureResult(output) { for (const candidate of candidates) { try { - return normalizeProcedureResult(JSON.parse(candidate)); + return normalizeProcedureResult(JSON.parse(candidate), procedureName); } catch { // SpacetimeDB CLI 在不同版本中可能附带说明文本，继续尝试后续候选。 } @@ -275,19 +279,55 @@ export async function assertReadableFile(filePath) { await access(path.resolve(filePath)); } -function normalizeProcedureResult(value) { +function normalizeProcedureResult(value, procedureName) { if (value && typeof value === 'object' && !Array.isArray(value)) { return value; } if (Array.isArray(value)) { - return normalizeSatsProduct(value); + return normalizeSatsProduct(value, procedureName); } throw new Error('procedure 返回值不是对象。'); } -function normalizeSatsProduct(value) { +function normalizeSatsProduct(value, procedureName) { + if ( + procedureName === 'backfill_external_generation_job_summaries_and_return' && + value.length === 8 + ) { + return { + ok: normalizeSatsValue(value[0]), + dry_run: normalizeSatsValue(value[1]), + scanned_count: normalizeSatsValue(value[2]), + selected_count: normalizeSatsValue(value[3]), + upserted_count: normalizeSatsValue(value[4]), + next_cursor_job_id: normalizeSatsOption(value[5]), + has_more: normalizeSatsValue(value[6]), + error_message: normalizeSatsOption(value[7]), + }; + } + + if ( + procedureName === 'compact_external_generation_job_payloads_and_return' && + value.length === 12 + ) { + return { + ok: normalizeSatsValue(value[0]), + dry_run: normalizeSatsValue(value[1]), + scanned_count: normalizeSatsValue(value[2]), + matched_count: normalizeSatsValue(value[3]), + updated_count: normalizeSatsValue(value[4]), + before_bytes: normalizeSatsValue(value[5]), + after_bytes: normalizeSatsValue(value[6]), + inline_media_count: normalizeSatsValue(value[7]), + invalid_json_count: normalizeSatsValue(value[8]), + next_cursor_job_id: normalizeSatsOption(value[9]), + has_more: normalizeSatsValue(value[10]), + error_message: normalizeSatsOption(value[11]), + }; + } + if (value.length === 3) { return { ok: normalizeSatsValue(value[0]), diff --git a/scripts/spacetime-migration-common.test.ts b/scripts/spacetime-migration-common.test.ts new file mode 100644 index 000000000..02219e50b --- /dev/null +++ b/scripts/spacetime-migration-common.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { + buildSpacetimeCallArgs, + encodeSpacetimeCliOption, +} from './spacetime-migration-common.mjs'; + +describe('SpacetimeDB CLI SATS option encoding', () => { + it('keeps absent options null and wraps present values as Some', () => { + expect(encodeSpacetimeCliOption(null)).toBeNull(); + expect(encodeSpacetimeCliOption(undefined)).toBeNull(); + expect(encodeSpacetimeCliOption('job-2')).toEqual([0, 'job-2']); + expect(encodeSpacetimeCliOption(123)).toEqual([0, 123]); + }); + + it('serializes non-empty maintenance cursors in the CLI procedure input', () => { + const args = buildSpacetimeCallArgs( + { + database: 'genarrative-prod', + passthrough: [], + serverUrl: 'http://127.0.0.1:3311', + }, + 'compact_external_generation_job_payloads_and_return', + { + dry_run: true, + limit: 1, + cursor_job_id: encodeSpacetimeCliOption('job-2'), + completed_before_micros: encodeSpacetimeCliOption(123), + }, + ); + + const input = JSON.parse(args.at(-2) ?? 'null'); + expect(input.cursor_job_id).toEqual([0, 'job-2']); + expect(input.completed_before_micros).toEqual([0, 123]); + }); +}); diff --git a/server-rs/crates/api-server/src/editor_generation_queue.rs b/server-rs/crates/api-server/src/editor_generation_queue.rs index 3d6a24b52..e58ea8dc9 100644 --- a/server-rs/crates/api-server/src/editor_generation_queue.rs +++ b/server-rs/crates/api-server/src/editor_generation_queue.rs @@ -1,6 +1,6 @@ use axum::http::StatusCode; use serde::Serialize; -use serde_json::json; +use serde_json::{Value, json}; use shared_contracts::external_generation::{ ExternalGenerationJobStatus, ExternalGenerationJobStatusRecord, }; @@ -25,6 +25,7 @@ pub(crate) const EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND: &str = pub(crate) const EDITOR_GENERATION_QUEUE_SOURCE_MODULE: &str = "editor-canvas"; const EDITOR_GENERATION_QUEUE_PROVIDER: &str = "editor-generation-worker"; +const MAX_EDITOR_GENERATION_JOB_PAYLOAD_BYTES: usize = 512 * 1024; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -46,12 +47,7 @@ where T: Serialize, { let job_id = build_prefixed_uuid_id("task-"); - let request_payload_json = serde_json::to_string(payload).map_err(|error| { - AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ - "provider": EDITOR_GENERATION_QUEUE_PROVIDER, - "message": format!("编辑器 worker 任务参数序列化失败：{error}"), - })) - })?; + let request_payload_json = serialize_editor_generation_job_payload(payload)?; let now_micros = current_utc_micros(); state .spacetime_client() @@ -78,6 +74,64 @@ where }) } +fn serialize_editor_generation_job_payload(payload: &T) -> Result +where + T: Serialize + ?Sized, +{ + let payload_value = serde_json::to_value(payload).map_err(payload_serialization_error)?; + if contains_inline_media_reference(&payload_value) { + return Err( + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": EDITOR_GENERATION_QUEUE_PROVIDER, + "message": "编辑器生成任务参数禁止包含 data: 或 blob: 内联媒体引用，请先将媒体上传到对象存储并改传 objectKey 或 resourceId。", + })), + ); + } + + let request_payload_json = + serde_json::to_string(&payload_value).map_err(payload_serialization_error)?; + let payload_bytes = request_payload_json.len(); + if payload_bytes > MAX_EDITOR_GENERATION_JOB_PAYLOAD_BYTES { + return Err( + AppError::from_status(StatusCode::PAYLOAD_TOO_LARGE).with_details(json!({ + "provider": EDITOR_GENERATION_QUEUE_PROVIDER, + "message": format!( + "编辑器生成任务 JSON 大小为 {payload_bytes} 字节，超过持久化上限 {MAX_EDITOR_GENERATION_JOB_PAYLOAD_BYTES} 字节；请移除冗余数据并改传 objectKey 或 resourceId。" + ), + "actualBytes": payload_bytes, + "maxBytes": MAX_EDITOR_GENERATION_JOB_PAYLOAD_BYTES, + })), + ); + } + + Ok(request_payload_json) +} + +fn payload_serialization_error(error: serde_json::Error) -> AppError { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "provider": EDITOR_GENERATION_QUEUE_PROVIDER, + "message": format!("编辑器 worker 任务参数序列化失败：{error}"), + })) +} + +fn contains_inline_media_reference(value: &Value) -> bool { + match value { + Value::String(value) => is_inline_media_reference(value), + Value::Array(values) => values.iter().any(contains_inline_media_reference), + Value::Object(values) => values.iter().any(|(key, value)| { + is_inline_media_reference(key) || contains_inline_media_reference(value) + }), + Value::Null | Value::Bool(_) | Value::Number(_) => false, + } +} + +fn is_inline_media_reference(value: &str) -> bool { + let prefix = value.trim_start().as_bytes().get(..5); + prefix.is_some_and(|prefix| { + prefix.eq_ignore_ascii_case(b"data:") || prefix.eq_ignore_ascii_case(b"blob:") + }) +} + pub(crate) fn editor_generation_queue_state( job: ExternalGenerationJobRecord, ) -> ExternalGenerationJobStatusRecord { @@ -106,3 +160,68 @@ pub(crate) fn editor_generation_source_entity_id( fn current_utc_micros() -> i64 { offset_datetime_to_unix_micros(time::OffsetDateTime::now_utc()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn serialize_payload_accepts_persistable_media_references() { + let payload = json!({ + "sourceImageObjectKey": "users/user-1/editor/source.png", + "resourceId": "resource-1", + "nested": [{ "prompt": "保留 data 与 blob 这两个普通单词" }], + }); + + let serialized = serialize_editor_generation_job_payload(&payload) + .expect("objectKey 和 resourceId 应允许进入持久任务 JSON"); + + assert_eq!( + serde_json::from_str::(&serialized).expect("应生成有效 JSON"), + payload + ); + } + + #[test] + fn serialize_payload_rejects_nested_data_url_case_insensitively() { + let payload = json!({ + "input": { + "references": [ + { "url": " \nDaTa:image/png;base64,AAAA" } + ] + } + }); + + let error = serialize_editor_generation_job_payload(&payload) + .expect_err("任意层级的 Data URL 都必须被拒绝"); + + assert_eq!(error.status_code(), StatusCode::BAD_REQUEST); + assert!(error.body_text().contains("禁止包含 data: 或 blob:")); + } + + #[test] + fn serialize_payload_rejects_nested_blob_url_case_insensitively() { + let payload = json!({ + "input": [{ "source": { "url": "\tBLOB:https://example.test/id" } }] + }); + + let error = serialize_editor_generation_job_payload(&payload) + .expect_err("任意层级的 Blob URL 都必须被拒绝"); + + assert_eq!(error.status_code(), StatusCode::BAD_REQUEST); + assert!(error.body_text().contains("禁止包含 data: 或 blob:")); + } + + #[test] + fn serialize_payload_rejects_json_larger_than_persistence_limit() { + let payload = json!({ + "prompt": "x".repeat(MAX_EDITOR_GENERATION_JOB_PAYLOAD_BYTES), + }); + + let error = serialize_editor_generation_job_payload(&payload) + .expect_err("超过上限的任务 JSON 必须被拒绝"); + + assert_eq!(error.status_code().as_u16(), 413); + assert!(error.body_text().contains("超过持久化上限")); + } +} diff --git a/server-rs/crates/api-server/src/external_generation.rs b/server-rs/crates/api-server/src/external_generation.rs index 0757cbc5a..5d3bacc4b 100644 --- a/server-rs/crates/api-server/src/external_generation.rs +++ b/server-rs/crates/api-server/src/external_generation.rs @@ -5,7 +5,7 @@ use axum::{ response::Response, }; use serde::Deserialize; -use serde_json::{Value, json}; +use serde_json::json; use shared_contracts::external_generation::{ ExternalGenerationJobStatus, ExternalGenerationJobStatusRecord, ExternalGenerationJobStatusResponse, ExternalGenerationQueueOverview, @@ -15,8 +15,8 @@ use shared_contracts::external_generation::{ }; use spacetime_client::{ ExternalGenerationJobAcknowledgeRecordInput, ExternalGenerationJobGetRecordInput, - ExternalGenerationJobListRecord, ExternalGenerationJobListRecordInput, - ExternalGenerationJobRecord, SpacetimeClientError, + ExternalGenerationJobListRecordInput, ExternalGenerationJobSummaryListRecord, + ExternalGenerationJobSummaryRecord, SpacetimeClientError, }; use crate::{ @@ -42,7 +42,7 @@ pub async fn get_external_generation_queue_overview( let owner_user_id = authenticated.claims().user_id().to_string(); let list = state .spacetime_client() - .list_external_generation_jobs(ExternalGenerationJobListRecordInput { + .list_external_generation_job_summaries(ExternalGenerationJobListRecordInput { owner_user_id, limit: 1, include_acknowledged_terminal: false, @@ -71,7 +71,7 @@ pub async fn list_external_generation_tasks( let statuses = external_generation_status_filter_input(&status_filter); let list = state .spacetime_client() - .list_external_generation_jobs(ExternalGenerationJobListRecordInput { + .list_external_generation_job_summaries(ExternalGenerationJobListRecordInput { owner_user_id, limit: requested_limit, include_acknowledged_terminal: query.include_acknowledged_terminal.unwrap_or(false), @@ -105,11 +105,13 @@ pub async fn acknowledge_external_generation_tasks( let owner_user_id = authenticated.claims().user_id().to_string(); let acknowledged = state .spacetime_client() - .acknowledge_external_generation_jobs(ExternalGenerationJobAcknowledgeRecordInput { - owner_user_id, - job_ids: payload.job_ids, - acknowledged_at_micros: current_utc_micros(), - }) + .acknowledge_external_generation_job_summaries( + ExternalGenerationJobAcknowledgeRecordInput { + owner_user_id, + job_ids: payload.job_ids, + acknowledged_at_micros: current_utc_micros(), + }, + ) .await .map_err(|error| external_generation_error_response(&request_context, error))?; @@ -134,7 +136,7 @@ pub async fn get_external_generation_job_status( let owner_user_id = authenticated.claims().user_id().to_string(); let job = state .spacetime_client() - .get_external_generation_job(ExternalGenerationJobGetRecordInput { + .get_external_generation_job_summary(ExternalGenerationJobGetRecordInput { job_id, owner_user_id, }) @@ -150,7 +152,7 @@ pub async fn get_external_generation_job_status( } fn map_external_generation_queue_overview( - list: &ExternalGenerationJobListRecord, + list: &ExternalGenerationJobSummaryListRecord, ) -> ExternalGenerationQueueOverview { ExternalGenerationQueueOverview { pending_count: list.pending_count, @@ -161,7 +163,7 @@ fn map_external_generation_queue_overview( } fn map_external_generation_job_status( - job: ExternalGenerationJobRecord, + job: ExternalGenerationJobSummaryRecord, ) -> ExternalGenerationJobStatusRecord { let (status, phase_detail, progress) = match job.status.as_str() { "completed" => (ExternalGenerationJobStatus::Completed, "生成已完成。", 100), @@ -182,17 +184,16 @@ fn map_external_generation_job_status( } fn map_external_generation_task_record( - job: ExternalGenerationJobRecord, + job: ExternalGenerationJobSummaryRecord, ) -> ExternalGenerationTaskRecord { let status_record = map_external_generation_job_status(job.clone()); - let request_prompt = extract_external_generation_request_prompt(&job.request_payload_json); ExternalGenerationTaskRecord { job_id: job.job_id, job_kind: job.job_kind, source_module: job.source_module, source_entity_id: job.source_entity_id, request_label: job.request_label, - request_prompt, + request_prompt: job.request_prompt, status: status_record.status, phase_label: status_record.phase_label, phase_detail: status_record.phase_detail, @@ -209,59 +210,6 @@ fn map_external_generation_task_record( } } -fn extract_external_generation_request_prompt(request_payload_json: &str) -> Option { - let payload: Value = serde_json::from_str(request_payload_json).ok()?; - for key in ["prompt", "promptText", "spritesheetLabel"] { - if let Some(prompt) = payload - .get(key) - .and_then(Value::as_str) - .and_then(normalize_external_generation_prompt_text) - { - return Some(prompt); - } - } - if let Some(prompt) = payload - .get("iconDescriptions") - .and_then(Value::as_array) - .map(|items| { - items - .iter() - .filter_map(Value::as_str) - .filter_map(normalize_external_generation_prompt_text) - .collect::>() - .join("、") - }) - .and_then(|value| normalize_external_generation_prompt_text(&value)) - { - return Some(prompt); - } - payload - .get("generationInputs") - .and_then(|value| value.get("fields")) - .and_then(Value::as_array) - .and_then(|fields| { - fields.iter().find_map(|field| { - let title = field.get("title").and_then(Value::as_str)?.trim(); - if !matches!(title, "prompt" | "gpt_description_prompt") { - return None; - } - field - .get("value") - .and_then(Value::as_str) - .and_then(normalize_external_generation_prompt_text) - }) - }) -} - -fn normalize_external_generation_prompt_text(value: &str) -> Option { - let trimmed = value.trim(); - if trimmed.is_empty() { - None - } else { - Some(trimmed.to_string()) - } -} - fn parse_external_generation_status_filter( statuses: Option<&str>, ) -> Vec { @@ -325,20 +273,29 @@ mod tests { } #[test] - fn extracts_external_generation_request_prompt_from_payload() { - assert_eq!( - extract_external_generation_request_prompt( - r#"{"prompt":" 发光主视觉 ","sourceImageSrc":"data:image/png;base64,secret"}"#, - ) - .as_deref(), - Some("发光主视觉"), - ); - assert_eq!( - extract_external_generation_request_prompt( - r#"{"iconDescriptions":["返回按钮"," 设置按钮 "],"referenceImageSrc":"data:image/png;base64,secret"}"#, - ) - .as_deref(), - Some("返回按钮、设置按钮"), - ); + fn maps_task_from_payload_free_summary_projection() { + let task = map_external_generation_task_record(ExternalGenerationJobSummaryRecord { + job_id: "task-1".to_string(), + job_kind: "editor_image_generation".to_string(), + owner_user_id: "user-1".to_string(), + source_module: "editor-canvas".to_string(), + source_entity_id: "project-1".to_string(), + request_label: "图片生成".to_string(), + request_prompt: Some("发光主视觉".to_string()), + status: "completed".to_string(), + last_error_message: None, + created_at: "2026-07-10T08:00:00Z".to_string(), + started_at: Some("2026-07-10T08:00:01Z".to_string()), + completed_at: Some("2026-07-10T08:00:10Z".to_string()), + updated_at: "2026-07-10T08:00:10Z".to_string(), + updated_at_micros: 1_000, + price_mud_points: 4, + refund_ledger_id: None, + notification_acknowledged_at: None, + notification_acknowledged_at_micros: None, + }); + + assert_eq!(task.request_prompt.as_deref(), Some("发光主视觉")); + assert_eq!(task.status, ExternalGenerationJobStatus::Completed); } } diff --git a/server-rs/crates/spacetime-client/src/external_generation.rs b/server-rs/crates/spacetime-client/src/external_generation.rs index fd217fb35..6d70087ea 100644 --- a/server-rs/crates/spacetime-client/src/external_generation.rs +++ b/server-rs/crates/spacetime-client/src/external_generation.rs @@ -307,6 +307,31 @@ impl SpacetimeClient { .await } + pub async fn get_external_generation_job_summary( + &self, + input: ExternalGenerationJobGetRecordInput, + ) -> Result { + let procedure_input = input.into(); + + self.call_after_connect( + "get_external_generation_job_summary_and_return", + move |connection, sender| { + connection + .procedures() + .get_external_generation_job_summary_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_job_summary_procedure_result); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + pub async fn list_external_generation_jobs( &self, input: ExternalGenerationJobListRecordInput, @@ -332,6 +357,31 @@ impl SpacetimeClient { .await } + pub async fn list_external_generation_job_summaries( + &self, + input: ExternalGenerationJobListRecordInput, + ) -> Result { + let procedure_input = input.into(); + + self.call_after_connect( + "list_external_generation_job_summaries_and_return", + move |connection, sender| { + connection + .procedures() + .list_external_generation_job_summaries_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_job_summary_list_result); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + pub async fn acknowledge_external_generation_jobs( &self, input: ExternalGenerationJobAcknowledgeRecordInput, @@ -357,6 +407,31 @@ impl SpacetimeClient { .await } + pub async fn acknowledge_external_generation_job_summaries( + &self, + input: ExternalGenerationJobAcknowledgeRecordInput, + ) -> Result { + let procedure_input = input.into(); + + self.call_after_connect( + "acknowledge_external_generation_job_summaries_and_return", + move |connection, sender| { + connection + .procedures() + .acknowledge_external_generation_job_summaries_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_job_summary_list_result); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + pub async fn get_external_generation_queue_stats( &self, ) -> Result { diff --git a/server-rs/crates/spacetime-client/src/lib.rs b/server-rs/crates/spacetime-client/src/lib.rs index dbbf9f247..2ae6f57ca 100644 --- a/server-rs/crates/spacetime-client/src/lib.rs +++ b/server-rs/crates/spacetime-client/src/lib.rs @@ -57,7 +57,8 @@ pub use mapper::{ ExternalGenerationJobEnqueueRecordInput, ExternalGenerationJobFailRecordInput, ExternalGenerationJobGetRecordInput, ExternalGenerationJobListRecord, ExternalGenerationJobListRecordInput, ExternalGenerationJobRecord, - ExternalGenerationJobRenewLeaseRecordInput, ExternalGenerationQueueStatsRecord, + ExternalGenerationJobRenewLeaseRecordInput, ExternalGenerationJobSummaryListRecord, + ExternalGenerationJobSummaryRecord, ExternalGenerationQueueStatsRecord, FeatureGateConfigRecord, JumpHopActionRequest, JumpHopActionResponse, JumpHopActionType, JumpHopCharacterAsset, JumpHopDifficulty, JumpHopDraftResponse, JumpHopGalleryCardResponse, JumpHopGalleryDetailResponse, JumpHopGalleryResponse, JumpHopGenerationStatus, diff --git a/server-rs/crates/spacetime-client/src/mapper.rs b/server-rs/crates/spacetime-client/src/mapper.rs index 36e68f966..ae78a9496 100644 --- a/server-rs/crates/spacetime-client/src/mapper.rs +++ b/server-rs/crates/spacetime-client/src/mapper.rs @@ -107,6 +107,7 @@ pub use self::external_generation::{ ExternalGenerationJobFailRecordInput, ExternalGenerationJobGetRecordInput, ExternalGenerationJobListRecord, ExternalGenerationJobListRecordInput, ExternalGenerationJobRecord, ExternalGenerationJobRenewLeaseRecordInput, + ExternalGenerationJobSummaryListRecord, ExternalGenerationJobSummaryRecord, ExternalGenerationQueueStatsRecord, }; pub use self::jump_hop::{ @@ -244,7 +245,9 @@ pub(crate) use self::external_api_key::{ }; pub(crate) use self::external_generation::{ map_external_generation_job_claim_result, map_external_generation_job_list_result, - map_external_generation_job_procedure_result, map_external_generation_queue_stats_result, + map_external_generation_job_procedure_result, map_external_generation_job_summary_list_result, + map_external_generation_job_summary_procedure_result, + map_external_generation_queue_stats_result, }; pub(crate) use self::inventory::{ map_runtime_inventory_state_procedure_result, map_runtime_item_reward_item_snapshot, diff --git a/server-rs/crates/spacetime-client/src/mapper/external_generation.rs b/server-rs/crates/spacetime-client/src/mapper/external_generation.rs index bbb775796..fa1901aac 100644 --- a/server-rs/crates/spacetime-client/src/mapper/external_generation.rs +++ b/server-rs/crates/spacetime-client/src/mapper/external_generation.rs @@ -146,6 +146,39 @@ pub(crate) fn map_external_generation_job_list_result( }) } +pub(crate) fn map_external_generation_job_summary_procedure_result( + result: ExternalGenerationJobSummaryProcedureResult, +) -> Result { + if !result.ok { + return Err(SpacetimeClientError::procedure_failed(result.error_message)); + } + + let job = result.job.ok_or_else(|| { + SpacetimeClientError::missing_snapshot("external_generation_job_summary 快照") + })?; + Ok(map_external_generation_job_summary_snapshot(job)) +} + +pub(crate) fn map_external_generation_job_summary_list_result( + result: ExternalGenerationJobSummaryProcedureResult, +) -> Result { + if !result.ok { + return Err(SpacetimeClientError::procedure_failed(result.error_message)); + } + + Ok(ExternalGenerationJobSummaryListRecord { + jobs: result + .jobs + .into_iter() + .map(map_external_generation_job_summary_snapshot) + .collect(), + pending_count: result.pending_count, + running_count: result.running_count, + unacknowledged_terminal_count: result.unacknowledged_terminal_count, + now_micros: result.now_micros, + }) +} + pub(crate) fn map_external_generation_queue_stats_result( result: ExternalGenerationQueueStatsProcedureResult, ) -> Result { @@ -207,6 +240,33 @@ fn map_external_generation_job_snapshot( } } +fn map_external_generation_job_summary_snapshot( + snapshot: ExternalGenerationJobSummarySnapshot, +) -> ExternalGenerationJobSummaryRecord { + ExternalGenerationJobSummaryRecord { + job_id: snapshot.job_id, + job_kind: snapshot.job_kind, + owner_user_id: snapshot.owner_user_id, + source_module: snapshot.source_module, + source_entity_id: snapshot.source_entity_id, + request_label: snapshot.request_label, + request_prompt: snapshot.request_prompt, + status: snapshot.status, + last_error_message: snapshot.last_error_message, + created_at: format_timestamp_micros(snapshot.created_at_micros), + started_at: snapshot.started_at_micros.map(format_timestamp_micros), + completed_at: snapshot.completed_at_micros.map(format_timestamp_micros), + updated_at: format_timestamp_micros(snapshot.updated_at_micros), + updated_at_micros: snapshot.updated_at_micros, + price_mud_points: snapshot.price_mud_points, + refund_ledger_id: snapshot.refund_ledger_id, + notification_acknowledged_at: snapshot + .notification_acknowledged_at_micros + .map(format_timestamp_micros), + notification_acknowledged_at_micros: snapshot.notification_acknowledged_at_micros, + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ExternalGenerationJobEnqueueRecordInput { pub job_id: String, @@ -320,6 +380,37 @@ pub struct ExternalGenerationJobListRecord { pub now_micros: i64, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationJobSummaryRecord { + pub job_id: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_prompt: Option, + pub status: String, + pub last_error_message: Option, + pub created_at: String, + pub started_at: Option, + pub completed_at: Option, + pub updated_at: String, + pub updated_at_micros: i64, + pub price_mud_points: u64, + pub refund_ledger_id: Option, + pub notification_acknowledged_at: Option, + pub notification_acknowledged_at_micros: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationJobSummaryListRecord { + pub jobs: Vec, + pub pending_count: u32, + pub running_count: u32, + pub unacknowledged_terminal_count: u32, + pub now_micros: i64, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ExternalGenerationQueueStatsRecord { pub pending_count: u32, diff --git a/server-rs/crates/spacetime-client/src/module_bindings.rs b/server-rs/crates/spacetime-client/src/module_bindings.rs index eb9675a45..4e3c143d3 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings.rs @@ -7,6 +7,7 @@ use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; pub mod accept_quest_reducer; +pub mod acknowledge_external_generation_job_summaries_and_return_procedure; pub mod acknowledge_external_generation_jobs_and_return_procedure; pub mod acknowledge_quest_completion_reducer; pub mod admin_disable_profile_redeem_code_procedure; @@ -114,6 +115,7 @@ pub mod auth_store_projection_user_type; pub mod auth_store_projection_view_type; pub mod authenticate_external_api_key_and_return_procedure; pub mod authorize_database_migration_operator_procedure; +pub mod backfill_external_generation_job_summaries_and_return_procedure; pub mod bark_battle_draft_config_row_type; pub mod bark_battle_draft_config_snapshot_type; pub mod bark_battle_draft_config_table; @@ -225,6 +227,7 @@ pub mod clear_platform_browse_history_and_return_procedure; pub mod click_match_3_d_item_procedure; pub mod close_profile_recharge_order_and_return_procedure; pub mod combat_outcome_type; +pub mod compact_external_generation_job_payloads_and_return_procedure; pub mod compile_big_fish_draft_procedure; pub mod compile_custom_world_published_profile_procedure; pub mod compile_jump_hop_draft_procedure; @@ -480,9 +483,17 @@ pub mod external_generation_job_event_type; pub mod external_generation_job_fail_input_type; pub mod external_generation_job_get_input_type; pub mod external_generation_job_list_input_type; +pub mod external_generation_job_payload_compaction_input_type; +pub mod external_generation_job_payload_compaction_procedure_result_type; pub mod external_generation_job_procedure_result_type; pub mod external_generation_job_renew_lease_input_type; pub mod external_generation_job_snapshot_type; +pub mod external_generation_job_summary_backfill_input_type; +pub mod external_generation_job_summary_backfill_procedure_result_type; +pub mod external_generation_job_summary_procedure_result_type; +pub mod external_generation_job_summary_snapshot_type; +pub mod external_generation_job_summary_table; +pub mod external_generation_job_summary_type; pub mod external_generation_job_table; pub mod external_generation_job_type; pub mod external_generation_queue_stats_procedure_result_type; @@ -524,6 +535,7 @@ pub mod get_editor_generation_pricing_config_and_return_procedure; pub mod get_editor_project_and_return_procedure; pub mod get_editor_showcase_campaign_config_and_return_procedure; pub mod get_external_generation_job_and_return_procedure; +pub mod get_external_generation_job_summary_and_return_procedure; pub mod get_external_generation_queue_stats_and_return_procedure; pub mod get_feature_gate_config_procedure; pub mod get_jump_hop_agent_session_procedure; @@ -639,6 +651,7 @@ pub mod list_custom_world_works_procedure; pub mod list_editor_agent_conversations_and_return_procedure; pub mod list_editor_projects_and_return_procedure; pub mod list_external_api_keys_and_return_procedure; +pub mod list_external_generation_job_summaries_and_return_procedure; pub mod list_external_generation_jobs_and_return_procedure; pub mod list_jump_hop_works_procedure; pub mod list_match_3_d_works_procedure; @@ -1329,6 +1342,7 @@ pub mod wooden_fish_works_list_input_type; pub mod wooden_fish_works_procedure_result_type; pub use accept_quest_reducer::accept_quest; +pub use acknowledge_external_generation_job_summaries_and_return_procedure::acknowledge_external_generation_job_summaries_and_return; pub use acknowledge_external_generation_jobs_and_return_procedure::acknowledge_external_generation_jobs_and_return; pub use acknowledge_quest_completion_reducer::acknowledge_quest_completion; pub use admin_disable_profile_redeem_code_procedure::admin_disable_profile_redeem_code; @@ -1436,6 +1450,7 @@ pub use auth_store_projection_user_type::AuthStoreProjectionUser; pub use auth_store_projection_view_type::AuthStoreProjectionView; pub use authenticate_external_api_key_and_return_procedure::authenticate_external_api_key_and_return; pub use authorize_database_migration_operator_procedure::authorize_database_migration_operator; +pub use backfill_external_generation_job_summaries_and_return_procedure::backfill_external_generation_job_summaries_and_return; pub use bark_battle_draft_config_row_type::BarkBattleDraftConfigRow; pub use bark_battle_draft_config_snapshot_type::BarkBattleDraftConfigSnapshot; pub use bark_battle_draft_config_table::*; @@ -1547,6 +1562,7 @@ pub use clear_platform_browse_history_and_return_procedure::clear_platform_brows pub use click_match_3_d_item_procedure::click_match_3_d_item; pub use close_profile_recharge_order_and_return_procedure::close_profile_recharge_order_and_return; pub use combat_outcome_type::CombatOutcome; +pub use compact_external_generation_job_payloads_and_return_procedure::compact_external_generation_job_payloads_and_return; pub use compile_big_fish_draft_procedure::compile_big_fish_draft; pub use compile_custom_world_published_profile_procedure::compile_custom_world_published_profile; pub use compile_jump_hop_draft_procedure::compile_jump_hop_draft; @@ -1802,9 +1818,17 @@ pub use external_generation_job_event_type::ExternalGenerationJobEvent; pub use external_generation_job_fail_input_type::ExternalGenerationJobFailInput; pub use external_generation_job_get_input_type::ExternalGenerationJobGetInput; pub use external_generation_job_list_input_type::ExternalGenerationJobListInput; +pub use external_generation_job_payload_compaction_input_type::ExternalGenerationJobPayloadCompactionInput; +pub use external_generation_job_payload_compaction_procedure_result_type::ExternalGenerationJobPayloadCompactionProcedureResult; pub use external_generation_job_procedure_result_type::ExternalGenerationJobProcedureResult; pub use external_generation_job_renew_lease_input_type::ExternalGenerationJobRenewLeaseInput; pub use external_generation_job_snapshot_type::ExternalGenerationJobSnapshot; +pub use external_generation_job_summary_backfill_input_type::ExternalGenerationJobSummaryBackfillInput; +pub use external_generation_job_summary_backfill_procedure_result_type::ExternalGenerationJobSummaryBackfillProcedureResult; +pub use external_generation_job_summary_procedure_result_type::ExternalGenerationJobSummaryProcedureResult; +pub use external_generation_job_summary_snapshot_type::ExternalGenerationJobSummarySnapshot; +pub use external_generation_job_summary_table::*; +pub use external_generation_job_summary_type::ExternalGenerationJobSummary; pub use external_generation_job_table::*; pub use external_generation_job_type::ExternalGenerationJob; pub use external_generation_queue_stats_procedure_result_type::ExternalGenerationQueueStatsProcedureResult; @@ -1846,6 +1870,7 @@ pub use get_editor_generation_pricing_config_and_return_procedure::get_editor_ge pub use get_editor_project_and_return_procedure::get_editor_project_and_return; pub use get_editor_showcase_campaign_config_and_return_procedure::get_editor_showcase_campaign_config_and_return; pub use get_external_generation_job_and_return_procedure::get_external_generation_job_and_return; +pub use get_external_generation_job_summary_and_return_procedure::get_external_generation_job_summary_and_return; pub use get_external_generation_queue_stats_and_return_procedure::get_external_generation_queue_stats_and_return; pub use get_feature_gate_config_procedure::get_feature_gate_config; pub use get_jump_hop_agent_session_procedure::get_jump_hop_agent_session; @@ -1961,6 +1986,7 @@ pub use list_custom_world_works_procedure::list_custom_world_works; pub use list_editor_agent_conversations_and_return_procedure::list_editor_agent_conversations_and_return; pub use list_editor_projects_and_return_procedure::list_editor_projects_and_return; pub use list_external_api_keys_and_return_procedure::list_external_api_keys_and_return; +pub use list_external_generation_job_summaries_and_return_procedure::list_external_generation_job_summaries_and_return; pub use list_external_generation_jobs_and_return_procedure::list_external_generation_jobs_and_return; pub use list_jump_hop_works_procedure::list_jump_hop_works; pub use list_match_3_d_works_procedure::list_match_3_d_works; @@ -2987,6 +3013,7 @@ pub struct DbUpdate { external_api_key: __sdk::TableUpdate, external_generation_job: __sdk::TableUpdate, external_generation_job_event: __sdk::TableUpdate, + external_generation_job_summary: __sdk::TableUpdate, feature_gate_config: __sdk::TableUpdate, inventory_slot: __sdk::TableUpdate, jump_hop_agent_session: __sdk::TableUpdate, @@ -3258,6 +3285,11 @@ impl TryFrom<__ws::v2::TransactionUpdate> for DbUpdate { "external_generation_job_event" => db_update.external_generation_job_event.append( external_generation_job_event_table::parse_table_update(table_update)?, ), + "external_generation_job_summary" => { + db_update.external_generation_job_summary.append( + external_generation_job_summary_table::parse_table_update(table_update)?, + ) + } "feature_gate_config" => db_update .feature_gate_config .append(feature_gate_config_table::parse_table_update(table_update)?), @@ -3813,6 +3845,12 @@ impl __sdk::DbUpdate for DbUpdate { &self.external_generation_job_event, ) .with_updates_by_pk(|row| &row.event_id); + diff.external_generation_job_summary = cache + .apply_diff_to_table::( + "external_generation_job_summary", + &self.external_generation_job_summary, + ) + .with_updates_by_pk(|row| &row.job_id); diff.feature_gate_config = cache .apply_diff_to_table::( "feature_gate_config", @@ -4412,6 +4450,9 @@ impl __sdk::DbUpdate for DbUpdate { "external_generation_job_event" => db_update .external_generation_job_event .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), + "external_generation_job_summary" => db_update + .external_generation_job_summary + .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), "feature_gate_config" => db_update .feature_gate_config .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), @@ -4836,6 +4877,9 @@ impl __sdk::DbUpdate for DbUpdate { "external_generation_job_event" => db_update .external_generation_job_event .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), + "external_generation_job_summary" => db_update + .external_generation_job_summary + .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), "feature_gate_config" => db_update .feature_gate_config .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), @@ -5159,6 +5203,7 @@ pub struct AppliedDiff<'r> { external_api_key: __sdk::TableAppliedDiff<'r, ExternalApiKey>, external_generation_job: __sdk::TableAppliedDiff<'r, ExternalGenerationJob>, external_generation_job_event: __sdk::TableAppliedDiff<'r, ExternalGenerationJobEvent>, + external_generation_job_summary: __sdk::TableAppliedDiff<'r, ExternalGenerationJobSummary>, feature_gate_config: __sdk::TableAppliedDiff<'r, FeatureGateConfig>, inventory_slot: __sdk::TableAppliedDiff<'r, InventorySlot>, jump_hop_agent_session: __sdk::TableAppliedDiff<'r, JumpHopAgentSessionRow>, @@ -5516,6 +5561,11 @@ impl<'r> __sdk::AppliedDiff<'r> for AppliedDiff<'r> { &self.external_generation_job_event, event, ); + callbacks.invoke_table_row_callbacks::( + "external_generation_job_summary", + &self.external_generation_job_summary, + event, + ); callbacks.invoke_table_row_callbacks::( "feature_gate_config", &self.feature_gate_config, @@ -6637,6 +6687,7 @@ impl __sdk::SpacetimeModule for RemoteModule { external_api_key_table::register_table(client_cache); external_generation_job_table::register_table(client_cache); external_generation_job_event_table::register_table(client_cache); + external_generation_job_summary_table::register_table(client_cache); feature_gate_config_table::register_table(client_cache); inventory_slot_table::register_table(client_cache); jump_hop_agent_session_table::register_table(client_cache); @@ -6776,6 +6827,7 @@ impl __sdk::SpacetimeModule for RemoteModule { "external_api_key", "external_generation_job", "external_generation_job_event", + "external_generation_job_summary", "feature_gate_config", "inventory_slot", "jump_hop_agent_session", diff --git a/server-rs/crates/spacetime-client/src/module_bindings/acknowledge_external_generation_job_summaries_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/acknowledge_external_generation_job_summaries_and_return_procedure.rs new file mode 100644 index 000000000..5ef0251b0 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/acknowledge_external_generation_job_summaries_and_return_procedure.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_acknowledge_input_type::ExternalGenerationJobAcknowledgeInput; +use super::external_generation_job_summary_procedure_result_type::ExternalGenerationJobSummaryProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct AcknowledgeExternalGenerationJobSummariesAndReturnArgs { + pub input: ExternalGenerationJobAcknowledgeInput, +} + +impl __sdk::InModule for AcknowledgeExternalGenerationJobSummariesAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `acknowledge_external_generation_job_summaries_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait acknowledge_external_generation_job_summaries_and_return { + fn acknowledge_external_generation_job_summaries_and_return( + &self, + input: ExternalGenerationJobAcknowledgeInput, + ) { + self.acknowledge_external_generation_job_summaries_and_return_then(input, |_, _| {}); + } + + fn acknowledge_external_generation_job_summaries_and_return_then( + &self, + input: ExternalGenerationJobAcknowledgeInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl acknowledge_external_generation_job_summaries_and_return for super::RemoteProcedures { + fn acknowledge_external_generation_job_summaries_and_return_then( + &self, + input: ExternalGenerationJobAcknowledgeInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobSummaryProcedureResult>( + "acknowledge_external_generation_job_summaries_and_return", + AcknowledgeExternalGenerationJobSummariesAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/backfill_external_generation_job_summaries_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/backfill_external_generation_job_summaries_and_return_procedure.rs new file mode 100644 index 000000000..8636a1587 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/backfill_external_generation_job_summaries_and_return_procedure.rs @@ -0,0 +1,61 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_summary_backfill_input_type::ExternalGenerationJobSummaryBackfillInput; +use super::external_generation_job_summary_backfill_procedure_result_type::ExternalGenerationJobSummaryBackfillProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct BackfillExternalGenerationJobSummariesAndReturnArgs { + pub input: ExternalGenerationJobSummaryBackfillInput, +} + +impl __sdk::InModule for BackfillExternalGenerationJobSummariesAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `backfill_external_generation_job_summaries_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait backfill_external_generation_job_summaries_and_return { + fn backfill_external_generation_job_summaries_and_return( + &self, + input: ExternalGenerationJobSummaryBackfillInput, + ) { + self.backfill_external_generation_job_summaries_and_return_then(input, |_, _| {}); + } + + fn backfill_external_generation_job_summaries_and_return_then( + &self, + input: ExternalGenerationJobSummaryBackfillInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl backfill_external_generation_job_summaries_and_return for super::RemoteProcedures { + fn backfill_external_generation_job_summaries_and_return_then( + &self, + input: ExternalGenerationJobSummaryBackfillInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp.invoke_procedure_with_callback::<_, ExternalGenerationJobSummaryBackfillProcedureResult>( + "backfill_external_generation_job_summaries_and_return", + BackfillExternalGenerationJobSummariesAndReturnArgs { input, }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/compact_external_generation_job_payloads_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/compact_external_generation_job_payloads_and_return_procedure.rs new file mode 100644 index 000000000..416c06161 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/compact_external_generation_job_payloads_and_return_procedure.rs @@ -0,0 +1,61 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_payload_compaction_input_type::ExternalGenerationJobPayloadCompactionInput; +use super::external_generation_job_payload_compaction_procedure_result_type::ExternalGenerationJobPayloadCompactionProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct CompactExternalGenerationJobPayloadsAndReturnArgs { + pub input: ExternalGenerationJobPayloadCompactionInput, +} + +impl __sdk::InModule for CompactExternalGenerationJobPayloadsAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `compact_external_generation_job_payloads_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait compact_external_generation_job_payloads_and_return { + fn compact_external_generation_job_payloads_and_return( + &self, + input: ExternalGenerationJobPayloadCompactionInput, + ) { + self.compact_external_generation_job_payloads_and_return_then(input, |_, _| {}); + } + + fn compact_external_generation_job_payloads_and_return_then( + &self, + input: ExternalGenerationJobPayloadCompactionInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl compact_external_generation_job_payloads_and_return for super::RemoteProcedures { + fn compact_external_generation_job_payloads_and_return_then( + &self, + input: ExternalGenerationJobPayloadCompactionInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp.invoke_procedure_with_callback::<_, ExternalGenerationJobPayloadCompactionProcedureResult>( + "compact_external_generation_job_payloads_and_return", + CompactExternalGenerationJobPayloadsAndReturnArgs { input, }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_payload_compaction_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_payload_compaction_input_type.rs new file mode 100644 index 000000000..06a2f380f --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_payload_compaction_input_type.rs @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobPayloadCompactionInput { + pub dry_run: bool, + pub limit: u32, + pub cursor_job_id: Option, + pub completed_before_micros: Option, +} + +impl __sdk::InModule for ExternalGenerationJobPayloadCompactionInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_payload_compaction_procedure_result_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_payload_compaction_procedure_result_type.rs new file mode 100644 index 000000000..134452fba --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_payload_compaction_procedure_result_type.rs @@ -0,0 +1,26 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobPayloadCompactionProcedureResult { + pub ok: bool, + pub dry_run: bool, + pub scanned_count: u64, + pub matched_count: u32, + pub updated_count: u32, + pub before_bytes: u64, + pub after_bytes: u64, + pub inline_media_count: u64, + pub invalid_json_count: u32, + pub next_cursor_job_id: Option, + pub has_more: bool, + pub error_message: Option, +} + +impl __sdk::InModule for ExternalGenerationJobPayloadCompactionProcedureResult { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_backfill_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_backfill_input_type.rs new file mode 100644 index 000000000..687b3f131 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_backfill_input_type.rs @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobSummaryBackfillInput { + pub owner_user_id: Option, + pub limit: u32, + pub cursor_job_id: Option, + pub dry_run: bool, +} + +impl __sdk::InModule for ExternalGenerationJobSummaryBackfillInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_backfill_procedure_result_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_backfill_procedure_result_type.rs new file mode 100644 index 000000000..9335eca2a --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_backfill_procedure_result_type.rs @@ -0,0 +1,22 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobSummaryBackfillProcedureResult { + pub ok: bool, + pub dry_run: bool, + pub scanned_count: u64, + pub selected_count: u32, + pub upserted_count: u32, + pub next_cursor_job_id: Option, + pub has_more: bool, + pub error_message: Option, +} + +impl __sdk::InModule for ExternalGenerationJobSummaryBackfillProcedureResult { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_procedure_result_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_procedure_result_type.rs new file mode 100644 index 000000000..b67d01d12 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_procedure_result_type.rs @@ -0,0 +1,24 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_summary_snapshot_type::ExternalGenerationJobSummarySnapshot; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobSummaryProcedureResult { + pub ok: bool, + pub job: Option, + pub jobs: Vec, + pub pending_count: u32, + pub running_count: u32, + pub unacknowledged_terminal_count: u32, + pub now_micros: i64, + pub error_message: Option, +} + +impl __sdk::InModule for ExternalGenerationJobSummaryProcedureResult { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_snapshot_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_snapshot_type.rs new file mode 100644 index 000000000..492102908 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_snapshot_type.rs @@ -0,0 +1,30 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobSummarySnapshot { + pub job_id: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_prompt: Option, + pub status: String, + pub last_error_message: Option, + pub created_at_micros: i64, + pub started_at_micros: Option, + pub completed_at_micros: Option, + pub updated_at_micros: i64, + pub price_mud_points: u64, + pub refund_ledger_id: Option, + pub notification_acknowledged_at_micros: Option, +} + +impl __sdk::InModule for ExternalGenerationJobSummarySnapshot { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_table.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_table.rs new file mode 100644 index 000000000..55e14f63a --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_table.rs @@ -0,0 +1,169 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use super::external_generation_job_summary_type::ExternalGenerationJobSummary; +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +/// Table handle for the table `external_generation_job_summary`. +/// +/// Obtain a handle from the [`ExternalGenerationJobSummaryTableAccess::external_generation_job_summary`] method on [`super::RemoteTables`], +/// like `ctx.db.external_generation_job_summary()`. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.external_generation_job_summary().on_insert(...)`. +pub struct ExternalGenerationJobSummaryTableHandle<'ctx> { + imp: __sdk::TableHandle, + ctx: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the table `external_generation_job_summary`. +/// +/// Implemented for [`super::RemoteTables`]. +pub trait ExternalGenerationJobSummaryTableAccess { + #[allow(non_snake_case)] + /// Obtain a [`ExternalGenerationJobSummaryTableHandle`], which mediates access to the table `external_generation_job_summary`. + fn external_generation_job_summary(&self) -> ExternalGenerationJobSummaryTableHandle<'_>; +} + +impl ExternalGenerationJobSummaryTableAccess for super::RemoteTables { + fn external_generation_job_summary(&self) -> ExternalGenerationJobSummaryTableHandle<'_> { + ExternalGenerationJobSummaryTableHandle { + imp: self + .imp + .get_table::("external_generation_job_summary"), + ctx: std::marker::PhantomData, + } + } +} + +pub struct ExternalGenerationJobSummaryInsertCallbackId(__sdk::CallbackId); +pub struct ExternalGenerationJobSummaryDeleteCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::Table for ExternalGenerationJobSummaryTableHandle<'ctx> { + type Row = ExternalGenerationJobSummary; + type EventContext = super::EventContext; + + fn count(&self) -> u64 { + self.imp.count() + } + fn iter(&self) -> impl Iterator + '_ { + self.imp.iter() + } + + type InsertCallbackId = ExternalGenerationJobSummaryInsertCallbackId; + + fn on_insert( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> ExternalGenerationJobSummaryInsertCallbackId { + ExternalGenerationJobSummaryInsertCallbackId(self.imp.on_insert(Box::new(callback))) + } + + fn remove_on_insert(&self, callback: ExternalGenerationJobSummaryInsertCallbackId) { + self.imp.remove_on_insert(callback.0) + } + + type DeleteCallbackId = ExternalGenerationJobSummaryDeleteCallbackId; + + fn on_delete( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> ExternalGenerationJobSummaryDeleteCallbackId { + ExternalGenerationJobSummaryDeleteCallbackId(self.imp.on_delete(Box::new(callback))) + } + + fn remove_on_delete(&self, callback: ExternalGenerationJobSummaryDeleteCallbackId) { + self.imp.remove_on_delete(callback.0) + } +} + +pub struct ExternalGenerationJobSummaryUpdateCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::TableWithPrimaryKey for ExternalGenerationJobSummaryTableHandle<'ctx> { + type UpdateCallbackId = ExternalGenerationJobSummaryUpdateCallbackId; + + fn on_update( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row, &Self::Row) + Send + 'static, + ) -> ExternalGenerationJobSummaryUpdateCallbackId { + ExternalGenerationJobSummaryUpdateCallbackId(self.imp.on_update(Box::new(callback))) + } + + fn remove_on_update(&self, callback: ExternalGenerationJobSummaryUpdateCallbackId) { + self.imp.remove_on_update(callback.0) + } +} + +/// Access to the `job_id` unique index on the table `external_generation_job_summary`, +/// which allows point queries on the field of the same name +/// via the [`ExternalGenerationJobSummaryJobIdUnique::find`] method. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.external_generation_job_summary().job_id().find(...)`. +pub struct ExternalGenerationJobSummaryJobIdUnique<'ctx> { + imp: __sdk::UniqueConstraintHandle, + phantom: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +impl<'ctx> ExternalGenerationJobSummaryTableHandle<'ctx> { + /// Get a handle on the `job_id` unique index on the table `external_generation_job_summary`. + pub fn job_id(&self) -> ExternalGenerationJobSummaryJobIdUnique<'ctx> { + ExternalGenerationJobSummaryJobIdUnique { + imp: self.imp.get_unique_constraint::("job_id"), + phantom: std::marker::PhantomData, + } + } +} + +impl<'ctx> ExternalGenerationJobSummaryJobIdUnique<'ctx> { + /// Find the subscribed row whose `job_id` column value is equal to `col_val`, + /// if such a row is present in the client cache. + pub fn find(&self, col_val: &String) -> Option { + self.imp.find(col_val) + } +} + +#[doc(hidden)] +pub(super) fn register_table(client_cache: &mut __sdk::ClientCache) { + let _table = client_cache + .get_or_make_table::("external_generation_job_summary"); + _table.add_unique_constraint::("job_id", |row| &row.job_id); +} + +#[doc(hidden)] +pub(super) fn parse_table_update( + raw_updates: __ws::v2::TableUpdate, +) -> __sdk::Result<__sdk::TableUpdate> { + __sdk::TableUpdate::parse_table_update(raw_updates).map_err(|e| { + __sdk::InternalError::failed_parse( + "TableUpdate", + "TableUpdate", + ) + .with_cause(e) + .into() + }) +} + +#[allow(non_camel_case_types)] +/// Extension trait for query builder access to the table `ExternalGenerationJobSummary`. +/// +/// Implemented for [`__sdk::QueryTableAccessor`]. +pub trait external_generation_job_summaryQueryTableAccess { + #[allow(non_snake_case)] + /// Get a query builder for the table `ExternalGenerationJobSummary`. + fn external_generation_job_summary( + &self, + ) -> __sdk::__query_builder::Table; +} + +impl external_generation_job_summaryQueryTableAccess for __sdk::QueryTableAccessor { + fn external_generation_job_summary( + &self, + ) -> __sdk::__query_builder::Table { + __sdk::__query_builder::Table::new("external_generation_job_summary") + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_type.rs new file mode 100644 index 000000000..685be2b15 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_summary_type.rs @@ -0,0 +1,103 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobSummary { + pub job_id: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_prompt: Option, + pub status: String, + pub last_error_message: Option, + pub created_at: __sdk::Timestamp, + pub started_at: Option<__sdk::Timestamp>, + pub completed_at: Option<__sdk::Timestamp>, + pub updated_at: __sdk::Timestamp, + pub price_mud_points: u64, + pub refund_ledger_id: Option, + pub notification_acknowledged_at: Option<__sdk::Timestamp>, +} + +impl __sdk::InModule for ExternalGenerationJobSummary { + type Module = super::RemoteModule; +} + +/// Column accessor struct for the table `ExternalGenerationJobSummary`. +/// +/// Provides typed access to columns for query building. +pub struct ExternalGenerationJobSummaryCols { + pub job_id: __sdk::__query_builder::Col, + pub job_kind: __sdk::__query_builder::Col, + pub owner_user_id: __sdk::__query_builder::Col, + pub source_module: __sdk::__query_builder::Col, + pub source_entity_id: __sdk::__query_builder::Col, + pub request_label: __sdk::__query_builder::Col, + pub request_prompt: __sdk::__query_builder::Col>, + pub status: __sdk::__query_builder::Col, + pub last_error_message: + __sdk::__query_builder::Col>, + pub created_at: __sdk::__query_builder::Col, + pub started_at: + __sdk::__query_builder::Col>, + pub completed_at: + __sdk::__query_builder::Col>, + pub updated_at: __sdk::__query_builder::Col, + pub price_mud_points: __sdk::__query_builder::Col, + pub refund_ledger_id: __sdk::__query_builder::Col>, + pub notification_acknowledged_at: + __sdk::__query_builder::Col>, +} + +impl __sdk::__query_builder::HasCols for ExternalGenerationJobSummary { + type Cols = ExternalGenerationJobSummaryCols; + fn cols(table_name: &'static str) -> Self::Cols { + ExternalGenerationJobSummaryCols { + job_id: __sdk::__query_builder::Col::new(table_name, "job_id"), + job_kind: __sdk::__query_builder::Col::new(table_name, "job_kind"), + owner_user_id: __sdk::__query_builder::Col::new(table_name, "owner_user_id"), + source_module: __sdk::__query_builder::Col::new(table_name, "source_module"), + source_entity_id: __sdk::__query_builder::Col::new(table_name, "source_entity_id"), + request_label: __sdk::__query_builder::Col::new(table_name, "request_label"), + request_prompt: __sdk::__query_builder::Col::new(table_name, "request_prompt"), + status: __sdk::__query_builder::Col::new(table_name, "status"), + last_error_message: __sdk::__query_builder::Col::new(table_name, "last_error_message"), + created_at: __sdk::__query_builder::Col::new(table_name, "created_at"), + started_at: __sdk::__query_builder::Col::new(table_name, "started_at"), + completed_at: __sdk::__query_builder::Col::new(table_name, "completed_at"), + updated_at: __sdk::__query_builder::Col::new(table_name, "updated_at"), + price_mud_points: __sdk::__query_builder::Col::new(table_name, "price_mud_points"), + refund_ledger_id: __sdk::__query_builder::Col::new(table_name, "refund_ledger_id"), + notification_acknowledged_at: __sdk::__query_builder::Col::new( + table_name, + "notification_acknowledged_at", + ), + } + } +} + +/// Indexed column accessor struct for the table `ExternalGenerationJobSummary`. +/// +/// Provides typed access to indexed columns for query building. +pub struct ExternalGenerationJobSummaryIxCols { + pub job_id: __sdk::__query_builder::IxCol, + pub owner_user_id: __sdk::__query_builder::IxCol, +} + +impl __sdk::__query_builder::HasIxCols for ExternalGenerationJobSummary { + type IxCols = ExternalGenerationJobSummaryIxCols; + fn ix_cols(table_name: &'static str) -> Self::IxCols { + ExternalGenerationJobSummaryIxCols { + job_id: __sdk::__query_builder::IxCol::new(table_name, "job_id"), + owner_user_id: __sdk::__query_builder::IxCol::new(table_name, "owner_user_id"), + } + } +} + +impl __sdk::__query_builder::CanBeLookupTable for ExternalGenerationJobSummary {} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_job_summary_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_job_summary_and_return_procedure.rs new file mode 100644 index 000000000..2b5703e72 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_job_summary_and_return_procedure.rs @@ -0,0 +1,59 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_get_input_type::ExternalGenerationJobGetInput; +use super::external_generation_job_summary_procedure_result_type::ExternalGenerationJobSummaryProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct GetExternalGenerationJobSummaryAndReturnArgs { + pub input: ExternalGenerationJobGetInput, +} + +impl __sdk::InModule for GetExternalGenerationJobSummaryAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `get_external_generation_job_summary_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait get_external_generation_job_summary_and_return { + fn get_external_generation_job_summary_and_return(&self, input: ExternalGenerationJobGetInput) { + self.get_external_generation_job_summary_and_return_then(input, |_, _| {}); + } + + fn get_external_generation_job_summary_and_return_then( + &self, + input: ExternalGenerationJobGetInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl get_external_generation_job_summary_and_return for super::RemoteProcedures { + fn get_external_generation_job_summary_and_return_then( + &self, + input: ExternalGenerationJobGetInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobSummaryProcedureResult>( + "get_external_generation_job_summary_and_return", + GetExternalGenerationJobSummaryAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/list_external_generation_job_summaries_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/list_external_generation_job_summaries_and_return_procedure.rs new file mode 100644 index 000000000..47e617b0d --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/list_external_generation_job_summaries_and_return_procedure.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_list_input_type::ExternalGenerationJobListInput; +use super::external_generation_job_summary_procedure_result_type::ExternalGenerationJobSummaryProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct ListExternalGenerationJobSummariesAndReturnArgs { + pub input: ExternalGenerationJobListInput, +} + +impl __sdk::InModule for ListExternalGenerationJobSummariesAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `list_external_generation_job_summaries_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait list_external_generation_job_summaries_and_return { + fn list_external_generation_job_summaries_and_return( + &self, + input: ExternalGenerationJobListInput, + ) { + self.list_external_generation_job_summaries_and_return_then(input, |_, _| {}); + } + + fn list_external_generation_job_summaries_and_return_then( + &self, + input: ExternalGenerationJobListInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl list_external_generation_job_summaries_and_return for super::RemoteProcedures { + fn list_external_generation_job_summaries_and_return_then( + &self, + input: ExternalGenerationJobListInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobSummaryProcedureResult>( + "list_external_generation_job_summaries_and_return", + ListExternalGenerationJobSummariesAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-module/src/external_generation.rs b/server-rs/crates/spacetime-module/src/external_generation.rs index e66f5ee62..c4bf6c665 100644 --- a/server-rs/crates/spacetime-module/src/external_generation.rs +++ b/server-rs/crates/spacetime-module/src/external_generation.rs @@ -1,4 +1,6 @@ use crate::*; +use std::cmp::Ordering; +use std::ops::RangeFrom; const EXTERNAL_GENERATION_STATUS_PENDING: &str = "pending"; const EXTERNAL_GENERATION_STATUS_RUNNING: &str = "running"; @@ -11,8 +13,15 @@ const EXTERNAL_GENERATION_EVENT_LEASE_RENEWED: &str = "lease_renewed"; const EXTERNAL_GENERATION_EVENT_COMPLETED: &str = "completed"; const EXTERNAL_GENERATION_EVENT_FAILED: &str = "failed"; const EXTERNAL_GENERATION_EVENT_ACKNOWLEDGED: &str = "acknowledged"; +const EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE: &str = "editor-canvas"; const EXTERNAL_GENERATION_FINAL_ATTEMPT_LEASE_EXPIRED_MESSAGE: &str = "worker 最终执行次数的 lease 已过期，任务已终止"; +const MAX_EXTERNAL_GENERATION_PAYLOAD_BYTES: usize = 512 * 1024; +const MAX_EXTERNAL_GENERATION_REQUEST_PROMPT_CHARS: usize = 2_048; +const MAX_EXTERNAL_GENERATION_ERROR_MESSAGE_CHARS: usize = 2_048; +const MAX_EXTERNAL_GENERATION_MAINTENANCE_BATCH_SIZE: u32 = 25; +const INLINE_MEDIA_REMOVED_PLACEHOLDER: &str = "[inline-media-removed]"; +const INLINE_MEDIA_ERROR_REDACTED_MESSAGE: &str = "外部生成失败（错误详情含内联媒体引用，已省略）"; #[spacetimedb::table( accessor = external_generation_job, @@ -31,6 +40,14 @@ const EXTERNAL_GENERATION_FINAL_ATTEMPT_LEASE_EXPIRED_MESSAGE: &str = index( accessor = by_external_generation_job_owner_user_id, btree(columns = [owner_user_id]) + ), + index( + accessor = by_external_generation_job_cursor, + btree(columns = [job_id, source_module]) + ), + index( + accessor = by_external_generation_job_source_cursor, + btree(columns = [source_module, job_id]) ) )] #[derive(Clone)] @@ -91,6 +108,34 @@ pub struct ExternalGenerationJobEvent { pub(crate) created_at: Timestamp, } +#[spacetimedb::table( + accessor = external_generation_job_summary, + index( + accessor = by_external_generation_job_summary_owner_user_id, + btree(columns = [owner_user_id]) + ) +)] +#[derive(Clone)] +pub struct ExternalGenerationJobSummary { + #[primary_key] + pub(crate) job_id: String, + pub(crate) job_kind: String, + pub(crate) owner_user_id: String, + pub(crate) source_module: String, + pub(crate) source_entity_id: String, + pub(crate) request_label: String, + pub(crate) request_prompt: Option, + pub(crate) status: String, + pub(crate) last_error_message: Option, + pub(crate) created_at: Timestamp, + pub(crate) started_at: Option, + pub(crate) completed_at: Option, + pub(crate) updated_at: Timestamp, + pub(crate) price_mud_points: u64, + pub(crate) refund_ledger_id: Option, + pub(crate) notification_acknowledged_at: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] pub struct ExternalGenerationJobEnqueueInput { pub job_id: String, @@ -165,6 +210,22 @@ pub struct ExternalGenerationJobAcknowledgeInput { pub acknowledged_at_micros: i64, } +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobSummaryBackfillInput { + pub owner_user_id: Option, + pub limit: u32, + pub cursor_job_id: Option, + pub dry_run: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobPayloadCompactionInput { + pub dry_run: bool, + pub limit: u32, + pub cursor_job_id: Option, + pub completed_before_micros: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] pub struct ExternalGenerationJobSnapshot { pub job_id: String, @@ -205,6 +266,66 @@ pub struct ExternalGenerationJobProcedureResult { pub error_message: Option, } +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobSummarySnapshot { + pub job_id: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_prompt: Option, + pub status: String, + pub last_error_message: Option, + pub created_at_micros: i64, + pub started_at_micros: Option, + pub completed_at_micros: Option, + pub updated_at_micros: i64, + pub price_mud_points: u64, + pub refund_ledger_id: Option, + pub notification_acknowledged_at_micros: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobSummaryProcedureResult { + pub ok: bool, + pub job: Option, + pub jobs: Vec, + pub pending_count: u32, + pub running_count: u32, + pub unacknowledged_terminal_count: u32, + pub now_micros: i64, + pub error_message: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobSummaryBackfillProcedureResult { + pub ok: bool, + pub dry_run: bool, + pub scanned_count: u64, + pub selected_count: u32, + pub upserted_count: u32, + pub next_cursor_job_id: Option, + pub has_more: bool, + pub error_message: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobPayloadCompactionProcedureResult { + pub ok: bool, + pub dry_run: bool, + pub scanned_count: u64, + pub matched_count: u32, + pub updated_count: u32, + pub before_bytes: u64, + pub after_bytes: u64, + pub inline_media_count: u64, + pub invalid_json_count: u32, + pub next_cursor_job_id: Option, + pub has_more: bool, + pub error_message: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] pub struct ExternalGenerationQueueStatsSnapshot { pub pending_count: u32, @@ -371,6 +492,93 @@ pub fn acknowledge_external_generation_jobs_and_return( } } +// 正式任务列表、详情与通知确认只返回轻量投影，禁止把持久任务 payload 带入 UI 读取链路。 +#[spacetimedb::procedure] +pub fn get_external_generation_job_summary_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobGetInput, +) -> ExternalGenerationJobSummaryProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + crate::editor_project_storage::require_editor_generation_runtime_service_identity( + tx, caller, + )?; + get_external_generation_job_summary_tx(tx, input.clone()) + }) { + Ok(job) => single_external_generation_job_summary_result(job), + Err(message) => failed_external_generation_job_summary_result(message), + } +} + +#[spacetimedb::procedure] +pub fn list_external_generation_job_summaries_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobListInput, +) -> ExternalGenerationJobSummaryProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + crate::editor_project_storage::require_editor_generation_runtime_service_identity( + tx, caller, + )?; + list_external_generation_job_summaries_tx(tx, input.clone()) + }) { + Ok(result) => result, + Err(message) => failed_external_generation_job_summary_result(message), + } +} + +#[spacetimedb::procedure] +pub fn acknowledge_external_generation_job_summaries_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobAcknowledgeInput, +) -> ExternalGenerationJobSummaryProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + crate::editor_project_storage::require_editor_generation_runtime_service_identity( + tx, caller, + )?; + acknowledge_external_generation_job_summaries_tx(tx, input.clone()) + }) { + Ok(result) => result, + Err(message) => failed_external_generation_job_summary_result(message), + } +} + +// 历史投影回填是显式维护动作；正式 list 不回扫大 payload 表。 +#[spacetimedb::procedure] +pub fn backfill_external_generation_job_summaries_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobSummaryBackfillInput, +) -> ExternalGenerationJobSummaryBackfillProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + crate::migration::require_migration_operator(tx, caller)?; + backfill_external_generation_job_summaries_tx(tx, input.clone()) + }) { + Ok(result) => result, + Err(message) => { + failed_external_generation_job_summary_backfill_result(input.dry_run, message) + } + } +} + +#[spacetimedb::procedure] +pub fn compact_external_generation_job_payloads_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobPayloadCompactionInput, +) -> ExternalGenerationJobPayloadCompactionProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + crate::migration::require_migration_operator(tx, caller)?; + compact_external_generation_job_payloads_tx(tx, input.clone()) + }) { + Ok(result) => result, + Err(message) => { + failed_external_generation_job_payload_compaction_result(input.dry_run, message) + } + } +} + #[spacetimedb::procedure] pub fn get_external_generation_queue_stats_and_return( ctx: &mut ProcedureContext, @@ -418,7 +626,8 @@ fn enqueue_external_generation_job_tx( "external_generation_job.request_label", &input.request_label, )?; - validate_required( + let request_payload_json = validate_external_generation_persisted_payload_for_source( + &input.source_module, "external_generation_job.request_payload_json", &input.request_payload_json, )?; @@ -429,6 +638,7 @@ fn enqueue_external_generation_job_tx( .dedupe_key() .find(&input.dedupe_key) { + persist_external_generation_job_summary(ctx, &row); return Ok(map_external_generation_job_row(row)); } if ctx @@ -451,7 +661,7 @@ fn enqueue_external_generation_job_tx( source_module: input.source_module.trim().to_string(), source_entity_id: input.source_entity_id.trim().to_string(), request_label: input.request_label.trim().to_string(), - request_payload_json: input.request_payload_json.trim().to_string(), + request_payload_json, status: EXTERNAL_GENERATION_STATUS_PENDING.to_string(), attempt: 0, max_attempts: input.max_attempts.max(1), @@ -469,7 +679,7 @@ fn enqueue_external_generation_job_tx( refund_ledger_id: None, notification_acknowledged_at: None, }; - ctx.db.external_generation_job().insert(row.clone()); + persist_external_generation_job_row(ctx, row.clone()); insert_external_generation_job_event( ctx, &row, @@ -619,11 +829,20 @@ fn complete_external_generation_job_tx( &input.worker_id, &input.lease_token, )?; + let result_payload_json = if is_external_generation_editor_source(&row.source_module) { + validate_optional_external_generation_payload_json( + "external_generation_job.result_payload_json", + input.result_payload_json.as_deref(), + )? + } else { + input + .result_payload_json + .as_deref() + .and_then(normalize_optional_text) + }; let completed_at = ctx.timestamp; row.status = EXTERNAL_GENERATION_STATUS_COMPLETED.to_string(); - row.result_payload_json = input - .result_payload_json - .and_then(|value| normalize_optional_text(value.as_str())); + row.result_payload_json = result_payload_json; row.lease_expires_at = None; row.completed_at = Some(completed_at); row.updated_at = completed_at; @@ -643,74 +862,141 @@ fn get_external_generation_job_tx( ctx: &ReducerContext, input: ExternalGenerationJobGetInput, ) -> Result { - validate_required("external_generation_job.job_id", &input.job_id)?; - validate_required( - "external_generation_job.owner_user_id", - &input.owner_user_id, - )?; - let row = ctx - .db - .external_generation_job() - .job_id() - .find(&input.job_id.trim().to_string()) - .ok_or_else(|| "external_generation_job 不存在".to_string())?; - if row.owner_user_id.trim() != input.owner_user_id.trim() { - return Err("external_generation_job 不存在".to_string()); - } - - Ok(map_external_generation_job_row(row)) + get_external_generation_job_summary_tx(ctx, input) + .map(map_external_generation_job_summary_to_compat_snapshot) } fn list_external_generation_jobs_tx( ctx: &ReducerContext, input: ExternalGenerationJobListInput, ) -> Result { + let result = list_external_generation_job_summaries_tx(ctx, input)?; + Ok(ExternalGenerationJobProcedureResult { + ok: result.ok, + job: None, + jobs: result + .jobs + .into_iter() + .map(map_external_generation_job_summary_to_compat_snapshot) + .collect(), + pending_count: result.pending_count, + running_count: result.running_count, + unacknowledged_terminal_count: result.unacknowledged_terminal_count, + now_micros: result.now_micros, + error_message: result.error_message, + }) +} + +fn acknowledge_external_generation_jobs_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobAcknowledgeInput, +) -> Result { + let result = acknowledge_external_generation_job_summaries_tx(ctx, input)?; + Ok(ExternalGenerationJobProcedureResult { + ok: result.ok, + job: None, + jobs: result + .jobs + .into_iter() + .map(map_external_generation_job_summary_to_compat_snapshot) + .collect(), + pending_count: result.pending_count, + running_count: result.running_count, + unacknowledged_terminal_count: result.unacknowledged_terminal_count, + now_micros: result.now_micros, + error_message: result.error_message, + }) +} + +fn get_external_generation_job_summary_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobGetInput, +) -> Result { + validate_required("external_generation_job.job_id", &input.job_id)?; + validate_required( + "external_generation_job.owner_user_id", + &input.owner_user_id, + )?; + let job_id = input.job_id.trim().to_string(); + let owner_user_id = input.owner_user_id.trim(); + + if let Some(summary) = ctx + .db + .external_generation_job_summary() + .job_id() + .find(&job_id) + { + if summary.owner_user_id.trim() != owner_user_id { + return Err("external_generation_job 不存在".to_string()); + } + return Ok(map_external_generation_job_summary_row(summary)); + } + + // 详情兼容旧任务时只允许按主键回填一行，绝不按 owner 扫描完整 payload 表。 + let row = ctx + .db + .external_generation_job() + .job_id() + .find(&job_id) + .ok_or_else(|| "external_generation_job 不存在".to_string())?; + if row.owner_user_id.trim() != owner_user_id { + return Err("external_generation_job 不存在".to_string()); + } + let summary = persist_external_generation_job_summary(ctx, &row); + Ok(map_external_generation_job_summary_row(summary)) +} + +fn list_external_generation_job_summaries_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobListInput, +) -> Result { validate_required( "external_generation_job.owner_user_id", &input.owner_user_id, )?; let owner_user_id = input.owner_user_id.trim().to_string(); let now_micros = ctx.timestamp.to_micros_since_unix_epoch(); - let (pending_count, running_count, unacknowledged_terminal_count) = - count_external_generation_jobs_for_owner(ctx, &owner_user_id); let status_filter = normalize_external_generation_job_status_filter(&input.statuses); - let mut rows = Vec::new(); + let limit = input.limit.clamp(1, 100) as usize; + let mut rows = Vec::with_capacity(limit); + let mut pending_count = 0u32; + let mut running_count = 0u32; + let mut unacknowledged_terminal_count = 0u32; + // 这里故意只读轻量投影，并在单次 owner 扫描中同时计数和维护固定大小 top-N； + // 历史任务由 operator maintenance procedure 显式分批回填。 for row in ctx .db - .external_generation_job() - .by_external_generation_job_owner_user_id() + .external_generation_job_summary() + .by_external_generation_job_summary_owner_user_id() .filter(&owner_user_id) { + match row.status.as_str() { + EXTERNAL_GENERATION_STATUS_PENDING => pending_count = pending_count.saturating_add(1), + EXTERNAL_GENERATION_STATUS_RUNNING => running_count = running_count.saturating_add(1), + EXTERNAL_GENERATION_STATUS_COMPLETED | EXTERNAL_GENERATION_STATUS_FAILED => { + if row.notification_acknowledged_at.is_none() { + unacknowledged_terminal_count = unacknowledged_terminal_count.saturating_add(1); + } + } + _ => {} + } let should_include = input.include_acknowledged_terminal - || !is_external_generation_job_terminal(&row) + || !is_external_generation_job_summary_terminal(&row) || row.notification_acknowledged_at.is_none(); let should_include_status = status_filter.is_empty() || status_filter.iter().any(|status| row.status == *status); if should_include && should_include_status { - rows.push(row); + retain_external_generation_job_summary_top_n(&mut rows, row, limit); } } - rows.sort_by(|left, right| { - external_generation_job_sort_bucket(left) - .cmp(&external_generation_job_sort_bucket(right)) - .then_with(|| { - external_generation_job_sort_time_micros(right) - .cmp(&external_generation_job_sort_time_micros(left)) - }) - .then_with(|| left.job_id.cmp(&right.job_id)) - }); - - let limit = input.limit.clamp(1, 100) as usize; - rows.truncate(limit); - - Ok(ExternalGenerationJobProcedureResult { + Ok(ExternalGenerationJobSummaryProcedureResult { ok: true, job: None, jobs: rows .into_iter() - .map(map_external_generation_job_row) + .map(map_external_generation_job_summary_row) .collect(), pending_count, running_count, @@ -720,41 +1006,31 @@ fn list_external_generation_jobs_tx( }) } -fn acknowledge_external_generation_jobs_tx( +fn acknowledge_external_generation_job_summaries_tx( ctx: &ReducerContext, input: ExternalGenerationJobAcknowledgeInput, -) -> Result { +) -> Result { validate_required( "external_generation_job.owner_user_id", &input.owner_user_id, )?; - if input.job_ids.is_empty() { - return Ok(ExternalGenerationJobProcedureResult { - ok: true, - job: None, - jobs: Vec::new(), - pending_count: 0, - running_count: 0, - unacknowledged_terminal_count: 0, - now_micros: ctx.timestamp.to_micros_since_unix_epoch(), - error_message: None, - }); - } - let owner_user_id = input.owner_user_id.trim().to_string(); let acknowledged_at = Timestamp::from_micros_since_unix_epoch(input.acknowledged_at_micros); let mut acknowledged = Vec::new(); for job_id in input.job_ids.iter().take(100) { + let normalized_job_id = job_id.trim().to_string(); let Some(mut row) = ctx .db - .external_generation_job() + .external_generation_job_summary() .job_id() - .find(&job_id.trim().to_string()) + .find(&normalized_job_id) else { continue; }; - if row.owner_user_id.trim() != owner_user_id || !is_external_generation_job_terminal(&row) { + if row.owner_user_id.trim() != owner_user_id + || !is_external_generation_job_summary_terminal(&row) + { continue; } if row.notification_acknowledged_at.is_some() { @@ -762,21 +1038,23 @@ fn acknowledge_external_generation_jobs_tx( } row.notification_acknowledged_at = Some(acknowledged_at); row.updated_at = acknowledged_at; - persist_external_generation_job_row(ctx, row.clone()); - insert_external_generation_job_event( + ctx.db + .external_generation_job_summary() + .job_id() + .update(row.clone()); + insert_external_generation_job_summary_event( ctx, &row, EXTERNAL_GENERATION_EVENT_ACKNOWLEDGED, Some("用户已确认任务通知".to_string()), - None, acknowledged_at, ); - acknowledged.push(map_external_generation_job_row(row)); + acknowledged.push(map_external_generation_job_summary_row(row)); } - let (pending_count, running_count, unacknowledged_terminal_count) = - count_external_generation_jobs_for_owner(ctx, &owner_user_id); - Ok(ExternalGenerationJobProcedureResult { + let (pending_count, running_count, unacknowledged_terminal_count) = + count_external_generation_job_summaries_for_owner(ctx, &owner_user_id); + Ok(ExternalGenerationJobSummaryProcedureResult { ok: true, job: None, jobs: acknowledged, @@ -788,6 +1066,178 @@ fn acknowledge_external_generation_jobs_tx( }) } +fn backfill_external_generation_job_summaries_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobSummaryBackfillInput, +) -> Result { + let owner_user_id = input + .owner_user_id + .as_deref() + .and_then(normalize_optional_text); + let cursor_job_id = input + .cursor_job_id + .as_deref() + .and_then(normalize_optional_text); + let limit = input + .limit + .clamp(1, MAX_EXTERNAL_GENERATION_MAINTENANCE_BATCH_SIZE) as usize; + let cursor_range = external_generation_job_maintenance_cursor_range(cursor_job_id.as_deref()); + let cursor_to_skip = cursor_job_id.clone(); + let rows = ctx + .db + .external_generation_job() + .by_external_generation_job_cursor() + .filter(cursor_range) + .filter(move |row| { + cursor_to_skip + .as_deref() + .is_none_or(|cursor| row.job_id != cursor) + }); + let (job_ids, next_cursor_job_id, has_more, scanned_count) = + select_external_generation_job_ids_for_maintenance(rows, limit, |row| { + owner_user_id + .as_deref() + .is_none_or(|owner| row.owner_user_id.trim() == owner) + && ctx + .db + .external_generation_job_summary() + .job_id() + .find(&row.job_id) + .is_none() + }); + + let mut upserted_count = 0u32; + if !input.dry_run { + for job_id in &job_ids { + if let Some(row) = ctx.db.external_generation_job().job_id().find(job_id) { + persist_external_generation_job_summary(ctx, &row); + upserted_count = upserted_count.saturating_add(1); + } + } + } + + Ok(ExternalGenerationJobSummaryBackfillProcedureResult { + ok: true, + dry_run: input.dry_run, + scanned_count, + selected_count: job_ids.len() as u32, + upserted_count, + next_cursor_job_id, + has_more, + error_message: None, + }) +} + +fn compact_external_generation_job_payloads_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobPayloadCompactionInput, +) -> Result { + let cursor_job_id = input + .cursor_job_id + .as_deref() + .and_then(normalize_optional_text); + let limit = input + .limit + .clamp(1, MAX_EXTERNAL_GENERATION_MAINTENANCE_BATCH_SIZE) as usize; + let cursor_range = external_generation_job_maintenance_cursor_range(cursor_job_id.as_deref()); + let cursor_to_skip = cursor_job_id.clone(); + let rows = ctx + .db + .external_generation_job() + .by_external_generation_job_source_cursor() + .filter((EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, cursor_range)) + .filter(move |row| { + cursor_to_skip + .as_deref() + .is_none_or(|cursor| row.job_id != cursor) + }); + let (job_ids, next_cursor_job_id, has_more, scanned_count) = + select_external_generation_job_ids_for_maintenance(rows, limit, |row| { + should_compact_external_generation_job_payloads(row, input.completed_before_micros) + }); + + let mut matched_count = 0u32; + let mut updated_count = 0u32; + let mut before_bytes = 0u64; + let mut after_bytes = 0u64; + let mut inline_media_count = 0u64; + let mut invalid_json_count = 0u32; + + for job_id in &job_ids { + let Some(mut row) = ctx.db.external_generation_job().job_id().find(job_id) else { + continue; + }; + if !should_compact_external_generation_job_payloads(&row, input.completed_before_micros) { + continue; + } + + let request_outcome = compact_external_generation_payload_json(&row.request_payload_json); + let result_outcome = row + .result_payload_json + .as_deref() + .map(compact_external_generation_payload_json); + invalid_json_count = invalid_json_count + .saturating_add(u32::from(request_outcome.invalid_json)) + .saturating_add(u32::from( + result_outcome + .as_ref() + .is_some_and(|outcome| outcome.invalid_json), + )); + let job_inline_media_count = request_outcome.inline_media_count.saturating_add( + result_outcome + .as_ref() + .map(|outcome| outcome.inline_media_count) + .unwrap_or(0), + ); + + if job_inline_media_count > 0 { + matched_count = matched_count.saturating_add(1); + inline_media_count = inline_media_count.saturating_add(job_inline_media_count); + for outcome in std::iter::once(&request_outcome).chain(result_outcome.iter()) { + if outcome.inline_media_count > 0 { + before_bytes = before_bytes.saturating_add(outcome.before_bytes); + after_bytes = after_bytes.saturating_add(outcome.after_bytes); + } + } + } + + if input.dry_run { + continue; + } + + if let Some(compacted_json) = request_outcome.compacted_json { + row.request_payload_json = compacted_json; + } + if let Some(outcome) = result_outcome { + if let Some(compacted_json) = outcome.compacted_json { + row.result_payload_json = Some(compacted_json); + } + } + if job_inline_media_count > 0 { + persist_external_generation_job_row(ctx, row); + updated_count = updated_count.saturating_add(1); + } else { + // 即使无需压缩，正式执行也顺带补齐该终态任务的轻量投影。 + persist_external_generation_job_summary(ctx, &row); + } + } + + Ok(ExternalGenerationJobPayloadCompactionProcedureResult { + ok: true, + dry_run: input.dry_run, + scanned_count, + matched_count, + updated_count, + before_bytes, + after_bytes, + inline_media_count, + invalid_json_count, + next_cursor_job_id, + has_more, + error_message: None, + }) +} + fn renew_external_generation_job_lease_tx( ctx: &ReducerContext, input: ExternalGenerationJobRenewLeaseInput, @@ -822,10 +1272,8 @@ fn fail_external_generation_job_tx( ctx: &ReducerContext, input: ExternalGenerationJobFailInput, ) -> Result { - let error_message = input.error_message.trim(); - if error_message.is_empty() { - return Err("external_generation_job.error_message 不能为空".to_string()); - } + let error_message = normalize_external_generation_error_message(&input.error_message) + .ok_or_else(|| "external_generation_job.error_message 不能为空".to_string())?; let mut row = get_worker_owned_external_generation_job( ctx, @@ -839,7 +1287,7 @@ fn fail_external_generation_job_tx( input.failed_at_micros, "external_generation_job.retry_delay", )?; - row.last_error_message = Some(error_message.to_string()); + row.last_error_message = Some(error_message.clone()); row.refund_ledger_id = input .refund_ledger_id .and_then(|value| normalize_optional_text(value.as_str())); @@ -861,7 +1309,7 @@ fn fail_external_generation_job_tx( ctx, &row, EXTERNAL_GENERATION_EVENT_FAILED, - Some(error_message.to_string()), + Some(error_message), Some(input.worker_id), failed_at, ); @@ -1030,7 +1478,68 @@ fn is_external_generation_job_terminal(row: &ExternalGenerationJob) -> bool { ) } -fn count_external_generation_jobs_for_owner( +fn is_external_generation_job_summary_terminal(row: &ExternalGenerationJobSummary) -> bool { + matches!( + row.status.as_str(), + EXTERNAL_GENERATION_STATUS_COMPLETED + | EXTERNAL_GENERATION_STATUS_FAILED + | EXTERNAL_GENERATION_STATUS_CANCELLED + ) +} + +fn should_compact_external_generation_job_payloads( + row: &ExternalGenerationJob, + completed_before_micros: Option, +) -> bool { + if !is_external_generation_editor_source(&row.source_module) + || !is_external_generation_job_terminal(row) + { + return false; + } + completed_before_micros.is_none_or(|cutoff| { + row.completed_at + .unwrap_or(row.updated_at) + .to_micros_since_unix_epoch() + <= cutoff + }) +} + +fn external_generation_job_maintenance_cursor_range( + cursor_job_id: Option<&str>, +) -> RangeFrom<&str> { + cursor_job_id.unwrap_or_default().. +} + +fn select_external_generation_job_ids_for_maintenance( + rows: impl Iterator, + limit: usize, + mut should_select: impl FnMut(&ExternalGenerationJob) -> bool, +) -> (Vec, Option, bool, u64) { + let mut rows = rows.peekable(); + let mut selected_job_ids = Vec::with_capacity(limit); + let mut next_cursor_job_id = None; + let mut scanned_count = 0u64; + + // 游标选择阶段最多反序列化 limit + 1 条大 payload row；apply 随后按主键逐条 + // 重新读取选中行，避免把整批大 payload 同时保留在事务内存中。 + for row in rows.by_ref().take(limit) { + scanned_count = scanned_count.saturating_add(1); + next_cursor_job_id = Some(row.job_id.clone()); + if should_select(&row) { + selected_job_ids.push(row.job_id); + } + } + + let has_more = rows.peek().is_some(); + ( + selected_job_ids, + next_cursor_job_id, + has_more, + scanned_count, + ) +} + +fn count_external_generation_job_summaries_for_owner( ctx: &ReducerContext, owner_user_id: &str, ) -> (u32, u32, u32) { @@ -1041,8 +1550,8 @@ fn count_external_generation_jobs_for_owner( for row in ctx .db - .external_generation_job() - .by_external_generation_job_owner_user_id() + .external_generation_job_summary() + .by_external_generation_job_summary_owner_user_id() .filter(&normalized_owner_user_id) { match row.status.as_str() { @@ -1060,6 +1569,7 @@ fn count_external_generation_jobs_for_owner( (pending_count, running_count, unacknowledged_terminal_count) } +#[cfg(test)] fn external_generation_job_sort_bucket(row: &ExternalGenerationJob) -> u8 { match row.status.as_str() { EXTERNAL_GENERATION_STATUS_RUNNING => 0, @@ -1076,6 +1586,7 @@ fn external_generation_job_sort_bucket(row: &ExternalGenerationJob) -> u8 { } } +#[cfg(test)] fn external_generation_job_sort_time_micros(row: &ExternalGenerationJob) -> i64 { if is_external_generation_job_terminal(row) { return row @@ -1086,6 +1597,65 @@ fn external_generation_job_sort_time_micros(row: &ExternalGenerationJob) -> i64 row.updated_at.to_micros_since_unix_epoch() } +fn external_generation_job_summary_sort_bucket(row: &ExternalGenerationJobSummary) -> u8 { + match row.status.as_str() { + EXTERNAL_GENERATION_STATUS_RUNNING => 0, + EXTERNAL_GENERATION_STATUS_PENDING => 1, + EXTERNAL_GENERATION_STATUS_COMPLETED | EXTERNAL_GENERATION_STATUS_FAILED + if row.notification_acknowledged_at.is_none() => + { + 2 + } + EXTERNAL_GENERATION_STATUS_COMPLETED | EXTERNAL_GENERATION_STATUS_FAILED => 3, + EXTERNAL_GENERATION_STATUS_CANCELLED if row.notification_acknowledged_at.is_none() => 4, + EXTERNAL_GENERATION_STATUS_CANCELLED => 5, + _ => 6, + } +} + +fn external_generation_job_summary_sort_time_micros(row: &ExternalGenerationJobSummary) -> i64 { + if is_external_generation_job_summary_terminal(row) { + return row + .completed_at + .unwrap_or(row.updated_at) + .to_micros_since_unix_epoch(); + } + row.updated_at.to_micros_since_unix_epoch() +} + +fn compare_external_generation_job_summaries( + left: &ExternalGenerationJobSummary, + right: &ExternalGenerationJobSummary, +) -> Ordering { + external_generation_job_summary_sort_bucket(left) + .cmp(&external_generation_job_summary_sort_bucket(right)) + .then_with(|| { + external_generation_job_summary_sort_time_micros(right) + .cmp(&external_generation_job_summary_sort_time_micros(left)) + }) + .then_with(|| left.job_id.cmp(&right.job_id)) +} + +fn retain_external_generation_job_summary_top_n( + rows: &mut Vec, + row: ExternalGenerationJobSummary, + limit: usize, +) { + if limit == 0 { + return; + } + let insert_at = rows + .binary_search_by(|existing| compare_external_generation_job_summaries(existing, &row)) + .unwrap_or_else(|index| index); + if insert_at >= limit { + return; + } + rows.insert(insert_at, row); + if rows.len() > limit { + rows.pop(); + } +} + fn normalize_external_generation_job_status_filter(statuses: &[String]) -> Vec<&'static str> { statuses .iter() @@ -1123,7 +1693,183 @@ fn persist_external_generation_job_row(ctx: &ReducerContext, row: ExternalGenera .external_generation_job() .job_id() .delete(&row.job_id); - ctx.db.external_generation_job().insert(row); + ctx.db.external_generation_job().insert(row.clone()); + persist_external_generation_job_summary(ctx, &row); +} + +fn persist_external_generation_job_summary( + ctx: &ReducerContext, + row: &ExternalGenerationJob, +) -> ExternalGenerationJobSummary { + let existing = ctx + .db + .external_generation_job_summary() + .job_id() + .find(&row.job_id); + let cached_request_prompt = existing + .as_ref() + .map(|summary| summary.request_prompt.clone()); + let cached_notification_acknowledged_at = existing + .as_ref() + .and_then(|summary| summary.notification_acknowledged_at); + if existing.is_some() { + ctx.db + .external_generation_job_summary() + .job_id() + .delete(&row.job_id); + } + let mut summary = build_external_generation_job_summary_row(row, cached_request_prompt); + if summary.notification_acknowledged_at.is_none() { + summary.notification_acknowledged_at = cached_notification_acknowledged_at; + if let Some(acknowledged_at) = cached_notification_acknowledged_at { + summary.updated_at = summary.updated_at.max(acknowledged_at); + } + } + ctx.db + .external_generation_job_summary() + .insert(summary.clone()); + summary +} + +fn build_external_generation_job_summary_row( + row: &ExternalGenerationJob, + cached_request_prompt: Option>, +) -> ExternalGenerationJobSummary { + ExternalGenerationJobSummary { + job_id: row.job_id.clone(), + job_kind: row.job_kind.clone(), + owner_user_id: row.owner_user_id.clone(), + source_module: row.source_module.clone(), + source_entity_id: row.source_entity_id.clone(), + request_label: row.request_label.clone(), + request_prompt: match cached_request_prompt { + Some(prompt) => prompt + .as_deref() + .and_then(normalize_external_generation_request_prompt_text), + None => extract_external_generation_request_prompt(&row.request_payload_json), + }, + status: row.status.clone(), + last_error_message: row + .last_error_message + .as_deref() + .and_then(normalize_external_generation_error_message), + created_at: row.created_at, + started_at: row.started_at, + completed_at: row.completed_at, + updated_at: row.updated_at, + price_mud_points: row.price_mud_points, + refund_ledger_id: row.refund_ledger_id.clone(), + notification_acknowledged_at: row.notification_acknowledged_at, + } +} + +fn extract_external_generation_request_prompt(request_payload_json: &str) -> Option { + let payload: serde_json::Value = serde_json::from_str(request_payload_json).ok()?; + for key in ["prompt", "promptText", "spritesheetLabel"] { + if let Some(prompt) = payload + .get(key) + .and_then(serde_json::Value::as_str) + .and_then(normalize_external_generation_request_prompt_text) + { + return Some(prompt); + } + } + if let Some(prompt) = payload + .get("iconDescriptions") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .filter_map(serde_json::Value::as_str) + .filter_map(normalize_external_generation_request_prompt_text) + .collect::>() + .join("、") + }) + .and_then(|value| normalize_external_generation_request_prompt_text(&value)) + { + return Some(prompt); + } + payload + .get("generationInputs") + .and_then(|value| value.get("fields")) + .and_then(serde_json::Value::as_array) + .and_then(|fields| { + fields.iter().find_map(|field| { + let title = field + .get("title") + .and_then(serde_json::Value::as_str)? + .trim(); + if !matches!(title, "prompt" | "gpt_description_prompt") { + return None; + } + field + .get("value") + .and_then(serde_json::Value::as_str) + .and_then(normalize_external_generation_request_prompt_text) + }) + }) +} + +fn normalize_external_generation_request_prompt_text(value: &str) -> Option { + let trimmed = value.trim(); + if trimmed.is_empty() || is_external_generation_inline_media_reference(trimmed) { + return None; + } + let mut chars = trimmed.chars(); + let mut normalized = chars + .by_ref() + .take(MAX_EXTERNAL_GENERATION_REQUEST_PROMPT_CHARS) + .collect::(); + if chars.next().is_some() { + normalized.push('…'); + } + Some(normalized) +} + +fn normalize_external_generation_error_message(value: &str) -> Option { + let trimmed = value.trim(); + if trimmed.is_empty() { + return None; + } + if contains_external_generation_inline_media_reference_text(trimmed) { + return Some(INLINE_MEDIA_ERROR_REDACTED_MESSAGE.to_string()); + } + let mut chars = trimmed.chars(); + let mut normalized = chars + .by_ref() + .take(MAX_EXTERNAL_GENERATION_ERROR_MESSAGE_CHARS) + .collect::(); + if chars.next().is_some() { + normalized.push('…'); + } + Some(normalized) +} + +fn contains_external_generation_inline_media_reference_text(value: &str) -> bool { + let normalized = value.to_ascii_lowercase(); + ["data:", "blob:"].iter().any(|scheme| { + normalized.match_indices(scheme).any(|(index, _)| { + index == 0 + || normalized + .as_bytes() + .get(index - 1) + .is_some_and(|previous| { + matches!( + previous, + b' ' | b'\t' + | b'\r' + | b'\n' + | b'"' + | b'\'' + | b'=' + | b'(' + | b'[' + | b'{' + | b',' + ) + }) + }) + }) } fn insert_external_generation_job_event( @@ -1158,6 +1904,43 @@ fn insert_external_generation_job_event( }); } +fn insert_external_generation_job_summary_event( + ctx: &ReducerContext, + row: &ExternalGenerationJobSummary, + event_kind: &str, + message: Option, + created_at: Timestamp, +) { + let event_id = format!( + "{}:{}:{}:summary:{}", + row.job_id.trim(), + event_kind.trim(), + row.status.trim(), + created_at.to_micros_since_unix_epoch() + ); + if ctx + .db + .external_generation_job_event() + .event_id() + .find(&event_id) + .is_some() + { + return; + } + ctx.db + .external_generation_job_event() + .insert(ExternalGenerationJobEvent { + event_id, + job_id: row.job_id.clone(), + owner_user_id: row.owner_user_id.clone(), + event_kind: event_kind.to_string(), + status: row.status.clone(), + message, + worker_id: None, + created_at, + }); +} + fn map_external_generation_job_row(row: ExternalGenerationJob) -> ExternalGenerationJobSnapshot { let notification_acknowledged_at_micros = row .notification_acknowledged_at @@ -1196,6 +1979,71 @@ fn map_external_generation_job_row(row: ExternalGenerationJob) -> ExternalGenera } } +fn map_external_generation_job_summary_row( + row: ExternalGenerationJobSummary, +) -> ExternalGenerationJobSummarySnapshot { + ExternalGenerationJobSummarySnapshot { + job_id: row.job_id, + job_kind: row.job_kind, + owner_user_id: row.owner_user_id, + source_module: row.source_module, + source_entity_id: row.source_entity_id, + request_label: row.request_label, + request_prompt: row.request_prompt, + status: row.status, + last_error_message: row.last_error_message, + created_at_micros: row.created_at.to_micros_since_unix_epoch(), + started_at_micros: row + .started_at + .map(|value| value.to_micros_since_unix_epoch()), + completed_at_micros: row + .completed_at + .map(|value| value.to_micros_since_unix_epoch()), + updated_at_micros: row.updated_at.to_micros_since_unix_epoch(), + price_mud_points: row.price_mud_points, + refund_ledger_id: row.refund_ledger_id, + notification_acknowledged_at_micros: row + .notification_acknowledged_at + .map(|value| value.to_micros_since_unix_epoch()), + } +} + +fn map_external_generation_job_summary_to_compat_snapshot( + summary: ExternalGenerationJobSummarySnapshot, +) -> ExternalGenerationJobSnapshot { + let request_payload_json = summary + .request_prompt + .as_ref() + .map(|prompt| serde_json::json!({ "prompt": prompt }).to_string()) + .unwrap_or_else(|| "{}".to_string()); + ExternalGenerationJobSnapshot { + job_id: summary.job_id, + dedupe_key: String::new(), + job_kind: summary.job_kind, + owner_user_id: summary.owner_user_id, + source_module: summary.source_module, + source_entity_id: summary.source_entity_id, + request_label: summary.request_label, + request_payload_json, + status: summary.status, + attempt: 0, + max_attempts: 0, + last_error_message: summary.last_error_message, + worker_id: None, + lease_expires_at_micros: None, + available_at_micros: summary.updated_at_micros, + result_payload_json: None, + created_at_micros: summary.created_at_micros, + started_at_micros: summary.started_at_micros, + completed_at_micros: summary.completed_at_micros, + updated_at_micros: summary.updated_at_micros, + lease_token: None, + price_mud_points: summary.price_mud_points, + refund_ledger_id: summary.refund_ledger_id, + notification_acknowledged_at_micros: summary.notification_acknowledged_at_micros, + } +} + fn single_external_generation_job_result( job: ExternalGenerationJobSnapshot, ) -> ExternalGenerationJobProcedureResult { @@ -1224,6 +2072,72 @@ fn failed_external_generation_job_result(message: String) -> ExternalGenerationJ } } +fn single_external_generation_job_summary_result( + job: ExternalGenerationJobSummarySnapshot, +) -> ExternalGenerationJobSummaryProcedureResult { + ExternalGenerationJobSummaryProcedureResult { + ok: true, + job: Some(job), + jobs: Vec::new(), + pending_count: 0, + running_count: 0, + unacknowledged_terminal_count: 0, + now_micros: 0, + error_message: None, + } +} + +fn failed_external_generation_job_summary_result( + message: String, +) -> ExternalGenerationJobSummaryProcedureResult { + ExternalGenerationJobSummaryProcedureResult { + ok: false, + job: None, + jobs: Vec::new(), + pending_count: 0, + running_count: 0, + unacknowledged_terminal_count: 0, + now_micros: 0, + error_message: Some(message), + } +} + +fn failed_external_generation_job_summary_backfill_result( + dry_run: bool, + message: String, +) -> ExternalGenerationJobSummaryBackfillProcedureResult { + ExternalGenerationJobSummaryBackfillProcedureResult { + ok: false, + dry_run, + scanned_count: 0, + selected_count: 0, + upserted_count: 0, + next_cursor_job_id: None, + has_more: false, + error_message: Some(message), + } +} + +fn failed_external_generation_job_payload_compaction_result( + dry_run: bool, + message: String, +) -> ExternalGenerationJobPayloadCompactionProcedureResult { + ExternalGenerationJobPayloadCompactionProcedureResult { + ok: false, + dry_run, + scanned_count: 0, + matched_count: 0, + updated_count: 0, + before_bytes: 0, + after_bytes: 0, + inline_media_count: 0, + invalid_json_count: 0, + next_cursor_job_id: None, + has_more: false, + error_message: Some(message), + } +} + fn validate_required(field: &str, value: &str) -> Result<(), String> { if value.trim().is_empty() { return Err(format!("{field} 不能为空")); @@ -1231,6 +2145,170 @@ fn validate_required(field: &str, value: &str) -> Result<(), String> { Ok(()) } +fn validate_external_generation_persisted_payload_for_source( + source_module: &str, + field: &str, + value: &str, +) -> Result { + if is_external_generation_editor_source(source_module) { + validate_external_generation_payload_json(field, value) + } else { + validate_required(field, value)?; + Ok(value.trim().to_string()) + } +} + +fn is_external_generation_editor_source(source_module: &str) -> bool { + source_module.trim() == EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE +} + +fn validate_external_generation_payload_json(field: &str, value: &str) -> Result { + let normalized = value.trim(); + if normalized.is_empty() { + return Err(format!("{field} 不能为空")); + } + let payload_bytes = normalized.len(); + if payload_bytes > MAX_EXTERNAL_GENERATION_PAYLOAD_BYTES { + return Err(format!( + "{field} JSON 大小为 {payload_bytes} 字节，超过持久化上限 {MAX_EXTERNAL_GENERATION_PAYLOAD_BYTES} 字节；请移除冗余数据并改传 objectKey 或 resourceId" + )); + } + let payload = serde_json::from_str::(normalized) + .map_err(|error| format!("{field} 不是合法 JSON: {error}"))?; + if contains_external_generation_inline_media_reference(&payload) { + return Err(format!( + "{field} 禁止包含 data: 或 blob: 内联媒体引用，请先上传对象存储并改传 objectKey 或 resourceId" + )); + } + Ok(normalized.to_string()) +} + +fn validate_optional_external_generation_payload_json( + field: &str, + value: Option<&str>, +) -> Result, String> { + let Some(value) = value else { + return Ok(None); + }; + if value.trim().is_empty() { + return Ok(None); + } + validate_external_generation_payload_json(field, value).map(Some) +} + +fn contains_external_generation_inline_media_reference(value: &serde_json::Value) -> bool { + match value { + serde_json::Value::String(value) => is_external_generation_inline_media_reference(value), + serde_json::Value::Array(values) => values + .iter() + .any(contains_external_generation_inline_media_reference), + serde_json::Value::Object(values) => values.iter().any(|(key, value)| { + is_external_generation_inline_media_reference(key) + || contains_external_generation_inline_media_reference(value) + }), + serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => { + false + } + } +} + +fn is_external_generation_inline_media_reference(value: &str) -> bool { + value + .trim_start() + .as_bytes() + .get(..5) + .is_some_and(|prefix| { + prefix.eq_ignore_ascii_case(b"data:") || prefix.eq_ignore_ascii_case(b"blob:") + }) +} + +struct ExternalGenerationPayloadCompactionOutcome { + compacted_json: Option, + before_bytes: u64, + after_bytes: u64, + inline_media_count: u64, + invalid_json: bool, +} + +fn compact_external_generation_payload_json( + payload_json: &str, +) -> ExternalGenerationPayloadCompactionOutcome { + let before_bytes = payload_json.len() as u64; + let Ok(mut payload) = serde_json::from_str::(payload_json) else { + return ExternalGenerationPayloadCompactionOutcome { + compacted_json: None, + before_bytes, + after_bytes: before_bytes, + inline_media_count: 0, + invalid_json: true, + }; + }; + let inline_media_count = compact_external_generation_inline_media_references(&mut payload); + if inline_media_count == 0 { + return ExternalGenerationPayloadCompactionOutcome { + compacted_json: None, + before_bytes, + after_bytes: before_bytes, + inline_media_count: 0, + invalid_json: false, + }; + } + let Ok(compacted_json) = serde_json::to_string(&payload) else { + return ExternalGenerationPayloadCompactionOutcome { + compacted_json: None, + before_bytes, + after_bytes: before_bytes, + inline_media_count: 0, + invalid_json: true, + }; + }; + ExternalGenerationPayloadCompactionOutcome { + after_bytes: compacted_json.len() as u64, + compacted_json: Some(compacted_json), + before_bytes, + inline_media_count, + invalid_json: false, + } +} + +fn compact_external_generation_inline_media_references(value: &mut serde_json::Value) -> u64 { + match value { + serde_json::Value::String(text) => { + if is_external_generation_inline_media_reference(text) { + *text = INLINE_MEDIA_REMOVED_PLACEHOLDER.to_string(); + 1 + } else { + 0 + } + } + serde_json::Value::Array(values) => values.iter_mut().fold(0u64, |count, value| { + count.saturating_add(compact_external_generation_inline_media_references(value)) + }), + serde_json::Value::Object(values) => { + let source = std::mem::take(values); + let mut compacted = serde_json::Map::new(); + let mut count = 0u64; + for (index, (key, mut nested)) in source.into_iter().enumerate() { + let mut next_key = key; + if is_external_generation_inline_media_reference(&next_key) { + count = count.saturating_add(1); + next_key = format!("__inline_media_key_removed_{index}"); + while compacted.contains_key(&next_key) { + next_key.push('_'); + } + } + count = count.saturating_add(compact_external_generation_inline_media_references( + &mut nested, + )); + compacted.insert(next_key, nested); + } + *values = compacted; + count + } + serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => 0, + } +} + fn duration_between_micros(later: i64, earlier: i64, field: &str) -> Result { let duration = later.saturating_sub(earlier); if duration <= 0 { @@ -1519,6 +2597,232 @@ mod tests { assert!(duration_between_micros(1_000, 1_000, "duration").is_err()); } + #[test] + fn persisted_payload_validation_rejects_inline_media_recursively() { + let error = validate_external_generation_payload_json( + "external_generation_job.request_payload_json", + r#"{"prompt":"保留 data 和 blob 普通文本","nested":[{"url":" \nDaTa:image/png;base64,AAAA"}]}"#, + ) + .expect_err("嵌套 Data URL 不得进入持久任务"); + + assert!(error.contains("禁止包含 data: 或 blob:")); + assert!( + validate_external_generation_payload_json( + "external_generation_job.request_payload_json", + r#"{"sourceImageObjectKey":"users/user-1/source.png","resourceId":"resource-1"}"#, + ) + .is_ok() + ); + } + + #[test] + fn persisted_payload_validation_rejects_invalid_or_oversized_json() { + assert!( + validate_external_generation_payload_json( + "external_generation_job.request_payload_json", + "not-json", + ) + .expect_err("持久任务参数必须是合法 JSON") + .contains("不是合法 JSON") + ); + let oversized = format!( + r#"{{"prompt":"{}"}}"#, + "x".repeat(MAX_EXTERNAL_GENERATION_PAYLOAD_BYTES) + ); + assert!( + validate_external_generation_payload_json( + "external_generation_job.result_payload_json", + &oversized, + ) + .expect_err("request 和 result 使用同一持久化上限") + .contains("超过持久化上限") + ); + } + + #[test] + fn persisted_payload_guard_does_not_break_non_editor_transient_references() { + let puzzle_payload = r#"{"reference_image_src":"data:image/png;base64,AAAA"}"#; + + assert_eq!( + validate_external_generation_persisted_payload_for_source( + "puzzle", + "external_generation_job.request_payload_json", + puzzle_payload, + ), + Ok(puzzle_payload.to_string()) + ); + assert!( + validate_external_generation_persisted_payload_for_source( + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + "external_generation_job.request_payload_json", + puzzle_payload, + ) + .is_err() + ); + } + + #[test] + fn job_summary_contract_never_serializes_request_or_result_payload() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_COMPLETED); + row.request_payload_json = + r#"{"prompt":"一只橙色陶罐猫","source":"data:image/png;base64,AAAA"}"#.to_string(); + row.result_payload_json = Some(r#"{"image":"data:image/png;base64,BBBB"}"#.to_string()); + let summary_row = build_external_generation_job_summary_row(&row, None); + let summary = map_external_generation_job_summary_row(summary_row); + let serialized = + serde_json::to_value(spacetimedb::sats::ser::serde::SerializeWrapper(&summary)) + .expect("summary 应可序列化"); + + assert_eq!(summary.request_prompt.as_deref(), Some("一只橙色陶罐猫")); + assert!(serialized.get("request_payload_json").is_none()); + assert!(serialized.get("result_payload_json").is_none()); + for internal_field in [ + "dedupe_key", + "attempt", + "max_attempts", + "worker_id", + "lease_expires_at_micros", + "available_at_micros", + ] { + assert!( + serialized.get(internal_field).is_none(), + "summary 不应复制 worker 内部字段 {internal_field}" + ); + } + } + + #[test] + fn job_summary_never_copies_inline_media_as_request_prompt() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_COMPLETED); + row.request_payload_json = r#"{"prompt":"data:image/png;base64,AAAA"}"#.to_string(); + + let extracted = build_external_generation_job_summary_row(&row, None); + let cached = build_external_generation_job_summary_row( + &row, + Some(Some(" blob:https://example.test/id".to_string())), + ); + + assert!(extracted.request_prompt.is_none()); + assert!(cached.request_prompt.is_none()); + } + + #[test] + fn job_summary_bounds_and_redacts_error_messages() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_FAILED); + row.last_error_message = + Some(r#"provider response: {"image":"data:image/png;base64,AAAA"}"#.to_string()); + let redacted = build_external_generation_job_summary_row(&row, None); + assert_eq!( + redacted.last_error_message.as_deref(), + Some(INLINE_MEDIA_ERROR_REDACTED_MESSAGE) + ); + + row.last_error_message = Some("x".repeat(MAX_EXTERNAL_GENERATION_ERROR_MESSAGE_CHARS + 10)); + let bounded = build_external_generation_job_summary_row(&row, None) + .last_error_message + .expect("非空错误应保留有界摘要"); + assert_eq!( + bounded.chars().count(), + MAX_EXTERNAL_GENERATION_ERROR_MESSAGE_CHARS + 1 + ); + assert!(bounded.ends_with('…')); + assert_eq!( + normalize_external_generation_error_message("metadata: unavailable").as_deref(), + Some("metadata: unavailable") + ); + } + + #[test] + fn job_summary_top_n_keeps_memory_bounded_and_ordered() { + let mut rows = Vec::new(); + for index in 1..=5 { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_COMPLETED); + row.job_id = format!("extgen-{index}"); + row.completed_at = Some(micros(index)); + row.updated_at = micros(index); + retain_external_generation_job_summary_top_n( + &mut rows, + build_external_generation_job_summary_row(&row, None), + 2, + ); + assert!(rows.len() <= 2); + } + + assert_eq!( + rows.into_iter().map(|row| row.job_id).collect::>(), + vec!["extgen-5".to_string(), "extgen-4".to_string()] + ); + } + + #[test] + fn active_jobs_are_never_payload_compaction_candidates() { + for status in [ + EXTERNAL_GENERATION_STATUS_PENDING, + EXTERNAL_GENERATION_STATUS_RUNNING, + ] { + let mut row = external_generation_job_fixture(status); + row.source_module = EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE.to_string(); + assert!(!should_compact_external_generation_job_payloads(&row, None)); + } + + let mut completed = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_COMPLETED); + completed.source_module = EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE.to_string(); + completed.completed_at = Some(micros(2_000)); + assert!(should_compact_external_generation_job_payloads( + &completed, + Some(2_000) + )); + assert!(!should_compact_external_generation_job_payloads( + &completed, + Some(1_999) + )); + + let non_editor = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_COMPLETED); + assert!(!should_compact_external_generation_job_payloads( + &non_editor, + None + )); + } + + #[test] + fn maintenance_selector_bounds_scanned_rows_and_advances_by_last_scanned_job() { + let rows = (1..=4).map(|index| { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_COMPLETED); + row.job_id = format!("extgen-{index}"); + row + }); + + let (selected, next_cursor, has_more, scanned_count) = + select_external_generation_job_ids_for_maintenance(rows, 2, |row| { + row.job_id != "extgen-1" + }); + + assert_eq!(selected, vec!["extgen-2".to_string()]); + assert_eq!(next_cursor.as_deref(), Some("extgen-2")); + assert!(has_more); + assert_eq!(scanned_count, 2); + } + + #[test] + fn payload_compaction_replaces_nested_inline_media_and_preserves_prompt() { + let payload = r#"{"prompt":"保留这段展示提示","nested":[{"url":"data:image/png;base64,AAAA"},{"deep":{"source":" BLOB:https://example.test/id"}}]}"#; + let outcome = compact_external_generation_payload_json(payload); + let compacted = outcome + .compacted_json + .as_deref() + .expect("含内联媒体的 JSON 应生成压缩结果"); + let value: serde_json::Value = serde_json::from_str(compacted).expect("压缩结果仍是 JSON"); + + assert_eq!(outcome.inline_media_count, 2); + assert!(outcome.after_bytes < outcome.before_bytes); + assert_eq!(value["prompt"], "保留这段展示提示"); + assert_eq!(value["nested"][0]["url"], INLINE_MEDIA_REMOVED_PLACEHOLDER); + assert_eq!( + value["nested"][1]["deep"]["source"], + INLINE_MEDIA_REMOVED_PLACEHOLDER + ); + } + fn external_generation_job_fixture(status: &str) -> ExternalGenerationJob { ExternalGenerationJob { job_id: "extgen-test".to_string(), diff --git a/server-rs/crates/spacetime-module/src/migration.rs b/server-rs/crates/spacetime-module/src/migration.rs index 9b9d195d0..f2d294872 100644 --- a/server-rs/crates/spacetime-module/src/migration.rs +++ b/server-rs/crates/spacetime-module/src/migration.rs @@ -185,6 +185,7 @@ macro_rules! migration_tables { ai_result_reference, ai_task_event, external_generation_job, + external_generation_job_summary, external_generation_job_event, runtime_snapshot, runtime_setting, @@ -843,7 +844,10 @@ fn authorize_database_migration_operator_tx( Ok(()) } -fn require_migration_operator(ctx: &ReducerContext, caller: Identity) -> Result<(), String> { +pub(crate) fn require_migration_operator( + ctx: &ReducerContext, + caller: Identity, +) -> Result<(), String> { if is_database_migration_operator(ctx, caller) { Ok(()) } else { diff --git a/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx b/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx index af406df39..2261f329c 100644 --- a/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx +++ b/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx @@ -25,6 +25,7 @@ import type { UiAssetExtractionState } from './ImageCanvasUiAssetExtractionModel import { useCanvasGenerationDialogs } from './useCanvasGenerationDialogs'; import { applyQueuedEditorGenerationProject, + resolveEditorGenerationMediaReference, useImageCanvasGenerationSubmissionWorkflow, } from './useImageCanvasGenerationSubmissionWorkflow'; @@ -32,6 +33,7 @@ const resolveEditorImageReferenceDataUrlMock = vi.hoisted(() => vi.fn()); const resolveEditorImageReferenceDataUrlForGenerationMock = vi.hoisted(() => vi.fn(), ); +const uploadEditorMediaAssetFileMock = vi.hoisted(() => vi.fn()); const editEditorImageMock = vi.hoisted(() => vi.fn()); const extractEditorUiDesignAssetsMock = vi.hoisted(() => vi.fn()); const renderUiDesignAssetExtractionMarkedImageMock = vi.hoisted(() => vi.fn()); @@ -68,6 +70,10 @@ vi.mock('../../services/image-editor/editorProjectClient', async () => { }; }); +vi.mock('../../services/image-editor/editorMediaAssetUploadClient', () => ({ + uploadEditorMediaAssetFile: uploadEditorMediaAssetFileMock, +})); + vi.mock('../../services/external-generation', () => ({ getExternalGenerationJobStatus: getExternalGenerationJobStatusMock, })); @@ -466,6 +472,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { beforeEach(() => { resolveEditorImageReferenceDataUrlMock.mockReset(); resolveEditorImageReferenceDataUrlForGenerationMock.mockReset(); + uploadEditorMediaAssetFileMock.mockReset(); editEditorImageMock.mockReset(); extractEditorUiDesignAssetsMock.mockReset(); renderUiDesignAssetExtractionMarkedImageMock.mockReset(); @@ -483,11 +490,47 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { resolveEditorImageReferenceDataUrlForGenerationMock.mockImplementation( async (src: string) => src, ); + uploadEditorMediaAssetFileMock.mockResolvedValue({ + src: 'https://signed.example.test/generation-reference.png', + objectKey: + 'generated-character-drafts/editor/generation-references/reference.png', + assetObjectId: 'asset-object-generation-reference', + legacyPublicPath: + '/generated-character-drafts/editor/generation-references/reference.png', + }); renderUiDesignAssetExtractionMarkedImageMock.mockResolvedValue( - 'data:image/png;base64,marked-ui-design', + 'data:image/png;base64,bWFya2VkLXVpLWRlc2lnbg==', ); }); + it('uses distinct upload paths for concurrent inline references in the same millisecond', async () => { + const dateNowSpy = vi.spyOn(Date, 'now').mockReturnValue(1_752_144_000_000); + try { + await Promise.all([ + resolveEditorGenerationMediaReference( + { src: 'data:image/png;base64,YQ==' }, + 'image', + 'project-1', + ), + resolveEditorGenerationMediaReference( + { src: 'data:image/png;base64,Yg==' }, + 'image', + 'project-1', + ), + ]); + } finally { + dateNowSpy.mockRestore(); + } + + expect(uploadEditorMediaAssetFileMock).toHaveBeenCalledTimes(2); + const [firstFile, , firstOptions] = + uploadEditorMediaAssetFileMock.mock.calls[0] ?? []; + const [secondFile, , secondOptions] = + uploadEditorMediaAssetFileMock.mock.calls[1] ?? []; + expect((firstFile as File).name).not.toBe((secondFile as File).name); + expect(firstOptions.pathSegments).not.toEqual(secondOptions.pathSegments); + }); + it('submits quick edits and updates the source layer directly', async () => { editEditorImageMock.mockResolvedValueOnce( createGenerated({ prompt: '快速修图' }), @@ -506,7 +549,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { expect(editEditorImageMock).toHaveBeenCalledWith( expect.objectContaining({ prompt: '快速修图', - sourceImageSrc: 'data:image/png;base64,source.png', + sourceImageSrc: 'resource-source', size: '1024x768', model: 'gpt-image-2', }), @@ -516,6 +559,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { 'canvasCompletion', ); expect(generateEditorImageMock).not.toHaveBeenCalled(); + expect(resolveEditorImageReferenceDataUrlMock).not.toHaveBeenCalled(); await waitFor(() => { expect(screen.getByTestId('layers').textContent).toContain( 'layer-source:源图:resource-source:-:-:data:image/png;base64,generated', @@ -606,7 +650,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { expect(editEditorImageMock).toHaveBeenCalledWith( expect.objectContaining({ prompt: '把当前图改成雨天', - sourceImageSrc: 'data:image/png;base64,source.png', + sourceImageSrc: 'resource-source', size: '1024x768', model: 'gpt-image-2', sourceResourceId: 'resource-source', @@ -727,7 +771,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { />, ); renderUiDesignAssetExtractionMarkedImageMock.mockResolvedValueOnce( - 'data:image/png;base64,marked-source', + 'data:image/png;base64,bWFya2VkLXNvdXJjZQ==', ); fireEvent.click(screen.getByRole('button', { name: '提交快速编辑' })); @@ -751,13 +795,24 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { ); }); await waitFor(() => { + expect(uploadEditorMediaAssetFileMock).toHaveBeenCalledWith( + expect.any(File), + 'image', + expect.objectContaining({ + assetKind: 'editor_generation_reference_image', + }), + ); expect(editEditorImageMock).toHaveBeenCalledWith( expect.objectContaining({ - sourceImageSrc: 'data:image/png;base64,marked-source', + sourceImageSrc: + 'generated-character-drafts/editor/generation-references/reference.png', targetLayerId: 'layer-source', }), ); }); + expect( + uploadEditorMediaAssetFileMock.mock.invocationCallOrder[0], + ).toBeLessThan(editEditorImageMock.mock.invocationCallOrder[0] ?? 0); expect(editEditorImageMock.mock.calls[0]?.[0]).not.toHaveProperty( 'referenceImageSrcs', ); @@ -812,7 +867,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { />, ); renderUiDesignAssetExtractionMarkedImageMock.mockResolvedValueOnce( - 'data:image/png;base64,marked-source', + 'data:image/png;base64,bWFya2VkLXNvdXJjZQ==', ); fireEvent.click(screen.getByRole('button', { name: '设置初始对话' })); @@ -839,7 +894,8 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { await waitFor(() => { expect(editEditorImageMock).toHaveBeenCalledWith( expect.objectContaining({ - sourceImageSrc: 'data:image/png;base64,marked-source', + sourceImageSrc: + 'generated-character-drafts/editor/generation-references/reference.png', model: 'gpt-image-2', targetLayerId: 'layer-source', }), @@ -907,7 +963,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { imageSize: '1K', kind: 'character', model: 'gpt-image-2', - referenceImageSrcs: ['data:image/png;base64,source.png'], + referenceImageSrcs: ['resource-source'], assetKind: 'character', }), ); @@ -959,8 +1015,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { initialLayers={[ createLayer({ title: '源视频', - src: 'https://assets.example.test/source.mp4', - objectKey: 'generated-character-drafts/editor/videos/source.mp4', + src: 'data:video/mp4;base64,c291cmNlLXZpZGVv', mediaType: 'video', assetKind: 'video', model: 'gpt-image-2', @@ -1001,7 +1056,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { mode: 'std', sound: 'off', referenceVideoSrcs: [ - 'generated-character-drafts/editor/videos/source.mp4', + 'generated-character-drafts/editor/generation-references/reference.png', ], assetFolderId: 'project', assetLabel: '源视频 快速编辑', @@ -1026,6 +1081,66 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { }); }); + it('uploads inline video references and keeps image objectKey and asset URLs', async () => { + generateEditorVideoMock.mockResolvedValueOnce(createVideoGenerated()); + render( + , + ); + + fireEvent.click(screen.getByRole('button', { name: '设置初始对话' })); + fireEvent.click(screen.getByRole('button', { name: '提交当前生成' })); + + await waitFor(() => { + expect(uploadEditorMediaAssetFileMock).toHaveBeenCalledWith( + expect.any(File), + 'video', + expect.objectContaining({ + assetKind: 'editor_generation_reference_video', + }), + ); + expect(generateEditorVideoMock).toHaveBeenCalledWith( + expect.objectContaining({ + referenceImageSrcs: [ + 'generated-character-drafts/editor/refs/image.png', + ], + referenceVideoSrcs: [ + 'generated-character-drafts/editor/generation-references/reference.png', + ], + referenceAudioSrcs: ['asset://asset-audio-reference'], + }), + ); + }); + }); + it('submits character action quick edits through image edits', async () => { editEditorImageMock.mockResolvedValueOnce( createGenerated({ @@ -1075,7 +1190,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { expect(editEditorImageMock).toHaveBeenCalledWith( expect.objectContaining({ prompt: '让当前图跳得更高', - sourceImageSrc: 'data:image/png;base64,frame.png', + sourceImageSrc: 'resource-source', model: 'gpt-image-2', assetLabel: '角色动作 快速编辑', targetLayerId: 'layer-source', @@ -1117,7 +1232,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { expect(editEditorImageMock).toHaveBeenCalledWith( expect.objectContaining({ prompt: '修改当前图片', - sourceImageSrc: 'data:image/png;base64,source.png', + sourceImageSrc: 'resource-source', }), ); }); @@ -1341,10 +1456,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { }); }); - it('resolves uploaded objectKey references before submitting image generation', async () => { - resolveEditorImageReferenceDataUrlForGenerationMock.mockResolvedValueOnce( - 'data:image/jpeg;base64,normalized-ref', - ); + it('submits uploaded objectKey references without downloading or inlining them', async () => { generateEditorImageMock.mockResolvedValueOnce( createGenerated({ prompt: '参考图生成' }), ); @@ -1382,16 +1494,17 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { fireEvent.click(screen.getByRole('button', { name: '提交当前生成' })); await waitFor(() => { - expect( - resolveEditorImageReferenceDataUrlForGenerationMock, - ).toHaveBeenCalledWith('generated-character-drafts/editor/refs/ref.png'); expect(generateEditorImageMock).toHaveBeenCalledWith( expect.objectContaining({ prompt: '参考图生成', - referenceImageSrcs: ['data:image/jpeg;base64,normalized-ref'], + referenceImageSrcs: [ + 'generated-character-drafts/editor/refs/ref.png', + ], }), ); }); + expect(resolveEditorImageReferenceDataUrlMock).not.toHaveBeenCalled(); + expect(uploadEditorMediaAssetFileMock).not.toHaveBeenCalled(); }); it('refreshes the wallet balance after a queued generation reaches a terminal state', async () => { @@ -1525,10 +1638,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { 1, unresolvedProject, ); - expect(applyProjectSnapshot).toHaveBeenNthCalledWith( - 2, - completedProject, - ); + expect(applyProjectSnapshot).toHaveBeenNthCalledWith(2, completedProject); } finally { vi.useRealTimers(); } @@ -1691,7 +1801,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { { id: 'ref-1', label: '主视觉参考', - src: 'data:image/png;base64,large-ref', + src: 'DaTa:image/png;base64,bGFyZ2UtcmVm', }, ], placeholder: { @@ -1721,7 +1831,9 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { model: 'gpt-image-2', aspectRatio: '9:16', imageSize: '1K', - referenceImageSrcs: ['data:image/png;base64,large-ref'], + referenceImageSrcs: [ + 'generated-character-drafts/editor/generation-references/reference.png', + ], }), ); }); @@ -1742,10 +1854,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { ); }); - it('compresses publication references before submitting generation', async () => { - resolveEditorImageReferenceDataUrlForGenerationMock.mockResolvedValueOnce( - 'data:image/jpeg;base64,compressed-ref', - ); + it('submits publication resource ids without downloading or inlining them', async () => { generateEditorImageMock.mockResolvedValueOnce( createGenerated({ imageSrc: 'data:image/png;base64,publication', @@ -1773,6 +1882,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { id: 'ref-1', label: '主视觉参考', src: 'data:image/png;base64,large-original-ref', + resourceId: 'resource-publication-ref', }, ], }} @@ -1783,18 +1893,17 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { fireEvent.click(screen.getByRole('button', { name: '提交当前生成' })); await waitFor(() => { - expect( - resolveEditorImageReferenceDataUrlForGenerationMock, - ).toHaveBeenCalledWith('data:image/png;base64,large-original-ref'); expect(generateEditorImageMock).toHaveBeenCalledWith( expect.objectContaining({ kind: 'publication-material', aspectRatio: '4:3', imageSize: '1K', - referenceImageSrcs: ['data:image/jpeg;base64,compressed-ref'], + referenceImageSrcs: ['resource-publication-ref'], }), ); }); + expect(resolveEditorImageReferenceDataUrlMock).not.toHaveBeenCalled(); + expect(uploadEditorMediaAssetFileMock).not.toHaveBeenCalled(); }); it('submits icon spec objects without requiring an icon spec reference', async () => { @@ -1913,7 +2022,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { iconSpecReference: { id: 'spec', label: '图标规范', - src: 'data:image/png;base64,spec', + src: 'data:image/png;base64,c3BlYw==', }, iconDescriptions: ['旧返回', '旧设置'], placeholder: { @@ -1934,7 +2043,8 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { await waitFor(() => { expect(generateEditorIconSpritesheetMock).toHaveBeenCalledWith( expect.objectContaining({ - referenceImageSrc: 'data:image/png;base64,spec', + referenceImageSrc: + 'generated-character-drafts/editor/generation-references/reference.png', iconDescriptions: ['返回按钮', '设置按钮'], model: 'gpt-image-2', }), @@ -1992,7 +2102,8 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { id: 'ref-style', label: '风格参考', src: 'https://signed.example.test/icon-style.png?expires=1', - objectKey: 'generated-character-drafts/editor/refs/icon-style.png', + objectKey: + 'generated-character-drafts/editor/refs/icon-style.png', assetObjectId: 'assetobj-icon-style', }, ], @@ -2004,7 +2115,9 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { fireEvent.click(screen.getByRole('button', { name: '提交图标' })); await waitFor(() => { - expect(resolveEditorImageReferenceDataUrlForGenerationMock).not.toHaveBeenCalled(); + expect( + resolveEditorImageReferenceDataUrlForGenerationMock, + ).not.toHaveBeenCalled(); expect(generateEditorIconSpritesheetMock).toHaveBeenCalledWith( expect.objectContaining({ referenceImageSrc: @@ -2115,6 +2228,9 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { }); it('renders selected UI marks into the extraction reference image and sends derived extraction pricing', async () => { + renderUiDesignAssetExtractionMarkedImageMock.mockResolvedValueOnce( + 'data:image/png;base64,bWFya2VkLXVpLWRlc2lnbg==', + ); extractEditorUiDesignAssetsMock.mockResolvedValueOnce({ spritesheetImageSrc: 'data:image/png;base64,ui-sheet', spritesheetWidth: 512, @@ -2166,7 +2282,8 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { await waitFor(() => { expect(extractEditorUiDesignAssetsMock).toHaveBeenCalledWith( expect.objectContaining({ - sourceImageSrc: 'data:image/png;base64,marked-ui-design', + sourceImageSrc: + 'generated-character-drafts/editor/generation-references/reference.png', model: 'gemini-3.1-flash-image-preview', screenColor: 'auto', segModel: 'birefnet', @@ -2213,7 +2330,9 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { fireEvent.click(screen.getByRole('button', { name: '提取带参考图UI素材' })); await waitFor(() => { - expect(resolveEditorImageReferenceDataUrlForGenerationMock).not.toHaveBeenCalled(); + expect( + resolveEditorImageReferenceDataUrlForGenerationMock, + ).not.toHaveBeenCalled(); }); await waitFor(() => { expect(extractEditorUiDesignAssetsMock).toHaveBeenCalledWith( @@ -2290,7 +2409,7 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { createLayer({ id: 'character-layer', assetKind: 'character', - objectKey: 'generated/character.png', + src: 'data:image/png;base64,Y2hhcmFjdGVy', }), ]} initialCharacterAnimationPanel={{ @@ -2314,7 +2433,8 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { expect(generateEditorCharacterAnimationMock).toHaveBeenCalledWith( expect.objectContaining({ sourceLayerId: 'character-layer', - sourceImageSrc: 'generated/character.png', + sourceImageSrc: + 'generated-character-drafts/editor/generation-references/reference.png', promptText: '跑步', model: 'seedance2.0-fast', }), @@ -2450,7 +2570,12 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { ); render( { generateEditorCharacterAnimationMock.mockRejectedValueOnce(new Error('')); render( word.toString(16).padStart(8, '0')).join( + '', + ); + } + editorGenerationUploadFallbackCounter += 1; + return `${Date.now().toString(36)}-${editorGenerationUploadFallbackCounter.toString(36)}-${Math.random().toString(36).slice(2)}`; +} + +function resolveEditorGenerationMediaReferenceSource( + reference: EditorGenerationMediaReference, + allowRegisteredIds: boolean, +) { + const resourceId = reference.resourceId?.trim(); + return ( + reference.objectKey?.trim() || + (allowRegisteredIds && + resourceId && + !resourceId.startsWith('local-resource-') && + !resourceId.startsWith('generation-dialog:') + ? resourceId + : '') || + (allowRegisteredIds ? reference.sourceAssetId?.trim() : '') || + reference.src.trim() + ); +} + +function dataUrlToEditorGenerationFile(dataUrl: string, fileName: string) { + const [header = '', payload = ''] = dataUrl.split(','); + const mimeMatch = /^data:([^;]+)(;base64)?$/iu.exec(header); + if (!mimeMatch) { + throw new Error('生成参考图不是有效的 Data URL'); + } + const type = mimeMatch[1] ?? 'image/png'; + const binary = mimeMatch[2] ? atob(payload) : decodeURIComponent(payload); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index); + } + return new File([bytes], fileName, { type }); +} + +async function inlineMediaSourceToEditorGenerationFile( + source: string, + mediaType: EditorMediaAssetUploadType, + uploadId: string, +) { + const fileName = `generation-reference-${uploadId}.${ + mediaType === 'video' ? 'mp4' : mediaType === 'audio' ? 'mp3' : 'png' + }`; + if (/^data:/iu.test(source)) { + return dataUrlToEditorGenerationFile(source, fileName); + } + const response = await fetch(source); + if (!response.ok) { + throw new Error('读取本地生成参考素材失败'); + } + const blob = await response.blob(); + return new File([blob], fileName, { + type: blob.type || `${mediaType}/*`, + }); +} + +async function uploadEditorGenerationInlineMediaSource( + source: string, + mediaType: EditorMediaAssetUploadType, + projectId?: string | null, +) { + const normalizedProjectId = projectId?.trim() || 'unscoped'; + const uploadId = createEditorGenerationMediaUploadId(); + const uploaded = await uploadEditorMediaAssetFile( + await inlineMediaSourceToEditorGenerationFile(source, mediaType, uploadId), + mediaType, + { + assetKind: `editor_generation_reference_${mediaType}`, + pathSegments: [ + 'editor', + 'generation-references', + normalizedProjectId, + uploadId, + ], + entityId: normalizedProjectId, + ...(projectId?.trim() + ? { metadata: { editor_project_id: projectId.trim() } } + : {}), + }, + ); + return uploaded.objectKey; +} + +export async function resolveEditorGenerationMediaReference( + reference: EditorGenerationMediaReference, + mediaType: EditorMediaAssetUploadType, + projectId?: string | null, + options: EditorGenerationMediaReferenceOptions = {}, +) { + const source = resolveEditorGenerationMediaReferenceSource( + reference, + options.allowRegisteredIds !== false, + ); + const inlineSource = /^(?:data|blob):/iu.test(source); + const remoteImageSource = + options.requireImageObjectReference === true && + mediaType === 'image' && + !reference.objectKey?.trim() && + /^(?:https?:|\/api\/)/iu.test(source); + if (!inlineSource && !remoteImageSource) { + return source; + } + const uploadSource = remoteImageSource + ? await resolveEditorImageReferenceDataUrl(source) + : source; + return uploadEditorGenerationInlineMediaSource( + uploadSource, + mediaType, + projectId, + ); +} + +async function uploadEditorGenerationImageDataUrl( + dataUrl: string, + projectId?: string | null, +) { + return uploadEditorGenerationInlineMediaSource(dataUrl, 'image', projectId); +} + const EDITOR_GENERATION_QUEUE_POLL_INTERVAL_MS = 1600; const EDITOR_GENERATION_QUEUE_TIMEOUT_MS = 20 * 60 * 1000; const EDITOR_GENERATION_QUEUE_PROJECT_REFRESH_RETRY_MS = 650; @@ -152,6 +296,8 @@ type GenerationSubmissionWorkflowOptions = { async function normalizeImageGenerationReferenceImages( input: Parameters[0], + references: CharacterReferenceImage[], + projectId?: string | null, ) { if (!input.referenceImageSrcs?.length) { return input; @@ -160,13 +306,88 @@ async function normalizeImageGenerationReferenceImages( return { ...input, referenceImageSrcs: await Promise.all( - input.referenceImageSrcs.map((referenceImageSrc) => - resolveEditorImageReferenceDataUrlForGeneration(referenceImageSrc), + input.referenceImageSrcs.map((referenceImageSrc, index) => + resolveEditorGenerationMediaReference( + references[index] + ? { ...references[index], src: referenceImageSrc } + : { src: referenceImageSrc }, + 'image', + projectId, + ), ), ), }; } +function resolveImageGenerationDialogReferences( + dialog: GenerateDialogState, +): CharacterReferenceImage[] { + if (dialog.mode === 'spec') { + return dialog.specReference ? [dialog.specReference] : []; + } + if (dialog.mode === 'character') { + return [ + ...(dialog.characterSpecReference ? [dialog.characterSpecReference] : []), + ...(dialog.characterReferences ?? []), + ]; + } + if (dialog.mode === 'ui-design') { + return [ + ...(dialog.uiDesignSpecReference ? [dialog.uiDesignSpecReference] : []), + ...(dialog.generationReferences ?? []), + ]; + } + if (dialog.mode === 'publication') { + return dialog.publicationReferences ?? []; + } + return dialog.generationReferences ?? []; +} + +async function normalizeVideoGenerationReferences( + input: Parameters[0], + references: CharacterReferenceImage[], + projectId?: string | null, +) { + const normalizeSources = async ( + sources: string[] | undefined, + mediaType: EditorMediaAssetUploadType, + ) => { + if (!sources?.length) { + return undefined; + } + const matchingReferences = references.filter( + (reference) => + (reference.mediaType ?? 'image') === + (mediaType === 'image' ? 'image' : mediaType), + ); + return Promise.all( + sources.map((source, index) => + resolveEditorGenerationMediaReference( + matchingReferences[index] + ? { ...matchingReferences[index], src: source } + : { src: source }, + mediaType, + projectId, + { allowRegisteredIds: false }, + ), + ), + ); + }; + + const [referenceImageSrcs, referenceVideoSrcs, referenceAudioSrcs] = + await Promise.all([ + normalizeSources(input.referenceImageSrcs, 'image'), + normalizeSources(input.referenceVideoSrcs, 'video'), + normalizeSources(input.referenceAudioSrcs, 'audio'), + ]); + return { + ...input, + ...(referenceImageSrcs ? { referenceImageSrcs } : {}), + ...(referenceVideoSrcs ? { referenceVideoSrcs } : {}), + ...(referenceAudioSrcs ? { referenceAudioSrcs } : {}), + }; +} + function delay(ms: number) { return new Promise((resolve) => window.setTimeout(resolve, ms)); } @@ -251,7 +472,9 @@ export async function applyQueuedEditorGenerationProject( onQueuedGenerationTask?: () => void, onWalletBalanceMayHaveChanged?: () => void, completionDialogId?: string | null, - transformProjectSnapshot?: (project: EditorProjectSnapshot) => EditorProjectSnapshot, + transformProjectSnapshot?: ( + project: EditorProjectSnapshot, + ) => EditorProjectSnapshot, ) { const queueState = queuedStateFromResponse(response); if (!queueState) { @@ -778,14 +1001,26 @@ export function useImageCanvasGenerationSubmissionWorkflow({ } try { - let sourceImageSrc = sourceLayer.objectKey?.trim() || sourceLayer.src; + let sourceImageSrc: string; if (options.marks?.length) { - sourceImageSrc = - await resolveEditorImageReferenceDataUrl(sourceImageSrc); - sourceImageSrc = await renderUiDesignAssetExtractionMarkedImage({ - source: sourceImageSrc, - marks: options.marks, - }); + const renderSourceImageSrc = await resolveEditorImageReferenceDataUrl( + sourceLayer.objectKey?.trim() || sourceLayer.src, + ); + const markedImageDataUrl = + await renderUiDesignAssetExtractionMarkedImage({ + source: renderSourceImageSrc, + marks: options.marks, + }); + sourceImageSrc = await uploadEditorGenerationImageDataUrl( + markedImageDataUrl, + projectId, + ); + } else { + sourceImageSrc = await resolveEditorGenerationMediaReference( + sourceLayer, + 'image', + projectId, + ); } const screenColor = DEFAULT_EDITOR_GENERATION_BACKGROUND_COLOR; const segModel = DEFAULT_EDITOR_BGFILTER_SEG_MODEL; @@ -793,9 +1028,15 @@ export function useImageCanvasGenerationSubmissionWorkflow({ sourceLayer, options.references, ); - const referenceImageSrcs = (options.references ?? []) - .map((reference) => resolveImageReferenceSubmissionSource(reference)) - .filter((source) => source.trim().length > 0); + const referenceImageSrcs = await Promise.all( + (options.references ?? []).map((reference) => + resolveEditorGenerationMediaReference( + reference, + 'image', + projectId, + ), + ), + ); const extractionPlan = resolveUiAssetExtractionGenerationPlan( options.marks?.length ?? 0, ); @@ -910,6 +1151,10 @@ export function useImageCanvasGenerationSubmissionWorkflow({ if (!canvasDialog) { return; } + const iconSpecReference = dialog.iconSpecReference; + if (!iconSpecReference) { + return; + } setSubmittingIconDialog({ ...canvasDialog, @@ -922,9 +1167,25 @@ export function useImageCanvasGenerationSubmissionWorkflow({ try { const canvasCompletionPlaceholder = getGeneratingDialogPlaceholder(dialog); + const referenceImageSrc = await resolveEditorGenerationMediaReference( + iconSpecReference, + 'image', + projectId, + ); + const referenceImageSrcs = await Promise.all( + (dialog.generationReferences ?? []).map((reference) => + resolveEditorGenerationMediaReference( + reference, + 'image', + projectId, + ), + ), + ); const generated = await runEditorGenerationWithWalletRefresh( generateEditorIconSpritesheet({ ...submissionPlan.input, + referenceImageSrc, + ...(referenceImageSrcs.length ? { referenceImageSrcs } : {}), projectId, generationInputs: submissionPlan.generationInputs, assetFolderId, @@ -1019,7 +1280,8 @@ export function useImageCanvasGenerationSubmissionWorkflow({ const normalizedPrompt = panelMode === 'redraw' ? basePrompt : basePrompt; let quickEditDialogId: string | undefined; const shouldUseCanvasPlaceholder = - panelMode === 'quick-edit' && isVideoQuickEditSource(quickEditSourceLayer); + panelMode === 'quick-edit' && + isVideoQuickEditSource(quickEditSourceLayer); if (shouldUseCanvasPlaceholder) { const quickEditDraft = createQuickEditGenerationDialogDraft({ sourceLayer: quickEditSourceLayer, @@ -1070,8 +1332,10 @@ export function useImageCanvasGenerationSubmissionWorkflow({ const extraReferenceImageSrcs = needsExtraReferenceImageSrcs ? await Promise.all( quickEditReferences.map((reference) => - resolveEditorImageReferenceDataUrl( - reference.objectKey?.trim() || reference.src, + resolveEditorGenerationMediaReference( + reference, + 'image', + projectId, ), ), ) @@ -1099,8 +1363,12 @@ export function useImageCanvasGenerationSubmissionWorkflow({ const videoModel = DEFAULT_VIDEO_MODEL; const videoResolution = '480p' as const; const videoDurationSeconds = 4 as const; - const sourceVideoSrc = - quickEditSourceLayer.objectKey?.trim() || quickEditSourceLayer.src; + const sourceVideoSrc = await resolveEditorGenerationMediaReference( + quickEditSourceLayer, + 'video', + projectId, + { allowRegisteredIds: false }, + ); const generationInputs = buildVideoGenerationInputs( normalizedPrompt, quickEditReferences, @@ -1167,21 +1435,32 @@ export function useImageCanvasGenerationSubmissionWorkflow({ ); return; } - let sourceReferenceImageSrc = await resolveEditorImageReferenceDataUrl( - quickEditSourceLayer.objectKey?.trim() || quickEditSourceLayer.src, - ); const quickEditSelectionMarks = panelMode === 'quick-edit' && quickEditSelectionState?.sourceLayerId === quickEditSourceLayer.id ? quickEditSelectionState.marks : []; + let sourceReferenceImageSrc: string; if (quickEditSelectionMarks.length > 0) { - sourceReferenceImageSrc = + const renderSourceImageSrc = await resolveEditorImageReferenceDataUrl( + quickEditSourceLayer.objectKey?.trim() || quickEditSourceLayer.src, + ); + const markedImageDataUrl = await renderUiDesignAssetExtractionMarkedImage({ - source: sourceReferenceImageSrc, + source: renderSourceImageSrc, marks: quickEditSelectionMarks, showIndexLabels: true, }); + sourceReferenceImageSrc = await uploadEditorGenerationImageDataUrl( + markedImageDataUrl, + projectId, + ); + } else { + sourceReferenceImageSrc = await resolveEditorGenerationMediaReference( + quickEditSourceLayer, + 'image', + projectId, + ); } const generationInputs = panelMode === 'redraw' @@ -1377,9 +1656,10 @@ export function useImageCanvasGenerationSubmissionWorkflow({ nextGeneratedIndex: layerCounterRef.current + 1, }); if (submissionPlan.kind === 'edit') { - const referenceImageSrc = await resolveEditorImageReferenceDataUrl( - submissionPlan.sourceLayer.objectKey?.trim() || - submissionPlan.sourceLayer.src, + const referenceImageSrc = await resolveEditorGenerationMediaReference( + submissionPlan.sourceLayer, + 'image', + projectId, ); const canvasCompletionPlaceholder = getGeneratingDialogPlaceholder(dialog); @@ -1436,21 +1716,34 @@ export function useImageCanvasGenerationSubmissionWorkflow({ generationInputs: submissionPlan.generationInputs, }); } else if (submissionPlan.kind === 'quick-edit') { - let referenceImageSrc = await resolveEditorImageReferenceDataUrl( - submissionPlan.sourceLayer.objectKey?.trim() || - submissionPlan.sourceLayer.src, - ); const quickEditSelectionMarks = quickEditSelectionState?.sourceLayerId === submissionPlan.sourceLayer.id ? quickEditSelectionState.marks : []; + let referenceImageSrc: string; if (quickEditSelectionMarks.length > 0) { - referenceImageSrc = await renderUiDesignAssetExtractionMarkedImage({ - source: referenceImageSrc, - marks: quickEditSelectionMarks, - showIndexLabels: true, - }); + const renderSourceImageSrc = + await resolveEditorImageReferenceDataUrl( + submissionPlan.sourceLayer.objectKey?.trim() || + submissionPlan.sourceLayer.src, + ); + const markedImageDataUrl = + await renderUiDesignAssetExtractionMarkedImage({ + source: renderSourceImageSrc, + marks: quickEditSelectionMarks, + showIndexLabels: true, + }); + referenceImageSrc = await uploadEditorGenerationImageDataUrl( + markedImageDataUrl, + projectId, + ); + } else { + referenceImageSrc = await resolveEditorGenerationMediaReference( + submissionPlan.sourceLayer, + 'image', + projectId, + ); } const generated = await runEditorGenerationWithWalletRefresh( editEditorImage({ @@ -1494,9 +1787,14 @@ export function useImageCanvasGenerationSubmissionWorkflow({ } else if (submissionPlan.kind === 'video') { const canvasCompletionPlaceholder = getGeneratingDialogPlaceholder(dialog); + const videoGenerationInput = await normalizeVideoGenerationReferences( + submissionPlan.input, + dialog.generationReferences ?? [], + projectId, + ); const generated = await runEditorGenerationWithWalletRefresh( generateEditorVideo({ - ...submissionPlan.input, + ...videoGenerationInput, projectId, generationInputs: submissionPlan.result.generationInputs, assetFolderId, @@ -1605,7 +1903,11 @@ export function useImageCanvasGenerationSubmissionWorkflow({ ); } else { const imageGenerationInput = - await normalizeImageGenerationReferenceImages(submissionPlan.input); + await normalizeImageGenerationReferenceImages( + submissionPlan.input, + resolveImageGenerationDialogReferences(dialog), + projectId, + ); const resultTitle = submissionPlan.result.title ?? `生成图片 ${layerCounterRef.current + 1}`; @@ -1764,9 +2066,19 @@ export function useImageCanvasGenerationSubmissionWorkflow({ const canvasCompletionPlaceholder = canvasDialog ? getGeneratingDialogPlaceholder(canvasDialog) : undefined; + const sourceImageSrc = await resolveEditorGenerationMediaReference( + characterAnimationSourceLayer, + 'image', + projectId, + { + allowRegisteredIds: false, + requireImageObjectReference: true, + }, + ); const result = await runEditorGenerationWithWalletRefresh( generateEditorCharacterAnimation({ ...submissionPlan.input, + sourceImageSrc, projectId, generationInputs, sourceResourceId: characterAnimationSourceLayer.resourceId, diff --git a/src/components/image-editor/useImageCanvasGenerationWorkflow.test.tsx b/src/components/image-editor/useImageCanvasGenerationWorkflow.test.tsx index 37a37385d..d4a5eacfa 100644 --- a/src/components/image-editor/useImageCanvasGenerationWorkflow.test.tsx +++ b/src/components/image-editor/useImageCanvasGenerationWorkflow.test.tsx @@ -1,6 +1,12 @@ /* @vitest-environment jsdom */ -import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { + act, + fireEvent, + render, + screen, + waitFor, +} from '@testing-library/react'; import { useRef, useState } from 'react'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -12,9 +18,9 @@ import type { ImageContextMenuState, SidebarPanel, } from './ImageCanvasEditorTypes'; +import { IMAGE_MODEL_GPT_IMAGE_2 } from './ImageCanvasGenerationModel'; import { useCanvasGenerationDialogs } from './useCanvasGenerationDialogs'; import { useImageCanvasGenerationWorkflow } from './useImageCanvasGenerationWorkflow'; -import { IMAGE_MODEL_GPT_IMAGE_2 } from './ImageCanvasGenerationModel'; const generateEditorImageMock = vi.hoisted(() => vi.fn()); const generateEditorCharacterAnimationMock = vi.hoisted(() => vi.fn()); @@ -1268,7 +1274,7 @@ describe('useImageCanvasGenerationWorkflow', () => { assetFolderId: undefined, assetKind: undefined, assetLabel: '生成图片 1', - referenceImageSrcs: ['data:image/png;base64,source'], + referenceImageSrcs: ['resource-source'], generationInputs: { fields: [{ title: '生成提示词', value: '一张生成图' }], references: [ @@ -1514,6 +1520,14 @@ describe('useImageCanvasGenerationWorkflow', () => { }); it('submits spec generation with reference image and reference prompt semantics', async () => { + uploadEditorMediaAssetFileMock.mockResolvedValueOnce({ + src: 'https://signed.example.test/generation-reference.png', + objectKey: + 'generated-character-drafts/editor/generation-references/reference.png', + assetObjectId: 'asset-object-generation-reference', + legacyPublicPath: + '/generated-character-drafts/editor/generation-references/reference.png', + }); generateEditorImageMock.mockResolvedValueOnce( createGenerated({ prompt: 'UI规范图' }), ); @@ -1531,7 +1545,9 @@ describe('useImageCanvasGenerationWorkflow', () => { model: 'gpt-image-2', aspectRatio: '16:9', imageSize: '2K', - referenceImageSrcs: ['data:image/png;base64,ref'], + referenceImageSrcs: [ + 'generated-character-drafts/editor/generation-references/reference.png', + ], prompt: expect.stringContaining('参考图生成规范'), }), ); @@ -1630,7 +1646,7 @@ describe('useImageCanvasGenerationWorkflow', () => { expect(editEditorImageMock).toHaveBeenCalledWith( expect.objectContaining({ prompt: '快速修图', - sourceImageSrc: 'data:image/png;base64,source', + sourceImageSrc: 'resource-source', assetLabel: '源图 快速编辑', sourceResourceId: 'resource-source', generationInputs: { @@ -1789,7 +1805,7 @@ describe('useImageCanvasGenerationWorkflow', () => { expect(resolveEditorImageReferenceDataUrlMock).not.toHaveBeenCalled(); await waitFor(() => { expect(removeImageBackgroundMock).toHaveBeenCalledWith({ - sourceImageSrc: '/generated-character-drafts/editor/private.png', + sourceImageSrc: 'generated-character-drafts/editor/private.png', projectId: undefined, targetLayerId: 'layer-source', assetKind: 'character', @@ -1832,6 +1848,7 @@ describe('useImageCanvasGenerationWorkflow', () => { ); expect(removeImageBackgroundMock).toHaveBeenCalledWith( expect.objectContaining({ + sourceImageSrc: 'resource-source', projectId: 'project-1', targetLayerId: 'layer-source', assetLabel: '源图 去背景', @@ -1845,6 +1862,7 @@ describe('useImageCanvasGenerationWorkflow', () => { }), }), ); + expect(resolveEditorImageReferenceDataUrlMock).not.toHaveBeenCalled(); await act(async () => { resolveBackgroundRemoval?.({ imageSrc: diff --git a/src/components/image-editor/useImageCanvasGenerationWorkflow.ts b/src/components/image-editor/useImageCanvasGenerationWorkflow.ts index 2f5b7280c..75b120429 100644 --- a/src/components/image-editor/useImageCanvasGenerationWorkflow.ts +++ b/src/components/image-editor/useImageCanvasGenerationWorkflow.ts @@ -19,6 +19,7 @@ import { type EditorProjectResourceSnapshot, type EditorProjectSnapshot, } from '../../services/image-editor/editorProjectClient'; +import { resizeCropExpandFrame } from './ImageCanvasCropExpandModel'; import type { CanvasGenerationDialogState, CanvasLayer, @@ -36,7 +37,6 @@ import type { SpecFormValues, SpecGenerationType, } from './ImageCanvasEditorTypes'; -import { resizeCropExpandFrame } from './ImageCanvasCropExpandModel'; import { appendCharacterReference, appendGenerationReference, @@ -103,6 +103,8 @@ import { } from './ImageCanvasUiAssetExtractionModel'; import { applyQueuedEditorGenerationProject, + createEditorGenerationMediaUploadId, + resolveEditorGenerationMediaReference, useImageCanvasGenerationSubmissionWorkflow, } from './useImageCanvasGenerationSubmissionWorkflow'; @@ -222,7 +224,10 @@ function preserveSourceLayerInProjectSnapshot( resources: sourceResource ? [...project.resources, sourceResource] : project.resources, - layers: [createProjectLayerSnapshotFromLayer(sourceLayer), ...project.layers], + layers: [ + createProjectLayerSnapshotFromLayer(sourceLayer), + ...project.layers, + ], }; } @@ -1434,9 +1439,10 @@ export function useImageCanvasGenerationWorkflow({ cropExpandSourceLayer.resourceId; let cropExpandAssetKind = cropExpandSourceLayer.assetKind; if (cropExpandProjectId) { + const cropExpandUploadId = createEditorGenerationMediaUploadId(); const cropExpandFile = dataUrlToImageFile( result.imageSrc, - `crop-expand-${cropExpandIndex}.png`, + `crop-expand-${cropExpandIndex}-${cropExpandUploadId}.png`, ); const uploadedCropExpand = await uploadEditorMediaAssetFile( cropExpandFile, @@ -1447,7 +1453,7 @@ export function useImageCanvasGenerationWorkflow({ 'editor', 'crop-expand', cropExpandProjectId, - `${Date.now()}`, + cropExpandUploadId, ], entityId: cropExpandProjectId, metadata: { @@ -1480,8 +1486,7 @@ export function useImageCanvasGenerationWorkflow({ cropExpandObjectKey = cropExpandResource.objectKey ?? uploadedCropExpand.objectKey; cropExpandAssetObjectId = - cropExpandResource.assetObjectId ?? - uploadedCropExpand.assetObjectId; + cropExpandResource.assetObjectId ?? uploadedCropExpand.assetObjectId; cropExpandSourceResourceId = cropExpandResource.sourceResourceId ?? cropExpandSourceLayer.resourceId; @@ -1574,10 +1579,11 @@ export function useImageCanvasGenerationWorkflow({ } : undefined; try { - const sourceObjectKey = sourceLayer.objectKey?.trim(); - const sourceImageSrc = sourceObjectKey - ? `/${sourceObjectKey.replace(/^\/+/u, '')}` - : await resolveEditorImageReferenceDataUrl(sourceLayer.src); + const sourceImageSrc = await resolveEditorGenerationMediaReference( + sourceLayer, + 'image', + projectId, + ); const result = await removeImageBackground({ sourceImageSrc, projectId, @@ -1672,7 +1678,6 @@ export function useImageCanvasGenerationWorkflow({ openPlacedCanvasGenerationDialog, projectId, refreshTaskListForQueuedGeneration, - resolveEditorImageReferenceDataUrl, setActiveSidebarPanel, setImageContextMenu, setMetadataLayer,