diff --git a/.dockerignore b/.dockerignore index 422a9ea01..35a52aba6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,6 +22,7 @@ tmp .env.secrets.* spacetime.local.json deploy/container/api-server.env +deploy/container/worker-smoke server-rs/target server-rs/target-* diff --git a/.gitignore b/.gitignore index 2885233dc..9a953a92e 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ temp*build*/ .env.secrets.local spacetime.local.json deploy/container/api-server.env +deploy/container/worker-smoke/ # Local load-test data extracted from private migration files scripts/loadtest/data/*.local.json diff --git a/.hermes/shared-memory/decision-log.md b/.hermes/shared-memory/decision-log.md index 3d37d65ba..0570e1a69 100644 --- a/.hermes/shared-memory/decision-log.md +++ b/.hermes/shared-memory/decision-log.md @@ -16,6 +16,14 @@ --- +## 2026-06-12 外部生成 worker 扩展到跳一跳、拼消消和敲木鱼 + +- 背景：外部图片生成已从 HTTP 长请求迁到 `external_generation_job` 队列；跳一跳、拼消消和敲木鱼继续扩展时需要统一 job 粒度、前端等待展示和本地 / 生产验证口径。 +- 决策：队列 BFF 暴露用户可见队列概览 `GET /api/runtime/external-generation/queue-overview` 和单 job 状态 `GET /api/runtime/external-generation/jobs/{jobId}`；首版固定“单动作单 job”，不拆提示词 / 生图 / 切图 / 持久化等阶段 job。进入队列的范围为跳一跳 `compile-draft` / `regenerate-tiles`、拼消消 `compile-draft` / `regenerate-atlas`、敲木鱼 `compile-draft` / `regenerate-hit-object` 图片资产动作；非外部图片生成动作继续 inline。 +- 影响范围：外部生成 worker Module、api-server BFF、生成页等待展示、跳一跳 / 拼消消 / 敲木鱼创作与结果页生成动作、本地和生产验证文档。 +- 验证方式：本地 `npm run dev` 默认保留 inline 开发体验；验证 worker 队列、等待展示、lease 或扩缩容时显式使用 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue` 并启动 worker，或运行 `npm run container:worker-smoke -- smoke`。部署后确认 `/healthz`、`/readyz`、队列概览 BFF、单 job 状态和对应玩法 session/detail 状态都能收敛。 +- 关联文档：`docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`。 + ## 2026-06-11 本地服务器管理入口采用 SSH alias + egui 桌面面板 - 背景：release / dev 等服务器的日常巡检已有 systemd、健康巡检 timer 和 HTTP 探测口径，但开发者本地仍需要在多个 SSH alias 间手工切换命令并重复执行启停操作。 @@ -541,7 +549,7 @@ ## 2026-06-05 Server-Provision 全程在目标部署 agent 执行且不安装构建链 - 背景：`Genarrative-Server-Provision` 的 `DEPLOY_TARGET=development` 语义是部署到 dev 服务器，不是构建机 dry-run。旧流水线把 development 映射到 `linux && genarrative-build`，还先在 build 节点准备 `provision-tools/` 再 stash 给后续阶段，导致真实 dev 初始化可能跑到 Jenkins controller / build 节点；脚本还安装 clang / lld / pkg-config / OpenSSL headers / sccache 等构建链依赖，超出了服务器初始化职责。 -- 决策：Server-Provision 只做服务器初始化，全程运行在目标部署 agent：development 使用 `linux && genarrative-dev-deploy`，release 使用 `linux && genarrative-release-deploy`。`Prepare Provision Tools` 与 `Provision Server` 在同一个目标 agent workspace 顺序执行，不再切到 `linux && genarrative-build`，不再 `stash/unstash` 工具包。`scripts/jenkins-server-provision.sh` 不再安装 clang / lld / pkg-config / libssl-dev / sccache；非 dry-run 仍要求目标 dev / release agent 具备 root 权限，因为 provision 会写 systemd、Nginx、`/etc` 和系统用户。Job 的 `Pipeline script from SCM` 与 Jenkinsfile 参数 `SOURCE_GIT_REMOTE_URL` 都必须使用本机路径或目标 agent 可访问的内网 Git 源，不允许公网 Git fallback。 +- 决策：Server-Provision 只做服务器初始化，全程运行在目标部署 agent：development 使用 `linux && genarrative-dev-deploy`，release 使用 `linux && genarrative-release-deploy`。`Prepare Provision Tools` 与 `Provision Server` 在同一个目标 agent workspace 顺序执行，不再切到 `linux && genarrative-build`，不再 `stash/unstash` 工具包。`scripts/jenkins-server-provision.sh` 不再安装 clang / lld / pkg-config / libssl-dev / sccache；当前 OpenSSL 3.2 独立运行时自举会安装 `build-essential` 等最小工具，这是满足 api-server/libcurl 运行时符号的受控例外，不代表 provision 承担 api-server 构建职责。非 dry-run 仍要求目标 dev / release agent 具备 root 权限，因为 provision 会写 systemd、Nginx、`/etc` 和系统用户。Job 的 `Pipeline script from SCM` 与 Jenkinsfile 参数 `SOURCE_GIT_REMOTE_URL` 都必须使用本机路径或目标 agent 可访问的内网 Git 源，不允许公网 Git fallback。 - 追加决策（2026-06-10）：`Prepare Provision Tools` 必须先读取目标机现状，再准备需要的文件。目标机 `/usr/local/bin/otelcol-contrib` 版本匹配 `OTELCOL_VERSION` 时直接复用；`${SPACETIME_ROOT}/bin/current/spacetimedb-cli` 和 `spacetimedb-standalone` 存在且 CLI 版本匹配 `SPACETIME_EXPECTED_VERSION` 或 `SPACETIME_DOWNLOAD_ROOT` 中的版本时，直接复用当前安装生成 `provision-tools/`。只有目标机缺失、不可执行或版本不匹配时，才消费 `PROVISION_DOWNLOADS_DIR` 中的本地包或进入下载分支。 - 影响范围：`jenkins/Jenkinsfile.production-server-provision`、`scripts/jenkins-server-provision.sh`、生产运维文档、Server-Provision 排障口径。 - 验证方式：Jenkins 日志中 Server-Provision 的 `Prepare`、`Checkout Provision Files`、`Prepare Provision Tools` 和 `Provision Server` 都在目标 dev / release agent 上执行；日志不出现 `Running on Jenkins`、`linux && genarrative-build`、`stash 'server-provision-tools'`、`Git 主地址拉取失败...改用备用地址`、`https://git.genarrative.world/GenarrativeAI/Genarrative.git` 或构建依赖 / sccache 安装步骤；`bash -n scripts/jenkins-server-provision.sh` 和编码检查通过。 @@ -579,6 +587,24 @@ - 验证方式：关闭任一创作入口后，新建创作请求返回 `creation_entry_disabled`；公开作品列表 / 详情 / 启动 / 运行态动作不返回该错误；进入平台首页不弹“平台首页：creation_entry_disabled”；关闭态入口卡显示锁定状态且不显示 `10-20泥点数`。 - 关联文档：`docs/【玩法创作】平台入口与玩法链路-2026-05-15.md`。 +## 2026-06-03 外部内容生成改为持久队列加 worker 角色 + +- 背景：拼图首图、图集、音频等外部生成链路长期占用 `api-server` HTTP handler，导致扩容只能放大 API 进程，且 HTTP 超时和外部 provider 波动会直接影响创作入口。 +- 决策：外部生成任务统一进入 SpacetimeDB `external_generation_job` 持久队列，由 `api-server` 的 `external-generation-worker` 进程角色 claim lease 后执行；HTTP 角色只做鉴权、表单/状态初始化、入队和返回 `queued/running/completed/failed` 操作状态。生产通过 systemd worker 模板增加实例数或提高 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY` 动态扩缩容，`GENARRATIVE_PROCESS_ROLE=all` 仅用于本地 smoke。拼图 `compile_puzzle_draft`、结果页 `generate_puzzle_images` 与 `generate_puzzle_ui_background` 已接入 worker；业务写回必须在 SpacetimeDB transaction 内校验 `external_generation_job` 的 `job_id + worker_id + lease_token`、job kind、owner 和 source entity，其中首图 worker 的前置 `compile_puzzle_agent_draft` 也必须带 guard。worker 核心业务写回失败不能返回内存快照并把 job 标成 completed；失败态业务写回成功后才能把 job 标成 failed，失败态未写回则保留租约等待后续重领。拼图业务失败不自动重试，只保留 lease 过期后的崩溃重领，避免钱包扣退费幂等漂移。生产发布会启用默认 `genarrative-external-generation-worker@1.service` 并等待 worker active，worker 停机时停止 claim 新任务并 drain 当前任务。 +- 2026-06-07 追加：`GENARRATIVE_EXTERNAL_GENERATION_MODE` 使用 `queue|inline` 显式策略；生产和容器扩缩容验证保持 `queue`。本地开发若需要同步等待结果，应通过 `.env.local` 或本机环境显式配置为 `inline`，由 HTTP handler 复用同一 worker executor 直接返回 `completed`，不创建 `external_generation_job`，不支持 worker 动态扩缩容；脚本不得硬编码该策略。拼图写回 guard 字段改为可选，queue 路径仍必须完整校验 `job_id + worker_id + lease_token`；inline 路径只允许三项同时为空，半空 guard 仍拒绝。 +- 2026-06-11 追加：生产新增固定 `external-generation-controller` 进程角色和 `genarrative-external-generation-controller.service`。controller 只读取 `get_external_generation_queue_stats_and_return` 队列统计并管理 `genarrative-external-generation-worker@N.service`，不监听 HTTP、不执行外部生成任务；默认保留 `@1`，按 `claimable_pending + running_active + expired_running` 计算目标实例数，上限由 `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MAX_WORKERS` 控制，缩容需要连续空闲轮数且每轮只停最高编号一个实例。 +- 影响范围：`server-rs/crates/spacetime-module/src/external_generation.rs`、`server-rs/crates/spacetime-client/src/external_generation.rs`、`server-rs/crates/api-server/src/external_generation_worker.rs`、`server-rs/crates/api-server/src/external_generation_worker_controller.rs`、`deploy/systemd/genarrative-external-generation-worker@.service`、`deploy/systemd/genarrative-external-generation-controller.service`、`deploy/env/external-generation-controller.env.example`、`scripts/deploy/production-api-deploy.sh`、`scripts/jenkins-server-provision.sh`、拼图 `compile_puzzle_draft`、拼图 `generate_puzzle_images`、拼图 `generate_puzzle_ui_background`、生产 env 模板和运维文档。 +- 验证方式：`npm run spacetime:generate`、`npm run check:spacetime-schema`、`npm run check:server-rs-ddd`、`cargo check -p api-server --manifest-path server-rs/Cargo.toml`，并在 queue 模式下用 `GENARRATIVE_PROCESS_ROLE=all npm run dev` smoke 至少一次 queued -> worker 完成链路；本地 inline 排查只确认不创建 `external_generation_job`。 +- 关联文档：`docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`、`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`。 + +## 2026-06-03 外部生成 worker lease 使用 SpacetimeDB 时间和 token 栅栏 + +- 背景：外部生成 worker 支持多进程动态缩扩容后，长任务超过单次 lease、worker 本机时钟漂移或复用 worker id 都可能导致同一任务被重复领取并被过期执行者回写。 +- 决策：`external_generation_job` 新增末尾字段 `lease_token`；`claim` 使用 SpacetimeDB `ctx.timestamp` 计算 lease，生成本次 claim token；worker 执行期间调用 `renew_external_generation_job_lease_and_return` 续租；`complete/fail` 必须带 `worker_id + lease_token` 才能回写。拼图 `compile_puzzle_draft` 的 dedupe key 包含本次 `extgen-` job id，避免同一 session 的失败或完成 job 吞掉后续重新生成。拼图首图前置 `compile_puzzle_agent_draft`、图片保存、UI 背景与失败态业务写回同样必须携带 lease guard，并在 `compile_puzzle_agent_draft`、`save_puzzle_generated_images`、`save_puzzle_ui_background`、`mark_puzzle_draft_generation_failed`、`mark_puzzle_level_generation_failed` 的 SpacetimeDB 事务内校验。 +- 影响范围：`server-rs/crates/spacetime-module/src/external_generation.rs`、`server-rs/crates/spacetime-module/src/puzzle.rs`、`server-rs/crates/module-puzzle/src/commands.rs`、`server-rs/crates/spacetime-client/src/external_generation.rs`、`server-rs/crates/spacetime-client/src/puzzle.rs`、`server-rs/crates/api-server/src/external_generation_worker.rs`、`server-rs/crates/api-server/src/puzzle/handlers.rs`、`server-rs/crates/api-server/src/puzzle/draft.rs`、`server-rs/crates/api-server/src/puzzle/generation.rs`。 +- 验证方式：`npm run spacetime:generate`、`npm run check:spacetime-schema`、`cargo test -p spacetime-module external_generation --manifest-path server-rs/Cargo.toml`、`cargo test -p api-server external_generation_worker --manifest-path server-rs/Cargo.toml`、`GENARRATIVE_PROCESS_ROLE=all npm run dev` 后检查 `/healthz`。 +- 关联文档：`docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`。 + ## 2026-06-04 Draft Generation Shelf 剩余草稿打开 intent 收口 - 背景：拼图 / 抓大鹅草稿打开 intent 已归入 `platformDraftGenerationShelfModel.ts`，但方洞挑战、大鱼吃小鱼和视觉小说仍在平台壳层内联判断已发布详情、缺 session、active generating、当前结果页和普通草稿恢复。 @@ -1135,7 +1161,7 @@ - 背景：Windows 本机直连极高 VU 压测会放大本地连接与发送缓冲行为，和线上 Linux + Nginx + systemd 拓扑不一致；需要一个更接近生产网络层的模拟方案，但不能扰动当前生产发布链路。 - 决策：新增 `deploy/container/` 容器化方案，使用 Docker Compose 组合 Linux release `api-server`、容器 SpacetimeDB、容器 Nginx、`otelcol-contrib` debug exporter 和可选 k6。该方案只用于本机或预发压测模拟，不替换当前生产 `systemd + Nginx + Jenkins` 路径。 -- 服务器模拟参数：2026-05-18 通过 `ssh genarrative-release` 采样，目标机器为 2 vCPU / 约 2 GiB RAM / Ubuntu 24.04 / Nginx `worker_connections=768`；容器方案按待发布运行口径使用 `nofile=4096`，并在 compose 中限制 `spacetimedb cpus=1.0 mem_limit=768m`、`api-server cpus=2.0 mem_limit=1g`、`nginx cpus=0.25 mem_limit=128m`、`otelcol cpus=0.25 mem_limit=128m`、`k6 cpus=0.5 mem_limit=512m`；Collector 镜像默认使用 `otel/opentelemetry-collector-contrib:0.151.0`。 +- 服务器模拟参数：2026-05-18 通过 `ssh genarrative-release` 采样，目标机器为 2 vCPU / 约 2 GiB RAM / Ubuntu 24.04 / Nginx `worker_connections=768`；容器方案按待发布运行口径使用 `nofile=4096`，并在 compose 中限制 `spacetimedb cpus=1.0 mem_limit=896m`、`api-server cpus=2.0 mem_limit=1g`、`external-generation-worker cpus=2.0 mem_limit=1g`、`nginx cpus=0.5 mem_limit=128m`、`otelcol cpus=0.25 mem_limit=128m`、`k6 cpus=1.0 mem_limit=512m`；Collector 镜像默认使用 `otel/opentelemetry-collector-contrib:0.151.0`。 - 隔离边界：容器方案使用独立 `deploy/container/api-server.env`、独立 Nginx 配置、独立 compose 命令和默认 `18080` 端口；真实 token 不进入镜像、不提交 Git；生产 systemd 单元、Jenkins 发布脚本和 `deploy/nginx/` 模板仍是正式线上来源。 - 生产 Collector：server-provision 可安装 `otelcol-contrib.service` 和本机 debug exporter 配置，但二进制由 Jenkins 构建机先准备 `provision-tools/otelcol-contrib` 再上传到 release 部署 agent，目标机不从 GitHub 下载；api-server 是否发送 OTLP 仍由 `GENARRATIVE_OTEL_ENABLED` 控制。 - 影响范围：`deploy/container/`、`scripts/container-compose.mjs`、`package.json` 容器命令、开发运维文档和容器 build context 排除规则。 diff --git a/.hermes/shared-memory/development-workflow.md b/.hermes/shared-memory/development-workflow.md index 2e7cab4e1..ca86734cf 100644 --- a/.hermes/shared-memory/development-workflow.md +++ b/.hermes/shared-memory/development-workflow.md @@ -120,6 +120,16 @@ npm run server-manager:panel npm run dev:spacetime:logs ``` +本机隔离验证外部生成 worker 队列、API-only 更新和 worker 动态扩缩容时，优先使用： + +```bash +npm run container:worker-smoke -- smoke +``` + +该命令生成 `deploy/container/worker-smoke/` 下的 gitignored env 与端口 state，启动独立 compose project 和独立 SpacetimeDB，用 unsupported job 验证 worker claim / fail 回写；排查时用 `api-update` 确认 API 重建不触碰 worker，用 `scale ` 调整 worker 数量。 +`external_generation_job` 是 private table，worker-smoke 通过 worker 日志里的 job_id 和 unsupported 记录确认消费，不通过 CLI SQL 查询队列表。 +worker-smoke 默认把本机 `spacetime` CLI 打成轻量 SpacetimeDB 镜像，避免首次 smoke 依赖官方大镜像下载。若容器内 Cargo 下载依赖不稳定，追加 `--local-binary`，让容器内 Cargo 复用本机 Cargo 缓存构建当前 `api-server` 二进制，并把产物放进 Debian bookworm smoke runtime；可用 `GENARRATIVE_WORKER_SMOKE_LOCAL_BASE_IMAGE` 覆盖运行时基础镜像；隔离端口或库数据需要重建时追加 `--force`。 + 后台管理前端： ```bash diff --git a/.hermes/shared-memory/pitfalls.md b/.hermes/shared-memory/pitfalls.md index d26732d10..dc5a9fe3b 100644 --- a/.hermes/shared-memory/pitfalls.md +++ b/.hermes/shared-memory/pitfalls.md @@ -15,6 +15,38 @@ - 关联：相关文件、文档、提交或 Issue ``` +## 外部生成 worker 业务失败重试会撞上钱包扣退费幂等 + +- 现象：同一个外部生成 job 如果第一次业务失败后退款，再用同一个业务资源 ID 自动重试并成功，钱包 `consume` ledger 可能因为同 ID 已存在而跳过，最终出现“失败已退、成功不再扣”的余额漂移。 +- 原因：资产操作扣费和退款都用稳定 ledger id 做幂等；这能保护 lease 过期后的崩溃重领不重复扣费，但不适合“已明确失败且已退款”的自动业务重试。 +- 处理：拼图 `puzzle_compile_draft` 首期设置 `max_attempts=1`，业务失败直接 failed，只保留 running lease 过期后的崩溃重领。后续若要恢复自动 retry，必须先引入 attempt-aware billing 或可配对撤销的账本接口。 +- 验证：检查 `external_generation_job.max_attempts`、worker 失败回写和钱包 ledger；失败后草稿进入 failed，重试应由用户重新触发新任务，而不是旧 job 自动 pending。 +- 关联：`server-rs/crates/api-server/src/puzzle/handlers.rs`、`server-rs/crates/api-server/src/asset_billing.rs`、`server-rs/crates/spacetime-module/src/runtime/profile.rs`、`docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md`。 + +## 外部生成队列不再由 HTTP 进程兜底执行 + +- 现象：拼图首关生成接口返回 `queued`，但生成页长时间不完成，重启 `genarrative-api.service` 也没有推进任务。 +- 原因：HTTP 角色只入队，不再直接调用外部 provider；如果没有运行 `GENARRATIVE_PROCESS_ROLE=external-generation-worker` 或 `all` 的进程，`external_generation_job` 会停留在 `pending/running`，直到有 worker claim。 +- 处理：生产用 `systemctl enable --now genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service` 启动保底 worker 和 controller；首次 API deploy 会在默认 worker pattern 下自动启用并启动 `@1`、等待 worker active，并重启验活 controller。扩容默认交给 controller 按队列统计启动 `@2.service` 等实例，手动扩缩容只作为兜底；worker 收到停机信号后会停止 claim 新任务并等待当前任务完成。本地 smoke 可临时用 `GENARRATIVE_PROCESS_ROLE=all npm run dev`；本地若只想同步排查可通过 `.env.local` 或本机环境设置 `GENARRATIVE_EXTERNAL_GENERATION_MODE=inline`，但这不会创建 job，也不能验证 worker 扩缩容。 +- 验证：`systemctl status genarrative-external-generation-controller.service 'genarrative-external-generation-worker@*.service'` 能看到 controller 和 worker 实例；queue 模式下任务被 claim 后 `worker_id` 与 `lease_expires_at` 会更新，完成后 session 进入 ready 或 failed；inline 模式下不应产生新的 `external_generation_job`。 +- 关联：`deploy/systemd/genarrative-external-generation-worker@.service`、`deploy/systemd/genarrative-external-generation-controller.service`、`deploy/env/external-generation-controller.env.example`、`server-rs/crates/spacetime-module/src/external_generation.rs`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`。 + +## 外部生成 worker 业务写回必须同事务校验 lease guard + +- 现象：worker `complete/fail` 已校验 `worker_id + lease_token`，但如果玩法 session / work profile 写回在此之前单独调用，过期 worker 仍可能先写入业务状态，随后才在 job complete/fail 阶段失败；带计费包装的旧 worker 还可能因为 stale guard 错误触发补偿退款。 +- 原因：队列状态栅栏只保护 `external_generation_job` 自身，不会自动保护玩法 procedure。业务写回必须自己带 claim 后的 `job_id / worker_id / lease_token`，并在同一个 SpacetimeDB transaction 内校验 job 仍为 `running`、lease 未过期、job kind、owner 和 source entity 匹配。 +- 处理：拼图首图 worker 的前置 `compile_puzzle_agent_draft`、`save_puzzle_generated_images`、`save_puzzle_ui_background`、`mark_puzzle_draft_generation_failed` 和 `mark_puzzle_level_generation_failed` 已接入 `external_generation_job` lease guard；api-server 的资产扣费包装遇到这类 stale worker lease guard 错误时不执行补偿退款，错误文本包含 `external_generation_job 当前不是 running 状态` 或 `external_generation_job 不存在` 时也按 stale guard 处理。inline 模式只允许 `job_id / worker_id / lease_token` 三项同时为空，半空 guard 仍拒绝。后续迁移其它玩法 worker 时必须复用该模式，不能只在 worker 进程内保存一份 token。 +- 验证：`cargo test -p api-server external_generation_worker --manifest-path server-rs/Cargo.toml`、`cargo test -p api-server asset_operation_billing_does_not_refund_stale_worker_lease_errors --manifest-path server-rs/Cargo.toml`、`cargo check -p api-server --manifest-path server-rs/Cargo.toml`。 +- 关联：`server-rs/crates/spacetime-module/src/external_generation.rs`、`server-rs/crates/spacetime-module/src/puzzle.rs`、`server-rs/crates/api-server/src/external_generation_worker.rs`、`server-rs/crates/api-server/src/asset_billing.rs`、`docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md`。 + +## 外部生成 worker 核心业务写回失败不能完成 job + +- 现象：worker 已经生成图片并拿到本地合成 session 快照，但 SpacetimeDB 业务写回因连接、旧 wasm 或 lease guard 失败没有真实落库；如果此时仍把 `external_generation_job` 标成 `completed`，前端只会看到队列完成而 session 长时间不变化，后续也没有 worker 会重领修复。 +- 原因：同步 HTTP handler 的“外部 provider 已成功但 SpacetimeDB 短暂不可用时返回内存快照”降级语义，不能直接搬进异步 worker。worker 的完成状态必须代表核心业务事实已经持久化。 +- 处理：worker 路径的 `save_puzzle_generated_images` / `save_puzzle_ui_background` 等核心业务写回失败时直接返回错误；只有核心写回已经成功后的非关键投影回写才允许降级记录 warning。业务失败态也必须先写回 session / work profile，写回成功后才允许把队列 job 标为 failed；失败态未写回时保留租约，等待 lease 过期后重领。生产首装和首次 API deploy 都必须至少启用一个 worker 实例，例如 `systemctl enable --now genarrative-external-generation-worker@1.service`。 +- 验证：`cargo check -p api-server --manifest-path server-rs/Cargo.toml`、`cargo test -p api-server asset_operation_billing_does_not_refund_stale_worker_lease_errors --manifest-path server-rs/Cargo.toml`，并在 smoke 时确认 queued 任务被 worker 消费后 session 真实更新。 +- 关联：`server-rs/crates/api-server/src/puzzle/draft.rs`、`server-rs/crates/api-server/src/puzzle/generation.rs`、`server-rs/crates/api-server/src/external_generation_worker.rs`、`docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md`。 + ## 生产冷备份后 API 不能只依赖 SpacetimeDB 自恢复 - 现象：release 机器 `03:20` 冷备份后，`spacetimedb.service` 已恢复，但作品列表、创作入口配置或公开 gallery 继续超时 / 502 / 504，`genarrative-api.service` 保持 stopped。 diff --git a/deploy/container/README.md b/deploy/container/README.md index f2e5af9b1..0baccf5f9 100644 --- a/deploy/container/README.md +++ b/deploy/container/README.md @@ -9,12 +9,14 @@ Docker Compose ├─ spacetimedb :3101，独立数据卷，供 api-server 连接 ├─ nginx :80 -> api-server:8082，负责静态站点、/admin/、/api/ 反代、upstream timing log、连接限制 ├─ api-server :8082，Linux release 构建，连接 compose 内 SpacetimeDB +├─ external-generation-worker，独立 worker 进程，消费 external_generation_job 队列 ├─ otelcol :4317/4318，debug exporter，接收 traces / metrics / logs └─ k6 profile=loadtest 时临时启动，在 compose 网络内压 nginx ``` -当前容器模拟参数按 `genarrative-release` 服务器采样值收口为 2 vCPU / 2 GiB RAM / 4096 soft nofile / 768 worker_connections，并已在 compose 里落实到 `spacetimedb cpus=1.0 mem_limit=896m`、`api-server cpus=2.0 mem_limit=1g`、`nginx cpus=0.5 mem_limit=128m`、`otelcol cpus=0.25 mem_limit=128m`、`k6 cpus=1.0 mem_limit=512m`。SpacetimeDB 同时设置 `--page_pool_max_size=402653184`，给 reducer、订阅与运行时保留更多非 page pool 内存。 +当前容器模拟参数按 `genarrative-release` 服务器采样值收口为 2 vCPU / 2 GiB RAM / 4096 soft nofile / 768 worker_connections，并已在 compose 里落实到 `spacetimedb cpus=1.0 mem_limit=896m`、`api-server cpus=2.0 mem_limit=1g`、`external-generation-worker cpus=2.0 mem_limit=1g`、`nginx cpus=0.5 mem_limit=128m`、`otelcol cpus=0.25 mem_limit=128m`、`k6 cpus=1.0 mem_limit=512m`。SpacetimeDB 同时设置 `--page_pool_max_size=402653184`，给 reducer、订阅与运行时保留更多非 page pool 内存。 容器 `api-server` 默认 `GENARRATIVE_API_WORKER_THREADS=4`，用于让 Tokio 在 2 vCPU 配额内有更多 I/O 调度 worker；该值不会突破 compose 里的 `cpus=2.0` CPU 上限。 +容器默认 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue`，用于验证 `api-server -> external_generation_job -> external-generation-worker` 链路；如只想本地同步排查 provider/OSS/SpacetimeDB 写回，可在本机 env 临时改为 `inline`，但该模式不会覆盖 worker 动态扩缩容验证。 Collector 镜像使用 `otel/opentelemetry-collector-contrib:0.151.0`。 生产服务器若启用 Collector，则由 `deploy/systemd/otelcol-contrib.service` 和 `deploy/otelcol/genarrative-debug.yaml` 托管，不走容器镜像。 @@ -55,7 +57,7 @@ Linux Docker Engine 若要从宿主机 CLI 连到容器内服务，直接用 `ht ## 构建工具链 -`api-server` 容器镜像只构建 Linux release API 二进制，不构建 `spacetime-module`。当前 `api-server -> spacetime-client -> spacetimedb-sdk 2.4.1` 依赖链要求 Rust 1.93，因此 `deploy/container/api-server.Dockerfile` 的 Rust builder 固定为 `rust:1.93-bookworm`。如果本机 Docker Hub 拉取失败，可以先在本机准备同名本地 builder 镜像，但不要把临时 bootstrap 容器或私有 registry 凭据写入仓库。 +`api-server` 容器镜像只构建 Linux release API 二进制，不构建 `spacetime-module`。当前 `api-server -> spacetime-client -> spacetimedb-sdk 2.4.1` 依赖链要求 Rust 1.93，因此 `deploy/container/api-server.Dockerfile` 的 Rust builder 固定为 `rust:1.93-bookworm`。镜像构建阶段会同时复制 `public/`，用于满足 API 二进制里 `include_bytes!` 引用的内置素材；不要把 `public/generated-*` 放入镜像上下文。如果本机 Docker Hub 拉取失败，可以先在本机准备同名本地 builder 镜像，但不要把临时 bootstrap 容器或私有 registry 凭据写入仓库。 ## 启动与验证 @@ -74,6 +76,7 @@ curl -sS http://127.0.0.1:18080/api/runtime/puzzle/gallery ```bash npm run container:logs -- nginx npm run container:logs -- api-server +npm run container:logs -- external-generation-worker npm run container:logs -- otelcol ``` @@ -85,6 +88,73 @@ npm run container:config -- --print 如果 `deploy/container/api-server.env` 已写入真实 token，不要把完整展开结果贴到公开渠道。 +动态扩缩容外部生成 worker 时，只调整 `external-generation-worker` service： + +```bash +npm run container:up -- --scale external-generation-worker=3 external-generation-worker +npm run container:up -- --scale external-generation-worker=1 external-generation-worker +``` + +动态扩缩容验证必须保持 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue`；`inline` 模式下生成请求由 `api-server` 同步执行，不会被这些 worker 实例消费。 + +### 外部生成 Worker 隔离 Smoke + +如果只想在本机隔离验证 worker 模式，不复用 `deploy/container/api-server.env`，使用专用脚本： + +```bash +npm run container:worker-smoke -- smoke +``` + +该脚本会生成 gitignored 的 `deploy/container/worker-smoke/api-server.env` 与端口 state，使用独立 compose project、独立 SpacetimeDB 数据卷和独立 host 端口，完成 `build -> up-spacetime -> publish -> up -> enqueue -> api-update -> enqueue`。测试 job 使用 `worker_smoke_unsupported` 类型，不访问真实 VectorEngine、LLM 或 OSS；预期结果是 worker 领取队列任务后按“不支持的任务类型”执行失败分支，从而验证队列 claim、lease、失败回写路径和 API / worker 进程隔离。`external_generation_job` 是 private table，脚本通过 worker 日志里的 job_id 和 unsupported 记录确认消费，不通过 CLI SQL 绕过权限。`smoke` 默认只启动 `api-server` 与 `external-generation-worker`，避免无关前端 / Nginx 镜像构建；需要同时验证 Nginx 时可分步执行 `up --with-nginx`。 + +分步排查时可执行： + +```bash +npm run container:worker-smoke -- init --force +npm run container:worker-smoke -- build +npm run container:worker-smoke -- up-spacetime +npm run container:worker-smoke -- publish +npm run container:worker-smoke -- up +npm run container:worker-smoke -- enqueue before-update +npm run container:worker-smoke -- api-update +npm run container:worker-smoke -- enqueue after-update +npm run container:worker-smoke -- status +``` + +如果隔离端口或库数据需要重置： + +```bash +npm run container:worker-smoke -- smoke --force +``` + +`container:worker-smoke` 默认会把本机 `spacetime` 2.4.1 CLI 打成轻量 SpacetimeDB 镜像，避免首次 smoke 必须拉取官方大镜像；普通 `npm run container:*` 压测仍默认使用 `clockworklabs/spacetime:v2.4.1`。如果 Docker build 阶段在容器内拉取 crates.io 依赖不稳定，可让容器内 Cargo 复用本机 Cargo 缓存构建当前二进制，再打入临时 smoke 镜像。该模式默认使用 `rust:1.93-bookworm` 作为 builder、Debian bookworm smoke runtime 承载构建产物；需要换 builder 镜像时设置 `GENARRATIVE_WORKER_SMOKE_CARGO_IMAGE`，需要换运行时基础镜像时设置 `GENARRATIVE_WORKER_SMOKE_LOCAL_BASE_IMAGE`： + +```bash +npm run container:worker-smoke -- smoke --local-binary +``` + +`api-update` 只会 `--force-recreate api-server`，并校验 `external-generation-worker` 容器 ID 不变；如要同时重建 API 镜像，使用： + +```bash +npm run container:worker-smoke -- api-update --build +``` + +验证 worker 动态扩缩容： + +```bash +npm run container:worker-smoke -- scale 3 +npm run container:worker-smoke -- ps +npm run container:worker-smoke -- enqueue scaled-workers +npm run container:worker-smoke -- scale 1 +``` + +查看或清理隔离环境： + +```bash +npm run container:worker-smoke -- logs external-generation-worker +npm run container:worker-smoke -- down -v +``` + 停止： ```bash diff --git a/deploy/container/api-server.Dockerfile b/deploy/container/api-server.Dockerfile index 1f098dfdd..686df1999 100644 --- a/deploy/container/api-server.Dockerfile +++ b/deploy/container/api-server.Dockerfile @@ -2,6 +2,7 @@ FROM rust:1.93-bookworm AS rust-builder WORKDIR /workspace COPY server-rs ./server-rs +COPY public ./public RUN cargo build --release -p api-server --manifest-path server-rs/Cargo.toml && \ cp server-rs/target/release/api-server /tmp/api-server diff --git a/deploy/container/api-server.env.example b/deploy/container/api-server.env.example index 8414dc149..9c95130a4 100644 --- a/deploy/container/api-server.env.example +++ b/deploy/container/api-server.env.example @@ -8,6 +8,14 @@ GENARRATIVE_API_PORT=8082 GENARRATIVE_API_LOG=info,tower_http=info GENARRATIVE_API_LISTEN_BACKLOG=1024 GENARRATIVE_API_WORKER_THREADS=4 +# 容器 smoke 可临时设 all；压测或预发按 api / external-generation-worker 拆进程。 +GENARRATIVE_PROCESS_ROLE=api +# 默认 queue 进入 external_generation_job；本地/小流量同步排查可显式设 inline。 +GENARRATIVE_EXTERNAL_GENERATION_MODE=queue +GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID= +GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY=2 +GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS=2000 +GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=3600 GENARRATIVE_API_MAX_CONCURRENT_REQUESTS=512 GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS=320 GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS=64 diff --git a/deploy/container/docker-compose.loadtest.yml b/deploy/container/docker-compose.loadtest.yml index ea3a3c9af..2466617a0 100644 --- a/deploy/container/docker-compose.loadtest.yml +++ b/deploy/container/docker-compose.loadtest.yml @@ -2,7 +2,7 @@ name: genarrative-container-loadtest services: spacetimedb: - image: clockworklabs/spacetime:v2.4.1 + image: ${GENARRATIVE_CONTAINER_SPACETIME_IMAGE:-clockworklabs/spacetime:v2.4.1} user: root command: [ @@ -44,7 +44,7 @@ services: cpus: "2.0" mem_limit: 1g env_file: - - ./api-server.env + - ${GENARRATIVE_CONTAINER_API_ENV_FILE:-./api-server.env} environment: GENARRATIVE_API_HOST: 0.0.0.0 GENARRATIVE_API_PORT: 8082 @@ -69,6 +69,32 @@ services: retries: 12 start_period: 20s + external-generation-worker: + build: + context: ../.. + dockerfile: deploy/container/api-server.Dockerfile + target: api-runtime + cpus: "2.0" + mem_limit: 1g + env_file: + - ${GENARRATIVE_CONTAINER_API_ENV_FILE:-./api-server.env} + environment: + GENARRATIVE_PROCESS_ROLE: external-generation-worker + GENARRATIVE_TRACKING_OUTBOX_DIR: /var/lib/genarrative/tracking-outbox-worker + OTEL_EXPORTER_OTLP_ENDPOINT: http://otelcol:4318 + OTEL_SERVICE_NAME: genarrative-external-generation-worker + extra_hosts: + - "host.docker.internal:host-gateway" + ulimits: + nofile: + soft: 4096 + hard: 4096 + depends_on: + spacetimedb: + condition: service_healthy + otelcol: + condition: service_started + nginx: build: context: ../.. diff --git a/deploy/env/api-server.env.example b/deploy/env/api-server.env.example index 90b2378b1..b967c534d 100644 --- a/deploy/env/api-server.env.example +++ b/deploy/env/api-server.env.example @@ -7,6 +7,14 @@ GENARRATIVE_API_PORT=8082 GENARRATIVE_API_LOG=info,tower_http=info GENARRATIVE_API_LISTEN_BACKLOG=1024 GENARRATIVE_API_WORKER_THREADS=4 +# api 只监听 HTTP；外部生成 worker 用独立进程设置为 external-generation-worker 后横向扩缩。 +GENARRATIVE_PROCESS_ROLE=api +# 默认 queue 进入 external_generation_job；本地/小流量同步排查可显式设 inline。 +GENARRATIVE_EXTERNAL_GENERATION_MODE=queue +GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID= +GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY=2 +GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS=2000 +GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=3600 GENARRATIVE_API_MAX_CONCURRENT_REQUESTS=512 GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS=320 GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS=64 diff --git a/deploy/env/external-generation-controller.env.example b/deploy/env/external-generation-controller.env.example new file mode 100644 index 000000000..e6d0cecaa --- /dev/null +++ b/deploy/env/external-generation-controller.env.example @@ -0,0 +1,13 @@ +# 复制到 /etc/genarrative/external-generation-controller.env 后按机器容量调整。 +# controller 只管理 systemd worker 实例；SpacetimeDB、外部 provider 密钥继续复用 api-server.env。 +# systemd unit 会强制设置 GENARRATIVE_PROCESS_ROLE=external-generation-controller。 + +GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MIN_WORKERS=1 +GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MAX_WORKERS=8 +GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_TARGET_JOBS_PER_WORKER=2 +GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_POLL_INTERVAL_MS=10000 +GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_SCALE_DOWN_IDLE_ROUNDS=6 +GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_SERVICE_TEMPLATE=genarrative-external-generation-worker@{}.service +GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_DRY_RUN=false +GENARRATIVE_API_LOG=info,tower_http=info +OTEL_SERVICE_NAME=genarrative-external-generation-controller diff --git a/deploy/env/external-generation-worker.env.example b/deploy/env/external-generation-worker.env.example new file mode 100644 index 000000000..3ddd83726 --- /dev/null +++ b/deploy/env/external-generation-worker.env.example @@ -0,0 +1,11 @@ +# 复制到 /etc/genarrative/external-generation-worker.env 后按机器容量调整。 +# 该文件只覆盖 worker 专属参数；SpacetimeDB、外部 provider 密钥继续复用 api-server.env。 +# systemd 模板会强制设置 GENARRATIVE_PROCESS_ROLE=external-generation-worker +# 和 GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID=%H-%i，避免多实例 ID 冲突。 + +GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY=2 +GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS=2000 +# 单次 lease 会由 worker 自动续租；该值覆盖心跳抖动窗口即可。 +GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=3600 +GENARRATIVE_API_LOG=info,tower_http=info +OTEL_SERVICE_NAME=genarrative-external-generation-worker diff --git a/deploy/systemd/genarrative-external-generation-controller.service b/deploy/systemd/genarrative-external-generation-controller.service new file mode 100644 index 000000000..d6daf273e --- /dev/null +++ b/deploy/systemd/genarrative-external-generation-controller.service @@ -0,0 +1,28 @@ +[Unit] +Description=Genarrative External Generation Worker Controller +After=network-online.target spacetimedb.service +Wants=network-online.target +Requires=spacetimedb.service + +[Service] +Type=simple +WorkingDirectory=/opt/genarrative/current +EnvironmentFile=/etc/genarrative/api-server.env +EnvironmentFile=-/etc/genarrative/external-generation-controller.env +Environment="LD_LIBRARY_PATH=/opt/genarrative/openssl-3.2.0/lib64:/opt/genarrative/openssl-3.2.0/lib" +ExecStart=/usr/bin/env GENARRATIVE_PROCESS_ROLE=external-generation-controller GENARRATIVE_TRACKING_OUTBOX_DIR=/var/lib/genarrative/tracking-outbox/controller OTEL_SERVICE_NAME=genarrative-external-generation-controller /opt/genarrative/current/api-server +Restart=always +RestartSec=5 +KillSignal=SIGINT +TimeoutStopSec=120 +LimitNOFILE=65535 +TasksMax=512 + +# controller 需要调用 systemctl 管理 worker@N 实例，因此不降为 genarrative 用户。 +# 它只复用 api-server 发布包和 SpacetimeDB 配置，不直接执行外部生成任务。 +PrivateTmp=true +ProtectSystem=full +ReadWritePaths=/opt/genarrative /var/lib/genarrative + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/genarrative-external-generation-worker@.service b/deploy/systemd/genarrative-external-generation-worker@.service new file mode 100644 index 000000000..ee4a71e8a --- /dev/null +++ b/deploy/systemd/genarrative-external-generation-worker@.service @@ -0,0 +1,30 @@ +[Unit] +Description=Genarrative External Generation Worker %i +After=network-online.target spacetimedb.service +Wants=network-online.target +Requires=spacetimedb.service + +[Service] +Type=simple +User=genarrative +Group=genarrative +WorkingDirectory=/opt/genarrative/current +EnvironmentFile=/etc/genarrative/api-server.env +EnvironmentFile=-/etc/genarrative/external-generation-worker.env +Environment="LD_LIBRARY_PATH=/opt/genarrative/openssl-3.2.0/lib64:/opt/genarrative/openssl-3.2.0/lib" +ExecStart=/usr/bin/env GENARRATIVE_PROCESS_ROLE=external-generation-worker GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID=%H-%i GENARRATIVE_TRACKING_OUTBOX_DIR=/var/lib/genarrative/tracking-outbox/%H-%i OTEL_SERVICE_NAME=genarrative-external-generation-worker /opt/genarrative/current/api-server +Restart=always +RestartSec=5 +KillSignal=SIGINT +TimeoutStopSec=7200 +LimitNOFILE=65535 +TasksMax=2048 + +# worker 复用 api-server 发布目录；外部生成审计与临时运行态只写服务端私有目录。 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ReadWritePaths=/opt/genarrative /var/lib/genarrative + +[Install] +WantedBy=multi-user.target diff --git a/docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md b/docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md new file mode 100644 index 000000000..90eb4a190 --- /dev/null +++ b/docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md @@ -0,0 +1,204 @@ +# 外部生成 Worker 化方案 + +更新时间：`2026-06-12` + +## 背景 + +当前 VectorEngine `gpt-image-2`、音频、LLM 等外部生成链路多数由 `api-server` 的 HTTP handler 直接等待上游、OSS 持久化和 SpacetimeDB 回写完成。前端虽然有生成页和会话轮询，但 HTTP 进程仍承担长耗时副作用，导致接入更多玩法或大图生成时只能放大 API 进程，而不能单独扩展外部生成吞吐。 + +## 目标 + +- 默认 `queue` 模式下，`api-server` 的 HTTP 角色只负责鉴权、入参校验、扣费前置/状态初始化、任务入队和返回 `queued` 操作结果。 +- 外部生成副作用由独立 `external-generation-worker` 角色执行。 +- 多个 worker 进程通过 SpacetimeDB 任务表抢占任务，依赖 lease 超时恢复，支持按进程数和单进程并发动态缩扩容。 +- 本地或小流量同步排查可显式启用 `inline` 模式，由 HTTP handler 复用同一 worker executor 同步执行并返回 `completed`；该模式不创建队列任务，也不具备 worker 横向扩容能力。 +- SpacetimeDB reducer / procedure 只做任务状态流转，不做网络、文件系统或外部 provider I/O。 +- 已接入拼图 `compile_puzzle_draft`、结果页 `generate_puzzle_images` 与结果页 `generate_puzzle_ui_background`；本轮扩展到跳一跳、拼消消和敲木鱼的外部图片生成动作。后续玩法继续复用同一队列 Module，不再为每个玩法发明独立队列。 +- 第一版外部生成队列粒度固定为“单个用户动作对应单个 job”。例如草稿编译、结果页单槽重生、图集重生都各自入一个 job；job 内部可以串行或并行调用 provider、OSS、SpacetimeDB 写回，但不再拆成“提示词 / 生图 / 切图 / 去背景 / 持久化 / 回写”等阶段 job。阶段进度只作为 `request_payload_json` / 业务 session 的展示状态，不作为队列调度单位。 +- 不调用外部图片 / 音频 / LLM provider 的动作继续 inline 执行，不为了统一排队而进入 `external_generation_job`。 + +## Module 与 Interface + +新增深一点的 **外部生成任务 Module**，Interface 收敛为： + +- `enqueue_external_generation_job_and_return`：按 `dedupe_key` 幂等创建或返回现有任务。 +- `claim_external_generation_jobs_and_return`：worker 按 `worker_id`、`limit` 和 lease 时长抢占 `pending` 或 lease 过期的 `running` 任务，返回本次 claim 的 `lease_token`。 +- `renew_external_generation_job_lease_and_return`：worker 长任务执行期间按 `worker_id + lease_token` 续租，防止外部生成超过单次 lease 后被重复领取。 +- `complete_external_generation_job_and_return`：worker 成功后按 `worker_id + lease_token` 写入 `result_payload_json`，任务进入 `completed`。 +- `fail_external_generation_job_and_return`：worker 失败后按 `worker_id + lease_token` 回写错误，并按 `max_attempts` 决定回到 `pending` 重试或进入 `failed`。 +- `get_external_generation_queue_stats_and_return`：controller 读取队列积压、运行中任务和过期 lease 数量，用于计算 worker 目标实例数；该 procedure 只读 `external_generation_job`，不直接操作 systemd。 +- `get_external_generation_job_and_return`：按 `job_id` 读取单个任务状态，给 BFF 和生成页展示使用；必须只返回调用者有权读取的任务，不能暴露其它用户的 payload、错误详情或 worker 内部字段。 + +这个 Module 的 **Seam** 在 SpacetimeDB procedure + `spacetime-client` facade；`api-server` HTTP role 和 worker role 都只依赖这个 Interface。外部 provider、OSS、计费补偿、玩法草稿回写仍留在 `api-server` worker implementation 内，不进入 SpacetimeDB reducer。 + +## BFF 状态接口 + +队列状态对前端只通过 `api-server` BFF 暴露，不允许前端直接查询 SpacetimeDB private table： + +- `GET /api/runtime/external-generation/queue-overview`：队列概览，用于生成页、调试面板或后台观测当前用户可见的等待状态。返回 pending / running / completed / failed / cancelled 数量、最早等待时间、当前可见 job 摘要，以及是否存在过期 lease 需要等待 worker 重领。 +- `GET /api/runtime/external-generation/jobs/{jobId}`：单 job 状态，用于生成页轮询某次动作。返回 `jobId`、`jobKind`、`sourceModule`、`sourceEntityId`、`status`、`attempt`、`maxAttempts`、`createdAt`、`startedAt`、`completedAt`、`updatedAt`、可展示的 `requestLabel`、可展示的 `lastErrorMessage`、以及业务侧下一次轮询所需的 source 标识。 + +BFF 只做鉴权、授权裁剪、字段脱敏和契约映射；队列事实仍以 `external_generation_job` 为准，业务结果仍以玩法 session / work profile 为准。生成页展示“排队中 / 处理中 / 失败 / 完成”时，应优先用单 job 状态补充等待信息，再继续按原玩法 session/detail 接口收敛到 ready 或 failed。队列接口不替代玩法恢复接口，也不把 private `request_payload_json` 原样传给前端。 + +## 任务表 + +新增私有表 `external_generation_job`： + +| 字段 | 说明 | +| --- | --- | +| `job_id` | 主键，`extgen-` 前缀 UUID | +| `dedupe_key` | 唯一键，建议为 `play/action/session/scope` | +| `job_kind` | 执行类型，当前拼图为 `puzzle_compile_draft`、`puzzle_generate_images`、`puzzle_generate_ui_background` | +| `owner_user_id` | 触发用户 | +| `source_module` | 玩法或能力名，例如 `puzzle` | +| `source_entity_id` | session/profile/work 等作用域 | +| `request_label` | 排障标签 | +| `request_payload_json` | worker 执行入参 JSON | +| `status` | `pending/running/completed/failed/cancelled` | +| `attempt` / `max_attempts` | 当前尝试次数与最大尝试次数 | +| `last_error_message` | 最近失败原因 | +| `worker_id` | 当前 lease owner | +| `lease_expires_at` | lease 到期时间 | +| `lease_token` | 本次 claim 的 fencing token，用于阻止过期 worker 回写 | +| `available_at` | 下次可领取时间 | +| `result_payload_json` | 完成摘要 | +| `created_at/started_at/completed_at/updated_at` | 审计时间 | + +索引： + +- `by_external_generation_job_status_available(status, available_at)` +- `by_external_generation_job_worker_id(worker_id)` +- `by_external_generation_job_source(source_module, source_entity_id)` +- `by_external_generation_job_owner_user_id(owner_user_id)` + +## 状态机 + +```text +pending -> running -> completed +pending -> running -> pending (可重试失败) +pending -> running -> failed (达到最大重试次数) +pending/running -> cancelled (预留) +``` + +`claim` 只领取 `pending` 且 `available_at <= now` 的任务，或 `running` 且 `lease_expires_at <= now` 的任务。领取时递增 `attempt`、写入 `worker_id`、`started_at`、新的 `lease_expires_at` 和 `lease_token`。SpacetimeDB procedure 使用 `ctx.timestamp` 作为状态流转时间，只从 worker 入参读取“时长差值”，不信任 worker 本机绝对时间。worker 每次执行只处理自己 claim 到的任务；续租、完成或失败时必须带同一个 `worker_id + lease_token`，且当前 lease 尚未过期，防止过期 worker 覆盖新 lease。 + +玩法业务写回也必须在 SpacetimeDB 同一事务里校验 lease fencing。拼图的 `compile_puzzle_agent_draft` worker 调用、`save_puzzle_generated_images`、`save_puzzle_ui_background`、`mark_puzzle_draft_generation_failed` 和 `mark_puzzle_level_generation_failed` 在 `queue` 模式下会带 `external_generation_job_id / worker_id / lease_token`，并校验 job 仍为 `running`、token 未过期、`job_kind`、`owner_user_id`、`source_module` 和 `source_entity_id` 均匹配后才写 session / work profile。`inline` 模式不创建 `external_generation_job`，因此这三个 guard 字段必须同时为空；transaction 只把三项全空识别为 api-server 受控同步写回，三项半空仍按非法请求拒绝。worker 路径的核心业务写回失败不能返回内存快照并把 job 标为 `completed`；失败态业务回写成功后才允许把队列 job 标为 `failed`，失败态仍未写回时保留当前租约并等待后续 lease 过期重领，避免队列状态和真实 session 脱节。api-server 的资产扣费包装遇到这类 stale worker lease guard 错误时不执行补偿退款，避免旧 worker 冲掉后续合法 worker 的同一账本扣费。 + +## 执行模式与进程角色 + +外部生成执行模式由 `GENARRATIVE_EXTERNAL_GENERATION_MODE` 控制： + +- `queue`：默认值，HTTP handler 入队 `external_generation_job`，由 `external-generation-worker` 角色 claim lease 后执行；生产、预发和压测默认使用该模式。 +- `inline`：HTTP handler 直接调用同一个 worker executor，同步等待 provider、OSS 和 SpacetimeDB 写回完成后返回 `operation.status = completed`；只用于本地或低并发排查，不提供队列持久化、lease 重领和 worker 横向扩容。 + +同一个 Rust binary 通过 `GENARRATIVE_PROCESS_ROLE` 切换： + +- `api`：只启动 HTTP server。 +- `external-generation-worker`：只启动外部生成 worker，不监听 HTTP。 +- `external-generation-controller`：只启动 worker controller，不监听 HTTP，也不直接执行外部生成任务。 +- `all`：本地开发可同时启动 HTTP 与 worker。 + +worker 配置： + +- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID`：实例 ID；未配置时用 hostname/pid 派生。 +- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY`：单进程并发领取/执行数量。 +- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS`：空队列轮询间隔。 +- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS`：任务 lease 时长；worker 会按约三分之一 lease、最长 30 秒的间隔续租。该值应覆盖一次心跳网络抖动窗口，不需要大于完整外部生成链路耗时。 + +controller 配置： + +- `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MIN_WORKERS`：保底 worker 实例数，生产默认 `1`，controller 不会主动停止 `@1`。 +- `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MAX_WORKERS`：自动扩容上限，生产模板默认 `8`。 +- `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_TARGET_JOBS_PER_WORKER`：每个 worker 实例承担的目标未完成任务数，默认 `2`；目标实例数按 `claimable_pending + running_active + expired_running` 计算后夹在 min/max 之间，避免把已包含过期 running 的 `claimable_count` 重复计入。 +- `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_POLL_INTERVAL_MS`：controller 轮询队列统计的间隔，默认 `10000`。 +- `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_SCALE_DOWN_IDLE_ROUNDS`：连续多少轮无可领取、无运行中、无过期 running 后才允许缩容，默认 `6`；缩容每轮只停止最高编号的一个实例。 +- `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_SERVICE_TEMPLATE`：systemd worker 模板，默认 `genarrative-external-generation-worker@{}.service`。 +- `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_DRY_RUN`：只记录决策不执行 systemctl，默认 `false`。 + +动态缩扩容方式：生产默认由 `deploy/systemd/genarrative-external-generation-controller.service` 启动 `GENARRATIVE_PROCESS_ROLE=external-generation-controller`，controller 读取 `get_external_generation_queue_stats_and_return` 后对 `genarrative-external-generation-worker@N.service` 执行精确 `systemctl start/stop`；无需改变 HTTP 进程数。controller 只操作 `@1..@MAX` 中的缺口或最高编号多余实例，保留 `@1` 作为保底 worker。缩容或发布重启 worker 时，进程收到 SIGINT/SIGTERM 后会停止 claim 新任务并等待当前任务完成；若进程被硬杀、机器断电或超过 systemd `TimeoutStopSec`，未完成任务会在 lease 过期后被其它 worker 重新领取。容器链路已有独立 `external-generation-worker` compose service；扩 worker 必须扩这个 worker service，不能只扩 `api-server` HTTP service。 + +## 已接入的拼图纵切 + +### 拼图 + +`compile_puzzle_draft`： + +1. HTTP handler 保存拼图表单草稿；`queue` 模式下 `queued/running` 的持久事实源是 `external_generation_job`，不把 HTTP 进程变成外部生成执行者。 +2. `queue` 模式下 HTTP handler 入队 `puzzle_compile_draft`，返回 `operation.status = queued` 和当前 session。拼图 dedupe key 包含本次 `extgen-` job id，只保证同一任务行唯一，不把同一 session 后续重新生成吞掉。`inline` 模式下 HTTP handler 复用同一 executor 同步执行，成功后直接返回 `completed` 和最新 session。 +3. 前端保持 `puzzle-generating`，继续轮询 `getPuzzleAgentSession`；首期不把 `queued/running` 写回 `puzzle_agent_session`，因此刷新或跨设备恢复生成中状态仍是后续 read model 工作。 +4. worker claim 后执行原有 `compile_puzzle_draft_with_initial_cover` 或 `compile_puzzle_draft_with_uploaded_cover`；前置 `compile_puzzle_agent_draft` 也必须携带本次 `job_id / worker_id / lease_token`，防止过期 worker 先把草稿卡和 session 写到 ready。 +5. 成功后沿原有 SpacetimeDB 拼图会话/作品写回，前端轮询看到 `progressPercent >= 94/96/100` 和 ready 草稿。 +6. 失败后调用 `mark_puzzle_draft_generation_failed`，拼图首期业务失败直接进入 failed；只有失败态写回成功才把队列 job 标为 failed，失败态写回失败则保留租约等待重领。队列仍保留 lease 过期后的崩溃重领，避免 worker 退款后再次成功导致钱包账本漂移。前端通过现有失败草稿/弹窗机制展示来源错误。 + +`generate_puzzle_images`： + +1. HTTP handler 校验本次 `levelsJson` 快照；`queue` 模式下入队 `puzzle_generate_images` 并返回 `operation.status = queued/running/completed/failed`，`inline` 模式下同步执行原 worker executor 并在成功后返回 `completed`。 +2. worker 执行原结果页关卡图链路：自动命名、VectorEngine / 上传图直用、关卡场景图、UI spritesheet、关卡背景资产包、OSS 持久化和 SpacetimeDB 回写。 +3. 成功后 `save_puzzle_generated_images` 写回目标关卡和草稿卡；失败后 `mark_puzzle_level_generation_failed` 只标记目标关卡 `failed`，不污染已 ready 的其它关卡。队列 job 只有在目标关卡失败态写回成功后才进入 failed。 +4. 前端结果页对 `queued/running` 操作继续轮询 `getPuzzleAgentSession`，目标关卡变为 ready 或 failed 后收敛。 + +`generate_puzzle_ui_background`： + +1. HTTP handler 校验本次 `levelsJson` 快照；`queue` 模式下入队 `puzzle_generate_ui_background` 并返回 `operation.status = queued/running/completed/failed`，`inline` 模式下同步执行原 worker executor 并在成功后返回 `completed`。 +2. worker 执行原结果页 UI 背景链路：归一化提示词、VectorEngine 生成、OSS 持久化和 `save_puzzle_ui_background` 写回。 +3. 成功后目标关卡写入 `uiBackgroundPrompt/uiBackgroundImageSrc/uiBackgroundImageObjectKey`；失败后复用 `mark_puzzle_level_generation_failed` 标记目标关卡 `failed`，并在失败态写回成功后才终结队列 job，让前端轮询能收敛。 + +### 跳一跳、拼消消和敲木鱼扩展范围 + +以下动作按同一 worker 模式迁移。命名以现有玩法 action 为准，队列 `job_kind` 采用后端稳定 snake_case，不新增平行队列： + +- 跳一跳 `jump-hop` + - `compile-draft`：草稿编译阶段需要生成地块 / 视觉资产时入队，例如 `jump_hop_compile_draft`。 + - `regenerate-tiles`：结果页地块图集重生入队，例如 `jump_hop_regenerate_tiles`。 +- 拼消消 `puzzle-clear` + - `compile-draft`：草稿编译阶段需要生成场地底图和卡片 atlas 时入队，例如 `puzzle_clear_compile_draft`。 + - `regenerate-atlas`：结果页素材 atlas 重生入队，例如 `puzzle_clear_regenerate_atlas`。 +- 敲木鱼 `wooden-fish` + - `compile-draft`：草稿编译阶段需要生成背景、敲击物或其它图片资产时入队，例如 `wooden_fish_compile_draft`。 + - `regenerate-hit-object`：结果页敲击物图片重生入队，例如 `wooden_fish_regenerate_hit_object`。 + +这些动作首版都保持“单动作单 job”：一次 `compile-draft` 或一次 `regenerate-*` 请求只创建一个 job，worker 内部负责该动作所需的 provider 调用、素材处理、OSS 持久化、失败态写回和业务成功写回。非外部图片生成动作，例如纯元信息保存、标签编辑、发布、试玩启动、运行态动作、删除和公开 read model 读取，继续 inline 执行。 + +每个玩法迁移时必须同时接入业务写回 lease guard：worker 路径带 `external_generation_job_id / worker_id / lease_token`，inline 路径三项同时为空。过期 worker 不得写 session / work profile；业务失败态写回成功后才允许 job 进入 `failed`。 + +## 验收 + +基础检查： + +```bash +npm run spacetime:generate +npm run check:spacetime-schema +npm run check:server-rs-ddd +cargo check -p api-server --manifest-path server-rs/Cargo.toml +``` + +定向测试： + +```bash +cargo test -p spacetime-module external_generation --manifest-path server-rs/Cargo.toml +cargo test -p spacetime-module level_generation_failure --manifest-path server-rs/Cargo.toml +cargo test -p api-server external_generation_worker --manifest-path server-rs/Cargo.toml +npm run test -- src/components/puzzle-result/PuzzleResultView.test.tsx -t "keeps generation progress visible" +npm run test -- src/components/rpg-entry/RpgEntryFlowShell.agent.interaction.test.tsx -t "compile_puzzle_draft" +``` + +本地 smoke： + +```bash +GENARRATIVE_PROCESS_ROLE=all npm run dev +curl -f http://127.0.0.1:/healthz +``` + +本地 `npm run dev` 默认保持 `inline` 开发体验：未显式配置 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue` 时，普通本地联调可以同步确认 provider、OSS 和 SpacetimeDB 写回链路本身是否可行。需要验证 worker 队列、BFF 队列状态、lease 重领或扩缩容时，必须显式使用 `queue`，并启动 worker 角色；可以用 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue GENARRATIVE_PROCESS_ROLE=all npm run dev:api-server` 做临时单进程 smoke，也可以使用隔离容器 smoke。 + +生产 smoke 需要保持 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue`，并至少启动一个 `api` 角色、一个 `external-generation-worker` 角色和一个 `external-generation-controller` 角色；发布脚本会在默认 worker pattern 下自动启用并启动 `genarrative-external-generation-worker@1.service`，重启并验活 `genarrative-external-generation-controller.service`。若 worker 数量归零，生成任务会保持 `queued/running`，不会由 HTTP 进程偷偷执行。部署验证除 `/healthz` / `/readyz` 外，还要确认队列概览 BFF 可读、单 job 状态能从 `queued/running` 收敛到业务 session/detail 的 ready 或 failed。 + +systemd 生产 controller 与手动兜底示例： + +```bash +systemctl enable --now genarrative-external-generation-worker@1.service +systemctl enable --now genarrative-external-generation-controller.service +systemctl start genarrative-external-generation-worker@2.service +systemctl stop genarrative-external-generation-worker@2.service +systemctl status genarrative-external-generation-controller.service 'genarrative-external-generation-worker@*.service' +``` diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index b5d01d817..dd491a155 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -113,7 +113,7 @@ npm run check:server-rs-ddd - `server-rs/crates/api-server/src/state.rs` 中的 `PuzzleApiState` 是拼图 HTTP/BFF 的 Feature State，集中暴露 `SpacetimeClient`、`PuzzleGalleryCache`、OSS client、作者查询所需认证服务、拼图 LLM client 和少量 VectorEngine / Agent 配置快照。拼图 handler 只提取 `State`，不得重新改回 `State`。 - `server-rs/crates/api-server/src/puzzle.rs` 只作为聚合入口，保留共享 import / 常量、内部模块声明和 handler re-export，不继续承载大段实现。 - `server-rs/crates/api-server/src/puzzle/handlers.rs` 承接 Axum handler，负责 extract、鉴权上下文、调用 SpacetimeDB facade / 编排 helper，并返回 HTTP/SSE 响应。 -- `server-rs/crates/api-server/src/puzzle/draft.rs` 承接表单草稿保存、草稿编译、首关命名、UI 背景 prompt、降级 snapshot 和初始资产就绪校验。 +- `server-rs/crates/api-server/src/puzzle/draft.rs` 承接表单草稿保存、草稿编译、首关命名、UI 背景 prompt 和初始资产就绪校验。 - `server-rs/crates/api-server/src/puzzle/generation.rs` 承接拼图图片与 UI 背景的生成编排、计费包裹和 reference image 路径选择。 - `server-rs/crates/api-server/src/puzzle/vector_engine.rs` 承接 VectorEngine 请求体、HTTP 调用、下载 / base64 解码、OSS 写入、asset object / binding 持久化和上游错误归一。 - `server-rs/crates/api-server/src/puzzle/mappers.rs` 承接 SpacetimeDB record 到 shared-contracts DTO 的映射。 @@ -233,6 +233,12 @@ npm run check:server-rs-ddd - Rust 结构体：`AiTaskStage` - 源码：`server-rs/crates/spacetime-module/src/ai/stages.rs` +### `external_generation_job` + +- Rust 结构体：`ExternalGenerationJob` +- 源码：`server-rs/crates/spacetime-module/src/external_generation.rs` +- 用途：外部生成 worker 的持久任务队列；`GENARRATIVE_EXTERNAL_GENERATION_MODE=queue` 时，`api-server` HTTP 角色只入队，`external-generation-worker` 角色通过 claim lease 领取、续租、执行，并用 `lease_token` 栅栏回写完成 / 失败。拼图 `compile_puzzle_draft` 的前置 `compile_puzzle_agent_draft`、`generate_puzzle_images` 与 `generate_puzzle_ui_background` 的业务写回也在对应 SpacetimeDB transaction 内校验 `job_id + worker_id + lease_token`、job kind、owner 和 source entity，避免过期 worker 写 session / work profile；`GENARRATIVE_EXTERNAL_GENERATION_MODE=inline` 时不创建该队列行，三个 external generation guard 字段必须同时为空才允许 api-server 受控同步写回，半空 guard 仍会拒绝。worker 成功写回业务事实后才能 complete job；业务失败态写回成功后才能 fail job，失败态未写回时保留租约等待后续重领。 + ### `ai_text_chunk` - Rust 结构体：`AiTextChunk` diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 2d7792270..c2a7d1725 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -1,6 +1,6 @@ # 本地开发验证与生产运维 -更新时间：`2026-06-09` +更新时间：`2026-06-12` ## 标准开发流程 @@ -51,6 +51,14 @@ Linux 本机多用户并发开发时，`npm run dev` 和 `npm run dev:*` 单模 开发态 `npm run dev` 与 `npm run dev:api-server` 会默认注入 `GENARRATIVE_DEV_PASSWORD_ENTRY_AUTO_REGISTER_ENABLED=true`，因此密码登录在本地开发环境可直接注册未知手机号账号；生产环境仍按 `api-server` 配置默认关闭该开关。 +本地 `npm run dev` 和 `npm run dev:api-server` 默认保留 inline 开发体验：未显式设置 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue` 时，外部生成 handler 会同步复用 worker executor，完成后返回 `completed`，便于快速确认 provider、OSS 和 SpacetimeDB 写回链路。inline 不创建 `external_generation_job`，也不能验证 worker lease、队列等待展示或动态扩缩容。 + +本地排查外部内容生成 worker 队列时，必须显式使用 queue，例如 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue GENARRATIVE_PROCESS_ROLE=all npm run dev:api-server`，让同一 Rust 进程同时监听 HTTP 并消费 `external_generation_job` 队列；更接近生产的验证应分别启动 `api`、`external-generation-worker` 和 `external-generation-controller`。生产默认 `GENARRATIVE_PROCESS_ROLE=api`，外部生成任务由独立 `GENARRATIVE_PROCESS_ROLE=external-generation-worker` 进程消费；生产与容器扩缩容验证保持 `queue`。当前进入持久队列的外部图片生成动作包括：拼图 `compile_puzzle_draft` / `generate_puzzle_images` / `generate_puzzle_ui_background`，跳一跳 `compile-draft` / `regenerate-tiles`，拼消消 `compile-draft` / `regenerate-atlas`，敲木鱼 `compile-draft` / `regenerate-hit-object`。非外部图片生成动作继续 inline，不进入队列。worker 数量为 0 时，HTTP 只返回 queued/running，不会兜底执行外部 provider。 + +生成页或排障面板展示队列等待时，只读取 BFF 队列接口：`GET /api/runtime/external-generation/queue-overview` 查看当前用户可见队列概览，`GET /api/runtime/external-generation/jobs/{jobId}` 查看单 job 状态。队列接口只提供等待 / 运行 / 失败 / 完成状态补充，最终草稿、作品和结果页仍要轮询对应玩法 session/detail 接口收敛到 ready 或 failed；不要直接查询 `external_generation_job` private table，也不要把 worker 内部 payload 暴露到前端。 + +需要验证“更新 API 不停 worker”和“worker 是否持续消费队列”时，优先使用隔离容器 smoke：`npm run container:worker-smoke -- smoke`。该脚本生成 gitignored 的 `deploy/container/worker-smoke/api-server.env`，启动独立 compose project 与独立 SpacetimeDB，发布当前 `spacetime-module` 后写入 `worker_smoke_unsupported` 测试 job；预期 worker claim 后执行 unsupported 失败分支，再执行 API-only recreate 并确认 worker 容器 ID 不变，最后再次入队验证 API 更新后队列仍可消费。`external_generation_job` 是 private table，脚本通过 worker 日志确认 job_id 被消费，不用 CLI SQL 查询私表。该 smoke 不读取 `.env.local`，也不依赖真实 VectorEngine / OSS 密钥；真实生图链路联调再在本地私有 env 中补齐 provider 配置。worker-smoke 默认把本机 `spacetime` CLI 打成轻量 SpacetimeDB 镜像，避免本机首次 smoke 依赖官方大镜像下载。若容器内 Cargo 拉取 crates.io 依赖不稳定，可用 `npm run container:worker-smoke -- smoke --local-binary` 让容器内 Cargo 复用本机 Cargo 缓存构建当前二进制，再打入 Debian bookworm smoke runtime 临时镜像；可用 `GENARRATIVE_WORKER_SMOKE_LOCAL_BASE_IMAGE` 覆盖运行时基础镜像；若隔离端口或库数据需要重建，追加 `--force`。完成 queue 链路验证时，还要用队列概览 BFF 和单 job 状态接口确认 job 从 queued/running 收敛，并用对应玩法 session/detail 接口确认业务状态同步完成。 + 本地只做账号/UI smoke 且需要短信登录时，`SMS_AUTH_PROVIDER` 应显式设为 `mock`，并把 `SMS_AUTH_MOCK_VERIFY_CODE` 设为固定值（当前常用 `123456`），再重启 `npm run dev` 或 `npm run dev:api-server`。如果 `.env.local` 还保留 `SMS_AUTH_PROVIDER=aliyun`，`POST /api/auth/phone/login` 用 mock 验证码会稳定报“验证码错误”，不是前端表单问题。真实短信联调再切回 `aliyun` 并重启。 微信小程序虚拟支付使用 `WECHAT_MINI_PROGRAM_VIRTUAL_PAYMENT_OFFER_ID`、`WECHAT_MINI_PROGRAM_VIRTUAL_PAYMENT_APP_KEY`、`WECHAT_MINI_PROGRAM_VIRTUAL_PAYMENT_SANDBOX_APP_KEY` 和 `WECHAT_MINI_PROGRAM_VIRTUAL_PAYMENT_ENV` 配置。小程序充值统一走 `wechat_mp_virtual` / `wx.requestVirtualPayment`：泥点属于代币（`coin`），`buyQuantity` 按当前充值商品快照里的 `points_amount` 传；会员和后台新增道具类商品走 `short_series_goods`，`productId` 对应微信后台道具 ID。旧登录快照若缺 `session_key`，需要用户在小程序内重新登录后再支付；客户端成功回调不是最终到账，仍以后端通知或查询确认订单为准。详细口径见 `docs/【技术方案】微信虚拟支付接入-2026-05-26.md`。 @@ -202,7 +210,7 @@ UI 相关修改要重点验证： ## SpacetimeDB 操作规则 -1. 不在人工命令、本地联调或文档示例中使用 `spacetime --root-dir`。 +1. 不在人工命令、本地联调或文档示例中使用 `spacetime --root-dir`；CI/CD 脚本内部为隔离运行用户登录态的受控用法例外，但不得写成手工排障命令。 2. 本地开发使用项目脚本维护数据目录；需要清空本地数据时先确认可丢弃，再停止服务并处理本地数据目录。 3. 发布目标必须显式 `--server` / `--server-url`。 4. 身份问题先查 `spacetime login show`、`spacetime server list` 和目标库权限，不通过切回旧 Node / PostgreSQL 绕过。 @@ -212,7 +220,7 @@ UI 相关修改要重点验证： ### SpacetimeDB 数据目录 OSS 备份 -数据库备份不放进 `spacetime-module` reducer / procedure：备份属于文件系统与 OSS 外部副作用，必须由运维脚本在 SpacetimeDB 宿主外执行。当前统一脚本为；生产 provision 还会安装 `genarrative-database-backup.timer`，每天 `03:20` 左右自动执行一次 OSS 冷备份： +数据库备份不放进 `spacetime-module` reducer / procedure：备份属于文件系统与 OSS 外部副作用，必须由运维脚本在 SpacetimeDB 宿主外执行。当前统一脚本为 `scripts/database-backup-to-oss.mjs`（npm 命令 `npm run database:backup:oss`）；生产 provision 还会安装 `genarrative-database-backup.timer`，每天 `03:20` 左右自动执行一次 OSS 冷备份： ```bash npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service @@ -260,11 +268,12 @@ Jenkins 按 web / api / Spacetime module / build / deploy / publish 拆分 `Genarrative-Server-Provision` 会安装并启用 `genarrative-health-patrol.timer`，默认每 5 分钟运行一次 `genarrative-health-patrol.service`。巡检脚本随 API release 归档到 `/opt/genarrative/current/scripts/ops/production-health-patrol.mjs`，只读检查： -- `genarrative-api.service`、`spacetimedb.service`、`nginx.service` 是否 active。 +- `genarrative-api.service`、`genarrative-external-generation-controller.service`、`spacetimedb.service`、`nginx.service` 是否 active。 +- 至少一个 `genarrative-external-generation-worker@*.service` 实例是否 active；如果 controller 存活但 worker 全部退出，巡检直接返回 `CRITICAL`，避免外部生成队列长期无人消费。 - API 直连 `/healthz`、`/readyz`。 - SpacetimeDB 直连 `/v1/ping`。 - 默认直连 API 端口检查 `/api/creation-entry/config`、`/api/runtime/puzzle/gallery`、`/api/runtime/custom-world-gallery`；如需走 Nginx / 公网域名，在 `/etc/genarrative/health-patrol.env` 配置 `GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=https://<域名>`。 -- 最近 15 分钟 `genarrative-api.service`、`spacetimedb.service`、`nginx.service` 的 `err..alert` 日志。 +- 最近 15 分钟 `genarrative-api.service`、`genarrative-external-generation-controller.service`、`genarrative-external-generation-worker@*.service`、`spacetimedb.service`、`nginx.service` 的 `err..alert` 日志。 巡检输出总状态 `OK / WARNING / CRITICAL`；只有 `CRITICAL` 默认让 systemd service 失败，`WARNING` 只写日志和状态文件，避免历史日志噪声把 timer 长期打成失败。最近一次结果写入 `/var/lib/genarrative/health-patrol/status.json`。手动执行： @@ -302,7 +311,9 @@ dev 服务器上的 Gitea 内网入口固定为 `http://10.2.0.10/GenarrativeAI/ 生产环境变量模板：`deploy/env/api-server.env.example`。真实密钥只放服务器，不提交 Git，不写入文档示例。 -`Genarrative-Server-Provision` 会安装 systemd 模板和 Nginx 站点模板，不再安装 clang / lld / pkg-config / OpenSSL headers / sccache 等通用构建链依赖。因 VectorEngine 图片上游 POST 已改用 `libcurl`，当前 Linux release 构建出的 `api-server` 运行时需要 `OPENSSL_3.2.0` 符号；Ubuntu 24.04 apt 默认只提供 OpenSSL 3.0.x，不能直接满足该符号版本。Provision 会把 OpenSSL `3.2.0` 独立安装到 `/opt/genarrative/openssl-3.2.0`，校验官方 tarball SHA256，并只通过 `genarrative-api.service` 的 `LD_LIBRARY_PATH=/opt/genarrative/openssl-3.2.0/lib64:/opt/genarrative/openssl-3.2.0/lib` 让 api-server 使用，避免替换系统 OpenSSL 或影响 ssh / nginx / apt。Ubuntu / apt 目标机会额外安装 `libnginx-mod-http-brotli-filter` 与 `libnginx-mod-http-brotli-static`，随后由 `scripts/jenkins-server-provision.sh` 通过临时 `nginx -t` 配置探测 Brotli 指令是否可用；该临时配置必须先 `include /etc/nginx/modules-enabled/*.conf`，因为 apt 安装的 Brotli 是动态模块，不会出现在普通 `nginx -V` 编译参数里。探测成功才在渲染后的 `deploy/nginx/genarrative.conf` / `genarrative-dev-http.conf` 中启用 Brotli，避免未安装模块的机器直接写入无效配置。Provision 写入 Genarrative Nginx 站点时会把 `/etc/nginx/sites-enabled/default*` 移到 `/etc/nginx/sites-disabled/`，避免 Debian / Certbot 默认站点继续占用 `genarrative.world` / `www.genarrative.world` 并在 `nginx -T` 中出现 `conflicting server name ... ignored`。如果 `nginx -t` 失败，脚本会恢复写入前的 Genarrative 配置和被移动的默认站点。 +`api-server` 进程角色由 `GENARRATIVE_PROCESS_ROLE` 控制：`api` 只监听 HTTP，`external-generation-worker` 只消费外部生成队列，`external-generation-controller` 只管理 worker systemd 实例，`all` 仅用于本地或临时 smoke，不隐式启动 controller。外部生成策略由 `GENARRATIVE_EXTERNAL_GENERATION_MODE` 控制；生产和容器压测默认保持 `queue`，本地 `npm run dev` 默认保留 `inline` 开发体验，只有显式配置 `queue` 才会落 `external_generation_job`。`inline` 只用于本地或低并发同步排查，HTTP handler 会直接复用 worker executor，完成后返回 `completed`，但不会落 `external_generation_job`，也不能通过增加 worker 进程扩吞吐。外部生成 worker 使用同一发布包和同一套 SpacetimeDB 配置，按实例数和 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY` 动态扩缩；生产默认由 `genarrative-external-generation-controller.service` 读取 `get_external_generation_queue_stats_and_return`，按 `claimable_pending + running_active + expired_running` 计算目标 worker 数，并对 `genarrative-external-generation-worker@N.service` 精确执行 `systemctl start/stop`。controller 参数模板是 `deploy/env/external-generation-controller.env.example`：默认保底 `MIN_WORKERS=1`、上限 `MAX_WORKERS=8`、每 worker 目标 `TARGET_JOBS_PER_WORKER=2`、`POLL_INTERVAL_MS=10000`、连续 `SCALE_DOWN_IDLE_ROUNDS=6` 轮完全空闲才缩容；缩容每轮只停止最高编号的一个实例，且不主动停止 `@1`。worker 收到 SIGINT/SIGTERM 后会停止 claim 新任务并等待当前任务完成；若进程被硬杀、机器断电或超过 systemd `TimeoutStopSec`，未完成任务才会在 lease 过期后由其它 worker 重领。每个 worker 实例应设置唯一 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID`，默认会用主机名和 pid 兜底；systemd 生产模板 `deploy/systemd/genarrative-external-generation-worker@.service` 会用 `%H-%i` 生成实例 ID，并把 tracking outbox 隔离到 `/var/lib/genarrative/tracking-outbox/%H-%i`。`Genarrative-Server-Provision` 会安装 worker 模板、controller unit 和两份专属 env 模板，默认 enable 首个 `genarrative-external-generation-worker@1.service` 与 `genarrative-external-generation-controller.service`；首次 API deploy 会在默认 worker pattern 下自动 `enable --now genarrative-external-generation-worker@1.service` 并等待 worker active，同时重启并验活 controller。手动兜底扩容仍可用 `systemctl start genarrative-external-generation-worker@2.service` / `@3.service`，缩容用 `systemctl stop genarrative-external-generation-worker@N.service`；controller 下轮会按队列压力修正到目标实例数。worker 专属参数模板是 `deploy/env/external-generation-worker.env.example`，密钥与 SpacetimeDB 连接仍复用 `/etc/genarrative/api-server.env`。API 发布脚本默认会重启并验活 `genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service`；若本次只发 HTTP 且不希望滚动 worker，可传 `--no-worker-services`，若不希望重启 controller 可传 `--no-worker-controller`。`GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS` 控制空队列轮询间隔，`GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS` 控制单次 lease，worker 会约每三分之一 lease、最长 30 秒续租；该值应覆盖一次心跳网络抖动窗口，不需要大于完整外部生成链路耗时。SpacetimeDB 使用自身事务时间计算 claim/renew/complete/fail，完成和失败回写还会校验 `lease_token` 与未过期 lease，避免同一 job 被过期 worker 覆盖。首版 worker 粒度是单动作单 job，不拆阶段 job；当前外部图片生成动作覆盖拼图、跳一跳、拼消消和敲木鱼，纯元信息保存、发布、试玩启动、运行态动作和公开读取继续 inline。当前生成业务失败只做用户重新触发，不做自动业务重试，避免 worker 退款和重试成功之间产生钱包账本漂移。 + +`Genarrative-Server-Provision` 会安装 systemd 模板和 Nginx 站点模板，不再安装 clang / lld / pkg-config / OpenSSL headers / sccache 等通用构建链依赖。因 VectorEngine 图片上游 POST 已改用 `libcurl`，当前 Linux release 构建出的 `api-server` 运行时需要 `OPENSSL_3.2.0` 符号；Ubuntu 24.04 apt 默认只提供 OpenSSL 3.0.x，不能直接满足该符号版本。Provision 会把 OpenSSL `3.2.0` 独立安装到 `/opt/genarrative/openssl-3.2.0`，校验官方 tarball SHA256，并只通过 `genarrative-api.service` 的 `LD_LIBRARY_PATH=/opt/genarrative/openssl-3.2.0/lib64:/opt/genarrative/openssl-3.2.0/lib` 让 api-server 使用，避免替换系统 OpenSSL 或影响 ssh / nginx / apt。Ubuntu / apt 目标机为完成这一步会安装 `build-essential`、`ca-certificates`、`curl`、`perl`、`tar` 等 OpenSSL 运行时自举工具；这只服务于独立 OpenSSL 运行时安装，不代表 provision 重新承担 api-server 构建职责。Ubuntu / apt 目标机会额外安装 `libnginx-mod-http-brotli-filter` 与 `libnginx-mod-http-brotli-static`，随后由 `scripts/jenkins-server-provision.sh` 通过临时 `nginx -t` 配置探测 Brotli 指令是否可用；该临时配置必须先 `include /etc/nginx/modules-enabled/*.conf`，因为 apt 安装的 Brotli 是动态模块，不会出现在普通 `nginx -V` 编译参数里。探测成功才在渲染后的 `deploy/nginx/genarrative.conf` / `genarrative-dev-http.conf` 中启用 Brotli，避免未安装模块的机器直接写入无效配置。Provision 写入 Genarrative Nginx 站点时会把 `/etc/nginx/sites-enabled/default*` 移到 `/etc/nginx/sites-disabled/`，避免 Debian / Certbot 默认站点继续占用 `genarrative.world` / `www.genarrative.world` 并在 `nginx -T` 中出现 `conflicting server name ... ignored`。如果 `nginx -t` 失败，脚本会恢复写入前的 Genarrative 配置和被移动的默认站点。 50 HTTP req/s 首版压测优化口径： @@ -319,7 +330,7 @@ dev 服务器上的 Gitea 内网入口固定为 `http://10.2.0.10/GenarrativeAI/ - 作品列表短期继续由 `api-server` / BFF 订阅 SpacetimeDB 公开 read model 后读本地 cache，不让浏览器前端直接订阅完整列表；未来如新增 `public_work_gallery_entry` 等专用公开作品列表 read model，前端只可订阅稳定、低基数、公开的专用投影，禁止订阅 `puzzle_work_profile`、`custom_world_profile` 等玩法源表后自行 join、聚合或判断权限。前端直订阅落地前必须先补齐权限、字段契约、排序 / 分页、埋点和 BFF 回退策略。 - 50 HTTP req/s 验收目标为 `http_req_failed < 1%`、`p95 < 2s`、`dropped_iterations = 0`，同时压测窗口内 Nginx 无新增 502。2026-05-19 容器 2C / 2G 连续 10 轮不重启 SpacetimeDB 压测：`PEAK_RPS=2500` 等价约 5000 HTTP req/s，平均实际吞吐约 `4219 HTTP req/s`，10 轮总计 `1,897,357` 个 200、`212,542` 个 429、`0` 个 5xx，200 请求平均 `p95=123ms`、`p99=234ms`；该档会把 SpacetimeDB 容器内存从约 `366MiB` 推到约 `885MiB / 896MiB`，因此当前不要继续抬公开 gallery 入口并发，应优先处理 SpacetimeDB 侧连接 / 订阅 / tracking 写入后的内存高水位。 -容器化压测与隔离部署方案单独放在 `deploy/container/`，用于本机或预发模拟 Linux release + Nginx + OTLP Collector 拓扑，不替换当前生产 `systemd + Nginx + Jenkins` 发布路径。当前容器模拟参数按 `genarrative-release` 采样值收口为 2 vCPU / 2 GiB RAM / `nofile=4096` / `worker_connections=768`，并在 compose 里落实到 `spacetimedb cpus=1.0 mem_limit=768m`、`api-server cpus=2.0 mem_limit=1g`、`nginx cpus=0.25 mem_limit=128m`、`otelcol cpus=0.25 mem_limit=128m`、`k6 cpus=0.5 mem_limit=512m`。容器 `api-server` 默认 `GENARRATIVE_API_WORKER_THREADS=4`，只增加 Tokio worker 调度并发，不突破 `api-server cpus=2.0` 的 CPU 配额： +容器化压测与隔离部署方案单独放在 `deploy/container/`，用于本机或预发模拟 Linux release + Nginx + OTLP Collector 拓扑，不替换当前生产 `systemd + Nginx + Jenkins` 发布路径。当前容器模拟参数按 `genarrative-release` 采样值收口为 2 vCPU / 2 GiB RAM / `nofile=4096` / `worker_connections=768`，并在 compose 里落实到 `spacetimedb cpus=1.0 mem_limit=896m`、`api-server cpus=2.0 mem_limit=1g`、`external-generation-worker cpus=2.0 mem_limit=1g`、`nginx cpus=0.5 mem_limit=128m`、`otelcol cpus=0.25 mem_limit=128m`、`k6 cpus=1.0 mem_limit=512m`。容器 `api-server` 默认 `GENARRATIVE_API_WORKER_THREADS=4`，只增加 Tokio worker 调度并发，不突破 `api-server cpus=2.0` 的 CPU 配额；容器默认 `GENARRATIVE_EXTERNAL_GENERATION_MODE=queue`，可用 `npm run container:up -- --scale external-generation-worker=N external-generation-worker` 验证外部生成 worker 动态扩缩容，`inline` 模式不参与该验证： ```bash npm run container:init @@ -332,6 +343,7 @@ npm run container:down 容器方案默认暴露 `http://127.0.0.1:18080`，`api-server` 在容器内监听 `0.0.0.0:8082`，Nginx 通过 `api-server:8082` upstream 反代 `/api/` 和 `/admin/api/`。SpacetimeDB 也纳入 compose，容器内由 `spacetimedb:3101` 提供服务，宿主机通过 `http://127.0.0.1:13101` 进行模块发布；Collector 镜像使用 `otel/opentelemetry-collector-contrib:0.151.0`。生产 provision 侧现在由目标 dev / release agent 自己准备 `provision-tools/otelcol-contrib`，并安装本机 `otelcol-contrib.service`，真实库名、token 和外部服务密钥只写本地 `deploy/container/api-server.env`，不提交 Git。完整拓扑、端口、k6 参数和 OTLP debug exporter 使用方法见 `deploy/container/README.md`。 `npm run container:config` 默认只做 quiet 校验，避免把本地 env 中的 token 展开到终端；确需排查完整 compose 时再传 `-- --print`。 +隔离验证 worker 队列和 API-only 更新时使用 `npm run container:worker-smoke -- smoke`。该命令不复用 `deploy/container/api-server.env`，会在 `deploy/container/worker-smoke/` 生成本机专用 env 与端口 state，并使用 unsupported job 验证 worker claim / fail 回写，不需要真实外部生成密钥；本机 crates.io 网络不稳时使用 `--local-binary`，由容器内 Cargo 复用本机 Cargo 缓存构建，并把产物放进 Debian bookworm smoke runtime。 OpenTelemetry 现阶段默认开启 OTLP traces / metrics / logs，但本地日志与 Nginx 文件日志仍保留： diff --git a/package.json b/package.json index c4b39b8d8..de61e0f3a 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "container:ps": "node scripts/container-compose.mjs ps", "container:config": "node scripts/container-compose.mjs config", "container:k6": "node scripts/container-compose.mjs k6", + "container:worker-smoke": "node scripts/container-worker-smoke.mjs", "check": "npm run lint && npm run test && npm run build && npm run check:content", "check:data": "node scripts/run-tsx.cjs scripts/validate-content.ts", "check:overrides": "node scripts/run-tsx.cjs scripts/validate-overrides.ts", diff --git a/packages/shared/src/contracts/externalGeneration.ts b/packages/shared/src/contracts/externalGeneration.ts new file mode 100644 index 000000000..8fe24f6f8 --- /dev/null +++ b/packages/shared/src/contracts/externalGeneration.ts @@ -0,0 +1,29 @@ +export type ExternalGenerationJobStatus = + | 'queued' + | 'running' + | 'completed' + | 'failed'; + +export interface ExternalGenerationQueueOverview { + pendingCount: number; + runningCount: number; + updatedAtMicros: number; +} + +export interface ExternalGenerationQueueOverviewResponse { + overview: ExternalGenerationQueueOverview; +} + +export interface ExternalGenerationJobStatusRecord { + operationId: string; + status: ExternalGenerationJobStatus; + phaseLabel: string; + phaseDetail: string; + progress: number; + error?: string | null; + updatedAtMicros: number; +} + +export interface ExternalGenerationJobStatusResponse { + job: ExternalGenerationJobStatusRecord; +} diff --git a/packages/shared/src/contracts/jumpHop.ts b/packages/shared/src/contracts/jumpHop.ts index 9379baa0b..43ab01465 100644 --- a/packages/shared/src/contracts/jumpHop.ts +++ b/packages/shared/src/contracts/jumpHop.ts @@ -1,3 +1,5 @@ +import type { ExternalGenerationJobStatusRecord } from './externalGeneration'; + export type JumpHopDifficulty = 'easy' | 'standard' | 'advanced' | 'challenge'; export type JumpHopStylePreset = @@ -206,6 +208,7 @@ export interface JumpHopActionResponse { actionType: JumpHopActionType; session: JumpHopSessionSnapshotResponse; work: JumpHopWorkProfileResponse | null; + queueState?: ExternalGenerationJobStatusRecord | null; } export interface JumpHopWorkSummaryResponse { diff --git a/packages/shared/src/contracts/puzzleAgentActions.ts b/packages/shared/src/contracts/puzzleAgentActions.ts index 501b8cc4c..bf0f92eec 100644 --- a/packages/shared/src/contracts/puzzleAgentActions.ts +++ b/packages/shared/src/contracts/puzzleAgentActions.ts @@ -1,4 +1,5 @@ import type { PuzzleAgentSessionSnapshot } from './puzzleAgentSession'; +import type { ExternalGenerationJobStatusRecord } from './externalGeneration'; export type PuzzleAgentSuggestedActionType = | 'request_summary' @@ -41,6 +42,7 @@ export interface PuzzleAgentOperationRecord { phaseDetail: string; progress: number; error?: string | null; + queueState?: ExternalGenerationJobStatusRecord | null; } export type PuzzleAgentActionRequest = diff --git a/packages/shared/src/contracts/puzzleClear.ts b/packages/shared/src/contracts/puzzleClear.ts index 728c93f94..e5e6845f1 100644 --- a/packages/shared/src/contracts/puzzleClear.ts +++ b/packages/shared/src/contracts/puzzleClear.ts @@ -1,3 +1,5 @@ +import type { ExternalGenerationJobStatusRecord } from './externalGeneration'; + export type PuzzleClearGenerationStatus = 'draft' | 'generating' | 'ready' | 'failed'; export type PuzzleClearShapeKind = '1x2' | '1x3' | '2x2' | '2x3'; @@ -109,6 +111,7 @@ export interface PuzzleClearActionResponse { actionType: PuzzleClearActionType; session: PuzzleClearSessionSnapshotResponse; work: PuzzleClearWorkProfileResponse | null; + queueState?: ExternalGenerationJobStatusRecord | null; } export interface PuzzleClearWorkSummaryResponse { diff --git a/packages/shared/src/contracts/woodenFish.ts b/packages/shared/src/contracts/woodenFish.ts index 040866f8c..033e7a6f6 100644 --- a/packages/shared/src/contracts/woodenFish.ts +++ b/packages/shared/src/contracts/woodenFish.ts @@ -1,3 +1,5 @@ +import type { ExternalGenerationJobStatusRecord } from './externalGeneration'; + export type WoodenFishGenerationStatus = | 'draft' | 'generating' @@ -104,6 +106,7 @@ export interface WoodenFishActionResponse { actionType: WoodenFishActionType; session: WoodenFishSessionSnapshotResponse; work: WoodenFishWorkProfileResponse | null; + queueState?: ExternalGenerationJobStatusRecord | null; } export interface WoodenFishWorkSummaryResponse { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index cfffc0a4b..cc84c6f11 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -8,6 +8,7 @@ export type * from './contracts/creativeAgent'; export type * from './contracts/customWorldAgent'; export * from './contracts/edutainmentBabyDrawing'; export * from './contracts/edutainmentBabyObject'; +export * from './contracts/externalGeneration'; export type * from './contracts/hyper3d'; export * from './contracts/match3dAgent'; export * from './contracts/match3dRuntime'; diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 6a8537f0a..89ce254ec 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -23,6 +23,46 @@ const checks = [ includes: 'genarrative-health-patrol.timer', reason: 'Server-Provision 必须安装并启用健康巡检 timer。', }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'genarrative-external-generation-controller.service', + reason: 'Server-Provision 必须安装并启用外部生成 worker controller。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'genarrative-external-generation-worker@1.service', + reason: 'Server-Provision 必须启用外部生成保底 worker 实例。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'ensure_default_worker_service', + reason: 'API Deploy 必须在缺少 worker 实例时补启动默认外部生成 worker。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'wait_for_worker_services', + reason: 'API Deploy 必须等待外部生成 worker 实例 active。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'wait_for_worker_controller_service', + reason: 'API Deploy 必须重启并验活外部生成 worker controller。', + }, + { + file: 'deploy/systemd/genarrative-external-generation-worker@.service', + includes: 'GENARRATIVE_PROCESS_ROLE=external-generation-worker', + reason: '外部生成 worker 模板必须作为独立 worker 进程角色运行。', + }, + { + file: 'deploy/systemd/genarrative-external-generation-controller.service', + includes: 'GENARRATIVE_PROCESS_ROLE=external-generation-controller', + reason: '外部生成 worker controller 必须作为独立进程角色运行。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: 'checkActiveWorkerInstances', + reason: '生产健康巡检必须检查至少一个外部生成 worker 实例 active。', + }, { file: 'scripts/build-production-release.sh', includes: 'production-health-patrol.mjs', diff --git a/scripts/check-spacetime-schema-guard.mjs b/scripts/check-spacetime-schema-guard.mjs index a935012a0..1f55c1e0d 100644 --- a/scripts/check-spacetime-schema-guard.mjs +++ b/scripts/check-spacetime-schema-guard.mjs @@ -475,14 +475,14 @@ function loadBaseSources(baseRef) { function getChangedFiles(baseRef) { const diffOutput = tryGit(['diff', '--name-only', '-z', baseRef, '--']) ?? ''; - const untrackedOutput = + const untrackedModuleOutput = tryGit(['ls-files', '--others', '--exclude-standard', '-z', moduleSrcRoot]) ?? ''; const untrackedBindingsOutput = tryGit(['ls-files', '--others', '--exclude-standard', '-z', bindingsRoot]) ?? ''; return new Set( [ ...diffOutput.split(/\u0000/u), - ...untrackedOutput.split(/\u0000/u), + ...untrackedModuleOutput.split(/\u0000/u), ...untrackedBindingsOutput.split(/\u0000/u), ] .map(normalizePath) diff --git a/scripts/container-worker-smoke.mjs b/scripts/container-worker-smoke.mjs new file mode 100644 index 000000000..48da3a5ee --- /dev/null +++ b/scripts/container-worker-smoke.mjs @@ -0,0 +1,839 @@ +import {spawn} from 'node:child_process'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + readFileSync, + writeFileSync, +} from 'node:fs'; +import net from 'node:net'; +import path from 'node:path'; + +const [, , rawCommand = 'help', ...rawArgs] = process.argv; + +const projectRoot = process.cwd(); +const composeFile = path.join('deploy', 'container', 'docker-compose.loadtest.yml'); +const smokeDir = path.join('deploy', 'container', 'worker-smoke'); +const envPath = path.join(smokeDir, 'api-server.env'); +const statePath = path.join(smokeDir, 'state.json'); +const localImageDir = path.join(smokeDir, 'image'); +const localImageDockerfilePath = path.join(localImageDir, 'Dockerfile.local'); +const localImageBinaryPath = path.join(localImageDir, 'api-server'); +const localCargoTargetDir = path.join('server-rs', 'target-worker-smoke'); +const localSpacetimeImageDir = path.join(smokeDir, 'spacetimedb-image'); +const localSpacetimeDockerfilePath = path.join(localSpacetimeImageDir, 'Dockerfile.local'); +const localSpacetimeBinaryPath = path.join(localSpacetimeImageDir, 'spacetime'); +const localSpacetimeStandalonePath = path.join( + localSpacetimeImageDir, + 'spacetimedb-standalone', +); +const projectName = process.env.GENARRATIVE_WORKER_SMOKE_PROJECT || 'genarrative-worker-smoke'; +const defaultDatabase = + process.env.GENARRATIVE_WORKER_SMOKE_DATABASE || 'genarrative-worker-smoke'; + +const command = rawCommand.trim(); +const supportedCommands = new Set([ + 'help', + 'init', + 'build', + 'up-spacetime', + 'publish', + 'up', + 'enqueue', + 'status', + 'api-update', + 'scale', + 'logs', + 'ps', + 'down', + 'smoke', +]); + +if (!supportedCommands.has(command)) { + printHelp(true); + process.exit(1); +} + +try { + await main(); +} catch (error) { + console.error(`[worker-smoke] ${error.message}`); + process.exit(1); +} + +async function main() { + switch (command) { + case 'help': + printHelp(false); + return; + case 'init': + await ensureStateAndEnv({force: rawArgs.includes('--force')}); + return; + case 'build': + await ensureStateAndEnv(); + await buildRuntimeImages(); + return; + case 'up-spacetime': + await ensureStateAndEnv(); + await ensureSpacetimeImage(); + await dockerCompose(['up', '-d', 'spacetimedb', 'otelcol']); + await waitForSpacetime(); + return; + case 'publish': + await ensureStateAndEnv(); + await publishModule(); + return; + case 'up': + await ensureStateAndEnv(); + await upRuntime(); + await waitForApi(); + return; + case 'enqueue': + await ensureStateAndEnv(); + await enqueueSmokeJob(); + return; + case 'status': + await ensureStateAndEnv(); + await printQueueStatus(); + return; + case 'api-update': + await ensureStateAndEnv(); + await apiOnlyUpdate({build: rawArgs.includes('--build')}); + return; + case 'scale': + await ensureStateAndEnv(); + await scaleWorkers(rawArgs[0] ?? '1'); + return; + case 'logs': + await ensureStateAndEnv(); + await dockerCompose(['logs', ...rawArgs]); + return; + case 'ps': + await ensureStateAndEnv(); + await dockerCompose(['ps', ...rawArgs]); + return; + case 'down': + await ensureStateAndEnv({create: false}); + await dockerCompose(['down', ...rawArgs]); + return; + case 'smoke': + await runSmoke(); + return; + default: + throw new Error(`未知命令: ${command}`); + } +} + +async function runSmoke() { + if (rawArgs.includes('--force')) { + await ensureStateAndEnv(); + await dockerComposeCapture(['down', '-v'], {allowFailure: true}); + } + const state = await ensureStateAndEnv({force: rawArgs.includes('--force')}); + await assertSavedPortsAvailableForNewProject(state); + console.log( + `[worker-smoke] 使用隔离环境 project=${projectName} database=${state.database}`, + ); + await buildRuntimeImages(); + await ensureSpacetimeImage(); + await dockerCompose(['up', '-d', 'spacetimedb', 'otelcol']); + await waitForSpacetime(); + await publishModule(); + await upRuntime(); + await waitForApi(); + await assertWorkersRunning(); + + const beforeWorkerIds = await getContainerIds('external-generation-worker'); + console.log(`[worker-smoke] worker 容器: ${beforeWorkerIds.join(', ')}`); + + const firstJobId = await enqueueSmokeJob({label: 'before-api-update'}); + await waitForJobConsumed(firstJobId); + + await apiOnlyUpdate({build: false}); + const afterWorkerIds = await getContainerIds('external-generation-worker'); + if (beforeWorkerIds.join('\n') !== afterWorkerIds.join('\n')) { + throw new Error( + `api-update 后 worker 容器发生变化: before=${beforeWorkerIds.join(',')} after=${afterWorkerIds.join(',')}`, + ); + } + console.log('[worker-smoke] api-only 更新未重建 worker 容器。'); + + const secondJobId = await enqueueSmokeJob({label: 'after-api-update'}); + await waitForJobConsumed(secondJobId); + await printQueueStatus(); + console.log('[worker-smoke] smoke 通过：worker 独立消费队列，API-only 更新未停止 worker。'); +} + +async function buildRuntimeImages() { + const imageMode = resolveImageMode(); + if (imageMode === 'local-binary') { + await buildLocalBinaryRuntimeImages(); + return; + } + await dockerCompose(['build', 'api-server', 'external-generation-worker']); +} + +function resolveImageMode() { + if (rawArgs.includes('--local-binary')) { + return 'local-binary'; + } + const envMode = process.env.GENARRATIVE_WORKER_SMOKE_IMAGE_MODE; + if (!envMode || envMode === 'dockerfile') { + return 'dockerfile'; + } + if (envMode === 'local-binary') { + return 'local-binary'; + } + throw new Error( + `GENARRATIVE_WORKER_SMOKE_IMAGE_MODE 仅支持 dockerfile 或 local-binary: ${envMode}`, + ); +} + +async function buildLocalBinaryRuntimeImages() { + const profile = + rawArgs.includes('--release') || + process.env.GENARRATIVE_WORKER_SMOKE_CARGO_PROFILE === 'release' + ? 'release' + : 'debug'; + const buildArgs = ['build', '-p', 'api-server', '--manifest-path', 'server-rs/Cargo.toml']; + if (profile === 'release') { + buildArgs.push('--release'); + } + const cargoImage = resolveLocalBinaryCargoImage(); + const cargoHome = resolveLocalBinaryCargoHome(); + mkdirSync(cargoHome, {recursive: true}); + + console.log( + `[worker-smoke] 使用 ${cargoImage} 复用本机 Cargo 缓存构建 ${profile} api-server 二进制。`, + ); + await run('docker', [ + 'run', + '--rm', + '-u', + currentUserSpec(), + '-v', + `${projectRoot}:/workspace`, + '-v', + `${cargoHome}:/cargo-home`, + '-w', + '/workspace', + '-e', + 'HOME=/cargo-home', + '-e', + 'CARGO_HOME=/cargo-home', + '-e', + `CARGO_TARGET_DIR=/workspace/${toContainerPath(localCargoTargetDir)}`, + cargoImage, + 'cargo', + '--config', + 'build.rustc-wrapper=""', + '--config', + 'target.x86_64-unknown-linux-gnu.linker="cc"', + '--config', + 'target.x86_64-unknown-linux-gnu.rustflags=[]', + ...buildArgs, + ]); + + const sourceBinaryPath = path.join(localCargoTargetDir, profile, 'api-server'); + if (!existsSync(sourceBinaryPath)) { + throw new Error(`未找到 worker smoke api-server 二进制: ${sourceBinaryPath}`); + } + + mkdirSync(localImageDir, {recursive: true}); + copyFileSync(sourceBinaryPath, localImageBinaryPath); + chmodSync(localImageBinaryPath, 0o755); + + const baseImage = await resolveLocalBinaryBaseImage(); + writeFileSync(localImageDockerfilePath, buildLocalBinaryDockerfile(baseImage), 'utf8'); + + await run('docker', [ + 'build', + '-f', + localImageDockerfilePath, + '-t', + `${projectName}-api-server`, + '-t', + `${projectName}-external-generation-worker`, + localImageDir, + ]); +} + +function resolveLocalBinaryCargoImage() { + return process.env.GENARRATIVE_WORKER_SMOKE_CARGO_IMAGE || 'rust:1.93-bookworm'; +} + +function resolveLocalBinaryCargoHome() { + if (process.env.GENARRATIVE_WORKER_SMOKE_CARGO_HOME) { + return path.resolve(process.env.GENARRATIVE_WORKER_SMOKE_CARGO_HOME); + } + if (!process.env.HOME) { + throw new Error('未找到 HOME，无法挂载本机 Cargo 缓存。'); + } + return path.join(process.env.HOME, '.cargo'); +} + +function currentUserSpec() { + if (typeof process.getuid === 'function' && typeof process.getgid === 'function') { + return `${process.getuid()}:${process.getgid()}`; + } + return '0:0'; +} + +async function ensureSpacetimeImage() { + if (process.env.GENARRATIVE_WORKER_SMOKE_SPACETIME_IMAGE_MODE === 'official') { + return; + } + const imageName = localSpacetimeImageName(); + const existingImage = await runCapture('docker', ['image', 'inspect', imageName], { + allowFailure: true, + quiet: true, + }); + if (existingImage.code === 0 && !rawArgs.includes('--force')) { + return; + } + + const spacetimePath = await resolveSpacetimeBinaryPath(); + if (!spacetimePath) { + throw new Error('未找到本机 spacetime CLI，无法构建隔离 SpacetimeDB 镜像。'); + } + + mkdirSync(localSpacetimeImageDir, {recursive: true}); + copyFileSync(spacetimePath, localSpacetimeBinaryPath); + chmodSync(localSpacetimeBinaryPath, 0o755); + const standalonePath = path.join(path.dirname(spacetimePath), 'spacetimedb-standalone'); + if (!existsSync(standalonePath)) { + throw new Error(`未找到本机 spacetimedb-standalone: ${standalonePath}`); + } + copyFileSync(standalonePath, localSpacetimeStandalonePath); + chmodSync(localSpacetimeStandalonePath, 0o755); + writeFileSync(localSpacetimeDockerfilePath, buildLocalSpacetimeDockerfile(), 'utf8'); + + console.log(`[worker-smoke] 使用本机 spacetime CLI 构建隔离镜像: ${imageName}`); + await run('docker', [ + 'build', + '-f', + localSpacetimeDockerfilePath, + '-t', + imageName, + localSpacetimeImageDir, + ]); +} + +function buildLocalSpacetimeDockerfile() { + return `FROM debian:bookworm-slim +WORKDIR /var/lib/spacetimedb +RUN apt-get update && \\ + apt-get install -y --no-install-recommends ca-certificates libstdc++6 zlib1g && \\ + rm -rf /var/lib/apt/lists/* +COPY spacetime /usr/local/bin/spacetime +COPY spacetimedb-standalone /usr/local/bin/spacetimedb-standalone +RUN chmod 0755 /usr/local/bin/spacetime /usr/local/bin/spacetimedb-standalone +ENTRYPOINT ["spacetime"] +`; +} + +async function resolveSpacetimeBinaryPath() { + if (process.env.GENARRATIVE_WORKER_SMOKE_SPACETIME_BIN) { + return process.env.GENARRATIVE_WORKER_SMOKE_SPACETIME_BIN; + } + const versionResult = await runCapture('spacetime', ['--version'], {quiet: true}); + const pathMatch = versionResult.stdout.match(/^spacetime Path:\s*(.+)$/mu); + if (pathMatch?.[1]) { + return pathMatch[1].trim(); + } + const whichResult = await runCapture('which', ['spacetime'], {quiet: true}); + return whichResult.stdout.trim(); +} + +async function resolveLocalBinaryBaseImage() { + if (process.env.GENARRATIVE_WORKER_SMOKE_LOCAL_BASE_IMAGE) { + return process.env.GENARRATIVE_WORKER_SMOKE_LOCAL_BASE_IMAGE; + } + return 'debian:bookworm-slim'; +} + +function buildLocalBinaryDockerfile(baseImage) { + return `FROM ${baseImage} +WORKDIR /srv/genarrative +RUN apt-get update && \\ + apt-get install -y --no-install-recommends ca-certificates curl libssl3 zlib1g libzstd1 && \\ + rm -rf /var/lib/apt/lists/* && \\ + (id -u genarrative >/dev/null 2>&1 || useradd --system --create-home --home-dir /srv/genarrative --shell /usr/sbin/nologin genarrative) +COPY api-server /usr/local/bin/api-server +RUN chmod 0755 /usr/local/bin/api-server && \\ + mkdir -p /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox && \\ + chown -R genarrative:genarrative /srv/genarrative /var/lib/genarrative +USER genarrative +EXPOSE 8082 +ENV GENARRATIVE_ENV=container \\ + GENARRATIVE_API_HOST=0.0.0.0 \\ + GENARRATIVE_API_PORT=8082 \\ + GENARRATIVE_TRACKING_OUTBOX_DIR=/var/lib/genarrative/tracking-outbox +CMD ["api-server"] +`; +} + +function toContainerPath(localPath) { + return localPath.split(path.sep).join('/'); +} + +async function upRuntime() { + const services = ['api-server', 'external-generation-worker']; + if (rawArgs.includes('--with-nginx')) { + services.push('nginx'); + } + await dockerCompose(['up', '-d', ...services]); +} + +async function ensureStateAndEnv(options = {}) { + const {force = false, create = true} = options; + if (!create && !existsSync(statePath)) { + return defaultState(); + } + mkdirSync(smokeDir, {recursive: true}); + + if (!existsSync(statePath) || force) { + const state = { + database: defaultDatabase, + spacetimePort: await findAvailablePort( + Number(process.env.GENARRATIVE_WORKER_SMOKE_SPACETIME_PORT || 19101), + ), + httpPort: await findAvailablePort( + Number(process.env.GENARRATIVE_WORKER_SMOKE_HTTP_PORT || 19080), + ), + otlpGrpcPort: await findAvailablePort( + Number(process.env.GENARRATIVE_WORKER_SMOKE_OTLP_GRPC_PORT || 15317), + ), + otlpHttpPort: await findAvailablePort( + Number(process.env.GENARRATIVE_WORKER_SMOKE_OTLP_HTTP_PORT || 15318), + ), + createdAt: new Date().toISOString(), + }; + writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8'); + } + + const state = readState(); + if (!existsSync(envPath) || force) { + writeFileSync(envPath, buildSmokeEnv(state), 'utf8'); + } + console.log(`[worker-smoke] env=${envPath}`); + console.log(`[worker-smoke] state=${statePath}`); + console.log(`[worker-smoke] SpacetimeDB=http://127.0.0.1:${state.spacetimePort}`); + console.log(`[worker-smoke] Nginx=http://127.0.0.1:${state.httpPort}`); + return state; +} + +function buildSmokeEnv(state) { + return `# 本文件由 scripts/container-worker-smoke.mjs 生成，仅用于本机隔离 worker smoke。 +# 不要在这里写真实生产密钥；目录 deploy/container/worker-smoke/ 已被 gitignore。 +GENARRATIVE_ENV=container-worker-smoke +GENARRATIVE_API_HOST=0.0.0.0 +GENARRATIVE_API_PORT=8082 +GENARRATIVE_API_LOG=info,tower_http=info +GENARRATIVE_API_LISTEN_BACKLOG=256 +GENARRATIVE_API_WORKER_THREADS=2 +GENARRATIVE_PROCESS_ROLE=api +GENARRATIVE_EXTERNAL_GENERATION_MODE=queue +GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID= +GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY=1 +GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS=500 +GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=60 +GENARRATIVE_API_MAX_CONCURRENT_REQUESTS=64 +GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS=32 +GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS=16 +GENARRATIVE_API_ADMIN_MAX_CONCURRENT_REQUESTS=8 +GENARRATIVE_TRACKING_OUTBOX_ENABLED=false +GENARRATIVE_TRACKING_OUTBOX_DIR=/var/lib/genarrative/tracking-outbox + +GENARRATIVE_OTEL_ENABLED=false +OTEL_SERVICE_NAME=genarrative-worker-smoke-api +OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4318 +OTEL_RESOURCE_ATTRIBUTES=deployment.environment=worker-smoke,service.namespace=genarrative + +GENARRATIVE_INTERNAL_API_SECRET=worker-smoke-internal-secret +GENARRATIVE_JWT_ISSUER=genarrative-worker-smoke +GENARRATIVE_JWT_SECRET=worker-smoke-jwt-secret +AUTH_REFRESH_COOKIE_SECURE=false +GENARRATIVE_DEV_PASSWORD_ENTRY_AUTO_REGISTER_ENABLED=true + +GENARRATIVE_SPACETIME_SERVER_URL=http://spacetimedb:3101 +GENARRATIVE_SPACETIME_DATABASE=${state.database} +GENARRATIVE_SPACETIME_TOKEN= +GENARRATIVE_SPACETIME_POOL_SIZE=2 +GENARRATIVE_SPACETIME_PROCEDURE_TIMEOUT_SECONDS=15 + +GENARRATIVE_LLM_PROVIDER=openai-compatible +GENARRATIVE_LLM_BASE_URL= +GENARRATIVE_LLM_API_KEY= +GENARRATIVE_LLM_MODEL= +VECTOR_ENGINE_BASE_URL= +VECTOR_ENGINE_API_KEY= +ALIYUN_OSS_BUCKET= +ALIYUN_OSS_ENDPOINT=oss-cn-shanghai.aliyuncs.com +ALIYUN_OSS_ACCESS_KEY_ID= +ALIYUN_OSS_ACCESS_KEY_SECRET= +WECHAT_MINIPROGRAM_MESSAGE_TOKEN= +WECHAT_MINIPROGRAM_MESSAGE_ENCODING_AES_KEY= +`; +} + +function defaultState() { + return { + database: defaultDatabase, + spacetimePort: 19101, + httpPort: 19080, + otlpGrpcPort: 15317, + otlpHttpPort: 15318, + }; +} + +function readState() { + if (!existsSync(statePath)) { + return defaultState(); + } + return JSON.parse(readFileSync(statePath, 'utf8')); +} + +async function findAvailablePort(startPort) { + for (let port = startPort; port < startPort + 100; port += 1) { + if (await isPortAvailable(port)) { + return port; + } + } + throw new Error(`未找到可用端口: ${startPort}-${startPort + 99}`); +} + +function isPortAvailable(port) { + return new Promise((resolve) => { + const server = net.createServer(); + server.once('error', () => resolve(false)); + server.once('listening', () => { + server.close(() => resolve(true)); + }); + server.listen(port, '127.0.0.1'); + }); +} + +async function publishModule() { + const state = readState(); + const serverUrl = spacetimeServerUrl(state); + const publishArgs = [ + 'publish', + state.database, + '--server', + serverUrl, + '--module-path', + 'server-rs/crates/spacetime-module', + '--delete-data=on-conflict', + '--anonymous', + '--yes=all', + '--no-config', + ]; + const buildOptions = process.env.GENARRATIVE_WORKER_SMOKE_STDB_BUILD_OPTIONS; + if (buildOptions) { + publishArgs.push('--build-options', buildOptions); + } + await run('spacetime', publishArgs); +} + +async function enqueueSmokeJob(options = {}) { + if (!rawArgs.includes('--no-worker-check')) { + await assertWorkersRunning(); + } + const state = readState(); + const nowMicros = Date.now() * 1000; + const suffix = `${Date.now()}-${Math.random().toString(16).slice(2, 8)}`; + const jobId = `extgen-smoke-${suffix}`; + const label = options.label || rawArgs[0] || 'manual'; + const input = { + job_id: jobId, + dedupe_key: `worker-smoke:${label}:${suffix}`, + job_kind: 'worker_smoke_unsupported', + owner_user_id: 'worker-smoke-user', + source_module: 'worker-smoke', + source_entity_id: `worker-smoke-entity-${suffix}`, + request_label: `worker-smoke ${label}`, + request_payload_json: JSON.stringify({label, suffix}), + max_attempts: 1, + available_at_micros: nowMicros, + created_at_micros: nowMicros, + }; + + await run('spacetime', [ + 'call', + '--server', + spacetimeServerUrl(state), + '--anonymous', + '--yes', + '--no-config', + state.database, + 'enqueue_external_generation_job_and_return', + JSON.stringify(input), + ]); + console.log(`[worker-smoke] 已入队测试 job: ${jobId}`); + return jobId; +} + +async function printQueueStatus() { + console.log('[worker-smoke] external_generation_job 是 private table，status 显示最近 worker 日志：'); + await printServiceLogs('external-generation-worker', 120); +} + +async function waitForJobConsumed(jobId) { + const deadline = Date.now() + 60_000; + let lastOutput = ''; + while (Date.now() < deadline) { + const result = await dockerComposeCapture( + ['logs', '--no-color', 'external-generation-worker'], + {allowFailure: true, quiet: true}, + ); + lastOutput = `${result.stdout}\n${result.stderr}`; + if (lastOutput.includes(jobId) && lastOutput.includes('暂不支持的任务类型')) { + console.log(`[worker-smoke] job ${jobId} 已被 worker 领取并执行到 unsupported 分支。`); + return; + } + await sleep(1000); + } + await printServiceLogs('external-generation-worker', 120); + throw new Error(`等待 worker 消费 job ${jobId} 超时，最后输出:\n${lastOutput}`); +} + +async function assertSavedPortsAvailableForNewProject(state) { + const existingContainers = await getProjectContainerIds(); + if (existingContainers.length > 0) { + return; + } + const ports = [ + ['SpacetimeDB', state.spacetimePort], + ['Nginx', state.httpPort], + ['OTLP gRPC', state.otlpGrpcPort], + ['OTLP HTTP', state.otlpHttpPort], + ]; + for (const [label, port] of ports) { + if (!(await isPortAvailable(port))) { + throw new Error( + `${label} 端口 ${port} 已被占用；可执行 npm run container:worker-smoke -- smoke --force 重新分配隔离端口。`, + ); + } + } +} + +async function getProjectContainerIds() { + const result = await dockerComposeCapture(['ps', '-q'], { + allowFailure: true, + quiet: true, + }); + if (result.code !== 0) { + return []; + } + return result.stdout + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); +} + +async function assertWorkersRunning() { + const result = await dockerComposeCapture( + ['ps', '--status', 'running', '-q', 'external-generation-worker'], + {allowFailure: true, quiet: true}, + ); + const workerIds = result.stdout + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); + if (result.code === 0 && workerIds.length > 0) { + return; + } + await printServiceLogs('external-generation-worker', 80); + throw new Error('external-generation-worker 未处于 running 状态，已输出最近日志。'); +} + +async function printServiceLogs(service, tail = 80) { + await dockerComposeCapture(['logs', '--tail', String(tail), service], { + allowFailure: true, + }); +} + +async function waitForSpacetime() { + const state = readState(); + const url = `${spacetimeServerUrl(state)}/v1/ping`; + await waitForHttp(url, 'SpacetimeDB'); +} + +async function waitForApi() { + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + const result = await dockerComposeCapture( + ['exec', '-T', 'api-server', 'curl', '-fsS', 'http://127.0.0.1:8082/healthz'], + {allowFailure: true, quiet: true}, + ); + if (result.code === 0) { + console.log('[worker-smoke] api-server 已就绪: api-server:8082/healthz'); + return; + } + await sleep(2000); + } + throw new Error('api-server 等待超时: api-server:8082/healthz'); +} + +async function waitForHttp(url, label) { + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + const result = await runCapture('curl', ['-fsS', '--max-time', '3', url], { + allowFailure: true, + }); + if (result.code === 0) { + console.log(`[worker-smoke] ${label} 已就绪: ${url}`); + return; + } + await sleep(2000); + } + throw new Error(`${label} 等待超时: ${url}`); +} + +async function apiOnlyUpdate({build}) { + const beforeWorkerIds = await getContainerIds('external-generation-worker'); + const args = ['up', '-d', '--no-deps', '--force-recreate']; + if (build) { + args.push('--build'); + } + args.push('api-server'); + await dockerCompose(args); + await waitForApi(); + const afterWorkerIds = await getContainerIds('external-generation-worker'); + if (beforeWorkerIds.join('\n') !== afterWorkerIds.join('\n')) { + throw new Error('API-only 更新不应重建 external-generation-worker 容器'); + } + console.log('[worker-smoke] API-only 更新完成，worker 容器保持不变。'); +} + +async function scaleWorkers(rawCount) { + const count = Number.parseInt(rawCount, 10); + if (!Number.isInteger(count) || count < 0 || count > 16) { + throw new Error(`worker 数量必须是 0-16 的整数: ${rawCount}`); + } + await dockerCompose([ + 'up', + '-d', + '--scale', + `external-generation-worker=${count}`, + 'external-generation-worker', + ]); +} + +async function getContainerIds(service) { + const result = await dockerComposeCapture(['ps', '-q', service]); + return result.stdout + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean) + .sort(); +} + +async function dockerCompose(args) { + await run('docker', composeArgs(args), {env: composeEnv()}); +} + +async function dockerComposeCapture(args, options = {}) { + return runCapture('docker', composeArgs(args), { + env: composeEnv(), + ...options, + }); +} + +function composeArgs(args) { + return ['compose', '-p', projectName, '-f', composeFile, ...args]; +} + +function composeEnv() { + const state = readState(); + return { + ...process.env, + GENARRATIVE_CONTAINER_API_ENV_FILE: './worker-smoke/api-server.env', + GENARRATIVE_CONTAINER_SPACETIME_IMAGE: + process.env.GENARRATIVE_CONTAINER_SPACETIME_IMAGE || localSpacetimeImageName(), + GENARRATIVE_CONTAINER_SPACETIME_PORT: String(state.spacetimePort), + GENARRATIVE_CONTAINER_HTTP_PORT: String(state.httpPort), + GENARRATIVE_CONTAINER_OTLP_GRPC_PORT: String(state.otlpGrpcPort), + GENARRATIVE_CONTAINER_OTLP_HTTP_PORT: String(state.otlpHttpPort), + }; +} + +function localSpacetimeImageName() { + return `${projectName}-spacetimedb:2.4.1`; +} + +function spacetimeServerUrl(state) { + return `http://127.0.0.1:${state.spacetimePort}`; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function run(commandName, args, options = {}) { + const result = await runCapture(commandName, args, options); + if (result.code !== 0 && !options.allowFailure) { + throw new Error(`${commandName} ${args.join(' ')} 失败，exit=${result.code}`); + } + return result; +} + +function runCapture(commandName, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(commandName, args, { + cwd: projectRoot, + env: options.env ?? process.env, + shell: false, + }); + let stdout = ''; + let stderr = ''; + child.stdout?.on('data', (chunk) => { + const text = chunk.toString(); + stdout += text; + if (!options.quiet) { + process.stdout.write(text); + } + }); + child.stderr?.on('data', (chunk) => { + const text = chunk.toString(); + stderr += text; + if (!options.quiet) { + process.stderr.write(text); + } + }); + child.on('error', reject); + child.on('exit', (code, signal) => { + if (signal) { + reject(new Error(`${commandName} 被信号终止: ${signal}`)); + return; + } + resolve({code: code ?? 0, stdout, stderr}); + }); + }); +} + +function printHelp(isError) { + const output = isError ? console.error : console.log; + output(`Usage: npm run container:worker-smoke -- + +Commands: + init [--force] 生成隔离 env 与端口 state + build [--local-binary] [--release] + 构建 api-server / worker 镜像；--local-binary 让容器内 Cargo 复用本机缓存 + up-spacetime 启动隔离 SpacetimeDB 与 otelcol + publish 向隔离 SpacetimeDB 发布 spacetime-module + up [--with-nginx] 启动 api-server / worker；需要 Nginx 时显式加 --with-nginx + enqueue [label] [--no-worker-check] + 写入一个 unsupported 测试 job，验证 worker claim/fail + status 查看最近 worker 日志；external_generation_job 是 private table + api-update [--build] 仅重建/重启 api-server，不触碰 worker + scale 调整 external-generation-worker 实例数 + ps 查看隔离 compose 状态 + logs [service] 查看隔离 compose 日志 + down [-v] 停止隔离 compose，-v 会清理数据卷 + smoke [--force] [--local-binary] [--release] + 一键执行 build -> publish -> up -> enqueue -> api-update -> enqueue +`); +} diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index 14029c520..e215b3e90 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -5,10 +5,11 @@ set -euo pipefail usage() { cat <<'EOF' 用法: - ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] + ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] 说明: 进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。 + 默认同时重启外部生成 worker controller 和已加载的 worker 实例；未启用 worker 单元时会自动跳过。 若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。 失败时保留维护模式。 EOF @@ -223,12 +224,144 @@ ensure_runtime_env_and_dirs() { fi } +list_worker_services() { + local pattern="$1" + + if [[ -z "${pattern}" ]]; then + return 0 + fi + + systemctl list-units --all --plain --no-legend "${pattern}" 2>/dev/null | awk '{print $1}' | sort -u +} + +ensure_default_worker_service() { + local pattern="$1" + local default_service="genarrative-external-generation-worker@1.service" + local template_service="genarrative-external-generation-worker@.service" + local services=() + + if [[ -z "${pattern}" ]]; then + return 0 + fi + + if [[ "${pattern}" != "genarrative-external-generation-worker@*.service" ]]; then + return 0 + fi + + if ! systemctl cat "${template_service}" >/dev/null 2>&1; then + echo "[production-api-deploy] 缺少外部生成 worker systemd 模板: ${template_service}" >&2 + return 1 + fi + + mapfile -t services < <(list_worker_services "${pattern}") + if [[ "${#services[@]}" -gt 0 ]]; then + return 0 + fi + + echo "[production-api-deploy] 未发现外部生成 worker 实例，启用并启动默认实例: ${default_service}" + systemctl enable --now "${default_service}" +} + +restart_worker_services() { + local pattern="$1" + local services=() + + if [[ -z "${pattern}" ]]; then + echo "[production-api-deploy] 跳过外部生成 worker 重启。" + return 0 + fi + + ensure_default_worker_service "${pattern}" + mapfile -t services < <(list_worker_services "${pattern}") + if [[ "${#services[@]}" -eq 0 ]]; then + echo "[production-api-deploy] 未发现已加载的外部生成 worker 单元: ${pattern}" >&2 + return 1 + fi + + echo "[production-api-deploy] 重启外部生成 worker: ${services[*]}" + systemctl restart "${services[@]}" +} + +wait_for_worker_services() { + local pattern="$1" + local services=() + local all_active + + if [[ -z "${pattern}" ]]; then + return 0 + fi + + mapfile -t services < <(list_worker_services "${pattern}") + if [[ "${#services[@]}" -eq 0 ]]; then + echo "[production-api-deploy] 外部生成 worker 单元不存在，发布失败: ${pattern}" >&2 + return 1 + fi + + echo "[production-api-deploy] 等待外部生成 worker active: ${services[*]}" + for _ in {1..30}; do + all_active=1 + for service in "${services[@]}"; do + if ! systemctl is-active --quiet "${service}"; then + all_active=0 + break + fi + done + if [[ "${all_active}" -eq 1 ]]; then + return 0 + fi + sleep 2 + done + + systemctl --no-pager --full status "${services[@]}" || true + echo "[production-api-deploy] 外部生成 worker 未在超时时间内进入 active，发布失败。" >&2 + return 1 +} + +ensure_worker_controller_service() { + local service="$1" + + if [[ -z "${service}" ]]; then + return 0 + fi + + if ! systemctl cat "${service}" >/dev/null 2>&1; then + echo "[production-api-deploy] 缺少外部生成 worker controller systemd 单元: ${service}" >&2 + return 1 + fi + + echo "[production-api-deploy] 启用并重启外部生成 worker controller: ${service}" + systemctl enable "${service}" + systemctl restart "${service}" +} + +wait_for_worker_controller_service() { + local service="$1" + + if [[ -z "${service}" ]]; then + return 0 + fi + + echo "[production-api-deploy] 等待外部生成 worker controller active: ${service}" + for _ in {1..30}; do + if systemctl is-active --quiet "${service}"; then + return 0 + fi + sleep 2 + done + + systemctl --no-pager --full status "${service}" || true + echo "[production-api-deploy] 外部生成 worker controller 未在超时时间内进入 active，发布失败。" >&2 + return 1 +} + SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" SOURCE_DIR="" VERSION="" RELEASE_ROOT="/opt/genarrative/releases" CURRENT_LINK="/opt/genarrative/current" SERVICE_NAME="genarrative-api.service" +WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service" +WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service" HEALTH_URL="http://127.0.0.1:8082/readyz" API_ENV_FILE="/etc/genarrative/api-server.env" DATABASE="" @@ -261,6 +394,22 @@ while [[ $# -gt 0 ]]; do SERVICE_NAME="${2:?缺少 --service 的值}" shift 2 ;; + --worker-service-pattern) + WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}" + shift 2 + ;; + --no-worker-services) + WORKER_SERVICE_PATTERN="" + shift + ;; + --worker-controller-service) + WORKER_CONTROLLER_SERVICE="${2:?缺少 --worker-controller-service 的值}" + shift 2 + ;; + --no-worker-controller) + WORKER_CONTROLLER_SERVICE="" + shift + ;; --health-url) HEALTH_URL="${2:?缺少 --health-url 的值}" shift 2 @@ -383,6 +532,10 @@ ln -sfn "${RELEASE_DIR}" "${CURRENT_LINK}" echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}" systemctl restart "${SERVICE_NAME}" +restart_worker_services "${WORKER_SERVICE_PATTERN}" +wait_for_worker_services "${WORKER_SERVICE_PATTERN}" +ensure_worker_controller_service "${WORKER_CONTROLLER_SERVICE}" +wait_for_worker_controller_service "${WORKER_CONTROLLER_SERVICE}" echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}" for _ in {1..30}; do diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 9f399e84e..9cc623974 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -4,6 +4,8 @@ set -euo pipefail PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}" SPACETIME_BIN_SOURCE="${SPACETIME_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/spacetime/spacetime}" OTELCOL_BIN_SOURCE="${OTELCOL_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/otelcol-contrib}" +WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}" +CONTROLLER_ENV_FILE="${CONTROLLER_ENV_FILE:-/etc/genarrative/external-generation-controller.env}" GENARRATIVE_OPENSSL_VERSION="${GENARRATIVE_OPENSSL_VERSION:-3.2.0}" GENARRATIVE_OPENSSL_PREFIX="${GENARRATIVE_OPENSSL_PREFIX:-/opt/genarrative/openssl-3.2.0}" GENARRATIVE_OPENSSL_SOURCE_URL="${GENARRATIVE_OPENSSL_SOURCE_URL:-https://github.com/openssl/openssl/releases/download/openssl-${GENARRATIVE_OPENSSL_VERSION}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz}" @@ -458,6 +460,7 @@ ensure_spacetime_owner_client_token() { echo "[server-provision] 已生成 SpacetimeDB client identity 并写入 GENARRATIVE_SPACETIME_TOKEN: ${identity_preview}..." fi + # 中文注释：这里是 provision 内部为 spacetimedb 运行用户隔离 CLI 登录态的受控用法，不作为人工 spacetime 命令示例。 if ! login_output="$(runuser -u spacetimedb -- "${cli_path}" --root-dir "${SPACETIME_ROOT}" login --token "${token}" 2>&1)"; then echo "[server-provision] 使用 GENARRATIVE_SPACETIME_TOKEN 登录 SpacetimeDB CLI 失败。" >&2 printf "%s\\n" "${login_output}" | sed -E "s/[A-Za-z0-9_.=-]{24,}/[REDACTED]/g" >&2 @@ -536,6 +539,14 @@ render_api_env_example() { deploy/env/api-server.env.example } +render_external_generation_worker_env_example() { + cat deploy/env/external-generation-worker.env.example +} + +render_external_generation_controller_env_example() { + cat deploy/env/external-generation-controller.env.example +} + render_otelcol_service() { cat deploy/systemd/otelcol-contrib.service } @@ -722,6 +733,30 @@ render_api_service() { deploy/systemd/genarrative-api.service } +render_external_generation_worker_service() { + local current_escaped api_env_escaped worker_env_escaped + current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" + api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")" + worker_env_escaped="$(escape_sed_replacement "${WORKER_ENV_FILE}")" + sed \ + -e "s|/opt/genarrative/current|${current_escaped}|g" \ + -e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \ + -e "s|/etc/genarrative/external-generation-worker.env|${worker_env_escaped}|g" \ + deploy/systemd/genarrative-external-generation-worker@.service +} + +render_external_generation_controller_service() { + local current_escaped api_env_escaped controller_env_escaped + current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" + api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")" + controller_env_escaped="$(escape_sed_replacement "${CONTROLLER_ENV_FILE}")" + sed \ + -e "s|/opt/genarrative/current|${current_escaped}|g" \ + -e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \ + -e "s|/etc/genarrative/external-generation-controller.env|${controller_env_escaped}|g" \ + deploy/systemd/genarrative-external-generation-controller.service +} + render_database_backup_service() { local current_escaped env_escaped current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" @@ -742,6 +777,8 @@ render_health_patrol_service() { require_path deploy/systemd/spacetimedb.service require_path deploy/systemd/genarrative-api.service +require_path deploy/systemd/genarrative-external-generation-worker@.service +require_path deploy/systemd/genarrative-external-generation-controller.service require_path deploy/systemd/genarrative-database-backup.service require_path deploy/systemd/genarrative-database-backup.timer require_path deploy/systemd/genarrative-health-patrol.service @@ -752,6 +789,8 @@ require_path deploy/nginx/genarrative.conf require_path deploy/nginx/genarrative-dev-http.conf require_path deploy/nginx/snippets/genarrative-maintenance.conf require_path deploy/env/api-server.env.example +require_path deploy/env/external-generation-worker.env.example +require_path deploy/env/external-generation-controller.env.example require_path scripts/deploy/maintenance-on.sh require_path scripts/deploy/maintenance-off.sh require_path scripts/deploy/maintenance-status.sh @@ -795,19 +834,25 @@ sync_spacetime_install "${SPACETIME_ROOT}" spacetimedb_service="$(mktemp)" api_service="$(mktemp)" +external_generation_worker_service="$(mktemp)" +external_generation_controller_service="$(mktemp)" database_backup_service="$(mktemp)" health_patrol_service="$(mktemp)" render_spacetimedb_service >"${spacetimedb_service}" render_api_service >"${api_service}" +render_external_generation_worker_service >"${external_generation_worker_service}" +render_external_generation_controller_service >"${external_generation_controller_service}" render_database_backup_service >"${database_backup_service}" render_health_patrol_service >"${health_patrol_service}" install_file "${spacetimedb_service}" /etc/systemd/system/spacetimedb.service 0644 install_file "${api_service}" /etc/systemd/system/genarrative-api.service 0644 +install_file "${external_generation_worker_service}" /etc/systemd/system/genarrative-external-generation-worker@.service 0644 +install_file "${external_generation_controller_service}" /etc/systemd/system/genarrative-external-generation-controller.service 0644 install_file "${database_backup_service}" /etc/systemd/system/genarrative-database-backup.service 0644 install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644 install_file "${health_patrol_service}" /etc/systemd/system/genarrative-health-patrol.service 0644 install_file deploy/systemd/genarrative-health-patrol.timer /etc/systemd/system/genarrative-health-patrol.timer 0644 -rm -f "${spacetimedb_service}" "${api_service}" "${database_backup_service}" "${health_patrol_service}" +rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${external_generation_controller_service}" "${database_backup_service}" "${health_patrol_service}" if [[ ! -f "${API_ENV_FILE}" ]]; then echo "+ create ${API_ENV_FILE} from example" @@ -821,6 +866,28 @@ else fi ensure_api_runtime_env_defaults +if [[ ! -f "${WORKER_ENV_FILE}" ]]; then + echo "+ create ${WORKER_ENV_FILE} from example" + if [[ "${DRY_RUN}" != "true" ]]; then + render_external_generation_worker_env_example >"${WORKER_ENV_FILE}" + chmod 0600 "${WORKER_ENV_FILE}" + chown root:root "${WORKER_ENV_FILE}" + fi +else + echo "[server-provision] 已存在 worker 环境文件，保留不覆盖: ${WORKER_ENV_FILE}" +fi + +if [[ ! -f "${CONTROLLER_ENV_FILE}" ]]; then + echo "+ create ${CONTROLLER_ENV_FILE} from example" + if [[ "${DRY_RUN}" != "true" ]]; then + render_external_generation_controller_env_example >"${CONTROLLER_ENV_FILE}" + chmod 0600 "${CONTROLLER_ENV_FILE}" + chown root:root "${CONTROLLER_ENV_FILE}" + fi +else + echo "[server-provision] 已存在 controller 环境文件，保留不覆盖: ${CONTROLLER_ENV_FILE}" +fi + if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then sync_otelcol_install otelcol_service="$(mktemp)" @@ -842,7 +909,7 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then run_cmd systemctl enable otelcol-contrib.service fi - run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-health-patrol.timer + run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service genarrative-health-patrol.timer if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then run_cmd systemctl restart otelcol-contrib.service fi @@ -851,8 +918,12 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then ensure_spacetime_owner_client_token if [[ -x "${CURRENT_LINK}/api-server" ]]; then run_cmd systemctl restart genarrative-api.service + run_cmd systemctl enable --now genarrative-external-generation-worker@1.service + run_cmd systemctl restart genarrative-external-generation-worker@1.service + run_cmd systemctl enable --now genarrative-external-generation-controller.service + run_cmd systemctl restart genarrative-external-generation-controller.service else - echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server，跳过 api-server 首次启动。后续 API deploy 会重启服务。" + echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server，跳过 api-server、外部生成 worker 和 controller 首次启动。后续 API deploy 会启用并启动默认 worker 与 controller。" fi fi diff --git a/scripts/ops/production-health-patrol.mjs b/scripts/ops/production-health-patrol.mjs index 219d8e295..01a1265c7 100644 --- a/scripts/ops/production-health-patrol.mjs +++ b/scripts/ops/production-health-patrol.mjs @@ -20,9 +20,11 @@ const DEFAULT_PUBLIC_PATHS = [ const DEFAULT_SERVICES = [ 'genarrative-api.service', + 'genarrative-external-generation-controller.service', 'spacetimedb.service', 'nginx.service', ]; +const WORKER_SERVICE_PATTERN = 'genarrative-external-generation-worker@*.service'; function usage() { console.log(`Usage: @@ -216,6 +218,61 @@ async function checkService(serviceName, timeoutMs) { ); } +async function checkActiveWorkerInstances(config) { + const result = await runCommand( + 'systemctl', + [ + 'list-units', + WORKER_SERVICE_PATTERN, + '--type=service', + '--state=active', + '--no-legend', + '--plain', + '--no-pager', + ], + config.timeoutMs, + ); + if (result.code !== 0) { + return checkResult( + 'service:external-generation-workers', + 'CRITICAL', + '无法枚举外部生成 worker 实例', + { + command: result.command, + stderr: result.stderr.trim() || result.error, + }, + ); + } + + const services = result.stdout + .split('\n') + .map((line) => line.trim().split(/\s+/u)[0]) + .filter((service) => + /^genarrative-external-generation-worker@.+\.service$/u.test(service), + ); + + if (services.length === 0) { + return checkResult( + 'service:external-generation-workers', + 'CRITICAL', + '没有 active 的外部生成 worker 实例', + { + command: result.command, + }, + ); + } + + return checkResult( + 'service:external-generation-workers', + 'OK', + `${services.length} 个 worker active`, + { + command: result.command, + services, + }, + ); +} + function requestUrl(url, timeoutMs) { return new Promise((resolve) => { const startedAt = Date.now(); @@ -310,6 +367,10 @@ async function checkRecentJournal(config) { '-u', 'genarrative-api.service', '-u', + 'genarrative-external-generation-controller.service', + '-u', + WORKER_SERVICE_PATTERN, + '-u', 'spacetimedb.service', '-u', 'nginx.service', @@ -426,6 +487,7 @@ async function main() { for (const serviceName of DEFAULT_SERVICES) { checks.push(await checkService(serviceName, config.timeoutMs)); } + checks.push(await checkActiveWorkerInstances(config)); checks.push(await checkHttp('api:/healthz', joinUrl(config.apiBaseUrl, '/healthz'), config)); checks.push(await checkHttp('api:/readyz', joinUrl(config.apiBaseUrl, '/readyz'), config)); diff --git a/server-rs/crates/api-server/Cargo.toml b/server-rs/crates/api-server/Cargo.toml index dc38ad00e..93df64598 100644 --- a/server-rs/crates/api-server/Cargo.toml +++ b/server-rs/crates/api-server/Cargo.toml @@ -56,7 +56,7 @@ shared-kernel = { workspace = true } shared-logging = { workspace = true } socket2 = { workspace = true } spacetime-client = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "time", "sync", "fs", "io-util", "signal"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "time", "sync", "fs", "io-util", "signal", "process"] } tokio-stream = { workspace = true } futures-util = { workspace = true } time = { workspace = true, features = ["formatting"] } diff --git a/server-rs/crates/api-server/src/app.rs b/server-rs/crates/api-server/src/app.rs index 5e58ea9e2..bebae2366 100644 --- a/server-rs/crates/api-server/src/app.rs +++ b/server-rs/crates/api-server/src/app.rs @@ -44,6 +44,7 @@ pub fn build_router(state: AppState) -> Router { .merge(modules::profile::router(state.clone())) .merge(modules::assets::router(state.clone())) .merge(modules::platform::router(state.clone())) + .merge(modules::external_generation::router(state.clone())) .merge(modules::play_flow::router(state.clone())) .route( "/api/profile/recharge/wechat/notify", diff --git a/server-rs/crates/api-server/src/asset_billing.rs b/server-rs/crates/api-server/src/asset_billing.rs index c6430554a..bc2dc80b8 100644 --- a/server-rs/crates/api-server/src/asset_billing.rs +++ b/server-rs/crates/api-server/src/asset_billing.rs @@ -52,7 +52,7 @@ where match operation.await { Ok(value) => Ok(value), Err(error) => { - if points_consumed { + if points_consumed && should_refund_asset_operation_error(&error) { refund_asset_operation_points( state, owner_user_id, @@ -67,6 +67,20 @@ where } } +pub(crate) fn should_refund_asset_operation_error(error: &AppError) -> bool { + let message = error.body_text(); + // 中文注释：worker lease guard 拒绝表示当前进程已失去队列写权限； + // 这类 stale worker 失败不能补偿退款，否则可能冲掉后续合法 worker 的同一账本扣费。 + !(message.contains("external_generation_job") + && (message.contains("lease") + || message.contains("worker") + || message.contains("job_kind") + || message.contains("source_") + || message.contains("owner_user_id") + || message.contains("不存在") + || message.contains("不是 running 状态"))) +} + /// 资产操作统一预扣泥点；扣费流水 ID 由业务资源 ID 参与构造，保证重试幂等。 async fn consume_asset_operation_points( state: &AppState, @@ -249,4 +263,31 @@ mod tests { &SpacetimeClientError::Procedure("泥点余额不足".to_string()), )); } + + #[test] + fn asset_operation_billing_does_not_refund_stale_worker_lease_errors() { + let stale_error = AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "spacetimedb", + "message": "external_generation_job lease 已过期", + })); + let completed_job_error = + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "spacetimedb", + "message": "external_generation_job 当前不是 running 状态", + })); + let missing_job_error = + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "spacetimedb", + "message": "external_generation_job 不存在", + })); + let ordinary_error = AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "vector-engine", + "message": "图片生成失败", + })); + + assert!(!should_refund_asset_operation_error(&stale_error)); + assert!(!should_refund_asset_operation_error(&completed_job_error)); + assert!(!should_refund_asset_operation_error(&missing_job_error)); + assert!(should_refund_asset_operation_error(&ordinary_error)); + } } diff --git a/server-rs/crates/api-server/src/config.rs b/server-rs/crates/api-server/src/config.rs index b23bb7829..b4af828a9 100644 --- a/server-rs/crates/api-server/src/config.rs +++ b/server-rs/crates/api-server/src/config.rs @@ -22,6 +22,19 @@ pub struct AppConfig { pub bind_port: u16, pub listen_backlog: i32, pub worker_threads: Option, + pub process_role: ProcessRole, + pub external_generation_mode: ExternalGenerationMode, + pub external_generation_worker_id: String, + pub external_generation_worker_concurrency: usize, + pub external_generation_worker_poll_interval: Duration, + pub external_generation_worker_lease: Duration, + pub external_generation_controller_min_workers: usize, + pub external_generation_controller_max_workers: usize, + pub external_generation_controller_target_jobs_per_worker: usize, + pub external_generation_controller_poll_interval: Duration, + pub external_generation_controller_scale_down_idle_rounds: u32, + pub external_generation_controller_service_template: String, + pub external_generation_controller_dry_run: bool, pub max_concurrent_requests: Option, pub gallery_max_concurrent_requests: Option, pub detail_max_concurrent_requests: Option, @@ -171,6 +184,56 @@ pub struct AppConfig { pub slow_request_threshold_ms: u64, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProcessRole { + Api, + ExternalGenerationWorker, + ExternalGenerationController, + All, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExternalGenerationMode { + Inline, + Queue, +} + +impl ExternalGenerationMode { + pub fn as_str(self) -> &'static str { + match self { + Self::Inline => "inline", + Self::Queue => "queue", + } + } + + pub fn is_inline(self) -> bool { + matches!(self, Self::Inline) + } +} + +impl ProcessRole { + pub fn as_str(self) -> &'static str { + match self { + Self::Api => "api", + Self::ExternalGenerationWorker => "external-generation-worker", + Self::ExternalGenerationController => "external-generation-controller", + Self::All => "all", + } + } + + pub fn runs_http(self) -> bool { + matches!(self, Self::Api | Self::All) + } + + pub fn runs_external_generation_worker(self) -> bool { + matches!(self, Self::ExternalGenerationWorker | Self::All) + } + + pub fn runs_external_generation_controller(self) -> bool { + matches!(self, Self::ExternalGenerationController) + } +} + impl Default for AppConfig { fn default() -> Self { Self { @@ -178,6 +241,20 @@ impl Default for AppConfig { bind_port: 3000, listen_backlog: 1024, worker_threads: None, + process_role: ProcessRole::Api, + external_generation_mode: ExternalGenerationMode::Queue, + external_generation_worker_id: default_external_generation_worker_id(), + external_generation_worker_concurrency: 2, + external_generation_worker_poll_interval: Duration::from_millis(2_000), + external_generation_worker_lease: Duration::from_secs(3_600), + external_generation_controller_min_workers: 1, + external_generation_controller_max_workers: 8, + external_generation_controller_target_jobs_per_worker: 2, + external_generation_controller_poll_interval: Duration::from_millis(10_000), + external_generation_controller_scale_down_idle_rounds: 6, + external_generation_controller_service_template: + "genarrative-external-generation-worker@{}.service".to_string(), + external_generation_controller_dry_run: false, max_concurrent_requests: None, gallery_max_concurrent_requests: None, detail_max_concurrent_requests: None, @@ -374,6 +451,78 @@ impl AppConfig { if let Some(worker_threads) = read_first_usize_env(&["GENARRATIVE_API_WORKER_THREADS"]) { config.worker_threads = Some(worker_threads); } + if let Some(process_role) = read_first_process_role_env(&["GENARRATIVE_PROCESS_ROLE"]) { + config.process_role = process_role; + } + if let Some(external_generation_mode) = + read_first_external_generation_mode_env(&["GENARRATIVE_EXTERNAL_GENERATION_MODE"]) + { + config.external_generation_mode = external_generation_mode; + } + if let Some(worker_id) = + read_first_non_empty_env(&["GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID"]) + { + config.external_generation_worker_id = worker_id; + } + if let Some(concurrency) = + read_first_usize_env(&["GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY"]) + { + config.external_generation_worker_concurrency = concurrency.max(1); + } + if let Some(poll_interval_ms) = read_first_positive_u64_env(&[ + "GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS", + ]) { + config.external_generation_worker_poll_interval = + Duration::from_millis(poll_interval_ms); + } + if let Some(lease_seconds) = read_first_duration_seconds_env(&[ + "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS", + ]) { + config.external_generation_worker_lease = Duration::from_secs(lease_seconds.max(1)); + } + if let Some(min_workers) = + read_first_usize_env(&["GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MIN_WORKERS"]) + { + config.external_generation_controller_min_workers = min_workers; + } + if let Some(max_workers) = + read_first_usize_env(&["GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MAX_WORKERS"]) + { + config.external_generation_controller_max_workers = max_workers; + } + if config.external_generation_controller_max_workers + < config.external_generation_controller_min_workers + { + config.external_generation_controller_max_workers = + config.external_generation_controller_min_workers; + } + if let Some(target_jobs_per_worker) = read_first_usize_env(&[ + "GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_TARGET_JOBS_PER_WORKER", + ]) { + config.external_generation_controller_target_jobs_per_worker = + target_jobs_per_worker.max(1); + } + if let Some(poll_interval_ms) = read_first_positive_u64_env(&[ + "GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_POLL_INTERVAL_MS", + ]) { + config.external_generation_controller_poll_interval = + Duration::from_millis(poll_interval_ms); + } + if let Some(idle_rounds) = read_first_u32_env(&[ + "GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_SCALE_DOWN_IDLE_ROUNDS", + ]) { + config.external_generation_controller_scale_down_idle_rounds = idle_rounds; + } + if let Some(service_template) = read_first_non_empty_env(&[ + "GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_SERVICE_TEMPLATE", + ]) { + config.external_generation_controller_service_template = service_template; + } + if let Some(dry_run) = + read_first_bool_env(&["GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_DRY_RUN"]) + { + config.external_generation_controller_dry_run = dry_run; + } if let Some(max_concurrent_requests) = read_first_usize_env(&["GENARRATIVE_API_MAX_CONCURRENT_REQUESTS"]) { @@ -1053,6 +1202,22 @@ fn read_first_llm_provider_env(keys: &[&str]) -> Option { }) } +fn read_first_process_role_env(keys: &[&str]) -> Option { + keys.iter().find_map(|key| { + env::var(key) + .ok() + .and_then(|value| parse_process_role(&value)) + }) +} + +fn read_first_external_generation_mode_env(keys: &[&str]) -> Option { + keys.iter().find_map(|key| { + env::var(key) + .ok() + .and_then(|value| parse_external_generation_mode(&value)) + }) +} + fn read_first_positive_u32_env(keys: &[&str]) -> Option { keys.iter().find_map(|key| { env::var(key) @@ -1100,6 +1265,49 @@ fn read_first_u8_env(keys: &[&str]) -> Option { .find_map(|key| env::var(key).ok().and_then(|value| parse_u8(&value))) } +fn default_external_generation_worker_id() -> String { + let host = env::var("HOSTNAME") + .or_else(|_| env::var("COMPUTERNAME")) + .unwrap_or_else(|_| "local".to_string()); + format!("{}-{}", host.trim(), std::process::id()) +} + +fn parse_process_role(value: &str) -> Option { + match trim_quoted_env_value(value).to_ascii_lowercase().as_str() { + "api" => Some(ProcessRole::Api), + "external-generation-worker" | "external_generation_worker" | "worker" => { + Some(ProcessRole::ExternalGenerationWorker) + } + "external-generation-controller" | "external_generation_controller" | "controller" => { + Some(ProcessRole::ExternalGenerationController) + } + "all" => Some(ProcessRole::All), + _ => None, + } +} + +fn parse_external_generation_mode(value: &str) -> Option { + match trim_quoted_env_value(value).to_ascii_lowercase().as_str() { + "inline" | "sync" | "synchronous" => Some(ExternalGenerationMode::Inline), + "queue" | "queued" | "worker" | "async" | "asynchronous" => { + Some(ExternalGenerationMode::Queue) + } + _ => None, + } +} + +fn trim_quoted_env_value(raw: &str) -> &str { + let raw = raw.trim(); + raw.strip_prefix('"') + .and_then(|value| value.strip_suffix('"')) + .or_else(|| { + raw.strip_prefix('\'') + .and_then(|value| value.strip_suffix('\'')) + }) + .unwrap_or(raw) + .trim() +} + fn read_first_positive_u16_env(keys: &[&str]) -> Option { keys.iter().find_map(|key| { env::var(key) @@ -1220,7 +1428,8 @@ fn parse_positive_u16(raw: &str) -> Option { #[cfg(test)] mod tests { use super::{ - AppConfig, DEFAULT_VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS, LlmProvider, parse_bool, + AppConfig, DEFAULT_VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS, ExternalGenerationMode, + LlmProvider, ProcessRole, parse_bool, parse_external_generation_mode, parse_process_role, }; use std::sync::{Mutex, OnceLock}; @@ -1262,6 +1471,91 @@ mod tests { assert_eq!(parse_bool("'off'"), Some(false)); } + #[test] + fn process_role_controls_http_and_external_generation_worker_roles() { + assert_eq!(parse_process_role("api"), Some(ProcessRole::Api)); + assert_eq!( + parse_process_role("\"external-generation-worker\""), + Some(ProcessRole::ExternalGenerationWorker) + ); + assert_eq!( + parse_process_role("'external_generation_worker'"), + Some(ProcessRole::ExternalGenerationWorker) + ); + assert_eq!( + parse_process_role("worker"), + Some(ProcessRole::ExternalGenerationWorker) + ); + assert_eq!( + parse_process_role("controller"), + Some(ProcessRole::ExternalGenerationController) + ); + assert_eq!( + parse_process_role("'external_generation_controller'"), + Some(ProcessRole::ExternalGenerationController) + ); + assert_eq!(parse_process_role("all"), Some(ProcessRole::All)); + assert_eq!(parse_process_role("unknown"), None); + + assert!(ProcessRole::Api.runs_http()); + assert!(!ProcessRole::Api.runs_external_generation_worker()); + assert!(!ProcessRole::Api.runs_external_generation_controller()); + assert!(!ProcessRole::ExternalGenerationWorker.runs_http()); + assert!(ProcessRole::ExternalGenerationWorker.runs_external_generation_worker()); + assert!(!ProcessRole::ExternalGenerationWorker.runs_external_generation_controller()); + assert!(!ProcessRole::ExternalGenerationController.runs_http()); + assert!(!ProcessRole::ExternalGenerationController.runs_external_generation_worker()); + assert!(ProcessRole::ExternalGenerationController.runs_external_generation_controller()); + assert!(ProcessRole::All.runs_http()); + assert!(ProcessRole::All.runs_external_generation_worker()); + assert!(!ProcessRole::All.runs_external_generation_controller()); + } + + #[test] + fn external_generation_mode_parses_inline_and_queue_aliases() { + assert_eq!( + parse_external_generation_mode("inline"), + Some(ExternalGenerationMode::Inline) + ); + assert_eq!( + parse_external_generation_mode("'sync'"), + Some(ExternalGenerationMode::Inline) + ); + assert_eq!( + parse_external_generation_mode("\"queue\""), + Some(ExternalGenerationMode::Queue) + ); + assert_eq!( + parse_external_generation_mode("worker"), + Some(ExternalGenerationMode::Queue) + ); + assert_eq!(parse_external_generation_mode("unknown"), None); + + assert!(ExternalGenerationMode::Inline.is_inline()); + assert!(!ExternalGenerationMode::Queue.is_inline()); + } + + #[test] + fn from_env_reads_external_generation_mode() { + let _guard = ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock"); + unsafe { + std::env::set_var("GENARRATIVE_EXTERNAL_GENERATION_MODE", "inline"); + } + + let config = AppConfig::from_env(); + + assert_eq!( + config.external_generation_mode, + ExternalGenerationMode::Inline + ); + unsafe { + std::env::remove_var("GENARRATIVE_EXTERNAL_GENERATION_MODE"); + } + } + #[test] fn from_env_reads_sms_enabled_when_shell_value_keeps_quotes() { let _guard = ENV_LOCK diff --git a/server-rs/crates/api-server/src/external_generation.rs b/server-rs/crates/api-server/src/external_generation.rs new file mode 100644 index 000000000..345dbe493 --- /dev/null +++ b/server-rs/crates/api-server/src/external_generation.rs @@ -0,0 +1,108 @@ +use axum::{ + Json, + extract::{Extension, Path, State}, + http::StatusCode, + response::Response, +}; +use serde_json::json; +use shared_contracts::external_generation::{ + ExternalGenerationJobStatus, ExternalGenerationJobStatusRecord, + ExternalGenerationJobStatusResponse, ExternalGenerationQueueOverview, + ExternalGenerationQueueOverviewResponse, +}; +use spacetime_client::{ + ExternalGenerationJobGetRecordInput, ExternalGenerationJobRecord, + ExternalGenerationQueueStatsRecord, SpacetimeClientError, +}; + +use crate::{ + api_response::json_success_body, auth::AuthenticatedAccessToken, http_error::AppError, + request_context::RequestContext, state::AppState, +}; + +const EXTERNAL_GENERATION_PROVIDER: &str = "external_generation"; + +pub async fn get_external_generation_queue_overview( + State(state): State, + Extension(request_context): Extension, +) -> Result, Response> { + let stats = state + .spacetime_client() + .get_external_generation_queue_stats() + .await + .map_err(|error| external_generation_error_response(&request_context, error))?; + + Ok(json_success_body( + Some(&request_context), + ExternalGenerationQueueOverviewResponse { + overview: map_external_generation_queue_overview(stats), + }, + )) +} + +pub async fn get_external_generation_job_status( + State(state): State, + Extension(request_context): Extension, + Extension(authenticated): Extension, + Path(job_id): Path, +) -> Result, Response> { + let owner_user_id = authenticated.claims().user_id().to_string(); + let job = state + .spacetime_client() + .get_external_generation_job(ExternalGenerationJobGetRecordInput { + job_id, + owner_user_id, + }) + .await + .map_err(|error| external_generation_error_response(&request_context, error))?; + + Ok(json_success_body( + Some(&request_context), + ExternalGenerationJobStatusResponse { + job: map_external_generation_job_status(job), + }, + )) +} + +fn map_external_generation_queue_overview( + stats: ExternalGenerationQueueStatsRecord, +) -> ExternalGenerationQueueOverview { + ExternalGenerationQueueOverview { + pending_count: stats.pending_count, + running_count: stats.running_active_count, + updated_at_micros: stats.now_micros, + } +} + +fn map_external_generation_job_status( + job: ExternalGenerationJobRecord, +) -> ExternalGenerationJobStatusRecord { + let (status, phase_detail, progress) = match job.status.as_str() { + "completed" => (ExternalGenerationJobStatus::Completed, "生成已完成。", 100), + "running" => (ExternalGenerationJobStatus::Running, "正在生成。", 35), + "failed" => (ExternalGenerationJobStatus::Failed, "生成失败。", 0), + _ => (ExternalGenerationJobStatus::Queued, "排队中。", 8), + }; + + ExternalGenerationJobStatusRecord { + operation_id: job.job_id, + status, + phase_label: job.request_label, + phase_detail: phase_detail.to_string(), + progress, + error: job.last_error_message, + updated_at_micros: job.updated_at_micros, + } +} + +fn external_generation_error_response( + request_context: &RequestContext, + error: SpacetimeClientError, +) -> Response { + AppError::from_status(StatusCode::BAD_GATEWAY) + .with_details(json!({ + "provider": EXTERNAL_GENERATION_PROVIDER, + "message": error.to_string(), + })) + .into_response_with_context(Some(request_context)) +} diff --git a/server-rs/crates/api-server/src/external_generation_worker.rs b/server-rs/crates/api-server/src/external_generation_worker.rs new file mode 100644 index 000000000..ec90016af --- /dev/null +++ b/server-rs/crates/api-server/src/external_generation_worker.rs @@ -0,0 +1,750 @@ +use std::{future::Future, io, pin::Pin, time::Duration}; + +use axum::extract::FromRef; +use serde_json::json; +use shared_kernel::offset_datetime_to_unix_micros; +use spacetime_client::{ + ExternalGenerationJobClaimRecordInput, ExternalGenerationJobCompleteRecordInput, + ExternalGenerationJobFailRecordInput, ExternalGenerationJobRecord, + ExternalGenerationJobRenewLeaseRecordInput, +}; +use tokio::{ + task::JoinSet, + time::{Instant, sleep}, +}; +use tracing::{error, info, warn}; + +use crate::{ + jump_hop::{ + JUMP_HOP_COMPILE_DRAFT_JOB_KIND, JumpHopCompileDraftWorkerPayload, + execute_jump_hop_compile_draft_worker_job, + }, + puzzle::{ + ExternalGenerationWriteLeaseGuard, PuzzleCompileDraftWorkerPayload, + PuzzleGenerateImagesWorkerPayload, PuzzleGenerateUiBackgroundWorkerPayload, + execute_puzzle_compile_draft_worker_job, execute_puzzle_generate_images_worker_job, + execute_puzzle_generate_ui_background_worker_job, release_puzzle_compile_background_claim, + }, + puzzle_clear::{ + PUZZLE_CLEAR_COMPILE_DRAFT_JOB_KIND, PuzzleClearCompileDraftWorkerPayload, + execute_puzzle_clear_compile_draft_worker_job, + }, + request_context::RequestContext, + state::{AppState, PuzzleApiState}, + wooden_fish::{ + WOODEN_FISH_GENERATE_IMAGE_ASSETS_JOB_KIND, WoodenFishGenerateImageAssetsWorkerPayload, + execute_wooden_fish_generate_image_assets_worker_job, + }, +}; + +pub(crate) const PUZZLE_COMPILE_DRAFT_JOB_KIND: &str = "puzzle_compile_draft"; +pub(crate) const PUZZLE_GENERATE_IMAGES_JOB_KIND: &str = "puzzle_generate_images"; +pub(crate) const PUZZLE_GENERATE_UI_BACKGROUND_JOB_KIND: &str = "puzzle_generate_ui_background"; + +pub(crate) async fn run_external_generation_worker(state: AppState) -> Result<(), io::Error> { + let worker_id = state.config.external_generation_worker_id.clone(); + let concurrency = state.config.external_generation_worker_concurrency.max(1); + let poll_interval = state.config.external_generation_worker_poll_interval; + let lease = state.config.external_generation_worker_lease; + let mut tasks = JoinSet::new(); + let mut shutdown = external_generation_worker_shutdown_signal(); + + info!( + worker_id, + concurrency, + poll_interval_ms = poll_interval.as_millis(), + lease_seconds = lease.as_secs(), + "external generation worker 已启动" + ); + + loop { + while tasks.len() >= concurrency { + if await_worker_task_or_shutdown(&mut tasks, &mut shutdown).await { + drain_external_generation_worker_tasks(&mut tasks).await; + return Ok(()); + } + } + + let available = concurrency.saturating_sub(tasks.len()).max(1); + let now_micros = current_utc_micros(); + let lease_expires_at_micros = now_micros.saturating_add(duration_micros_i64(lease)); + + let claim_jobs = state.spacetime_client().claim_external_generation_jobs( + ExternalGenerationJobClaimRecordInput { + worker_id: worker_id.clone(), + limit: available.min(u32::MAX as usize) as u32, + lease_expires_at_micros, + claimed_at_micros: now_micros, + }, + ); + tokio::pin!(claim_jobs); + let jobs = match tokio::select! { + _ = shutdown.as_mut() => { + drain_external_generation_worker_tasks(&mut tasks).await; + return Ok(()); + } + result = &mut claim_jobs => result + } { + Ok(jobs) => jobs, + Err(error) => { + error!(error = %error, "领取外部生成任务失败，等待下一轮重试"); + if await_one_task_or_sleep_or_shutdown( + &mut tasks, + sleep(poll_interval), + &mut shutdown, + ) + .await + { + drain_external_generation_worker_tasks(&mut tasks).await; + return Ok(()); + } + continue; + } + }; + + if jobs.is_empty() { + if await_one_task_or_sleep_or_shutdown(&mut tasks, sleep(poll_interval), &mut shutdown) + .await + { + drain_external_generation_worker_tasks(&mut tasks).await; + return Ok(()); + } + continue; + } + + for job in jobs { + let state = state.clone(); + let worker_id = worker_id.clone(); + tasks.spawn(async move { + if let Err(error) = + process_external_generation_job(state, worker_id, lease, job).await + { + error!(error = %error, "external generation worker 执行任务失败"); + } + }); + } + } +} + +type ExternalGenerationShutdownSignal = Pin + Send>>; + +fn external_generation_worker_shutdown_signal() -> ExternalGenerationShutdownSignal { + Box::pin(async { + wait_for_external_generation_worker_shutdown_signal().await; + }) +} + +#[cfg(unix)] +async fn wait_for_external_generation_worker_shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + + let mut sigterm = signal(SignalKind::terminate()).ok(); + tokio::select! { + result = tokio::signal::ctrl_c() => { + if let Err(error) = result { + warn!(error = %error, "external generation worker 监听 SIGINT 失败"); + } + } + _ = async { + if let Some(sigterm) = sigterm.as_mut() { + sigterm.recv().await; + } else { + std::future::pending::<()>().await; + } + } => {} + } +} + +#[cfg(not(unix))] +async fn wait_for_external_generation_worker_shutdown_signal() { + if let Err(error) = tokio::signal::ctrl_c().await { + warn!(error = %error, "external generation worker 监听 Ctrl-C 失败"); + } +} + +async fn await_worker_task(tasks: &mut JoinSet<()>) { + if let Some(result) = tasks.join_next().await + && let Err(error) = result + { + error!(error = %error, "external generation worker 子任务 panic"); + } +} + +async fn await_worker_task_or_shutdown( + tasks: &mut JoinSet<()>, + shutdown: &mut ExternalGenerationShutdownSignal, +) -> bool { + tokio::select! { + _ = shutdown.as_mut() => true, + _ = await_worker_task(tasks) => false, + } +} + +async fn await_one_task_or_sleep_or_shutdown( + tasks: &mut JoinSet<()>, + sleeper: impl Future, + shutdown: &mut ExternalGenerationShutdownSignal, +) -> bool { + tokio::pin!(sleeper); + if tasks.is_empty() { + tokio::select! { + _ = shutdown.as_mut() => true, + _ = &mut sleeper => false, + } + } else { + tokio::select! { + _ = shutdown.as_mut() => true, + _ = &mut sleeper => false, + result = tasks.join_next() => { + if let Some(Err(error)) = result { + error!(error = %error, "external generation worker 子任务 panic"); + } + false + } + } + } +} + +async fn drain_external_generation_worker_tasks(tasks: &mut JoinSet<()>) { + info!( + in_flight_jobs = tasks.len(), + "external generation worker 收到停机信号，停止领取新任务并等待当前任务完成" + ); + while !tasks.is_empty() { + await_worker_task(tasks).await; + } + info!("external generation worker 已完成优雅停机"); +} + +async fn process_external_generation_job( + state: AppState, + worker_id: String, + lease: Duration, + job: ExternalGenerationJobRecord, +) -> Result<(), String> { + let heartbeat_interval = external_generation_worker_heartbeat_interval(lease); + let work = process_external_generation_job_once(state.clone(), worker_id.clone(), job.clone()); + tokio::pin!(work); + let heartbeat = sleep(heartbeat_interval); + tokio::pin!(heartbeat); + + loop { + tokio::select! { + biased; + result = &mut work => return result, + _ = &mut heartbeat => { + renew_job_lease(&state, &worker_id, &job, lease).await?; + heartbeat.as_mut().reset(Instant::now() + heartbeat_interval); + } + } + } +} + +async fn process_external_generation_job_once( + state: AppState, + worker_id: String, + job: ExternalGenerationJobRecord, +) -> Result<(), String> { + match job.job_kind.as_str() { + PUZZLE_COMPILE_DRAFT_JOB_KIND => { + let payload = match serde_json::from_str::( + job.request_payload_json.as_str(), + ) { + Ok(payload) => payload, + Err(error) => { + let message = format!("拼图生成任务参数解析失败：{error}"); + fail_job(&state, &worker_id, &job, message.clone()).await?; + return Err(message); + } + }; + let request_context = RequestContext::new( + format!("external-generation-worker-{}", job.job_id), + format!("external-generation-worker {}", job.job_kind), + std::time::Duration::ZERO, + false, + ); + let puzzle_state = PuzzleApiState::from_ref(&state); + let write_guard = build_external_generation_write_lease_guard(&worker_id, &job)?; + match execute_puzzle_compile_draft_worker_job( + &puzzle_state, + &request_context, + payload.clone(), + write_guard, + ) + .await + { + Ok(session) => { + let result = complete_job( + &state, + &worker_id, + &job, + Some( + json!({ + "sessionId": session.session_id, + "progressPercent": session.progress_percent, + }) + .to_string(), + ), + ) + .await; + if result.is_ok() { + release_puzzle_compile_background_claim(&puzzle_state, &payload); + } + result + } + Err(error) => { + let message = error.body_text(); + let should_release_claim = error.should_fail_queue_job(); + let result = fail_queue_job_after_worker_error( + &state, &worker_id, &job, &error, &message, + ) + .await; + if result.is_ok() && should_release_claim { + release_puzzle_compile_background_claim(&puzzle_state, &payload); + } + result?; + Err(message) + } + } + } + PUZZLE_GENERATE_IMAGES_JOB_KIND => { + let payload = match serde_json::from_str::( + job.request_payload_json.as_str(), + ) { + Ok(payload) => payload, + Err(error) => { + let message = format!("拼图关卡图片生成任务参数解析失败：{error}"); + fail_job(&state, &worker_id, &job, message.clone()).await?; + return Err(message); + } + }; + let request_context = RequestContext::new( + format!("external-generation-worker-{}", job.job_id), + format!("external-generation-worker {}", job.job_kind), + std::time::Duration::ZERO, + false, + ); + let puzzle_state = PuzzleApiState::from_ref(&state); + let write_guard = build_external_generation_write_lease_guard(&worker_id, &job)?; + match execute_puzzle_generate_images_worker_job( + &puzzle_state, + &request_context, + payload, + write_guard, + ) + .await + { + Ok(session) => { + complete_job( + &state, + &worker_id, + &job, + Some( + json!({ + "sessionId": session.session_id, + "progressPercent": session.progress_percent, + }) + .to_string(), + ), + ) + .await + } + Err(error) => { + let message = error.body_text(); + fail_queue_job_after_worker_error(&state, &worker_id, &job, &error, &message) + .await?; + Err(message) + } + } + } + PUZZLE_GENERATE_UI_BACKGROUND_JOB_KIND => { + let payload = match serde_json::from_str::( + job.request_payload_json.as_str(), + ) { + Ok(payload) => payload, + Err(error) => { + let message = format!("拼图 UI 背景图生成任务参数解析失败：{error}"); + fail_job(&state, &worker_id, &job, message.clone()).await?; + return Err(message); + } + }; + let request_context = RequestContext::new( + format!("external-generation-worker-{}", job.job_id), + format!("external-generation-worker {}", job.job_kind), + std::time::Duration::ZERO, + false, + ); + let puzzle_state = PuzzleApiState::from_ref(&state); + let write_guard = build_external_generation_write_lease_guard(&worker_id, &job)?; + match execute_puzzle_generate_ui_background_worker_job( + &puzzle_state, + &request_context, + payload, + write_guard, + ) + .await + { + Ok(session) => { + complete_job( + &state, + &worker_id, + &job, + Some( + json!({ + "sessionId": session.session_id, + "progressPercent": session.progress_percent, + }) + .to_string(), + ), + ) + .await + } + Err(error) => { + let message = error.body_text(); + fail_queue_job_after_worker_error(&state, &worker_id, &job, &error, &message) + .await?; + Err(message) + } + } + } + JUMP_HOP_COMPILE_DRAFT_JOB_KIND => { + let payload = match serde_json::from_str::( + job.request_payload_json.as_str(), + ) { + Ok(payload) => payload, + Err(error) => { + let message = format!("跳一跳生成任务参数解析失败：{error}"); + fail_job(&state, &worker_id, &job, message.clone()).await?; + return Err(message); + } + }; + let request_context = RequestContext::new( + format!("external-generation-worker-{}", job.job_id), + format!("external-generation-worker {}", job.job_kind), + std::time::Duration::ZERO, + false, + ); + match execute_jump_hop_compile_draft_worker_job(&state, &request_context, payload).await + { + Ok(session) => { + complete_job( + &state, + &worker_id, + &job, + Some( + json!({ + "sessionId": session.session_id, + "status": session.status, + }) + .to_string(), + ), + ) + .await + } + Err(response) => { + let message = response_error_message(response).await; + fail_job(&state, &worker_id, &job, message.clone()).await?; + Err(message) + } + } + } + PUZZLE_CLEAR_COMPILE_DRAFT_JOB_KIND => { + let payload = match serde_json::from_str::( + job.request_payload_json.as_str(), + ) { + Ok(payload) => payload, + Err(error) => { + let message = format!("拼消消生成任务参数解析失败：{error}"); + fail_job(&state, &worker_id, &job, message.clone()).await?; + return Err(message); + } + }; + let request_context = RequestContext::new( + format!("external-generation-worker-{}", job.job_id), + format!("external-generation-worker {}", job.job_kind), + std::time::Duration::ZERO, + false, + ); + match execute_puzzle_clear_compile_draft_worker_job(&state, &request_context, payload) + .await + { + Ok(session) => { + complete_job( + &state, + &worker_id, + &job, + Some( + json!({ + "sessionId": session.session_id, + "status": session.status, + }) + .to_string(), + ), + ) + .await + } + Err(response) => { + let message = response_error_message(response).await; + fail_job(&state, &worker_id, &job, message.clone()).await?; + Err(message) + } + } + } + WOODEN_FISH_GENERATE_IMAGE_ASSETS_JOB_KIND => { + let payload = match serde_json::from_str::( + job.request_payload_json.as_str(), + ) { + Ok(payload) => payload, + Err(error) => { + let message = format!("敲木鱼图片生成任务参数解析失败：{error}"); + fail_job(&state, &worker_id, &job, message.clone()).await?; + return Err(message); + } + }; + let request_context = RequestContext::new( + format!("external-generation-worker-{}", job.job_id), + format!("external-generation-worker {}", job.job_kind), + std::time::Duration::ZERO, + false, + ); + match execute_wooden_fish_generate_image_assets_worker_job( + &state, + &request_context, + payload, + ) + .await + { + Ok(session) => { + complete_job( + &state, + &worker_id, + &job, + Some( + json!({ + "sessionId": session.session_id, + "status": session.status, + }) + .to_string(), + ), + ) + .await + } + Err(response) => { + let message = response_error_message(response).await; + fail_job(&state, &worker_id, &job, message.clone()).await?; + Err(message) + } + } + } + unknown => { + warn!( + job_id = job.job_id, + job_kind = unknown, + "external generation worker 收到暂不支持的任务类型" + ); + fail_job( + &state, + &worker_id, + &job, + format!("暂不支持的外部生成任务类型：{unknown}"), + ) + .await + } + } +} + +async fn response_error_message(response: axum::response::Response) -> String { + use axum::body::to_bytes; + let status = response.status(); + let body_bytes = match to_bytes(response.into_body(), 64 * 1024).await { + Ok(bytes) => bytes, + Err(error) => { + return format!("外部生成任务失败：{status}，响应读取失败：{error}"); + } + }; + let body_text = String::from_utf8_lossy(&body_bytes).trim().to_string(); + if body_text.is_empty() { + return format!("外部生成任务失败：{status}"); + } + if let Ok(body_json) = serde_json::from_str::(&body_text) + && let Some(message) = body_json + .get("error") + .and_then(|error| error.get("message")) + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|message| !message.is_empty()) + { + return message.to_string(); + } + body_text +} + +async fn fail_queue_job_after_worker_error( + state: &AppState, + worker_id: &str, + job: &ExternalGenerationJobRecord, + error: &crate::puzzle::PuzzleExternalGenerationWorkerError, + message: &str, +) -> Result<(), String> { + if error.should_fail_queue_job() { + return fail_job(state, worker_id, job, message.to_string()).await; + } + + warn!( + job_id = job.job_id, + job_kind = job.job_kind, + "external generation worker 业务失败态尚未写回，保留任务租约等待后续重试" + ); + Ok(()) +} + +async fn complete_job( + state: &AppState, + worker_id: &str, + job: &ExternalGenerationJobRecord, + result_payload_json: Option, +) -> Result<(), String> { + state + .spacetime_client() + .complete_external_generation_job(ExternalGenerationJobCompleteRecordInput { + job_id: job.job_id.clone(), + worker_id: worker_id.to_string(), + lease_token: require_job_lease_token(job)?, + result_payload_json, + completed_at_micros: current_utc_micros(), + }) + .await + .map(|_| ()) + .map_err(|error| error.to_string()) +} + +async fn fail_job( + state: &AppState, + worker_id: &str, + job: &ExternalGenerationJobRecord, + error_message: String, +) -> Result<(), String> { + let now_micros = current_utc_micros(); + state + .spacetime_client() + .fail_external_generation_job(ExternalGenerationJobFailRecordInput { + job_id: job.job_id.clone(), + worker_id: worker_id.to_string(), + lease_token: require_job_lease_token(job)?, + error_message, + retry_after_micros: now_micros.saturating_add(60_000_000), + failed_at_micros: now_micros, + }) + .await + .map(|_| ()) + .map_err(|error| error.to_string()) +} + +async fn renew_job_lease( + state: &AppState, + worker_id: &str, + job: &ExternalGenerationJobRecord, + lease: Duration, +) -> Result<(), String> { + let now_micros = current_utc_micros(); + state + .spacetime_client() + .renew_external_generation_job_lease(ExternalGenerationJobRenewLeaseRecordInput { + job_id: job.job_id.clone(), + worker_id: worker_id.to_string(), + lease_token: require_job_lease_token(job)?, + lease_expires_at_micros: now_micros.saturating_add(duration_micros_i64(lease)), + renewed_at_micros: now_micros, + }) + .await + .map(|_| ()) + .map_err(|error| error.to_string()) +} + +fn require_job_lease_token(job: &ExternalGenerationJobRecord) -> Result { + job.lease_token + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| format!("external_generation_job {} 缺少 lease token", job.job_id)) +} + +fn build_external_generation_write_lease_guard( + worker_id: &str, + job: &ExternalGenerationJobRecord, +) -> Result { + Ok(ExternalGenerationWriteLeaseGuard::from_claimed_job( + job.job_id.clone(), + worker_id.to_string(), + require_job_lease_token(job)?, + )) +} + +fn duration_micros_i64(duration: Duration) -> i64 { + duration.as_micros().min(i64::MAX as u128) as i64 +} + +fn external_generation_worker_heartbeat_interval(lease: Duration) -> Duration { + let heartbeat_millis = (lease.as_millis() / 3).clamp(250, 30_000) as u64; + Duration::from_millis(heartbeat_millis) +} + +fn current_utc_micros() -> i64 { + offset_datetime_to_unix_micros(time::OffsetDateTime::now_utc()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn worker_write_guard_uses_claimed_job_lease_token() { + let job = external_generation_job_record_fixture(Some("lease-1")); + + let guard = build_external_generation_write_lease_guard("worker-a", &job) + .expect("guard should build"); + + assert_eq!(guard.job_id.as_deref(), Some("extgen-1")); + assert_eq!(guard.worker_id.as_deref(), Some("worker-a")); + assert_eq!(guard.lease_token.as_deref(), Some("lease-1")); + } + + #[test] + fn worker_write_guard_requires_claimed_job_lease_token() { + let job = external_generation_job_record_fixture(None); + + let error = build_external_generation_write_lease_guard("worker-a", &job) + .expect_err("missing token should fail"); + + assert!(error.contains("缺少 lease token")); + } + + fn external_generation_job_record_fixture( + lease_token: Option<&str>, + ) -> ExternalGenerationJobRecord { + ExternalGenerationJobRecord { + job_id: "extgen-1".to_string(), + dedupe_key: "puzzle:generate_puzzle_images:session-1:extgen-1".to_string(), + job_kind: PUZZLE_GENERATE_IMAGES_JOB_KIND.to_string(), + owner_user_id: "user-1".to_string(), + source_module: "puzzle".to_string(), + source_entity_id: "session-1:puzzle-level-1".to_string(), + request_label: "拼图关卡图片生成".to_string(), + request_payload_json: "{}".to_string(), + status: "running".to_string(), + attempt: 1, + max_attempts: 1, + last_error_message: None, + worker_id: Some("worker-a".to_string()), + lease_expires_at: Some("2026-06-03T00:00:00Z".to_string()), + available_at: "2026-06-03T00:00:00Z".to_string(), + result_payload_json: None, + created_at: "2026-06-03T00:00:00Z".to_string(), + started_at: Some("2026-06-03T00:00:00Z".to_string()), + completed_at: None, + updated_at: "2026-06-03T00:00:00Z".to_string(), + lease_token: lease_token.map(ToOwned::to_owned), + } + } +} diff --git a/server-rs/crates/api-server/src/external_generation_worker_controller.rs b/server-rs/crates/api-server/src/external_generation_worker_controller.rs new file mode 100644 index 000000000..3c4e588cc --- /dev/null +++ b/server-rs/crates/api-server/src/external_generation_worker_controller.rs @@ -0,0 +1,465 @@ +use std::{collections::BTreeSet, future::Future, io, pin::Pin, process::Stdio, time::Duration}; + +use spacetime_client::ExternalGenerationQueueStatsRecord; +use tokio::{ + process::Command, + time::{Instant, sleep}, +}; +use tracing::{error, info, warn}; + +use crate::state::AppState; + +#[derive(Clone, Debug)] +struct ExternalGenerationWorkerControllerConfig { + min_workers: usize, + max_workers: usize, + target_jobs_per_worker: usize, + poll_interval: Duration, + scale_down_idle_rounds: u32, + service_template: String, + dry_run: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct ExternalGenerationWorkerControllerDecision { + desired_workers: usize, + should_scale_down: bool, + idle_rounds: u32, +} + +#[derive(Debug, Default)] +struct ExternalGenerationWorkerControllerState { + idle_rounds: u32, +} + +pub(crate) async fn run_external_generation_worker_controller( + state: AppState, +) -> Result<(), io::Error> { + let config = ExternalGenerationWorkerControllerConfig::from_state(&state); + let mut controller_state = ExternalGenerationWorkerControllerState::default(); + let mut shutdown = external_generation_controller_shutdown_signal(); + + info!( + min_workers = config.min_workers, + max_workers = config.max_workers, + target_jobs_per_worker = config.target_jobs_per_worker, + poll_interval_ms = config.poll_interval.as_millis(), + scale_down_idle_rounds = config.scale_down_idle_rounds, + service_template = config.service_template, + dry_run = config.dry_run, + "external generation worker controller 已启动" + ); + + loop { + let tick = run_external_generation_controller_tick(&state, &config, &mut controller_state); + tokio::select! { + _ = shutdown.as_mut() => { + info!("external generation worker controller 收到停机信号"); + return Ok(()); + } + result = tick => { + if let Err(error) = result { + error!(error = %error, "external generation worker controller 本轮扩缩容失败"); + } + } + } + + let next_tick = sleep(config.poll_interval); + tokio::pin!(next_tick); + tokio::select! { + _ = shutdown.as_mut() => { + info!("external generation worker controller 收到停机信号"); + return Ok(()); + } + _ = &mut next_tick => {} + } + } +} + +async fn run_external_generation_controller_tick( + state: &AppState, + config: &ExternalGenerationWorkerControllerConfig, + controller_state: &mut ExternalGenerationWorkerControllerState, +) -> Result<(), String> { + let stats = state + .spacetime_client() + .get_external_generation_queue_stats() + .await + .map_err(|error| format!("读取 external_generation_job 队列统计失败：{error}"))?; + let active_instances = list_active_external_generation_worker_instances(config).await?; + let current_workers = active_instances.len(); + let decision = decide_external_generation_worker_target( + &stats, + current_workers, + controller_state.idle_rounds, + config, + ); + controller_state.idle_rounds = decision.idle_rounds; + + info!( + pending = stats.pending_count, + delayed_pending = stats.delayed_pending_count, + claimable = stats.claimable_count, + running_active = stats.running_active_count, + expired_running = stats.expired_running_count, + oldest_claimable_age_ms = stats.oldest_claimable_age_micros.unwrap_or(0) / 1_000, + current_workers, + desired_workers = decision.desired_workers, + idle_rounds = decision.idle_rounds, + "external generation worker controller 完成队列评估" + ); + + reconcile_external_generation_worker_instances(config, &active_instances, &decision).await +} + +fn decide_external_generation_worker_target( + stats: &ExternalGenerationQueueStatsRecord, + current_workers: usize, + previous_idle_rounds: u32, + config: &ExternalGenerationWorkerControllerConfig, +) -> ExternalGenerationWorkerControllerDecision { + let pressure = stats + .claimable_pending_count + .saturating_add(stats.running_active_count) + .saturating_add(stats.expired_running_count); + let desired_from_pressure = + ceil_div_usize(pressure as usize, config.target_jobs_per_worker.max(1)); + let desired_workers = desired_from_pressure.clamp(config.min_workers, config.max_workers); + let is_idle = stats.claimable_count == 0 + && stats.expired_running_count == 0 + && stats.running_active_count == 0 + && desired_workers <= config.min_workers; + let idle_rounds = if is_idle { + previous_idle_rounds.saturating_add(1) + } else { + 0 + }; + let should_scale_down = current_workers > desired_workers + && idle_rounds >= config.scale_down_idle_rounds + && config.scale_down_idle_rounds > 0; + + ExternalGenerationWorkerControllerDecision { + desired_workers, + should_scale_down, + idle_rounds, + } +} + +async fn reconcile_external_generation_worker_instances( + config: &ExternalGenerationWorkerControllerConfig, + active_instances: &BTreeSet, + decision: &ExternalGenerationWorkerControllerDecision, +) -> Result<(), String> { + let current_workers = active_instances.len(); + let mut started = 0usize; + for instance in 1..=config.max_workers { + if current_workers.saturating_add(started) >= decision.desired_workers { + break; + } + if !active_instances.contains(&instance) { + systemctl_worker_instance(config, "start", instance).await?; + started = started.saturating_add(1); + } + } + + if decision.desired_workers > current_workers && started == 0 { + warn!( + current_workers, + desired_workers = decision.desired_workers, + "external generation worker controller 未找到可启动的缺口实例" + ); + } + if started > 0 { + return Ok(()); + } + + if decision.should_scale_down && decision.desired_workers < current_workers { + if let Some(instance) = active_instances + .iter() + .rev() + .copied() + .find(|instance| *instance > config.min_workers.max(1)) + { + systemctl_worker_instance(config, "stop", instance).await?; + } + } + + Ok(()) +} + +async fn list_active_external_generation_worker_instances( + config: &ExternalGenerationWorkerControllerConfig, +) -> Result, String> { + let mut active_instances = BTreeSet::new(); + for instance in 1..=config.max_workers { + if is_external_generation_worker_instance_active(config, instance).await? { + active_instances.insert(instance); + } + } + Ok(active_instances) +} + +async fn is_external_generation_worker_instance_active( + config: &ExternalGenerationWorkerControllerConfig, + instance: usize, +) -> Result { + let service = format_worker_service_name(&config.service_template, instance)?; + if config.dry_run { + return Ok(instance <= config.min_workers); + } + + let output = Command::new("systemctl") + .arg("is-active") + .arg("--quiet") + .arg(&service) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .output() + .await + .map_err(|error| format!("执行 systemctl is-active {service} 失败：{error}"))?; + Ok(output.status.success()) +} + +async fn systemctl_worker_instance( + config: &ExternalGenerationWorkerControllerConfig, + action: &str, + instance: usize, +) -> Result<(), String> { + let service = format_worker_service_name(&config.service_template, instance)?; + if config.dry_run { + info!( + action, + service, "external generation worker controller dry-run 跳过 systemctl" + ); + return Ok(()); + } + + let started_at = Instant::now(); + let output = Command::new("systemctl") + .arg(action) + .arg(&service) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|error| format!("执行 systemctl {action} {service} 失败：{error}"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!( + "systemctl {action} {service} 返回失败 status={} stderr={}", + output.status, stderr + )); + } + + info!( + action, + service, + elapsed_ms = started_at.elapsed().as_millis(), + "external generation worker controller 已执行 systemctl" + ); + Ok(()) +} + +fn format_worker_service_name(template: &str, instance: usize) -> Result { + let instance = instance.to_string(); + if template.contains("{}") { + return Ok(template.replacen("{}", &instance, 1)); + } + if template.contains("%i") { + return Ok(template.replacen("%i", &instance, 1)); + } + Err("external generation controller service template 必须包含 {} 或 %i".to_string()) +} + +fn ceil_div_usize(value: usize, divisor: usize) -> usize { + if value == 0 { + 0 + } else { + value.saturating_add(divisor.saturating_sub(1)) / divisor.max(1) + } +} + +impl ExternalGenerationWorkerControllerConfig { + fn from_state(state: &AppState) -> Self { + let min_workers = state.config.external_generation_controller_min_workers; + let max_workers = state + .config + .external_generation_controller_max_workers + .max(min_workers); + Self { + min_workers, + max_workers, + target_jobs_per_worker: state + .config + .external_generation_controller_target_jobs_per_worker + .max(1), + poll_interval: state.config.external_generation_controller_poll_interval, + scale_down_idle_rounds: state + .config + .external_generation_controller_scale_down_idle_rounds, + service_template: state + .config + .external_generation_controller_service_template + .clone(), + dry_run: state.config.external_generation_controller_dry_run, + } + } +} + +type ExternalGenerationControllerShutdownSignal = Pin + Send>>; + +fn external_generation_controller_shutdown_signal() -> ExternalGenerationControllerShutdownSignal { + Box::pin(async { + wait_for_external_generation_controller_shutdown_signal().await; + }) +} + +#[cfg(unix)] +async fn wait_for_external_generation_controller_shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + + let mut sigterm = signal(SignalKind::terminate()).ok(); + tokio::select! { + result = tokio::signal::ctrl_c() => { + if let Err(error) = result { + warn!(error = %error, "external generation worker controller 监听 SIGINT 失败"); + } + } + _ = async { + if let Some(sigterm) = sigterm.as_mut() { + sigterm.recv().await; + } else { + std::future::pending::<()>().await; + } + } => {} + } +} + +#[cfg(not(unix))] +async fn wait_for_external_generation_controller_shutdown_signal() { + if let Err(error) = tokio::signal::ctrl_c().await { + warn!(error = %error, "external generation worker controller 监听 Ctrl-C 失败"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scales_up_to_max_when_queue_pressure_is_high() { + let config = controller_config_fixture(); + let stats = stats_fixture(120, 0, 8); + + let decision = decide_external_generation_worker_target(&stats, 1, 0, &config); + + assert_eq!(decision.desired_workers, 8); + assert!(!decision.should_scale_down); + assert_eq!(decision.idle_rounds, 0); + } + + #[test] + fn scale_down_requires_consecutive_idle_rounds() { + let config = controller_config_fixture(); + let stats = stats_fixture(0, 0, 0); + + let first = decide_external_generation_worker_target(&stats, 5, 0, &config); + let ready = decide_external_generation_worker_target( + &stats, + 5, + config.scale_down_idle_rounds.saturating_sub(1), + &config, + ); + + assert_eq!(first.desired_workers, config.min_workers); + assert!(!first.should_scale_down); + assert!(ready.should_scale_down); + } + + #[test] + fn running_jobs_hold_capacity_before_scale_down() { + let config = controller_config_fixture(); + let stats = stats_fixture(0, 6, 0); + + let decision = decide_external_generation_worker_target(&stats, 5, 5, &config); + + assert_eq!(decision.desired_workers, 3); + assert!(!decision.should_scale_down); + assert_eq!(decision.idle_rounds, 0); + } + + #[test] + fn expired_running_jobs_are_not_counted_twice_as_claimable_pressure() { + let config = controller_config_fixture(); + let stats = stats_fixture(0, 0, 3); + + let decision = decide_external_generation_worker_target(&stats, 1, 0, &config); + + assert_eq!(decision.desired_workers, 2); + assert!(!decision.should_scale_down); + } + + #[test] + fn formats_worker_service_name_with_supported_templates() { + assert_eq!( + format_worker_service_name("genarrative-external-generation-worker@{}.service", 3) + .expect("format"), + "genarrative-external-generation-worker@3.service" + ); + assert_eq!( + format_worker_service_name("worker@%i.service", 7).expect("format"), + "worker@7.service" + ); + assert!(format_worker_service_name("worker.service", 1).is_err()); + } + + #[tokio::test] + async fn dry_run_reconcile_does_not_start_low_number_gaps_when_capacity_is_enough() { + let config = controller_config_fixture(); + let active_instances = BTreeSet::from([3usize, 4usize]); + let decision = ExternalGenerationWorkerControllerDecision { + desired_workers: 2, + should_scale_down: false, + idle_rounds: 0, + }; + + let result = + reconcile_external_generation_worker_instances(&config, &active_instances, &decision) + .await; + + assert!(result.is_ok()); + } + + fn controller_config_fixture() -> ExternalGenerationWorkerControllerConfig { + ExternalGenerationWorkerControllerConfig { + min_workers: 1, + max_workers: 8, + target_jobs_per_worker: 2, + poll_interval: Duration::from_secs(10), + scale_down_idle_rounds: 3, + service_template: "genarrative-external-generation-worker@{}.service".to_string(), + dry_run: true, + } + } + + fn stats_fixture( + claimable_pending_count: u32, + running_active_count: u32, + expired_running_count: u32, + ) -> ExternalGenerationQueueStatsRecord { + let claimable_count = claimable_pending_count.saturating_add(expired_running_count); + ExternalGenerationQueueStatsRecord { + pending_count: claimable_pending_count, + delayed_pending_count: 0, + claimable_pending_count, + running_active_count, + expired_running_count, + terminal_count: 0, + claimable_count, + oldest_claimable_age_micros: None, + now_micros: 0, + } + } +} diff --git a/server-rs/crates/api-server/src/jump_hop.rs b/server-rs/crates/api-server/src/jump_hop.rs index 015a510ed..8200eab26 100644 --- a/server-rs/crates/api-server/src/jump_hop.rs +++ b/server-rs/crates/api-server/src/jump_hop.rs @@ -9,7 +9,11 @@ use module_assets::{ generate_asset_binding_id, generate_asset_object_id, }; use platform_oss::{LegacyAssetPrefix, OssHeadObjectRequest, OssObjectAccess}; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use shared_contracts::external_generation::{ + ExternalGenerationJobStatus, ExternalGenerationJobStatusRecord, +}; use shared_contracts::jump_hop::{ JumpHopActionRequest, JumpHopActionType, JumpHopCharacterAsset, JumpHopDraftResponse, JumpHopGalleryDetailResponse, JumpHopGenerationStatus, JumpHopJumpRequest, JumpHopJumpResponse, @@ -20,7 +24,9 @@ use shared_contracts::jump_hop::{ JumpHopWorksResponse, JumpHopWorkspaceCreateRequest, }; use shared_kernel::{build_prefixed_uuid_id, format_timestamp_micros}; -use spacetime_client::SpacetimeClientError; +use spacetime_client::{ + ExternalGenerationJobEnqueueRecordInput, ExternalGenerationJobRecord, SpacetimeClientError, +}; use std::{ collections::BTreeMap, time::{SystemTime, UNIX_EPOCH}, @@ -49,6 +55,7 @@ use crate::{ }; const JUMP_HOP_TILE_ITEM_COUNT: usize = 18; +pub(crate) const JUMP_HOP_COMPILE_DRAFT_JOB_KIND: &str = "jump_hop_compile_draft"; const JUMP_HOP_PROVIDER: &str = "jump-hop"; const JUMP_HOP_CREATION_PROVIDER: &str = "jump-hop-creation"; @@ -72,6 +79,14 @@ const JUMP_HOP_BACK_BUTTON_IMAGE_SIZE: &str = "1024*1024"; const JUMP_HOP_BACK_BUTTON_IMAGE_WIDTH: u32 = 1024; const JUMP_HOP_BACK_BUTTON_IMAGE_HEIGHT: u32 = 1024; +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct JumpHopCompileDraftWorkerPayload { + pub session_id: String, + pub owner_user_id: String, + pub payload: JumpHopActionRequest, +} + #[derive(Clone, Debug, PartialEq, Eq)] struct JumpHopTileAtlasSlice { tile_type: JumpHopTileType, @@ -174,6 +189,37 @@ pub async fn execute_jump_hop_action( let owner_user_id = authenticated.claims().user_id().to_string(); let mut payload = payload; let is_compile_draft = matches!(payload.action_type, JumpHopActionType::CompileDraft); + let should_queue_generation = matches!( + payload.action_type, + JumpHopActionType::CompileDraft | JumpHopActionType::RegenerateTiles + ) && !state.config.external_generation_mode.is_inline(); + if should_queue_generation { + let mut queued_response = state + .spacetime_client() + .mark_jump_hop_generation_queued( + session_id.clone(), + owner_user_id.clone(), + payload.clone(), + ) + .await + .map_err(|error| { + jump_hop_error_response( + &request_context, + JUMP_HOP_CREATION_PROVIDER, + map_jump_hop_client_error(error), + ) + })?; + let queue_job = enqueue_jump_hop_compile_draft_job( + &state, + &request_context, + &session_id, + owner_user_id.as_str(), + payload, + ) + .await?; + queued_response.queue_state = Some(map_jump_hop_queue_job_status(queue_job)); + return Ok(json_success_body(Some(&request_context), queued_response)); + } let generation_points_cost = if is_compile_draft { resolve_jump_hop_generation_points_cost(&state).await } else { @@ -246,6 +292,99 @@ pub async fn execute_jump_hop_action( } } +async fn enqueue_jump_hop_compile_draft_job( + state: &AppState, + request_context: &RequestContext, + session_id: &str, + owner_user_id: &str, + payload: JumpHopActionRequest, +) -> Result { + let job_id = build_prefixed_uuid_id("extgen-"); + let now_micros = current_utc_micros(); + let request_payload_json = serde_json::to_string(&JumpHopCompileDraftWorkerPayload { + session_id: session_id.to_string(), + owner_user_id: owner_user_id.to_string(), + payload, + }) + .map_err(|error| { + jump_hop_error_response( + request_context, + JUMP_HOP_CREATION_PROVIDER, + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "message": format!("跳一跳 worker 任务参数序列化失败：{error}"), + })), + ) + })?; + state + .spacetime_client() + .enqueue_external_generation_job(ExternalGenerationJobEnqueueRecordInput { + dedupe_key: format!("jump-hop:compile-draft:{session_id}:{job_id}"), + job_id, + job_kind: JUMP_HOP_COMPILE_DRAFT_JOB_KIND.to_string(), + owner_user_id: owner_user_id.to_string(), + source_module: "jump-hop".to_string(), + source_entity_id: session_id.to_string(), + request_label: "跳一跳草稿生成".to_string(), + request_payload_json, + max_attempts: 1, + available_at_micros: now_micros, + created_at_micros: now_micros, + }) + .await + .map_err(|error| { + jump_hop_error_response( + request_context, + JUMP_HOP_CREATION_PROVIDER, + map_jump_hop_client_error(error), + ) + }) +} + +fn map_jump_hop_queue_job_status( + job: ExternalGenerationJobRecord, +) -> ExternalGenerationJobStatusRecord { + ExternalGenerationJobStatusRecord { + operation_id: job.job_id, + status: ExternalGenerationJobStatus::Queued, + phase_label: job.request_label, + phase_detail: "排队中。".to_string(), + progress: 8, + error: job.last_error_message, + updated_at_micros: job.updated_at_micros, + } +} + +pub(crate) async fn execute_jump_hop_compile_draft_worker_job( + state: &AppState, + request_context: &RequestContext, + mut worker_payload: JumpHopCompileDraftWorkerPayload, +) -> Result { + maybe_generate_jump_hop_assets( + state, + request_context, + worker_payload.session_id.as_str(), + worker_payload.owner_user_id.as_str(), + &mut worker_payload.payload, + ) + .await?; + let response = state + .spacetime_client() + .execute_jump_hop_action( + worker_payload.session_id, + worker_payload.owner_user_id, + worker_payload.payload, + ) + .await + .map_err(|error| { + jump_hop_error_response( + request_context, + JUMP_HOP_CREATION_PROVIDER, + map_jump_hop_client_error(error), + ) + })?; + Ok(response.session) +} + async fn resolve_jump_hop_generation_points_cost(state: &AppState) -> u64 { crate::creation_entry_config::resolve_creation_entry_mud_point_cost( state, @@ -1005,15 +1144,8 @@ fn slice_jump_hop_tile_atlas( let y1 = (row.saturating_add(1)).saturating_mul(height) / JUMP_HOP_TILE_ATLAS_ROWS; let tile_width = x1.saturating_sub(x0).max(1); let tile_height = y1.saturating_sub(y0).max(1); - let faces = slice_jump_hop_tile_uv_faces( - &source, - x0, - y0, - tile_width, - tile_height, - row, - col, - )?; + let faces = + slice_jump_hop_tile_uv_faces(&source, x0, y0, tile_width, tile_height, row, col)?; slices.push(JumpHopTileAtlasSlice { tile_type: jump_hop_tile_type_by_index(index), source_atlas_cell: format!("row-{}-col-{}", row + 1, col + 1), @@ -1043,22 +1175,70 @@ fn slice_jump_hop_tile_uv_faces( Ok(JumpHopTileFaceSlices { top: slice_jump_hop_tile_uv_face( - source, uv_x, uv_y, face_side, atlas_row, atlas_col, JumpHopTileFaceKey::Top, 1, 0, + source, + uv_x, + uv_y, + face_side, + atlas_row, + atlas_col, + JumpHopTileFaceKey::Top, + 1, + 0, )?, front: slice_jump_hop_tile_uv_face( - source, uv_x, uv_y, face_side, atlas_row, atlas_col, JumpHopTileFaceKey::Front, 1, 1, + source, + uv_x, + uv_y, + face_side, + atlas_row, + atlas_col, + JumpHopTileFaceKey::Front, + 1, + 1, )?, right: slice_jump_hop_tile_uv_face( - source, uv_x, uv_y, face_side, atlas_row, atlas_col, JumpHopTileFaceKey::Right, 2, 1, + source, + uv_x, + uv_y, + face_side, + atlas_row, + atlas_col, + JumpHopTileFaceKey::Right, + 2, + 1, )?, back: slice_jump_hop_tile_uv_face( - source, uv_x, uv_y, face_side, atlas_row, atlas_col, JumpHopTileFaceKey::Back, 3, 1, + source, + uv_x, + uv_y, + face_side, + atlas_row, + atlas_col, + JumpHopTileFaceKey::Back, + 3, + 1, )?, left: slice_jump_hop_tile_uv_face( - source, uv_x, uv_y, face_side, atlas_row, atlas_col, JumpHopTileFaceKey::Left, 0, 1, + source, + uv_x, + uv_y, + face_side, + atlas_row, + atlas_col, + JumpHopTileFaceKey::Left, + 0, + 1, )?, bottom: slice_jump_hop_tile_uv_face( - source, uv_x, uv_y, face_side, atlas_row, atlas_col, JumpHopTileFaceKey::Bottom, 1, 2, + source, + uv_x, + uv_y, + face_side, + atlas_row, + atlas_col, + JumpHopTileFaceKey::Bottom, + 1, + 2, )?, }) } @@ -1095,12 +1275,7 @@ fn slice_jump_hop_tile_uv_face( Ok(JumpHopTileFaceSlice { face, - source_atlas_cell: format!( - "row-{}-col-{}/{}", - atlas_row + 1, - atlas_col + 1, - face_label - ), + source_atlas_cell: format!("row-{}-col-{}/{}", atlas_row + 1, atlas_col + 1, face_label), bytes: cursor.into_inner(), }) } @@ -1827,7 +2002,9 @@ mod tests { assert!(prompt.contains("18个用于跳一跳地板的立方体主题物体 UV 展开包装图")); assert!(prompt.contains("按三列六行均匀排布")); assert!(prompt.contains("每个大单元格代表一个完整的 1x1x1 立方体方块物体")); - assert!(prompt.contains("该单元内的六张面贴图精确贴到 Three.js 标准极小倒角立方体的六个面上")); + assert!( + prompt.contains("该单元内的六张面贴图精确贴到 Three.js 标准极小倒角立方体的六个面上") + ); assert!(prompt.contains("cube object UV unwrap atlas / 立方体主题物体六面展开图集")); assert!(prompt.contains("不是单纯平铺材质、不是抽象纹理、不是只把主题颜色铺满")); assert!(prompt.contains("游戏界面或图标集页面")); @@ -1850,7 +2027,9 @@ mod tests { assert!(prompt.contains("full-bleed opaque square face texture")); assert!(prompt.contains("四角、边缘和中心都要有可识别内容")); assert!(prompt.contains("不留透明、不留空白、不留实底背景")); - assert!(prompt.contains("允许大面积水果切面、果柄叶片、剥皮条带、籽点、条纹和轮廓图案作为包装身份锚点")); + assert!(prompt.contains( + "允许大面积水果切面、果柄叶片、剥皮条带、籽点、条纹和轮廓图案作为包装身份锚点" + )); assert!(prompt.contains("不要把一个小水果、小叶片、小石头或小物体放在面中央")); assert!(prompt.contains("这不是透视渲染图")); assert!(prompt.contains("不要画摄像机视角、透视块、已烘焙侧壁")); @@ -1868,14 +2047,18 @@ mod tests { assert!(prompt.contains("小贴纸图标、小物体居中、纯果皮材质、纯果肉纹理")); assert!(prompt.contains("English guardrail")); assert!(prompt.contains("one vertical 1024x1536 image")); - assert!(prompt.contains("exactly 18 cube object UV unwraps in a 3 columns by 6 rows atlas")); + assert!( + prompt.contains("exactly 18 cube object UV unwraps in a 3 columns by 6 rows atlas") + ); assert!(prompt.contains("row1 col2 top")); assert!(prompt.contains("row2 col1 left")); assert!(prompt.contains("row2 col2 front")); assert!(prompt.contains("row2 col3 right")); assert!(prompt.contains("row2 col4 back")); assert!(prompt.contains("row3 col2 bottom")); - assert!(prompt.contains("six different face textures that stitch into one recognizable cubified theme object")); + assert!(prompt.contains( + "six different face textures that stitch into one recognizable cubified theme object" + )); assert!(prompt.contains("no generic flat material")); assert!(prompt.contains("no small centered stickers")); assert!(prompt.contains("every face is full-bleed opaque square texture")); @@ -2022,7 +2205,9 @@ mod tests { "科幻芯片主题的俯视角清爽游戏化立体感平台素材", ); - assert!(prompt.contains("画面内容是科幻芯片主题的正交平面清爽游戏化立方体主题身份方块包装贴图")); + assert!( + prompt.contains("画面内容是科幻芯片主题的正交平面清爽游戏化立方体主题身份方块包装贴图") + ); assert!(!prompt.contains("画面内容是科幻芯片主题的俯视角清爽游戏化立体感平台素材")); assert!(!prompt.contains("画面内容是科幻芯片主题的俯视角")); @@ -2118,12 +2303,10 @@ mod tests { .max(1); let tile_x = atlas_col.saturating_mul(cell_width); let tile_y = atlas_row.saturating_mul(cell_height); - let uv_x = tile_x.saturating_add( - cell_width.saturating_sub(face_side * JUMP_HOP_TILE_UV_FACE_COLS) / 2, - ); - let uv_y = tile_y.saturating_add( - cell_height.saturating_sub(face_side * JUMP_HOP_TILE_UV_FACE_ROWS) / 2, - ); + let uv_x = tile_x + .saturating_add(cell_width.saturating_sub(face_side * JUMP_HOP_TILE_UV_FACE_COLS) / 2); + let uv_y = tile_y + .saturating_add(cell_height.saturating_sub(face_side * JUMP_HOP_TILE_UV_FACE_ROWS) / 2); for y in uv_y + face_row * face_side..uv_y + (face_row + 1) * face_side { for x in uv_x + face_col * face_side..uv_x + (face_col + 1) * face_side { atlas.put_pixel(x, y, color); @@ -2159,14 +2342,8 @@ mod tests { ), "{message}" ); - assert!( - decoded.pixels().any(|pixel| pixel.0 == color), - "{message}" - ); - assert!( - decoded.pixels().all(|pixel| pixel.0[3] == 255), - "{message}" - ); + assert!(decoded.pixels().any(|pixel| pixel.0 == color), "{message}"); + assert!(decoded.pixels().all(|pixel| pixel.0[3] == 255), "{message}"); } #[test] diff --git a/server-rs/crates/api-server/src/main.rs b/server-rs/crates/api-server/src/main.rs index 3b7d8e8c4..38e0f23b9 100644 --- a/server-rs/crates/api-server/src/main.rs +++ b/server-rs/crates/api-server/src/main.rs @@ -40,6 +40,9 @@ mod edutainment_baby_drawing; mod edutainment_baby_object; mod error_middleware; mod external_api_audit; +mod external_generation; +mod external_generation_worker; +mod external_generation_worker_controller; pub(crate) mod generated_asset_sheets; mod generated_image_assets; mod health; @@ -114,6 +117,8 @@ use tracing::{error, info, warn}; use crate::{ app::{build_router, build_spacetime_unavailable_router}, config::AppConfig, + external_generation_worker::run_external_generation_worker, + external_generation_worker_controller::run_external_generation_worker_controller, state::{AppState, AppStateInitError}, tracking_outbox::TrackingOutbox, wallet_refund_outbox::WalletRefundOutbox, @@ -167,24 +172,57 @@ async fn run_server(config: AppConfig) -> Result<(), io::Error> { process_metrics::register_process_metrics(); telemetry::register_http_runtime_metrics(); + if !config.process_role.runs_http() { + return run_worker_only(config).await; + } + + run_http_role(config).await +} + +async fn run_worker_only(config: AppConfig) -> Result<(), io::Error> { + let process_role = config.process_role; + let state = restore_app_state_for_startup(config) + .await + .map_err(|error| { + io::Error::other(format!( + "初始化 external generation worker 状态失败：{error}" + )) + })?; + spawn_app_state_background_workers(&state); + info!( + process_role = process_role.as_str(), + "api-server 以非 HTTP 角色启动" + ); + if process_role.runs_external_generation_worker() { + run_external_generation_worker(state).await + } else if process_role.runs_external_generation_controller() { + run_external_generation_worker_controller(state).await + } else { + Err(io::Error::other(format!( + "不支持的非 HTTP 进程角色：{}", + process_role.as_str() + ))) + } +} + +async fn run_http_role(config: AppConfig) -> Result<(), io::Error> { let bind_address = config.bind_socket_addr(); let listen_backlog = config.listen_backlog; let worker_threads = config.worker_threads; let otel_enabled = config.otel_enabled; + let process_role = config.process_role; let outbox_flush_timeout = config.shutdown_outbox_flush_timeout; let listener = build_tcp_listener(bind_address, listen_backlog)?; - let (router, shutdown_context) = match restore_app_state_for_startup(config).await { + let (router, shutdown_context, worker_state) = match restore_app_state_for_startup(config).await + { Ok(state) => { - state.puzzle_gallery_cache().spawn_cleanup_task(); + spawn_app_state_background_workers(&state); let tracking_outbox = state.tracking_outbox(); - if let Some(outbox) = tracking_outbox.clone() { - outbox.spawn_worker(); - } let wallet_refund_outbox = state.wallet_refund_outbox(); - if let Some(outbox) = wallet_refund_outbox.clone() { - outbox.spawn_worker(); - } + let worker_state = process_role + .runs_external_generation_worker() + .then(|| state.clone()); ( build_router(state.clone()), ShutdownContext { @@ -193,6 +231,7 @@ async fn run_server(config: AppConfig) -> Result<(), io::Error> { wallet_refund_outbox, outbox_flush_timeout, }, + worker_state, ) } Err(AppStateInitError::DependencyUnavailable(message)) => ( @@ -203,6 +242,7 @@ async fn run_server(config: AppConfig) -> Result<(), io::Error> { wallet_refund_outbox: None, outbox_flush_timeout, }, + None, ), Err(error) => { return Err(std::io::Error::other(format!( @@ -216,12 +256,20 @@ async fn run_server(config: AppConfig) -> Result<(), io::Error> { listen_backlog, worker_threads = worker_threads.unwrap_or(0), otel_enabled, + process_role = process_role.as_str(), "api-server 已完成 tracing 初始化并开始监听" ); - let result = axum::serve(listener, router) - .with_graceful_shutdown(shutdown_signal(shutdown_context.clone())) - .await; + let http_server = axum::serve(listener, router) + .with_graceful_shutdown(shutdown_signal(shutdown_context.clone())); + let result = if let Some(worker_state) = worker_state { + tokio::select! { + result = http_server => result, + result = run_external_generation_worker(worker_state) => result, + } + } else { + http_server.await + }; finalize_shutdown(shutdown_context).await; result } @@ -332,6 +380,16 @@ async fn finalize_shutdown(context: ShutdownContext) { } } +fn spawn_app_state_background_workers(state: &AppState) { + state.puzzle_gallery_cache().spawn_cleanup_task(); + if let Some(outbox) = state.tracking_outbox() { + outbox.spawn_worker(); + } + if let Some(outbox) = state.wallet_refund_outbox() { + outbox.spawn_worker(); + } +} + fn build_tcp_listener( bind_address: SocketAddr, listen_backlog: i32, diff --git a/server-rs/crates/api-server/src/modules.rs b/server-rs/crates/api-server/src/modules.rs index 88caf30d2..f79ec8340 100644 --- a/server-rs/crates/api-server/src/modules.rs +++ b/server-rs/crates/api-server/src/modules.rs @@ -5,6 +5,7 @@ pub mod bark_battle; pub mod big_fish; pub mod custom_world; pub mod edutainment; +pub mod external_generation; pub mod health; pub mod internal; pub mod jump_hop; diff --git a/server-rs/crates/api-server/src/modules/external_generation.rs b/server-rs/crates/api-server/src/modules/external_generation.rs new file mode 100644 index 000000000..b65c832d9 --- /dev/null +++ b/server-rs/crates/api-server/src/modules/external_generation.rs @@ -0,0 +1,26 @@ +use axum::{Router, middleware, routing::get}; + +use crate::{ + auth::require_bearer_auth, + external_generation::{ + get_external_generation_job_status, get_external_generation_queue_overview, + }, + state::AppState, +}; + +pub fn router(state: AppState) -> Router { + Router::new() + .route( + "/api/runtime/external-generation/queue-overview", + get(get_external_generation_queue_overview).route_layer( + middleware::from_fn_with_state(state.clone(), require_bearer_auth), + ), + ) + .route( + "/api/runtime/external-generation/jobs/{job_id}", + get(get_external_generation_job_status).route_layer(middleware::from_fn_with_state( + state.clone(), + require_bearer_auth, + )), + ) +} diff --git a/server-rs/crates/api-server/src/puzzle.rs b/server-rs/crates/api-server/src/puzzle.rs index 6581676c4..cc9d0237e 100644 --- a/server-rs/crates/api-server/src/puzzle.rs +++ b/server-rs/crates/api-server/src/puzzle.rs @@ -52,22 +52,22 @@ use shared_contracts::{ }; use shared_kernel::{build_prefixed_uuid_id, format_timestamp_micros}; use spacetime_client::{ - PuzzleAgentMessageRecord, PuzzleAgentMessageSubmitRecordInput, - PuzzleAgentSessionCreateRecordInput, PuzzleAgentSessionRecord, - PuzzleAgentSuggestedActionRecord, PuzzleAnchorItemRecord, PuzzleAnchorPackRecord, - PuzzleAudioAssetRecord, PuzzleBackgroundCompileTaskClaimRecordInput, + ExternalGenerationJobEnqueueRecordInput, PuzzleAgentMessageRecord, + PuzzleAgentMessageSubmitRecordInput, PuzzleAgentSessionCreateRecordInput, + PuzzleAgentSessionRecord, PuzzleAgentSuggestedActionRecord, PuzzleAnchorItemRecord, + PuzzleAnchorPackRecord, PuzzleAudioAssetRecord, PuzzleBackgroundCompileTaskClaimRecordInput, PuzzleBackgroundCompileTaskReleaseRecordInput, PuzzleCreatorIntentRecord, PuzzleDraftCompileFailureRecordInput, PuzzleDraftLevelRecord, PuzzleFormDraftRecord, PuzzleFormDraftSaveRecordInput, PuzzleGeneratedImageCandidateRecord, PuzzleGeneratedImagesSaveRecordInput, PuzzleLeaderboardEntryRecord, - PuzzleLeaderboardSubmitRecordInput, PuzzlePublishRecordInput, PuzzleRecommendedNextWorkRecord, - PuzzleResultDraftRecord, PuzzleResultPreviewBlockerRecord, PuzzleResultPreviewFindingRecord, - PuzzleResultPreviewRecord, PuzzleRunDragRecordInput, PuzzleRunPauseRecordInput, - PuzzleRunPropRecordInput, PuzzleRunRecord, PuzzleRunStartRecordInput, PuzzleRunSwapRecordInput, - PuzzleSelectCoverImageRecordInput, PuzzleUiBackgroundSaveRecordInput, - PuzzleWorkLikeReportRecordInput, PuzzleWorkPointIncentiveClaimRecordInput, - PuzzleWorkProfileRecord, PuzzleWorkRemixRecordInput, PuzzleWorkUpsertRecordInput, - SpacetimeClientError, + PuzzleLeaderboardSubmitRecordInput, PuzzleLevelGenerationFailureRecordInput, + PuzzlePublishRecordInput, PuzzleRecommendedNextWorkRecord, PuzzleResultDraftRecord, + PuzzleResultPreviewBlockerRecord, PuzzleResultPreviewFindingRecord, PuzzleResultPreviewRecord, + PuzzleRunDragRecordInput, PuzzleRunPauseRecordInput, PuzzleRunPropRecordInput, PuzzleRunRecord, + PuzzleRunStartRecordInput, PuzzleRunSwapRecordInput, PuzzleSelectCoverImageRecordInput, + PuzzleUiBackgroundSaveRecordInput, PuzzleWorkLikeReportRecordInput, + PuzzleWorkPointIncentiveClaimRecordInput, PuzzleWorkProfileRecord, PuzzleWorkRemixRecordInput, + PuzzleWorkUpsertRecordInput, SpacetimeClientError, }; use std::convert::Infallible; @@ -79,6 +79,10 @@ use crate::{ should_skip_asset_operation_billing_for_connectivity, }, auth::{AuthenticatedAccessToken, RuntimePrincipal}, + external_generation_worker::{ + PUZZLE_COMPILE_DRAFT_JOB_KIND, PUZZLE_GENERATE_IMAGES_JOB_KIND, + PUZZLE_GENERATE_UI_BACKGROUND_JOB_KIND, + }, generated_asset_sheets::apply_generated_asset_sheet_green_screen_alpha, http_error::AppError, llm_model_routing::{CREATION_TEMPLATE_LLM_MODEL, PUZZLE_LEVEL_NAME_VISION_LLM_MODEL}, @@ -185,6 +189,25 @@ async fn release_claimed_puzzle_background_compile_task( } } +pub(crate) fn spawn_release_claimed_puzzle_background_compile_task( + state: PuzzleApiState, + task_id: String, + claim_id: String, + session_id: String, + owner_user_id: String, +) { + tokio::spawn(async move { + release_claimed_puzzle_background_compile_task( + &state, + &task_id, + &claim_id, + &session_id, + &owner_user_id, + ) + .await; + }); +} + fn has_puzzle_cover_image_src(value: &Option) -> bool { value .as_deref() @@ -215,6 +238,65 @@ fn mark_puzzle_initial_generation_started_snapshot( session } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ExternalGenerationWriteLeaseGuard { + pub(crate) job_id: Option, + pub(crate) worker_id: Option, + pub(crate) lease_token: Option, +} + +impl ExternalGenerationWriteLeaseGuard { + pub(crate) fn inline() -> Self { + Self { + job_id: None, + worker_id: None, + lease_token: None, + } + } + + pub(crate) fn from_claimed_job(job_id: String, worker_id: String, lease_token: String) -> Self { + Self { + job_id: Some(job_id), + worker_id: Some(worker_id), + lease_token: Some(lease_token), + } + } +} + +#[derive(Debug)] +pub(crate) struct PuzzleExternalGenerationWorkerError { + error: AppError, + should_fail_queue_job: bool, +} + +impl PuzzleExternalGenerationWorkerError { + pub(crate) fn with_failure_state_written(error: AppError) -> Self { + Self { + error, + should_fail_queue_job: true, + } + } + + pub(crate) fn with_failure_state_pending(error: AppError) -> Self { + Self { + error, + should_fail_queue_job: false, + } + } + + pub(crate) fn body_text(&self) -> String { + self.error.body_text() + } + + pub(crate) fn into_app_error(self) -> AppError { + self.error + } + + pub(crate) fn should_fail_queue_job(&self) -> bool { + self.should_fail_queue_job + } +} + pub(crate) fn format_puzzle_reference_image_upload_bytes(bytes: usize) -> String { format!("{:.1}MB", bytes as f64 / 1024.0 / 1024.0) } @@ -237,7 +319,7 @@ mod mappers; use self::mappers::*; mod draft; -use self::draft::*; +pub(crate) use self::draft::*; mod tags; @@ -246,7 +328,7 @@ use self::tags::*; mod generation; mod vector_engine; -use self::generation::*; +pub(crate) use self::generation::*; use self::vector_engine::*; #[cfg(test)] diff --git a/server-rs/crates/api-server/src/puzzle/draft.rs b/server-rs/crates/api-server/src/puzzle/draft.rs index ddb1d6ea9..0f6e949e1 100644 --- a/server-rs/crates/api-server/src/puzzle/draft.rs +++ b/server-rs/crates/api-server/src/puzzle/draft.rs @@ -137,6 +137,213 @@ pub(crate) async fn create_seeded_puzzle_session_when_form_save_missing( Ok(replacement.session_id) } +fn default_puzzle_image_generation_points_cost() -> u64 { + PUZZLE_IMAGE_GENERATION_POINTS_COST +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PuzzleCompileDraftWorkerPayload { + pub session_id: String, + pub owner_user_id: String, + pub billing_asset_id: String, + pub ai_redraw: bool, + #[serde(default = "default_puzzle_image_generation_points_cost")] + pub billing_points_cost: u64, + #[serde(default)] + pub prompt_text: Option, + #[serde(default)] + pub reference_image_src: Option, + #[serde(default)] + pub image_model: Option, + pub requested_at_micros: i64, + #[serde(default)] + pub background_task_id: Option, + #[serde(default)] + pub background_claim_id: Option, +} + +pub(crate) async fn execute_puzzle_compile_draft_worker_job( + state: &PuzzleApiState, + request_context: &RequestContext, + payload: PuzzleCompileDraftWorkerPayload, + external_generation_guard: ExternalGenerationWriteLeaseGuard, +) -> Result { + let now = current_utc_micros(); + let session = if payload.ai_redraw { + execute_billable_asset_operation_with_cost( + state.root_state(), + &payload.owner_user_id, + "puzzle_initial_image", + &payload.billing_asset_id, + payload.billing_points_cost, + async { + compile_puzzle_draft_with_initial_cover( + state, + request_context, + payload.session_id.clone(), + payload.owner_user_id.clone(), + payload.prompt_text.as_deref(), + payload.reference_image_src.as_deref(), + payload.image_model.as_deref(), + now, + &external_generation_guard, + ) + .await + }, + ) + .await + } else { + compile_puzzle_draft_with_uploaded_cover( + state, + request_context, + payload.session_id.clone(), + payload.owner_user_id.clone(), + payload.prompt_text.as_deref(), + payload.reference_image_src.as_deref(), + now, + &external_generation_guard, + ) + .await + }; + + match session { + Ok(session) => { + if session + .draft + .as_ref() + .is_some_and(|draft| draft.generation_status == "ready") + { + send_generation_result_subscribe_message_after_completion( + state.root_state(), + GenerationResultSubscribeMessage { + owner_user_id: payload.owner_user_id.clone(), + task_name: Some("拼图".to_string()), + work_name: session.draft.as_ref().map(|draft| draft.work_title.clone()), + status: GenerationResultSubscribeMessageStatus::Succeeded, + consumed_points: if payload.ai_redraw { + payload.billing_points_cost + } else { + 0 + }, + completed_at_micros: current_utc_micros(), + page: Some("/pages/web-view/index".to_string()), + }, + ) + .await; + } + release_inline_puzzle_compile_background_claim( + state, + &payload, + &external_generation_guard, + ); + Ok(session) + } + Err(error) => { + match mark_puzzle_compile_failure_for_worker( + state, + &payload.session_id, + &payload.owner_user_id, + error.body_text(), + now, + &external_generation_guard, + ) + .await + { + Ok(()) => { + send_generation_result_subscribe_message_after_completion( + state.root_state(), + GenerationResultSubscribeMessage { + owner_user_id: payload.owner_user_id.clone(), + task_name: Some("拼图".to_string()), + work_name: None, + status: GenerationResultSubscribeMessageStatus::Failed, + consumed_points: 0, + completed_at_micros: now, + page: Some("/pages/web-view/index".to_string()), + }, + ) + .await; + release_inline_puzzle_compile_background_claim( + state, + &payload, + &external_generation_guard, + ); + Err(PuzzleExternalGenerationWorkerError::with_failure_state_written(error)) + } + Err(mark_error) => { + Err(PuzzleExternalGenerationWorkerError::with_failure_state_pending(mark_error)) + } + } + } + } +} + +fn release_inline_puzzle_compile_background_claim( + state: &PuzzleApiState, + payload: &PuzzleCompileDraftWorkerPayload, + external_generation_guard: &ExternalGenerationWriteLeaseGuard, +) { + if external_generation_guard.job_id.is_some() { + return; + } + release_puzzle_compile_background_claim(state, payload); +} + +pub(crate) fn release_puzzle_compile_background_claim( + state: &PuzzleApiState, + payload: &PuzzleCompileDraftWorkerPayload, +) { + let (Some(task_id), Some(claim_id)) = ( + payload.background_task_id.as_ref(), + payload.background_claim_id.as_ref(), + ) else { + return; + }; + + spawn_release_claimed_puzzle_background_compile_task( + state.clone(), + task_id.clone(), + claim_id.clone(), + payload.session_id.clone(), + payload.owner_user_id.clone(), + ); +} + +pub(crate) async fn mark_puzzle_compile_failure_for_worker( + state: &PuzzleApiState, + session_id: &str, + owner_user_id: &str, + error_message: String, + failed_at_micros: i64, + external_generation_guard: &ExternalGenerationWriteLeaseGuard, +) -> Result<(), AppError> { + let result = state + .spacetime_client() + .mark_puzzle_draft_generation_failed(PuzzleDraftCompileFailureRecordInput { + session_id: session_id.to_string(), + owner_user_id: owner_user_id.to_string(), + error_message, + failed_at_micros, + external_generation_job_id: external_generation_guard.job_id.clone(), + external_generation_worker_id: external_generation_guard.worker_id.clone(), + external_generation_lease_token: external_generation_guard.lease_token.clone(), + }) + .await; + if let Err(error) = result { + tracing::warn!( + provider = PUZZLE_AGENT_API_BASE_PROVIDER, + session_id, + owner_user_id, + message = %error, + "拼图 worker 草稿失败态回写失败" + ); + return Err(map_puzzle_client_error(error)); + } + + Ok(()) +} + pub(crate) fn select_puzzle_level_for_api( draft: &PuzzleResultDraftRecord, level_id: Option<&str>, @@ -1163,6 +1370,44 @@ pub(crate) fn find_puzzle_level_for_initial_asset_check<'a>( .or_else(|| levels.first()) } +pub(crate) async fn compile_puzzle_draft_with_initial_cover( + state: &PuzzleApiState, + request_context: &RequestContext, + session_id: String, + owner_user_id: String, + prompt_text: Option<&str>, + reference_image_src: Option<&str>, + image_model: Option<&str>, + now: i64, + external_generation_guard: &ExternalGenerationWriteLeaseGuard, +) -> Result { + let compiled_session = state + .spacetime_client() + .compile_puzzle_agent_draft_with_external_generation_guard( + session_id, + owner_user_id.clone(), + now, + external_generation_guard.job_id.clone(), + external_generation_guard.worker_id.clone(), + external_generation_guard.lease_token.clone(), + ) + .await + .map_err(map_puzzle_compile_error)?; + + generate_puzzle_initial_cover_from_compiled_session( + state, + request_context, + compiled_session, + owner_user_id, + prompt_text, + reference_image_src, + image_model, + now, + external_generation_guard, + ) + .await +} + pub(crate) async fn generate_puzzle_initial_cover_from_compiled_session( state: &PuzzleApiState, request_context: &RequestContext, @@ -1172,6 +1417,7 @@ pub(crate) async fn generate_puzzle_initial_cover_from_compiled_session( reference_image_src: Option<&str>, image_model: Option<&str>, now: i64, + external_generation_guard: &ExternalGenerationWriteLeaseGuard, ) -> Result { let draft = compiled_session.draft.clone().ok_or_else(|| { AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ @@ -1322,6 +1568,9 @@ pub(crate) async fn generate_puzzle_initial_cover_from_compiled_session( levels_json: levels_json_with_generated_name.clone(), candidates_json, saved_at_micros: current_utc_micros(), + external_generation_job_id: external_generation_guard.job_id.clone(), + external_generation_worker_id: external_generation_guard.worker_id.clone(), + external_generation_lease_token: external_generation_guard.lease_token.clone(), }) .await .map_err(map_puzzle_client_error) @@ -1435,6 +1684,7 @@ pub(crate) async fn compile_puzzle_draft_with_uploaded_cover( prompt_text: Option<&str>, reference_image_src: Option<&str>, now: i64, + external_generation_guard: &ExternalGenerationWriteLeaseGuard, ) -> Result { let uploaded_image_src = reference_image_src .map(str::trim) @@ -1469,7 +1719,14 @@ pub(crate) async fn compile_puzzle_draft_with_uploaded_cover( })?; let compiled_session = state .spacetime_client() - .compile_puzzle_agent_draft(session_id.clone(), owner_user_id.clone(), now) + .compile_puzzle_agent_draft_with_external_generation_guard( + session_id.clone(), + owner_user_id.clone(), + now, + external_generation_guard.job_id.clone(), + external_generation_guard.worker_id.clone(), + external_generation_guard.lease_token.clone(), + ) .await .map_err(map_puzzle_compile_error)?; let draft = compiled_session.draft.clone().ok_or_else(|| { @@ -1618,6 +1875,9 @@ pub(crate) async fn compile_puzzle_draft_with_uploaded_cover( levels_json: levels_json_with_generated_name.clone(), candidates_json, saved_at_micros: current_utc_micros(), + external_generation_job_id: external_generation_guard.job_id.clone(), + external_generation_worker_id: external_generation_guard.worker_id.clone(), + external_generation_lease_token: external_generation_guard.lease_token.clone(), }) .await .map_err(map_puzzle_client_error) diff --git a/server-rs/crates/api-server/src/puzzle/generation.rs b/server-rs/crates/api-server/src/puzzle/generation.rs index 3713a6530..091ec04ea 100644 --- a/server-rs/crates/api-server/src/puzzle/generation.rs +++ b/server-rs/crates/api-server/src/puzzle/generation.rs @@ -22,6 +22,510 @@ pub(crate) fn should_use_uploaded_puzzle_image_directly( .is_some_and(|value| !value.is_empty()) } +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PuzzleGenerateImagesWorkerPayload { + pub session_id: String, + pub owner_user_id: String, + pub billing_asset_id: String, + #[serde(default)] + pub level_id: Option, + #[serde(default)] + pub prompt_text: Option, + #[serde(default)] + pub reference_image_src: Option, + #[serde(default)] + pub reference_image_srcs: Vec, + #[serde(default)] + pub reference_image_asset_object_id: Option, + #[serde(default)] + pub reference_image_asset_object_ids: Vec, + #[serde(default)] + pub image_model: Option, + #[serde(default)] + pub ai_redraw: Option, + #[serde(default)] + pub should_auto_name_level: Option, + #[serde(default)] + pub work_title: Option, + #[serde(default)] + pub work_description: Option, + #[serde(default)] + pub picture_description: Option, + #[serde(default)] + pub summary: Option, + #[serde(default)] + pub theme_tags: Option>, + #[serde(default)] + pub levels_json: Option, + pub requested_at_micros: i64, +} + +impl PuzzleGenerateImagesWorkerPayload { + fn to_action_request(&self) -> ExecutePuzzleAgentActionRequest { + ExecutePuzzleAgentActionRequest { + action: "generate_puzzle_images".to_string(), + prompt_text: self.prompt_text.clone(), + reference_image_src: self.reference_image_src.clone(), + reference_image_srcs: self.reference_image_srcs.clone(), + reference_image_asset_object_id: self.reference_image_asset_object_id.clone(), + reference_image_asset_object_ids: self.reference_image_asset_object_ids.clone(), + image_model: self.image_model.clone(), + ai_redraw: self.ai_redraw, + candidate_count: Some(1), + should_auto_name_level: self.should_auto_name_level, + candidate_id: None, + level_id: self.level_id.clone(), + work_title: self.work_title.clone(), + work_description: self.work_description.clone(), + picture_description: self.picture_description.clone(), + level_name: None, + summary: self.summary.clone(), + theme_tags: self.theme_tags.clone(), + levels_json: self.levels_json.clone(), + } + } +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PuzzleGenerateUiBackgroundWorkerPayload { + pub session_id: String, + pub owner_user_id: String, + pub billing_asset_id: String, + #[serde(default)] + pub level_id: Option, + #[serde(default)] + pub prompt_text: Option, + #[serde(default)] + pub levels_json: Option, + pub requested_at_micros: i64, +} + +impl PuzzleGenerateUiBackgroundWorkerPayload { + fn to_action_request(&self) -> ExecutePuzzleAgentActionRequest { + ExecutePuzzleAgentActionRequest { + action: "generate_puzzle_ui_background".to_string(), + prompt_text: self.prompt_text.clone(), + reference_image_src: None, + reference_image_srcs: Vec::new(), + reference_image_asset_object_id: None, + reference_image_asset_object_ids: Vec::new(), + image_model: None, + ai_redraw: None, + candidate_count: None, + should_auto_name_level: None, + candidate_id: None, + level_id: self.level_id.clone(), + work_title: None, + work_description: None, + picture_description: None, + level_name: None, + summary: None, + theme_tags: None, + levels_json: self.levels_json.clone(), + } + } +} + +pub(crate) async fn execute_puzzle_generate_images_worker_job( + state: &PuzzleApiState, + request_context: &RequestContext, + payload: PuzzleGenerateImagesWorkerPayload, + external_generation_guard: ExternalGenerationWriteLeaseGuard, +) -> Result { + let now = current_utc_micros(); + let session = execute_billable_asset_operation_with_cost( + state.root_state(), + &payload.owner_user_id, + "puzzle_generated_image", + &payload.billing_asset_id, + PUZZLE_IMAGE_GENERATION_POINTS_COST, + async { + execute_puzzle_generate_images_worker_job_inner( + state, + request_context, + &payload, + now, + &external_generation_guard, + ) + .await + }, + ) + .await; + + match session { + Ok(session) => Ok(session), + Err(error) => { + match mark_puzzle_level_generation_failure_for_worker( + state, + &payload, + error.body_text(), + now, + &external_generation_guard, + ) + .await + { + Ok(()) => { + Err(PuzzleExternalGenerationWorkerError::with_failure_state_written(error)) + } + Err(mark_error) => { + Err(PuzzleExternalGenerationWorkerError::with_failure_state_pending(mark_error)) + } + } + } + } +} + +pub(crate) async fn execute_puzzle_generate_ui_background_worker_job( + state: &PuzzleApiState, + request_context: &RequestContext, + payload: PuzzleGenerateUiBackgroundWorkerPayload, + external_generation_guard: ExternalGenerationWriteLeaseGuard, +) -> Result { + let now = current_utc_micros(); + let session = execute_billable_asset_operation_with_cost( + state.root_state(), + &payload.owner_user_id, + "puzzle_ui_background_image", + &payload.billing_asset_id, + PUZZLE_IMAGE_GENERATION_POINTS_COST, + async { + execute_puzzle_generate_ui_background_worker_job_inner( + state, + request_context, + &payload, + now, + &external_generation_guard, + ) + .await + }, + ) + .await; + + match session { + Ok(session) => Ok(session), + Err(error) => { + match mark_puzzle_level_generation_failure_for_external_generation( + state, + &payload.session_id, + &payload.owner_user_id, + payload.level_id.clone(), + payload.levels_json.clone(), + error.body_text(), + now, + &external_generation_guard, + ) + .await + { + Ok(()) => { + Err(PuzzleExternalGenerationWorkerError::with_failure_state_written(error)) + } + Err(mark_error) => { + Err(PuzzleExternalGenerationWorkerError::with_failure_state_pending(mark_error)) + } + } + } + } +} + +async fn execute_puzzle_generate_images_worker_job_inner( + state: &PuzzleApiState, + request_context: &RequestContext, + payload: &PuzzleGenerateImagesWorkerPayload, + now: i64, + external_generation_guard: &ExternalGenerationWriteLeaseGuard, +) -> Result { + let action_payload = payload.to_action_request(); + let target_level_id = payload.level_id.clone(); + let levels_json = payload.levels_json.clone(); + let session = get_puzzle_session_for_image_generation( + state, + payload.session_id.clone(), + payload.owner_user_id.clone(), + &action_payload, + levels_json.as_deref(), + now, + ) + .await?; + let mut draft = session.draft.clone().ok_or_else(|| { + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": "拼图结果页草稿尚未生成", + })) + })?; + if let Some(levels_json) = levels_json.as_ref() { + draft.levels = parse_puzzle_level_records_from_module_json(levels_json)?; + } + let mut target_level = select_puzzle_level_for_api(&draft, target_level_id.as_deref())?; + let prompt = resolve_puzzle_level_image_prompt( + payload.prompt_text.as_deref(), + &target_level.picture_description, + &draft.summary, + ); + let should_auto_name_level = payload + .should_auto_name_level + .unwrap_or_else(|| target_level.level_name.trim().is_empty()); + if should_auto_name_level { + let naming = + generate_puzzle_first_level_name(state, target_level.picture_description.as_str()) + .await; + target_level.level_name = naming.level_name.clone(); + target_level.ui_background_prompt = naming.ui_background_prompt.clone(); + } + let reference_image_sources = collect_puzzle_reference_image_sources( + payload.reference_image_src.as_deref(), + payload.reference_image_srcs.as_slice(), + payload.reference_image_asset_object_id.as_deref(), + payload.reference_image_asset_object_ids.as_slice(), + ); + let primary_reference_image_src = reference_image_sources.first().map(String::as_str); + // 中文注释：拼图结果页从多候选抽卡收口为单图替换，前端传入的旧 candidateCount 只做兼容忽略。 + let candidate_start_index = target_level.candidates.len(); + let ai_redraw = payload.ai_redraw.unwrap_or(true); + let mut candidates = + if should_use_uploaded_puzzle_image_directly(primary_reference_image_src, ai_redraw) { + vec![ + create_uploaded_puzzle_image_candidate( + state, + payload.owner_user_id.as_str(), + &session.session_id, + &target_level.level_name, + &prompt, + primary_reference_image_src.expect("checked reference image"), + candidate_start_index, + ) + .await?, + ] + } else { + let (_, profile_id) = build_stable_puzzle_work_ids(&session.session_id); + generate_puzzle_image_candidates( + state, + payload.owner_user_id.as_str(), + Some(profile_id.as_str()), + &session.session_id, + &target_level.level_name, + &prompt, + primary_reference_image_src, + ai_redraw, + payload.image_model.as_deref(), + 1, + candidate_start_index, + ) + .await + .map_err(map_puzzle_generation_endpoint_error)? + }; + if candidates.is_empty() { + return Err( + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": "拼图候选图生成结果为空", + })), + ); + } + if let Some(refined_naming) = generate_puzzle_first_level_name_from_image( + state, + target_level.picture_description.as_str(), + &candidates[0].downloaded_image, + ) + .await + .filter(|_| should_auto_name_level) + { + target_level.level_name = refined_naming.level_name.clone(); + if refined_naming.ui_background_prompt.is_some() { + target_level.ui_background_prompt = refined_naming.ui_background_prompt.clone(); + } + } + let mut updated_levels = + build_puzzle_levels_with_primary_update(&draft, &target_level, primary_reference_image_src); + for candidate in &mut candidates { + candidate.record.prompt = prompt.clone(); + } + let selected_candidate = candidates + .iter() + .find(|candidate| candidate.record.selected) + .or_else(|| candidates.first()) + .ok_or_else(|| { + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": "拼图候选图生成结果为空", + })) + })?; + let asset_bundle = generate_puzzle_level_asset_bundle_required( + state, + request_context, + payload.owner_user_id.as_str(), + &session.session_id, + &target_level, + &selected_candidate.downloaded_image, + ) + .await?; + attach_puzzle_level_asset_bundle( + &mut updated_levels, + target_level.level_id.as_str(), + asset_bundle, + ); + attach_selected_puzzle_candidate_to_levels( + &mut updated_levels, + target_level.level_id.as_str(), + &selected_candidate.record, + ); + let levels_json_with_generated_name = + Some(serialize_puzzle_level_records_for_module(&updated_levels)?); + let candidates_json = serde_json::to_string( + &candidates + .iter() + .map(|candidate| to_puzzle_generated_image_candidate(&candidate.record)) + .collect::>(), + ) + .map_err(|error| { + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": format!("拼图候选图序列化失败：{error}"), + })) + })?; + state + .spacetime_client() + .save_puzzle_generated_images(PuzzleGeneratedImagesSaveRecordInput { + session_id: session.session_id.clone(), + owner_user_id: payload.owner_user_id.clone(), + level_id: Some(target_level.level_id.clone()), + levels_json: levels_json_with_generated_name, + candidates_json, + saved_at_micros: now, + external_generation_job_id: external_generation_guard.job_id.clone(), + external_generation_worker_id: external_generation_guard.worker_id.clone(), + external_generation_lease_token: external_generation_guard.lease_token.clone(), + }) + .await + .map_err(map_puzzle_client_error) +} + +async fn execute_puzzle_generate_ui_background_worker_job_inner( + state: &PuzzleApiState, + request_context: &RequestContext, + payload: &PuzzleGenerateUiBackgroundWorkerPayload, + now: i64, + external_generation_guard: &ExternalGenerationWriteLeaseGuard, +) -> Result { + let action_payload = payload.to_action_request(); + let target_level_id = payload.level_id.clone(); + let levels_json = payload.levels_json.clone(); + let session = get_puzzle_session_for_image_generation( + state, + payload.session_id.clone(), + payload.owner_user_id.clone(), + &action_payload, + levels_json.as_deref(), + now, + ) + .await?; + let mut draft = session.draft.clone().ok_or_else(|| { + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": "拼图结果页草稿尚未生成", + })) + })?; + if let Some(levels_json) = levels_json.as_ref() { + draft.levels = parse_puzzle_level_records_from_module_json(levels_json)?; + } + let target_level = select_puzzle_level_for_api(&draft, target_level_id.as_deref())?; + let raw_prompt = payload + .prompt_text + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or_default() + .to_string(); + let resolved_prompt = + normalize_puzzle_ui_background_prompt(raw_prompt.as_str(), &draft, &target_level); + let generated = generate_puzzle_ui_background_image( + state, + request_context, + payload.owner_user_id.as_str(), + &session.session_id, + &target_level.level_name, + resolved_prompt.as_str(), + ) + .await + .map_err(map_puzzle_generation_endpoint_error)?; + state + .spacetime_client() + .save_puzzle_ui_background(PuzzleUiBackgroundSaveRecordInput { + session_id: session.session_id.clone(), + owner_user_id: payload.owner_user_id.clone(), + level_id: Some(target_level.level_id.clone()), + levels_json, + prompt: resolved_prompt.clone(), + image_src: generated.image_src.clone(), + image_object_key: Some(generated.object_key.clone()), + saved_at_micros: now, + external_generation_job_id: external_generation_guard.job_id.clone(), + external_generation_worker_id: external_generation_guard.worker_id.clone(), + external_generation_lease_token: external_generation_guard.lease_token.clone(), + }) + .await + .map_err(map_puzzle_client_error) +} + +pub(crate) async fn mark_puzzle_level_generation_failure_for_worker( + state: &PuzzleApiState, + payload: &PuzzleGenerateImagesWorkerPayload, + error_message: String, + failed_at_micros: i64, + external_generation_guard: &ExternalGenerationWriteLeaseGuard, +) -> Result<(), AppError> { + mark_puzzle_level_generation_failure_for_external_generation( + state, + &payload.session_id, + &payload.owner_user_id, + payload.level_id.clone(), + payload.levels_json.clone(), + error_message, + failed_at_micros, + external_generation_guard, + ) + .await +} + +async fn mark_puzzle_level_generation_failure_for_external_generation( + state: &PuzzleApiState, + session_id: &str, + owner_user_id: &str, + level_id: Option, + levels_json: Option, + error_message: String, + failed_at_micros: i64, + external_generation_guard: &ExternalGenerationWriteLeaseGuard, +) -> Result<(), AppError> { + let result = state + .spacetime_client() + .mark_puzzle_level_generation_failed(PuzzleLevelGenerationFailureRecordInput { + session_id: session_id.to_string(), + owner_user_id: owner_user_id.to_string(), + level_id, + levels_json, + error_message, + failed_at_micros, + external_generation_job_id: external_generation_guard.job_id.clone(), + external_generation_worker_id: external_generation_guard.worker_id.clone(), + external_generation_lease_token: external_generation_guard.lease_token.clone(), + }) + .await; + if let Err(error) = result { + tracing::warn!( + provider = PUZZLE_AGENT_API_BASE_PROVIDER, + session_id = %session_id, + owner_user_id = %owner_user_id, + message = %error, + "拼图 worker 关卡生图失败态回写失败" + ); + return Err(map_puzzle_client_error(error)); + } + + Ok(()) +} + pub(crate) async fn create_uploaded_puzzle_image_candidate( state: &PuzzleApiState, owner_user_id: &str, diff --git a/server-rs/crates/api-server/src/puzzle/handlers.rs b/server-rs/crates/api-server/src/puzzle/handlers.rs index d7a2b0bda..e5e24ddad 100644 --- a/server-rs/crates/api-server/src/puzzle/handlers.rs +++ b/server-rs/crates/api-server/src/puzzle/handlers.rs @@ -589,7 +589,6 @@ pub async fn execute_puzzle_agent_action( let now = current_utc_micros(); let action = payload.action.trim().to_string(); let billing_asset_id = format!("{}:{}:{}", session_id, action, request_context.request_id()); - let mut operation_consumed_points = 0; tracing::info!( provider = PUZZLE_AGENT_API_BASE_PROVIDER, session_id = %session_id, @@ -610,50 +609,6 @@ pub async fn execute_puzzle_agent_action( "拼图 Agent action 开始执行" ); - let mark_puzzle_compile_failure = |error: &AppError, compile_session_id: &str| { - let state = state.clone(); - let owner_user_id = owner_user_id.clone(); - let error_message = error.body_text(); - let session_id = compile_session_id.to_string(); - let log_session_id = session_id.clone(); - let log_owner_user_id = owner_user_id.clone(); - async move { - let failed_at_micros = current_utc_micros(); - let result = state - .spacetime_client() - .mark_puzzle_draft_generation_failed(PuzzleDraftCompileFailureRecordInput { - session_id, - owner_user_id: owner_user_id.clone(), - error_message, - failed_at_micros, - }) - .await; - if let Err(error) = result { - tracing::warn!( - provider = PUZZLE_AGENT_API_BASE_PROVIDER, - session_id = %log_session_id, - owner_user_id = %log_owner_user_id, - message = %error, - "拼图草稿失败态回写失败，继续返回原始错误" - ); - } else { - send_generation_result_subscribe_message_after_completion( - state.root_state(), - GenerationResultSubscribeMessage { - owner_user_id, - task_name: Some("拼图".to_string()), - work_name: None, - status: GenerationResultSubscribeMessageStatus::Failed, - consumed_points: 0, - completed_at_micros: failed_at_micros, - page: Some("/pages/web-view/index".to_string()), - }, - ) - .await; - } - } - }; - let (operation_type, phase_label, phase_detail, session) = match action.as_str() { "compile_puzzle_draft" => { let ai_redraw = payload.ai_redraw.unwrap_or(true); @@ -667,7 +622,6 @@ pub async fn execute_puzzle_agent_action( } else { 0 }; - operation_consumed_points = puzzle_draft_generation_points_cost; let reference_image_sources = collect_puzzle_reference_image_sources( payload.reference_image_src.as_deref(), payload.reference_image_srcs.as_slice(), @@ -694,7 +648,20 @@ pub async fn execute_puzzle_agent_action( Ok(next_session_id) => next_session_id, Err(response) => return Err(response), }; - let session = if ai_redraw { + let worker_payload = PuzzleCompileDraftWorkerPayload { + session_id: compile_session_id.clone(), + owner_user_id: owner_user_id.clone(), + billing_asset_id: billing_asset_id.clone(), + ai_redraw, + billing_points_cost: puzzle_draft_generation_points_cost, + prompt_text: prompt_text.map(ToOwned::to_owned), + reference_image_src: primary_reference_image_src.map(ToOwned::to_owned), + image_model: payload.image_model.clone(), + requested_at_micros: now, + background_task_id: None, + background_claim_id: None, + }; + let worker_payload = if ai_redraw { let background_task_id = build_puzzle_background_compile_task_id(&compile_session_id); let background_claim_id = build_puzzle_background_compile_claim_id( @@ -713,223 +680,202 @@ pub async fn execute_puzzle_agent_action( }, ) .await - .map_err(map_puzzle_client_error); - match claim_result { - Ok(false) => { - tracing::info!( - provider = PUZZLE_AGENT_API_BASE_PROVIDER, - session_id = %compile_session_id, - owner_user_id = %owner_user_id, - task_id = %background_task_id, - "拼图首图后台生成任务已存在，本次 action 直接返回生成中会话" - ); - state - .spacetime_client() - .get_puzzle_agent_session( - compile_session_id.clone(), - owner_user_id.clone(), + .map_err(|error| { + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + map_puzzle_client_error(error), + ) + })?; + if !claim_result { + tracing::info!( + provider = PUZZLE_AGENT_API_BASE_PROVIDER, + session_id = %compile_session_id, + owner_user_id = %owner_user_id, + task_id = %background_task_id, + "拼图首图后台生成任务已存在，本次 action 直接返回生成中会话" + ); + let session = state + .spacetime_client() + .get_puzzle_agent_session(compile_session_id.clone(), owner_user_id.clone()) + .await + .map(mark_puzzle_initial_generation_started_snapshot) + .map_err(|error| { + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + map_puzzle_client_error(error), ) - .await - .map(mark_puzzle_initial_generation_started_snapshot) - .map_err(map_puzzle_client_error) - } - Ok(true) => { - let compiled_session = state - .spacetime_client() - .compile_puzzle_agent_draft( - compile_session_id.clone(), - owner_user_id.clone(), - now, - ) - .await - .map_err(map_puzzle_compile_error); - match compiled_session { - Ok(compiled_session) => { - let response_session = - mark_puzzle_initial_generation_started_snapshot( - compiled_session.clone(), - ); - let background_state = state.clone(); - let background_request_context = request_context.clone(); - let background_session_id = compile_session_id.clone(); - let background_owner_user_id = owner_user_id.clone(); - let background_task_id = background_task_id.clone(); - let background_claim_id = background_claim_id.clone(); - let background_prompt_text = prompt_text.map(str::to_string); - let background_reference_image_src = - primary_reference_image_src.map(str::to_string); - let background_image_model = payload.image_model.clone(); - let background_points_cost = puzzle_draft_generation_points_cost; - let background_work_name = compiled_session - .draft - .as_ref() - .map(|draft| draft.work_title.clone()); - let background_billing_asset_id = format!( - "{background_session_id}:compile_puzzle_draft:{}", - background_request_context.request_id() - ); - tokio::spawn(async move { - let operation_owner_user_id = background_owner_user_id.clone(); - let background_root_state = - background_state.root_state().clone(); - let operation_state = background_state.clone(); - let result = execute_billable_asset_operation_with_cost( - &background_root_state, - &background_owner_user_id, - "puzzle_initial_image", - &background_billing_asset_id, - background_points_cost, - async move { - generate_puzzle_initial_cover_from_compiled_session( - &operation_state, - &background_request_context, - compiled_session, - operation_owner_user_id, - background_prompt_text.as_deref(), - background_reference_image_src.as_deref(), - background_image_model.as_deref(), - current_utc_micros(), - ) - .await - }, - ) - .await; - match result { - Ok(session) => { - send_generation_result_subscribe_message_after_completion( - &background_root_state, - GenerationResultSubscribeMessage { - owner_user_id: background_owner_user_id.clone(), - task_name: Some("拼图".to_string()), - work_name: session - .draft - .as_ref() - .map(|draft| draft.work_title.clone()), - status: - GenerationResultSubscribeMessageStatus::Succeeded, - consumed_points: background_points_cost, - completed_at_micros: current_utc_micros(), - page: Some("/pages/web-view/index".to_string()), - }, - ) - .await; - tracing::info!( - provider = PUZZLE_AGENT_API_BASE_PROVIDER, - session_id = %session.session_id, - owner_user_id = %background_owner_user_id, - "拼图首图后台生成任务完成" - ); - } - Err(error) => { - let error_message = error.body_text(); - let failed_at_micros = current_utc_micros(); - let failure_result = background_state - .spacetime_client() - .mark_puzzle_draft_generation_failed( - PuzzleDraftCompileFailureRecordInput { - session_id: background_session_id.clone(), - owner_user_id: background_owner_user_id - .clone(), - error_message: error_message.clone(), - failed_at_micros, - }, - ) - .await; - if let Err(mark_error) = failure_result { - tracing::warn!( - provider = PUZZLE_AGENT_API_BASE_PROVIDER, - session_id = %background_session_id, - owner_user_id = %background_owner_user_id, - message = %mark_error, - "拼图首图后台生成失败态回写失败" - ); - } else { - send_generation_result_subscribe_message_after_completion( - &background_root_state, - GenerationResultSubscribeMessage { - owner_user_id: background_owner_user_id - .clone(), - task_name: Some("拼图".to_string()), - work_name: background_work_name.clone(), - status: - GenerationResultSubscribeMessageStatus::Failed, - consumed_points: 0, - completed_at_micros: failed_at_micros, - page: Some( - "/pages/web-view/index".to_string(), - ), - }, - ) - .await; - } - tracing::warn!( - provider = PUZZLE_AGENT_API_BASE_PROVIDER, - session_id = %background_session_id, - owner_user_id = %background_owner_user_id, - message = %error_message, - "拼图首图后台生成任务失败" - ); - } - } - release_claimed_puzzle_background_compile_task( - &background_state, - &background_task_id, - &background_claim_id, - &background_session_id, - &background_owner_user_id, - ) - .await; - }); - Ok(response_session) - } - Err(error) => { - release_claimed_puzzle_background_compile_task( - &state, - &background_task_id, - &background_claim_id, - &compile_session_id, - &owner_user_id, - ) - .await; - Err(error) - } - } - } - Err(error) => Err(error), + })?; + return Ok(json_success_body( + Some(&request_context), + PuzzleAgentActionResponse { + operation: PuzzleAgentOperationResponse { + operation_id: background_task_id, + operation_type: "compile_puzzle_draft".to_string(), + status: "running".to_string(), + phase_label: "首关拼图草稿".to_string(), + phase_detail: "首关草稿生成已在后台处理中。".to_string(), + progress: session.progress_percent.max(10), + error: None, + }, + session: map_puzzle_agent_session_response(session), + }, + )); + } + + PuzzleCompileDraftWorkerPayload { + background_task_id: Some(background_task_id), + background_claim_id: Some(background_claim_id), + ..worker_payload } } else { - compile_puzzle_draft_with_uploaded_cover( + worker_payload + }; + if state + .root_state() + .config + .external_generation_mode + .is_inline() + { + tracing::info!( + provider = PUZZLE_AGENT_API_BASE_PROVIDER, + session_id = %compile_session_id, + owner_user_id = %owner_user_id, + external_generation_mode = state.root_state().config.external_generation_mode.as_str(), + "拼图首关草稿生成使用 inline 模式同步执行" + ); + let session = execute_puzzle_compile_draft_worker_job( &state, &request_context, - compile_session_id.clone(), - owner_user_id.clone(), - prompt_text, - primary_reference_image_src, - now, + worker_payload.clone(), + ExternalGenerationWriteLeaseGuard::inline(), ) .await - }; - let session = match session { - Ok(session) => Ok(session), - Err(error) => { - mark_puzzle_compile_failure(&error, &compile_session_id).await; - Err(puzzle_error_response( + .map_err(|error| { + puzzle_error_response( &request_context, PUZZLE_AGENT_API_BASE_PROVIDER, - error, - )) + error.into_app_error(), + ) + })?; + return Ok(json_success_body( + Some(&request_context), + PuzzleAgentActionResponse { + operation: PuzzleAgentOperationResponse { + operation_id: build_prefixed_uuid_id("extgen-inline-"), + operation_type: "compile_puzzle_draft".to_string(), + status: "completed".to_string(), + phase_label: "首关拼图草稿".to_string(), + phase_detail: if ai_redraw { + "首关草稿生成已完成。".to_string() + } else { + "首关草稿编译已完成。".to_string() + }, + progress: 100, + error: None, + }, + session: map_puzzle_agent_session_response(session), + }, + )); + } + let request_payload_json = serde_json::to_string(&worker_payload).map_err(|error| { + if let (Some(task_id), Some(claim_id)) = ( + worker_payload.background_task_id.as_deref(), + worker_payload.background_claim_id.as_deref(), + ) { + spawn_release_claimed_puzzle_background_compile_task( + state.clone(), + task_id.to_string(), + claim_id.to_string(), + compile_session_id.clone(), + owner_user_id.clone(), + ); } + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": format!("拼图生成任务参数序列化失败：{error}"), + })), + ) + })?; + let external_generation_job_id = build_prefixed_uuid_id("extgen-"); + let job = state + .spacetime_client() + .enqueue_external_generation_job(ExternalGenerationJobEnqueueRecordInput { + job_id: external_generation_job_id.clone(), + dedupe_key: format!( + "puzzle:compile_puzzle_draft:{compile_session_id}:{external_generation_job_id}" + ), + job_kind: PUZZLE_COMPILE_DRAFT_JOB_KIND.to_string(), + owner_user_id: owner_user_id.clone(), + source_module: "puzzle".to_string(), + source_entity_id: compile_session_id.clone(), + request_label: "拼图首关草稿生成".to_string(), + request_payload_json, + max_attempts: 1, + available_at_micros: now, + created_at_micros: now, + }) + .await + .map_err(|error| { + if let (Some(task_id), Some(claim_id)) = ( + worker_payload.background_task_id.as_deref(), + worker_payload.background_claim_id.as_deref(), + ) { + spawn_release_claimed_puzzle_background_compile_task( + state.clone(), + task_id.to_string(), + claim_id.to_string(), + compile_session_id.clone(), + owner_user_id.clone(), + ); + } + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + map_puzzle_client_error(error), + ) + })?; + let session = state + .spacetime_client() + .get_puzzle_agent_session(compile_session_id.clone(), owner_user_id.clone()) + .await + .map(mark_puzzle_initial_generation_started_snapshot) + .map_err(|error| { + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + map_puzzle_client_error(error), + ) + })?; + let (status, progress) = match job.status.as_str() { + "completed" => ("completed", 100), + "running" => ("running", session.progress_percent.max(10)), + "failed" => ("failed", session.progress_percent), + _ => ("queued", session.progress_percent.max(5)), }; - ( - "compile_puzzle_draft", - "首关拼图草稿", - if ai_redraw { - "已编译首关草稿，并启动首关画面和 UI 资产后台生成。" - } else { - "已编译首关草稿，并直接应用上传图片、生成 UI 背景为第一关图片。" + return Ok(json_success_body( + Some(&request_context), + PuzzleAgentActionResponse { + operation: PuzzleAgentOperationResponse { + operation_id: job.job_id, + operation_type: "compile_puzzle_draft".to_string(), + status: status.to_string(), + phase_label: "首关拼图草稿".to_string(), + phase_detail: if ai_redraw { + "首关草稿生成已进入后台队列。".to_string() + } else { + "首关草稿编译已进入后台队列。".to_string() + }, + progress, + error: job.last_error_message, + }, + session: map_puzzle_agent_session_response(session), }, - session, - ) + )); } "save_puzzle_form_draft" => { let seed_text = build_puzzle_form_seed_text_from_parts( @@ -991,367 +937,291 @@ pub async fn execute_puzzle_agent_action( payload.levels_json.as_deref(), ) .map_err(|message| { - AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ - "provider": PUZZLE_AGENT_API_BASE_PROVIDER, - "message": message, - })) - }); - let session = execute_billable_asset_operation_with_cost( - state.root_state(), - &owner_user_id, - "puzzle_generated_image", - &billing_asset_id, - PUZZLE_IMAGE_GENERATION_POINTS_COST, - async { - let levels_json = levels_json?; - let session = get_puzzle_session_for_image_generation( - &state, - session_id.clone(), - owner_user_id.clone(), - &payload, - levels_json.as_deref(), - now, - ) - .await?; - let mut draft = session.draft.clone().ok_or_else(|| { - AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ - "provider": PUZZLE_AGENT_API_BASE_PROVIDER, - "message": "拼图结果页草稿尚未生成", - })) - })?; - if let Some(levels_json) = levels_json.as_ref() { - draft.levels = parse_puzzle_level_records_from_module_json(levels_json)?; - } - let mut target_level = - select_puzzle_level_for_api(&draft, target_level_id.as_deref())?; - let fallback_level_name = target_level.level_name.clone(); - let prompt = resolve_puzzle_level_image_prompt( - payload.prompt_text.as_deref(), - &target_level.picture_description, - &draft.summary, - ); - let should_auto_name_level = payload - .should_auto_name_level - .unwrap_or_else(|| target_level.level_name.trim().is_empty()); - let mut generated_naming = if should_auto_name_level { - let naming = generate_puzzle_first_level_name( - &state, - target_level.picture_description.as_str(), - ) - .await; - target_level.level_name = naming.level_name.clone(); - target_level.ui_background_prompt = naming.ui_background_prompt.clone(); - Some(naming) - } else { - None - }; - let reference_image_sources = collect_puzzle_reference_image_sources( - payload.reference_image_src.as_deref(), - payload.reference_image_srcs.as_slice(), - payload.reference_image_asset_object_id.as_deref(), - payload.reference_image_asset_object_ids.as_slice(), - ); - let primary_reference_image_src = - reference_image_sources.first().map(String::as_str); - // 拼图结果页从多候选抽卡收口为单图替换，前端传入的旧 candidateCount 只做兼容忽略。 - let candidate_start_index = target_level.candidates.len(); - let ai_redraw = payload.ai_redraw.unwrap_or(true); - let mut candidates = if should_use_uploaded_puzzle_image_directly( - primary_reference_image_src, - ai_redraw, - ) { - vec![ - create_uploaded_puzzle_image_candidate( - &state, - owner_user_id.as_str(), - &session.session_id, - &target_level.level_name, - &prompt, - primary_reference_image_src.expect("checked reference image"), - candidate_start_index, - ) - .await?, - ] - } else { - let (_, profile_id) = build_stable_puzzle_work_ids(&session.session_id); - generate_puzzle_image_candidates( - &state, - owner_user_id.as_str(), - Some(profile_id.as_str()), - &session.session_id, - &target_level.level_name, - &prompt, - primary_reference_image_src, - ai_redraw, - payload.image_model.as_deref(), - 1, - candidate_start_index, - ) - .await - .map_err(map_puzzle_generation_endpoint_error)? - }; - if candidates.is_empty() { - return Err(AppError::from_status(StatusCode::BAD_GATEWAY).with_details( - json!({ - "provider": PUZZLE_AGENT_API_BASE_PROVIDER, - "message": "拼图候选图生成结果为空", - }), - )); - } - if let Some(refined_naming) = generate_puzzle_first_level_name_from_image( - &state, - target_level.picture_description.as_str(), - &candidates[0].downloaded_image, - ) - .await - .filter(|_| should_auto_name_level) - { - target_level.level_name = refined_naming.level_name.clone(); - if refined_naming.ui_background_prompt.is_some() { - target_level.ui_background_prompt = - refined_naming.ui_background_prompt.clone(); - } - generated_naming = Some(refined_naming); - } - let generated_level_name = target_level.level_name.clone(); - let mut updated_levels = build_puzzle_levels_with_primary_update( - &draft, - &target_level, - primary_reference_image_src, - ); - for candidate in &mut candidates { - candidate.record.prompt = prompt.clone(); - } - let selected_candidate = candidates - .iter() - .find(|candidate| candidate.record.selected) - .or_else(|| candidates.first()) - .ok_or_else(|| { - AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ - "provider": PUZZLE_AGENT_API_BASE_PROVIDER, - "message": "拼图候选图生成结果为空", - })) - })?; - let asset_bundle = generate_puzzle_level_asset_bundle_required( - &state, + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": message, + })), + ) + })?; + let worker_payload = PuzzleGenerateImagesWorkerPayload { + session_id: session_id.clone(), + owner_user_id: owner_user_id.clone(), + billing_asset_id: billing_asset_id.clone(), + level_id: target_level_id.clone(), + prompt_text: payload.prompt_text.clone(), + reference_image_src: payload.reference_image_src.clone(), + reference_image_srcs: payload.reference_image_srcs.clone(), + reference_image_asset_object_id: payload.reference_image_asset_object_id.clone(), + reference_image_asset_object_ids: payload.reference_image_asset_object_ids.clone(), + image_model: payload.image_model.clone(), + ai_redraw: payload.ai_redraw, + should_auto_name_level: payload.should_auto_name_level, + work_title: payload.work_title.clone(), + work_description: payload.work_description.clone(), + picture_description: payload.picture_description.clone(), + summary: payload.summary.clone(), + theme_tags: payload.theme_tags.clone(), + levels_json, + requested_at_micros: now, + }; + if state + .root_state() + .config + .external_generation_mode + .is_inline() + { + tracing::info!( + provider = PUZZLE_AGENT_API_BASE_PROVIDER, + session_id = %session_id, + owner_user_id = %owner_user_id, + external_generation_mode = state.root_state().config.external_generation_mode.as_str(), + "拼图关卡图片生成使用 inline 模式同步执行" + ); + let session = execute_puzzle_generate_images_worker_job( + &state, + &request_context, + worker_payload, + ExternalGenerationWriteLeaseGuard::inline(), + ) + .await + .map_err(|error| { + puzzle_error_response( &request_context, - owner_user_id.as_str(), - &session.session_id, - &target_level, - &selected_candidate.downloaded_image, + PUZZLE_AGENT_API_BASE_PROVIDER, + error.into_app_error(), ) - .await?; - attach_puzzle_level_asset_bundle( - &mut updated_levels, - target_level.level_id.as_str(), - asset_bundle, - ); - attach_selected_puzzle_candidate_to_levels( - &mut updated_levels, - target_level.level_id.as_str(), - &selected_candidate.record, - ); - let levels_json_with_generated_name = - Some(serialize_puzzle_level_records_for_module(&updated_levels)?); - let candidates_json = serde_json::to_string( - &candidates - .iter() - .map(|candidate| to_puzzle_generated_image_candidate(&candidate.record)) - .collect::>(), + })?; + return Ok(json_success_body( + Some(&request_context), + PuzzleAgentActionResponse { + operation: PuzzleAgentOperationResponse { + operation_id: build_prefixed_uuid_id("extgen-inline-"), + operation_type: "generate_puzzle_images".to_string(), + status: "completed".to_string(), + phase_label: "拼图图片生成".to_string(), + phase_detail: "关卡图片生成已完成。".to_string(), + progress: 100, + error: None, + }, + session: map_puzzle_agent_session_response(session), + }, + )); + } + let request_payload_json = serde_json::to_string(&worker_payload).map_err(|error| { + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": format!("拼图关卡图片生成任务参数序列化失败：{error}"), + })), + ) + })?; + let external_generation_job_id = build_prefixed_uuid_id("extgen-"); + let source_entity_id = target_level_id + .as_deref() + .map(|level_id| format!("{session_id}:{level_id}")) + .unwrap_or_else(|| session_id.clone()); + let job = state + .spacetime_client() + .enqueue_external_generation_job(ExternalGenerationJobEnqueueRecordInput { + job_id: external_generation_job_id.clone(), + dedupe_key: format!( + "puzzle:generate_puzzle_images:{session_id}:{external_generation_job_id}" + ), + job_kind: PUZZLE_GENERATE_IMAGES_JOB_KIND.to_string(), + owner_user_id: owner_user_id.clone(), + source_module: "puzzle".to_string(), + source_entity_id, + request_label: "拼图关卡图片生成".to_string(), + request_payload_json, + max_attempts: 1, + available_at_micros: now, + created_at_micros: now, + }) + .await + .map_err(|error| { + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + map_puzzle_client_error(error), ) - .map_err(|error| { - AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ - "provider": PUZZLE_AGENT_API_BASE_PROVIDER, - "message": format!("拼图候选图序列化失败：{error}"), - })) - })?; - let save_result = state - .spacetime_client() - .save_puzzle_generated_images(PuzzleGeneratedImagesSaveRecordInput { - session_id: session.session_id.clone(), - owner_user_id: owner_user_id.clone(), - level_id: Some(target_level.level_id.clone()), - levels_json: levels_json_with_generated_name, - candidates_json, - saved_at_micros: now, - }) - .await; - match save_result { - Ok(session) => Ok(session), - Err(error) - if should_skip_asset_operation_billing_for_connectivity(&error) => - { - // 中文注释：VectorEngine/OSS 已生成真实图片时，SpacetimeDB 短暂 503 不应让前端看不到本次图片；先返回内存合成快照，待后续操作恢复正常持久化。 - tracing::warn!( - provider = PUZZLE_AGENT_API_BASE_PROVIDER, - session_id = %session.session_id, - owner_user_id = %owner_user_id, - error = %error, - "拼图图片已生成但 SpacetimeDB 草稿回写不可用，降级返回本次生成快照" - ); - let fallback_session = - replace_puzzle_session_draft_snapshot(session, draft, now); - let fallback_session = if should_auto_name_level { - apply_generated_puzzle_first_level_name_to_session_snapshot( - fallback_session, - target_level.level_id.as_str(), - generated_level_name.as_str(), - fallback_level_name.as_str(), - now, - ) - } else { - fallback_session - }; - let mut fallback_session = - apply_generated_puzzle_candidates_to_session_snapshot( - apply_generated_puzzle_levels_to_session_snapshot( - fallback_session, - updated_levels, - now, - ), - target_level.level_id.as_str(), - candidates.into_records(), - primary_reference_image_src, - now, - ); - if let Some(generated_naming) = generated_naming.as_ref() { - fallback_session = - apply_generated_puzzle_metadata_to_session_snapshot( - fallback_session, - target_level.level_id.as_str(), - generated_naming, - fallback_level_name.as_str(), - now, - ); - } - Ok(fallback_session) - } - Err(error) => Err(map_puzzle_client_error(error)), - } + })?; + let session = state + .spacetime_client() + .get_puzzle_agent_session(session_id.clone(), owner_user_id.clone()) + .await + .map_err(|error| { + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + map_puzzle_client_error(error), + ) + })?; + let (status, progress) = match job.status.as_str() { + "completed" => ("completed", 100), + "running" => ("running", 35), + "failed" => ("failed", 0), + _ => ("queued", 8), + }; + return Ok(json_success_body( + Some(&request_context), + PuzzleAgentActionResponse { + operation: PuzzleAgentOperationResponse { + operation_id: job.job_id, + operation_type: "generate_puzzle_images".to_string(), + status: status.to_string(), + phase_label: "拼图图片生成".to_string(), + phase_detail: "关卡图片生成已进入后台队列。".to_string(), + progress, + error: job.last_error_message, + }, + session: map_puzzle_agent_session_response(session), }, - ) - .await - .map_err(|error| { - puzzle_error_response(&request_context, PUZZLE_AGENT_API_BASE_PROVIDER, error) - }); - ( - "generate_puzzle_images", - "拼图图片生成", - "已生成并替换当前拼图图片。", - session, - ) + )); } "generate_puzzle_ui_background" => { let target_level_id = payload.level_id.clone(); - let raw_prompt = payload - .prompt_text - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or_default() - .to_string(); let levels_json = normalize_puzzle_levels_json_for_module( payload.levels_json.as_deref(), ) .map_err(|message| { - AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ - "provider": PUZZLE_AGENT_API_BASE_PROVIDER, - "message": message, - })) - }); - let session = execute_billable_asset_operation_with_cost( - state.root_state(), - &owner_user_id, - "puzzle_ui_background_image", - &billing_asset_id, - PUZZLE_IMAGE_GENERATION_POINTS_COST, - async { - let levels_json = levels_json?; - let session = get_puzzle_session_for_image_generation( - &state, - session_id.clone(), - owner_user_id.clone(), - &payload, - levels_json.as_deref(), - now, - ) - .await?; - let mut draft = session.draft.clone().ok_or_else(|| { - AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ - "provider": PUZZLE_AGENT_API_BASE_PROVIDER, - "message": "拼图结果页草稿尚未生成", - })) - })?; - if let Some(levels_json) = levels_json.as_ref() { - draft.levels = parse_puzzle_level_records_from_module_json(levels_json)?; - } - let target_level = - select_puzzle_level_for_api(&draft, target_level_id.as_deref())?; - let resolved_prompt = normalize_puzzle_ui_background_prompt( - raw_prompt.as_str(), - &draft, - &target_level, - ); - let generated = generate_puzzle_ui_background_image( - &state, + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": message, + })), + ) + })?; + let worker_payload = PuzzleGenerateUiBackgroundWorkerPayload { + session_id: session_id.clone(), + owner_user_id: owner_user_id.clone(), + billing_asset_id: billing_asset_id.clone(), + level_id: target_level_id.clone(), + prompt_text: payload.prompt_text.clone(), + levels_json, + requested_at_micros: now, + }; + if state + .root_state() + .config + .external_generation_mode + .is_inline() + { + tracing::info!( + provider = PUZZLE_AGENT_API_BASE_PROVIDER, + session_id = %session_id, + owner_user_id = %owner_user_id, + external_generation_mode = state.root_state().config.external_generation_mode.as_str(), + "拼图 UI 背景图生成使用 inline 模式同步执行" + ); + let session = execute_puzzle_generate_ui_background_worker_job( + &state, + &request_context, + worker_payload, + ExternalGenerationWriteLeaseGuard::inline(), + ) + .await + .map_err(|error| { + puzzle_error_response( &request_context, - owner_user_id.as_str(), - &session.session_id, - &target_level.level_name, - resolved_prompt.as_str(), + PUZZLE_AGENT_API_BASE_PROVIDER, + error.into_app_error(), ) - .await - .map_err(map_puzzle_generation_endpoint_error)?; - let save_result = state - .spacetime_client() - .save_puzzle_ui_background(PuzzleUiBackgroundSaveRecordInput { - session_id: session.session_id.clone(), - owner_user_id: owner_user_id.clone(), - level_id: Some(target_level.level_id.clone()), - levels_json, - prompt: resolved_prompt.clone(), - image_src: generated.image_src.clone(), - image_object_key: Some(generated.object_key.clone()), - saved_at_micros: now, - }) - .await; - match save_result { - Ok(session) => Ok(session), - Err(error) - if should_skip_asset_operation_billing_for_connectivity(&error) => - { - tracing::warn!( - provider = PUZZLE_AGENT_API_BASE_PROVIDER, - session_id = %session.session_id, - owner_user_id = %owner_user_id, - error = %error, - "拼图 UI 背景图已生成但 SpacetimeDB 草稿回写不可用，降级返回本次生成快照" - ); - let fallback_session = - replace_puzzle_session_draft_snapshot(session, draft, now); - Ok(apply_generated_puzzle_ui_background_to_session_snapshot( - fallback_session, - target_level.level_id.as_str(), - resolved_prompt, - generated.image_src, - Some(generated.object_key), - now, - )) - } - Err(error) => Err(map_puzzle_client_error(error)), - } + })?; + return Ok(json_success_body( + Some(&request_context), + PuzzleAgentActionResponse { + operation: PuzzleAgentOperationResponse { + operation_id: build_prefixed_uuid_id("extgen-inline-"), + operation_type: "generate_puzzle_ui_background".to_string(), + status: "completed".to_string(), + phase_label: "UI 背景图生成".to_string(), + phase_detail: "拼图 UI 背景图生成已完成。".to_string(), + progress: 100, + error: None, + }, + session: map_puzzle_agent_session_response(session), + }, + )); + } + let request_payload_json = serde_json::to_string(&worker_payload).map_err(|error| { + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": PUZZLE_AGENT_API_BASE_PROVIDER, + "message": format!("拼图 UI 背景图生成任务参数序列化失败：{error}"), + })), + ) + })?; + let external_generation_job_id = build_prefixed_uuid_id("extgen-"); + let source_entity_id = target_level_id + .as_deref() + .map(|level_id| format!("{session_id}:{level_id}")) + .unwrap_or_else(|| session_id.clone()); + let job = state + .spacetime_client() + .enqueue_external_generation_job(ExternalGenerationJobEnqueueRecordInput { + job_id: external_generation_job_id.clone(), + dedupe_key: format!( + "puzzle:generate_puzzle_ui_background:{session_id}:{external_generation_job_id}" + ), + job_kind: PUZZLE_GENERATE_UI_BACKGROUND_JOB_KIND.to_string(), + owner_user_id: owner_user_id.clone(), + source_module: "puzzle".to_string(), + source_entity_id, + request_label: "拼图 UI 背景图生成".to_string(), + request_payload_json, + max_attempts: 1, + available_at_micros: now, + created_at_micros: now, + }) + .await + .map_err(|error| { + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + map_puzzle_client_error(error), + ) + })?; + let session = state + .spacetime_client() + .get_puzzle_agent_session(session_id.clone(), owner_user_id.clone()) + .await + .map_err(|error| { + puzzle_error_response( + &request_context, + PUZZLE_AGENT_API_BASE_PROVIDER, + map_puzzle_client_error(error), + ) + })?; + let (status, progress) = match job.status.as_str() { + "completed" => ("completed", 100), + "running" => ("running", session.progress_percent.max(55)), + "failed" => ("failed", session.progress_percent), + _ => ("queued", session.progress_percent.max(12)), + }; + return Ok(json_success_body( + Some(&request_context), + PuzzleAgentActionResponse { + operation: PuzzleAgentOperationResponse { + operation_id: job.job_id, + operation_type: "generate_puzzle_ui_background".to_string(), + status: status.to_string(), + phase_label: "UI 背景图生成".to_string(), + phase_detail: "拼图 UI 背景图生成已进入后台队列。".to_string(), + progress, + error: job.last_error_message, + }, + session: map_puzzle_agent_session_response(session), }, - ) - .await - .map_err(|error| { - puzzle_error_response(&request_context, PUZZLE_AGENT_API_BASE_PROVIDER, error) - }); - ( - "generate_puzzle_ui_background", - "UI 背景图生成", - "已生成拼图 UI 背景图。", - session, - ) + )); } "generate_puzzle_tags" => { let work_title = payload @@ -1535,27 +1405,6 @@ pub async fn execute_puzzle_agent_action( }; let session = session?; - if operation_type == "compile_puzzle_draft" - && session - .draft - .as_ref() - .is_some_and(|draft| draft.generation_status == "ready") - { - send_generation_result_subscribe_message_after_completion( - state.root_state(), - GenerationResultSubscribeMessage { - owner_user_id: owner_user_id.clone(), - task_name: Some("拼图".to_string()), - work_name: session.draft.as_ref().map(|draft| draft.work_title.clone()), - status: GenerationResultSubscribeMessageStatus::Succeeded, - consumed_points: operation_consumed_points, - completed_at_micros: current_utc_micros(), - page: Some("/pages/web-view/index".to_string()), - }, - ) - .await; - } - Ok(json_success_body( Some(&request_context), PuzzleAgentActionResponse { diff --git a/server-rs/crates/api-server/src/puzzle/tests.rs b/server-rs/crates/api-server/src/puzzle/tests.rs index 31ccf74c2..dee3d4191 100644 --- a/server-rs/crates/api-server/src/puzzle/tests.rs +++ b/server-rs/crates/api-server/src/puzzle/tests.rs @@ -535,6 +535,108 @@ fn puzzle_image_generation_fallback_session_ready_when_asset_pack_complete() { assert_eq!(session.stage, "ready_to_publish"); } +#[test] +fn puzzle_generate_images_worker_payload_keeps_action_snapshot() { + let raw_levels_json = serde_json::to_string(&vec![json!({ + "levelId": "puzzle-level-2", + "levelName": "", + "pictureDescription": "新关卡里有一座发光钟楼。", + "candidates": [], + "selectedCandidateId": null, + "coverImageSrc": null, + "coverAssetId": null, + "generationStatus": "generating", + })]) + .expect("levels json"); + let levels_json = normalize_puzzle_levels_json_for_module(Some(raw_levels_json.as_str())) + .expect("levels should normalize") + .expect("levels json should exist"); + let payload = PuzzleGenerateImagesWorkerPayload { + session_id: "puzzle-session-1".to_string(), + owner_user_id: "user-1".to_string(), + billing_asset_id: "puzzle-session-1:123".to_string(), + level_id: Some("puzzle-level-2".to_string()), + prompt_text: Some("发光钟楼".to_string()), + reference_image_src: None, + reference_image_srcs: vec!["data:image/png;base64,abc".to_string()], + reference_image_asset_object_id: Some("asset-object-1".to_string()), + reference_image_asset_object_ids: vec!["asset-object-2".to_string()], + image_model: Some(PUZZLE_IMAGE_MODEL_GPT_IMAGE_2.to_string()), + ai_redraw: Some(true), + should_auto_name_level: Some(true), + work_title: Some("暖灯猫街作品".to_string()), + work_description: Some("一套雨夜猫街主题拼图。".to_string()), + picture_description: None, + summary: Some("一套雨夜猫街主题拼图。".to_string()), + theme_tags: Some(vec!["猫咪".to_string(), "雨夜".to_string()]), + levels_json: Some(levels_json.clone()), + requested_at_micros: 123, + }; + + let encoded = serde_json::to_string(&payload).expect("payload should serialize"); + let decoded: PuzzleGenerateImagesWorkerPayload = + serde_json::from_str(encoded.as_str()).expect("payload should deserialize"); + + assert_eq!(decoded.level_id.as_deref(), Some("puzzle-level-2")); + assert_eq!(decoded.reference_image_srcs.len(), 1); + assert_eq!( + decoded.reference_image_asset_object_ids, + vec!["asset-object-2".to_string()] + ); + assert_eq!(decoded.should_auto_name_level, Some(true)); + let records = parse_puzzle_level_records_from_module_json( + decoded.levels_json.as_deref().expect("levels json"), + ) + .expect("levels should parse as module json"); + assert_eq!(records[0].level_id, "puzzle-level-2"); + assert_eq!(records[0].generation_status, "generating"); +} + +#[test] +fn puzzle_generate_ui_background_worker_payload_keeps_action_snapshot() { + let raw_levels_json = serde_json::to_string(&vec![json!({ + "levelId": "puzzle-level-3", + "levelName": "钟楼回廊", + "pictureDescription": "新关卡里有一座发光钟楼。", + "uiBackgroundPrompt": "发光钟楼延展成竖屏回廊，远处有暖色窗光。", + "candidates": [], + "selectedCandidateId": null, + "coverImageSrc": null, + "coverAssetId": null, + "generationStatus": "generating", + })]) + .expect("levels json"); + let levels_json = normalize_puzzle_levels_json_for_module(Some(raw_levels_json.as_str())) + .expect("levels should normalize") + .expect("levels json should exist"); + let payload = PuzzleGenerateUiBackgroundWorkerPayload { + session_id: "puzzle-session-1".to_string(), + owner_user_id: "user-1".to_string(), + billing_asset_id: "puzzle-session-1:456".to_string(), + level_id: Some("puzzle-level-3".to_string()), + prompt_text: Some("发光钟楼延展成竖屏回廊".to_string()), + levels_json: Some(levels_json.clone()), + requested_at_micros: 456, + }; + + let encoded = serde_json::to_string(&payload).expect("payload should serialize"); + let decoded: PuzzleGenerateUiBackgroundWorkerPayload = + serde_json::from_str(encoded.as_str()).expect("payload should deserialize"); + + assert_eq!(decoded.level_id.as_deref(), Some("puzzle-level-3")); + assert_eq!( + decoded.prompt_text.as_deref(), + Some("发光钟楼延展成竖屏回廊") + ); + assert_eq!(decoded.requested_at_micros, 456); + let records = parse_puzzle_level_records_from_module_json( + decoded.levels_json.as_deref().expect("levels json"), + ) + .expect("levels should parse as module json"); + assert_eq!(records[0].level_id, "puzzle-level-3"); + assert_eq!(records[0].generation_status, "generating"); +} + #[test] fn puzzle_first_level_name_parser_accepts_json_and_normalizes_text() { assert_eq!( diff --git a/server-rs/crates/api-server/src/puzzle_clear.rs b/server-rs/crates/api-server/src/puzzle_clear.rs index 1221df47c..8c88dd71e 100644 --- a/server-rs/crates/api-server/src/puzzle_clear.rs +++ b/server-rs/crates/api-server/src/puzzle_clear.rs @@ -11,7 +11,11 @@ use module_assets::{ generate_asset_binding_id, generate_asset_object_id, }; use platform_oss::{LegacyAssetPrefix, OssHeadObjectRequest, OssObjectAccess}; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use shared_contracts::external_generation::{ + ExternalGenerationJobStatus, ExternalGenerationJobStatusRecord, +}; use shared_contracts::puzzle_clear::{ PuzzleClearActionRequest, PuzzleClearActionType, PuzzleClearCardAsset, PuzzleClearDraftResponse, PuzzleClearGenerationStatus, PuzzleClearImageAsset, @@ -22,7 +26,9 @@ use shared_contracts::puzzle_clear::{ PuzzleClearWorkspaceCreateRequest, }; use shared_kernel::{build_prefixed_uuid_id, format_timestamp_micros}; -use spacetime_client::SpacetimeClientError; +use spacetime_client::{ + ExternalGenerationJobEnqueueRecordInput, ExternalGenerationJobRecord, SpacetimeClientError, +}; use std::{ collections::BTreeMap, time::{SystemTime, UNIX_EPOCH}, @@ -51,6 +57,7 @@ const PUZZLE_CLEAR_CREATION_PROVIDER: &str = "puzzle-clear-creation"; const PUZZLE_CLEAR_RUNTIME_PROVIDER: &str = "puzzle-clear-runtime"; const PUZZLE_CLEAR_TEMPLATE_ID: &str = "puzzle-clear"; const PUZZLE_CLEAR_TEMPLATE_NAME: &str = "拼消消"; +pub(crate) const PUZZLE_CLEAR_COMPILE_DRAFT_JOB_KIND: &str = "puzzle_clear_compile_draft"; const PUZZLE_CLEAR_RUNTIME_RUNS_ROUTE: &str = "/api/runtime/puzzle-clear/runs"; const PUZZLE_CLEAR_ATLAS_CELL_SIZE: u32 = 256; const PUZZLE_CLEAR_SHEET_COLUMNS: u32 = 4; @@ -76,6 +83,15 @@ const PUZZLE_CLEAR_SHEET_INTERNAL_SEAM_SIDE_CONTRAST_THRESHOLD: f32 = 145.0; const PUZZLE_CLEAR_SHEET_INTERNAL_SEAM_SIDE_TEXTURE_MAX: f32 = 36.0; const PUZZLE_CLEAR_ATLAS_NEGATIVE_PROMPT: &str = "文字、Logo、水印、按钮、UI 字、网格线、编号、标签、边框、外轮廓框、白色描边、白色贴纸边、圆角框、阴影框、分隔线、裁切参考线、单格内部拼接线、内部竖切、内部横切、照片拼贴、相册拼贴、多场景拼贴、双联图、三联图、画中画、单格双图、单格多图、低清晰度、纯色背景、空白背景、白底商品图、孤立主体、单体素材、素材表、图标、贴纸、同品种重复、同一物体多角度、重复同款小图、主体跨格、主体贴边、拼贴、重影、不同图案互相穿插"; +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PuzzleClearCompileDraftWorkerPayload { + pub session_id: String, + pub owner_user_id: String, + pub author_display_name: String, + pub payload: PuzzleClearActionRequest, +} + pub async fn create_puzzle_clear_session( State(state): State, Extension(request_context): Extension, @@ -160,6 +176,39 @@ pub async fn execute_puzzle_clear_action( .unwrap_or("拼消消玩家") .to_string(); let mut payload = payload; + let should_queue_generation = matches!( + payload.action_type, + PuzzleClearActionType::CompileDraft | PuzzleClearActionType::RegenerateAtlas + ) && !state.config.external_generation_mode.is_inline(); + if should_queue_generation { + let mut queued_response = state + .spacetime_client() + .mark_puzzle_clear_generation_queued( + session_id.clone(), + owner_user_id.clone(), + author_display_name.clone(), + payload.clone(), + ) + .await + .map_err(|error| { + puzzle_clear_error_response( + &request_context, + PUZZLE_CLEAR_CREATION_PROVIDER, + map_puzzle_clear_client_error(error), + ) + })?; + let queue_job = enqueue_puzzle_clear_compile_draft_job( + &state, + &request_context, + &session_id, + owner_user_id.as_str(), + author_display_name.as_str(), + payload, + ) + .await?; + queued_response.queue_state = Some(map_puzzle_clear_queue_job_status(queue_job)); + return Ok(json_success_body(Some(&request_context), queued_response)); + } if let Err(response) = maybe_prepare_puzzle_clear_assets_inner( &state, &request_context, @@ -210,6 +259,129 @@ pub async fn execute_puzzle_clear_action( Ok(json_success_body(Some(&request_context), response)) } +async fn enqueue_puzzle_clear_compile_draft_job( + state: &AppState, + request_context: &RequestContext, + session_id: &str, + owner_user_id: &str, + author_display_name: &str, + payload: PuzzleClearActionRequest, +) -> Result { + let job_id = build_prefixed_uuid_id("extgen-"); + let now_micros = current_utc_micros(); + let request_payload_json = serde_json::to_string(&PuzzleClearCompileDraftWorkerPayload { + session_id: session_id.to_string(), + owner_user_id: owner_user_id.to_string(), + author_display_name: author_display_name.to_string(), + payload, + }) + .map_err(|error| { + puzzle_clear_error_response( + request_context, + PUZZLE_CLEAR_CREATION_PROVIDER, + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "message": format!("拼消消 worker 任务参数序列化失败：{error}"), + })), + ) + })?; + state + .spacetime_client() + .enqueue_external_generation_job(ExternalGenerationJobEnqueueRecordInput { + dedupe_key: format!("puzzle-clear:compile-draft:{session_id}:{job_id}"), + job_id, + job_kind: PUZZLE_CLEAR_COMPILE_DRAFT_JOB_KIND.to_string(), + owner_user_id: owner_user_id.to_string(), + source_module: "puzzle-clear".to_string(), + source_entity_id: session_id.to_string(), + request_label: "拼消消草稿生成".to_string(), + request_payload_json, + max_attempts: 1, + available_at_micros: now_micros, + created_at_micros: now_micros, + }) + .await + .map_err(|error| { + puzzle_clear_error_response( + request_context, + PUZZLE_CLEAR_CREATION_PROVIDER, + map_puzzle_clear_client_error(error), + ) + }) +} + +fn map_puzzle_clear_queue_job_status( + job: ExternalGenerationJobRecord, +) -> ExternalGenerationJobStatusRecord { + ExternalGenerationJobStatusRecord { + operation_id: job.job_id, + status: ExternalGenerationJobStatus::Queued, + phase_label: job.request_label, + phase_detail: "排队中。".to_string(), + progress: 8, + error: job.last_error_message, + updated_at_micros: job.updated_at_micros, + } +} + +pub(crate) async fn execute_puzzle_clear_compile_draft_worker_job( + state: &AppState, + request_context: &RequestContext, + mut worker_payload: PuzzleClearCompileDraftWorkerPayload, +) -> Result { + if let Err(response) = maybe_prepare_puzzle_clear_assets_inner( + state, + request_context, + worker_payload.session_id.as_str(), + worker_payload.owner_user_id.as_str(), + &mut worker_payload.payload, + ) + .await + { + let (error_message, response) = extract_puzzle_clear_response_error_message(response).await; + tracing::warn!( + provider = PUZZLE_CLEAR_CREATION_PROVIDER, + session_id = worker_payload.session_id, + error = %error_message, + "拼消消 worker 素材生成失败，准备回写 failed 状态" + ); + if let Err(writeback_error) = state + .spacetime_client() + .mark_puzzle_clear_generation_failed( + worker_payload.session_id.clone(), + worker_payload.owner_user_id.clone(), + worker_payload.author_display_name.clone(), + worker_payload.payload.clone(), + ) + .await + { + tracing::warn!( + provider = PUZZLE_CLEAR_CREATION_PROVIDER, + session_id = worker_payload.session_id, + error = %writeback_error, + "拼消消 worker 失败状态回写失败" + ); + } + return Err(response); + } + let response = state + .spacetime_client() + .execute_puzzle_clear_action( + worker_payload.session_id, + worker_payload.owner_user_id, + worker_payload.author_display_name, + worker_payload.payload, + ) + .await + .map_err(|error| { + puzzle_clear_error_response( + request_context, + PUZZLE_CLEAR_CREATION_PROVIDER, + map_puzzle_clear_client_error(error), + ) + })?; + Ok(response.session) +} + pub async fn list_puzzle_clear_works( State(state): State, Extension(request_context): Extension, diff --git a/server-rs/crates/api-server/src/wooden_fish.rs b/server-rs/crates/api-server/src/wooden_fish.rs index a8c466682..9f583dd66 100644 --- a/server-rs/crates/api-server/src/wooden_fish.rs +++ b/server-rs/crates/api-server/src/wooden_fish.rs @@ -14,7 +14,11 @@ use module_assets::{ build_asset_object_upsert_input, generate_asset_binding_id, generate_asset_object_id, }; use platform_oss::{LegacyAssetPrefix, OssHeadObjectRequest, OssObjectAccess}; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use shared_contracts::external_generation::{ + ExternalGenerationJobStatus, ExternalGenerationJobStatusRecord, +}; use shared_contracts::wooden_fish::{ WoodenFishActionRequest, WoodenFishAudioAsset, WoodenFishCheckpointRunRequest, WoodenFishDraftResponse, WoodenFishFinishRunRequest, WoodenFishGalleryDetailResponse, @@ -24,7 +28,9 @@ use shared_contracts::wooden_fish::{ WoodenFishWorkspaceCreateRequest, }; use shared_kernel::{build_prefixed_uuid_id, format_timestamp_micros}; -use spacetime_client::SpacetimeClientError; +use spacetime_client::{ + ExternalGenerationJobEnqueueRecordInput, ExternalGenerationJobRecord, SpacetimeClientError, +}; use crate::generated_image_assets::{ GeneratedImageAssetAdapter, GeneratedImageAssetDataUrl, @@ -54,6 +60,8 @@ const WOODEN_FISH_CREATION_PROVIDER: &str = "wooden-fish-creation"; const WOODEN_FISH_RUNTIME_PROVIDER: &str = "wooden-fish-runtime"; const WOODEN_FISH_TEMPLATE_ID: &str = "wooden-fish"; const WOODEN_FISH_TEMPLATE_NAME: &str = "敲木鱼"; +pub(crate) const WOODEN_FISH_GENERATE_IMAGE_ASSETS_JOB_KIND: &str = + "wooden_fish_generate_image_assets"; const DEFAULT_HIT_OBJECT_PROMPT: &str = "默认敲击物图案，圆润木质质感，透明背景"; const DEFAULT_HIT_OBJECT_ASSET_ID: &str = "wooden-fish-default-hit-object"; const DEFAULT_HIT_OBJECT_IMAGE_SRC: &str = "/wooden-fish/default-hit-object.png"; @@ -73,6 +81,15 @@ const DEFAULT_HIT_OBJECT_REFERENCE_BYTES: &[u8] = include_bytes!(concat!( )); const WOODEN_FISH_AUTHOR_FALLBACK_DISPLAY_NAME: &str = "玩家"; +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct WoodenFishGenerateImageAssetsWorkerPayload { + pub session_id: String, + pub owner_user_id: String, + pub author_display_name: String, + pub payload: WoodenFishActionRequest, +} + pub async fn create_wooden_fish_session( State(state): State, Extension(request_context): Extension, @@ -155,6 +172,40 @@ pub async fn execute_wooden_fish_action( payload.action_type, shared_contracts::wooden_fish::WoodenFishActionType::CompileDraft ); + let should_queue_generation = matches!( + payload.action_type, + shared_contracts::wooden_fish::WoodenFishActionType::CompileDraft + | shared_contracts::wooden_fish::WoodenFishActionType::RegenerateHitObject + ) && !state.config.external_generation_mode.is_inline(); + if should_queue_generation { + let mut queued_response = state + .spacetime_client() + .mark_wooden_fish_generation_queued( + session_id.clone(), + owner_user_id.clone(), + author_display_name.clone(), + payload.clone(), + ) + .await + .map_err(|error| { + wooden_fish_error_response( + &request_context, + WOODEN_FISH_CREATION_PROVIDER, + map_wooden_fish_client_error(error), + ) + })?; + let queue_job = enqueue_wooden_fish_generate_image_assets_job( + &state, + &request_context, + &session_id, + owner_user_id.as_str(), + author_display_name.as_str(), + payload, + ) + .await?; + queued_response.queue_state = Some(map_wooden_fish_queue_job_status(queue_job)); + return Ok(json_success_body(Some(&request_context), queued_response)); + } let generation_points_cost = if is_compile_draft { resolve_wooden_fish_generation_points_cost(&state).await } else { @@ -226,6 +277,70 @@ pub async fn execute_wooden_fish_action( Ok(json_success_body(Some(&request_context), response)) } +async fn enqueue_wooden_fish_generate_image_assets_job( + state: &AppState, + request_context: &RequestContext, + session_id: &str, + owner_user_id: &str, + author_display_name: &str, + payload: WoodenFishActionRequest, +) -> Result { + let job_id = build_prefixed_uuid_id("extgen-"); + let now_micros = current_utc_micros(); + let request_payload_json = serde_json::to_string(&WoodenFishGenerateImageAssetsWorkerPayload { + session_id: session_id.to_string(), + owner_user_id: owner_user_id.to_string(), + author_display_name: author_display_name.to_string(), + payload, + }) + .map_err(|error| { + wooden_fish_error_response( + request_context, + WOODEN_FISH_CREATION_PROVIDER, + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "message": format!("敲木鱼 worker 任务参数序列化失败：{error}"), + })), + ) + })?; + state + .spacetime_client() + .enqueue_external_generation_job(ExternalGenerationJobEnqueueRecordInput { + dedupe_key: format!("wooden-fish:generate-image-assets:{session_id}:{job_id}"), + job_id, + job_kind: WOODEN_FISH_GENERATE_IMAGE_ASSETS_JOB_KIND.to_string(), + owner_user_id: owner_user_id.to_string(), + source_module: "wooden-fish".to_string(), + source_entity_id: session_id.to_string(), + request_label: "敲木鱼图片素材生成".to_string(), + request_payload_json, + max_attempts: 1, + available_at_micros: now_micros, + created_at_micros: now_micros, + }) + .await + .map_err(|error| { + wooden_fish_error_response( + request_context, + WOODEN_FISH_CREATION_PROVIDER, + map_wooden_fish_client_error(error), + ) + }) +} + +fn map_wooden_fish_queue_job_status( + job: ExternalGenerationJobRecord, +) -> ExternalGenerationJobStatusRecord { + ExternalGenerationJobStatusRecord { + operation_id: job.job_id, + status: ExternalGenerationJobStatus::Queued, + phase_label: job.request_label, + phase_detail: "排队中。".to_string(), + progress: 8, + error: job.last_error_message, + updated_at_micros: job.updated_at_micros, + } +} + pub async fn publish_wooden_fish_work( State(state): State, Path(profile_id): Path, @@ -635,6 +750,40 @@ async fn execute_wooden_fish_action_with_generated_assets( }) } +pub(crate) async fn execute_wooden_fish_generate_image_assets_worker_job( + state: &AppState, + request_context: &RequestContext, + mut worker_payload: WoodenFishGenerateImageAssetsWorkerPayload, +) -> Result { + let result = execute_wooden_fish_action_with_generated_assets( + state, + request_context, + worker_payload.session_id.as_str(), + worker_payload.owner_user_id.as_str(), + worker_payload.author_display_name.as_str(), + &mut worker_payload.payload, + ) + .await; + if result.as_ref().err().is_some_and(|response| { + response.status().is_server_error() + && matches!( + worker_payload.payload.action_type, + shared_contracts::wooden_fish::WoodenFishActionType::CompileDraft + ) + }) { + mark_wooden_fish_generation_failed( + state, + request_context, + worker_payload.session_id.as_str(), + worker_payload.owner_user_id.as_str(), + worker_payload.author_display_name.as_str(), + ) + .await; + } + let response = result?; + Ok(response.session) +} + async fn resolve_wooden_fish_generation_points_cost(state: &AppState) -> u64 { crate::creation_entry_config::resolve_creation_entry_mud_point_cost( state, diff --git a/server-rs/crates/module-puzzle/src/commands.rs b/server-rs/crates/module-puzzle/src/commands.rs index 994ecd9e8..e38c60e47 100644 --- a/server-rs/crates/module-puzzle/src/commands.rs +++ b/server-rs/crates/module-puzzle/src/commands.rs @@ -66,6 +66,9 @@ pub struct PuzzleDraftCompileInput { pub session_id: String, pub owner_user_id: String, pub compiled_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } #[cfg_attr(feature = "spacetime-types", derive(SpacetimeType))] @@ -94,6 +97,23 @@ pub struct PuzzleDraftCompileFailureInput { pub owner_user_id: String, pub error_message: String, pub failed_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, +} + +#[cfg_attr(feature = "spacetime-types", derive(SpacetimeType))] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PuzzleLevelGenerationFailureInput { + pub session_id: String, + pub owner_user_id: String, + pub level_id: Option, + pub levels_json: Option, + pub error_message: String, + pub failed_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } #[cfg_attr(feature = "spacetime-types", derive(SpacetimeType))] @@ -105,6 +125,9 @@ pub struct PuzzleGeneratedImagesSaveInput { pub levels_json: Option, pub candidates_json: String, pub saved_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } #[cfg_attr(feature = "spacetime-types", derive(SpacetimeType))] @@ -118,6 +141,9 @@ pub struct PuzzleUiBackgroundSaveInput { pub image_src: String, pub image_object_key: Option, pub saved_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } #[cfg_attr(feature = "spacetime-types", derive(SpacetimeType))] diff --git a/server-rs/crates/shared-contracts/src/external_generation.rs b/server-rs/crates/shared-contracts/src/external_generation.rs new file mode 100644 index 000000000..86de93771 --- /dev/null +++ b/server-rs/crates/shared-contracts/src/external_generation.rs @@ -0,0 +1,42 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ExternalGenerationJobStatus { + Queued, + Running, + Completed, + Failed, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExternalGenerationQueueOverview { + pub pending_count: u32, + pub running_count: u32, + pub updated_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExternalGenerationQueueOverviewResponse { + pub overview: ExternalGenerationQueueOverview, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExternalGenerationJobStatusRecord { + pub operation_id: String, + pub status: ExternalGenerationJobStatus, + pub phase_label: String, + pub phase_detail: String, + pub progress: u8, + pub error: Option, + pub updated_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExternalGenerationJobStatusResponse { + pub job: ExternalGenerationJobStatusRecord, +} diff --git a/server-rs/crates/shared-contracts/src/jump_hop.rs b/server-rs/crates/shared-contracts/src/jump_hop.rs index 3bc629118..d750be624 100644 --- a/server-rs/crates/shared-contracts/src/jump_hop.rs +++ b/server-rs/crates/shared-contracts/src/jump_hop.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Serialize}; +use crate::external_generation::ExternalGenerationJobStatusRecord; + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub enum JumpHopDifficulty { @@ -311,6 +313,8 @@ pub struct JumpHopActionResponse { pub session: JumpHopSessionSnapshotResponse, #[serde(default)] pub work: Option, + #[serde(default)] + pub queue_state: Option, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] diff --git a/server-rs/crates/shared-contracts/src/lib.rs b/server-rs/crates/shared-contracts/src/lib.rs index 4faea2dac..8f21eacb1 100644 --- a/server-rs/crates/shared-contracts/src/lib.rs +++ b/server-rs/crates/shared-contracts/src/lib.rs @@ -11,6 +11,7 @@ pub mod creation_agent_document_input; pub mod creation_audio; pub mod creation_entry_config; pub mod creative_agent; +pub mod external_generation; pub mod hyper3d; pub mod jump_hop; pub mod llm; diff --git a/server-rs/crates/shared-contracts/src/puzzle_clear.rs b/server-rs/crates/shared-contracts/src/puzzle_clear.rs index 9d2af4f28..615fc5aed 100644 --- a/server-rs/crates/shared-contracts/src/puzzle_clear.rs +++ b/server-rs/crates/shared-contracts/src/puzzle_clear.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Serialize}; +use crate::external_generation::ExternalGenerationJobStatusRecord; + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum PuzzleClearGenerationStatus { @@ -141,6 +143,8 @@ pub struct PuzzleClearActionResponse { pub action_type: PuzzleClearActionType, pub session: PuzzleClearSessionSnapshotResponse, pub work: Option, + #[serde(default)] + pub queue_state: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/server-rs/crates/shared-contracts/src/wooden_fish.rs b/server-rs/crates/shared-contracts/src/wooden_fish.rs index 422ea6503..342d89ca8 100644 --- a/server-rs/crates/shared-contracts/src/wooden_fish.rs +++ b/server-rs/crates/shared-contracts/src/wooden_fish.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Serialize}; +use crate::external_generation::ExternalGenerationJobStatusRecord; + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub enum WoodenFishGenerationStatus { @@ -164,6 +166,8 @@ pub struct WoodenFishActionResponse { pub session: WoodenFishSessionSnapshotResponse, #[serde(default)] pub work: Option, + #[serde(default)] + pub queue_state: Option, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] diff --git a/server-rs/crates/spacetime-client/src/bark_battle.rs b/server-rs/crates/spacetime-client/src/bark_battle.rs index 994826842..22e0e71ce 100644 --- a/server-rs/crates/spacetime-client/src/bark_battle.rs +++ b/server-rs/crates/spacetime-client/src/bark_battle.rs @@ -104,9 +104,7 @@ impl SpacetimeClient { if result.ok { Ok(()) } else { - Err(SpacetimeClientError::procedure_failed( - result.error_message, - )) + Err(SpacetimeClientError::procedure_failed(result.error_message)) } }); send_once(&sender, mapped); diff --git a/server-rs/crates/spacetime-client/src/external_generation.rs b/server-rs/crates/spacetime-client/src/external_generation.rs new file mode 100644 index 000000000..7c2635ded --- /dev/null +++ b/server-rs/crates/spacetime-client/src/external_generation.rs @@ -0,0 +1,173 @@ +use super::*; +use crate::mapper::*; + +impl SpacetimeClient { + pub async fn enqueue_external_generation_job( + &self, + input: ExternalGenerationJobEnqueueRecordInput, + ) -> Result { + let procedure_input = input.into(); + + self.call_after_connect( + "enqueue_external_generation_job_and_return", + move |connection, sender| { + connection + .procedures() + .enqueue_external_generation_job_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_job_procedure_result); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + + pub async fn claim_external_generation_jobs( + &self, + input: ExternalGenerationJobClaimRecordInput, + ) -> Result, SpacetimeClientError> { + let procedure_input = input.into(); + + self.call_after_connect( + "claim_external_generation_jobs_and_return", + move |connection, sender| { + connection + .procedures() + .claim_external_generation_jobs_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_job_claim_result); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + + pub async fn complete_external_generation_job( + &self, + input: ExternalGenerationJobCompleteRecordInput, + ) -> Result { + let procedure_input = input.into(); + + self.call_after_connect( + "complete_external_generation_job_and_return", + move |connection, sender| { + connection + .procedures() + .complete_external_generation_job_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_job_procedure_result); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + + pub async fn renew_external_generation_job_lease( + &self, + input: ExternalGenerationJobRenewLeaseRecordInput, + ) -> Result { + let procedure_input = input.into(); + + self.call_after_connect( + "renew_external_generation_job_lease_and_return", + move |connection, sender| { + connection + .procedures() + .renew_external_generation_job_lease_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_job_procedure_result); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + + pub async fn fail_external_generation_job( + &self, + input: ExternalGenerationJobFailRecordInput, + ) -> Result { + let procedure_input = input.into(); + + self.call_after_connect( + "fail_external_generation_job_and_return", + move |connection, sender| { + connection + .procedures() + .fail_external_generation_job_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_job_procedure_result); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + + pub async fn get_external_generation_job( + &self, + input: ExternalGenerationJobGetRecordInput, + ) -> Result { + let procedure_input = input.into(); + + self.call_after_connect( + "get_external_generation_job_and_return", + move |connection, sender| { + connection + .procedures() + .get_external_generation_job_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_job_procedure_result); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + + pub async fn get_external_generation_queue_stats( + &self, + ) -> Result { + self.call_after_connect( + "get_external_generation_queue_stats_and_return", + move |connection, sender| { + connection + .procedures() + .get_external_generation_queue_stats_and_return_then(move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_external_generation_queue_stats_result); + send_once(&sender, mapped); + }); + }, + ) + .await + } +} diff --git a/server-rs/crates/spacetime-client/src/jump_hop.rs b/server-rs/crates/spacetime-client/src/jump_hop.rs index 6274315c6..7d9855932 100644 --- a/server-rs/crates/spacetime-client/src/jump_hop.rs +++ b/server-rs/crates/spacetime-client/src/jump_hop.rs @@ -113,6 +113,55 @@ impl SpacetimeClient { action_type: payload.action_type, session, work, + queue_state: None, + }) + } + + pub async fn mark_jump_hop_generation_queued( + &self, + session_id: String, + owner_user_id: String, + payload: JumpHopActionRequest, + ) -> Result { + let current = self + .get_jump_hop_session(session_id.clone(), owner_user_id.clone()) + .await?; + let action_type = payload.action_type.clone(); + let scope = match action_type { + JumpHopActionType::CompileDraft => JumpHopDraftMergeScope::CompileDraft, + JumpHopActionType::RegenerateTiles => JumpHopDraftMergeScope::RegenerateTiles, + _ => { + return Err(SpacetimeClientError::validation_failed( + "jump-hop queued generation 只支持 compile-draft/regenerate-tiles", + )); + } + }; + let mut base_draft = current.draft.clone(); + if matches!(action_type, JumpHopActionType::RegenerateTiles) + && let Some(draft) = base_draft.as_mut() + { + draft.tile_atlas_asset = None; + draft.tile_assets.clear(); + } + let mut draft = merge_action_into_draft(base_draft, &payload, scope)?; + let profile_id = resolve_jump_hop_profile_id(&draft, &action_type)?; + draft.profile_id = Some(profile_id.clone()); + draft.generation_status = JumpHopGenerationStatus::Generating; + let session = self + .compile_jump_hop_draft(build_generating_compile_input( + ¤t, + &owner_user_id, + &profile_id, + &draft, + current_unix_micros(), + )?) + .await?; + + Ok(JumpHopActionResponse { + action_type, + session, + work: None, + queue_state: None, }) } @@ -233,15 +282,14 @@ impl SpacetimeClient { }; self.call_after_connect("delete_jump_hop_work", move |connection, sender| { - connection.procedures().delete_jump_hop_work_then( - procedure_input, - move |_, result| { + connection + .procedures() + .delete_jump_hop_work_then(procedure_input, move |_, result| { let mapped = result .map_err(SpacetimeClientError::from_sdk_error) .and_then(map_jump_hop_works_procedure_result); send_once(&sender, mapped); - }, - ); + }); }) .await } @@ -805,6 +853,50 @@ fn build_compile_input( }) } +fn build_generating_compile_input( + current: &JumpHopSessionSnapshotResponse, + owner_user_id: &str, + profile_id: &str, + draft: &JumpHopDraftResponse, + now_micros: i64, +) -> Result { + Ok(JumpHopDraftCompileInput { + session_id: current.session_id.clone(), + owner_user_id: owner_user_id.to_string(), + profile_id: profile_id.to_string(), + author_display_name: "跳一跳玩家".to_string(), + seed_text: draft.work_title.clone(), + work_title: draft.work_title.clone(), + work_description: draft.work_description.clone(), + theme_tags_json: Some(json_string(&draft.theme_tags)?), + theme_text: Some(draft.theme_text.clone()), + difficulty: Some(difficulty_to_str(&draft.difficulty).to_string()), + style_preset: Some(style_to_str(&draft.style_preset).to_string()), + character_prompt: Some(draft.character_prompt.clone()), + tile_prompt: Some(draft.tile_prompt.clone()), + end_mood_prompt: draft.end_mood_prompt.clone(), + character_asset_json: draft + .character_asset + .as_ref() + .map(json_string) + .transpose()?, + tile_atlas_asset_json: draft + .tile_atlas_asset + .as_ref() + .map(json_string) + .transpose()?, + tile_assets_json: Some(json_string(&draft.tile_assets)?), + cover_composite: draft.cover_composite.clone(), + back_button_asset_json: draft + .back_button_asset + .as_ref() + .map(json_string) + .transpose()?, + generation_status: Some("generating".to_string()), + compiled_at_micros: now_micros, + }) +} + fn build_update_input( owner_user_id: &str, profile_id: &str, diff --git a/server-rs/crates/spacetime-client/src/lib.rs b/server-rs/crates/spacetime-client/src/lib.rs index fc4cd6072..4cf5cf16f 100644 --- a/server-rs/crates/spacetime-client/src/lib.rs +++ b/server-rs/crates/spacetime-client/src/lib.rs @@ -30,12 +30,16 @@ pub use mapper::{ CustomWorldPublishGateRecord, CustomWorldPublishWorldRecord, CustomWorldPublishWorldRecordInput, CustomWorldPublishedProfileCompileRecord, CustomWorldResultPreviewBlockerRecord, CustomWorldSupportedActionRecord, - CustomWorldWorkSummaryRecord, JumpHopActionRequest, JumpHopActionResponse, JumpHopActionType, - JumpHopCharacterAsset, JumpHopDifficulty, JumpHopDraftResponse, JumpHopGalleryCardResponse, - JumpHopGalleryDetailResponse, JumpHopGalleryResponse, JumpHopGenerationStatus, - JumpHopJumpRequest, JumpHopJumpResponse, JumpHopJumpResult, JumpHopLastJump, JumpHopPath, - JumpHopPlatform, JumpHopRestartRunRequest, JumpHopRunResponse, JumpHopRunStatus, - JumpHopRuntimeRunSnapshotResponse, JumpHopScoring, JumpHopSessionResponse, + CustomWorldWorkSummaryRecord, ExternalGenerationJobClaimRecordInput, + ExternalGenerationJobCompleteRecordInput, ExternalGenerationJobEnqueueRecordInput, + ExternalGenerationJobFailRecordInput, ExternalGenerationJobGetRecordInput, + ExternalGenerationJobRecord, ExternalGenerationJobRenewLeaseRecordInput, + ExternalGenerationQueueStatsRecord, JumpHopActionRequest, JumpHopActionResponse, + JumpHopActionType, JumpHopCharacterAsset, JumpHopDifficulty, JumpHopDraftResponse, + JumpHopGalleryCardResponse, JumpHopGalleryDetailResponse, JumpHopGalleryResponse, + JumpHopGenerationStatus, JumpHopJumpRequest, JumpHopJumpResponse, JumpHopJumpResult, + JumpHopLastJump, JumpHopPath, JumpHopPlatform, JumpHopRestartRunRequest, JumpHopRunResponse, + JumpHopRunStatus, JumpHopRuntimeRunSnapshotResponse, JumpHopScoring, JumpHopSessionResponse, JumpHopSessionSnapshotResponse, JumpHopStartRunRequest, JumpHopStylePreset, JumpHopTileAsset, JumpHopTileType, JumpHopWorkDetailResponse, JumpHopWorkMutationResponse, JumpHopWorkProfileResponse, JumpHopWorkSummaryResponse, JumpHopWorksResponse, @@ -65,12 +69,13 @@ pub use mapper::{ PuzzleDraftCompileFailureRecordInput, PuzzleDraftLevelRecord, PuzzleFormDraftRecord, PuzzleFormDraftSaveRecordInput, PuzzleGalleryCardRecord, PuzzleGeneratedImageCandidateRecord, PuzzleGeneratedImagesSaveRecordInput, PuzzleLeaderboardEntryRecord, - PuzzleLeaderboardSubmitRecordInput, PuzzleMergedGroupRecord, PuzzlePieceStateRecord, - PuzzlePublishRecordInput, PuzzleRecommendedNextWorkRecord, PuzzleResultDraftRecord, - PuzzleResultPreviewBlockerRecord, PuzzleResultPreviewFindingRecord, PuzzleResultPreviewRecord, - PuzzleRunDragRecordInput, PuzzleRunNextLevelRecordInput, PuzzleRunPauseRecordInput, - PuzzleRunPropRecordInput, PuzzleRunRecord, PuzzleRunStartRecordInput, PuzzleRunSwapRecordInput, - PuzzleRuntimeLevelRecord, PuzzleSelectCoverImageRecordInput, PuzzleUiBackgroundSaveRecordInput, + PuzzleLeaderboardSubmitRecordInput, PuzzleLevelGenerationFailureRecordInput, + PuzzleMergedGroupRecord, PuzzlePieceStateRecord, PuzzlePublishRecordInput, + PuzzleRecommendedNextWorkRecord, PuzzleResultDraftRecord, PuzzleResultPreviewBlockerRecord, + PuzzleResultPreviewFindingRecord, PuzzleResultPreviewRecord, PuzzleRunDragRecordInput, + PuzzleRunNextLevelRecordInput, PuzzleRunPauseRecordInput, PuzzleRunPropRecordInput, + PuzzleRunRecord, PuzzleRunStartRecordInput, PuzzleRunSwapRecordInput, PuzzleRuntimeLevelRecord, + PuzzleSelectCoverImageRecordInput, PuzzleUiBackgroundSaveRecordInput, PuzzleWorkLikeReportRecordInput, PuzzleWorkPointIncentiveClaimRecordInput, PuzzleWorkProfileRecord, PuzzleWorkRemixRecordInput, PuzzleWorkUpsertRecordInput, ResolveCombatActionRecord, ResolveNpcBattleInteractionInput, @@ -112,6 +117,7 @@ pub use bark_battle::{ pub mod big_fish; pub mod combat; pub mod custom_world; +pub mod external_generation; pub mod inventory; pub mod jump_hop; pub mod match3d; diff --git a/server-rs/crates/spacetime-client/src/mapper.rs b/server-rs/crates/spacetime-client/src/mapper.rs index 6ba49d7d3..1e6fae4f2 100644 --- a/server-rs/crates/spacetime-client/src/mapper.rs +++ b/server-rs/crates/spacetime-client/src/mapper.rs @@ -8,6 +8,7 @@ mod big_fish; mod combat; mod common; mod custom_world; +mod external_generation; mod inventory; mod jump_hop; mod match3d; @@ -68,6 +69,12 @@ pub use self::common::{ VisualNovelRunSnapshotRecordInput, VisualNovelRunStartRecordInput, VisualNovelWorkCompileRecordInput, }; +pub use self::external_generation::{ + ExternalGenerationJobClaimRecordInput, ExternalGenerationJobCompleteRecordInput, + ExternalGenerationJobEnqueueRecordInput, ExternalGenerationJobFailRecordInput, + ExternalGenerationJobGetRecordInput, ExternalGenerationJobRecord, + ExternalGenerationJobRenewLeaseRecordInput, ExternalGenerationQueueStatsRecord, +}; pub use self::jump_hop::{ JumpHopActionRequest, JumpHopActionResponse, JumpHopActionType, JumpHopCharacterAsset, JumpHopDifficulty, JumpHopDraftResponse, JumpHopGalleryCardResponse, @@ -106,13 +113,13 @@ pub use self::puzzle::{ PuzzleCreatorIntentRecord, PuzzleDraftCompileFailureRecordInput, PuzzleDraftLevelRecord, PuzzleFormDraftRecord, PuzzleFormDraftSaveRecordInput, PuzzleGalleryCardRecord, PuzzleGeneratedImageCandidateRecord, PuzzleGeneratedImagesSaveRecordInput, - PuzzleLeaderboardEntryRecord, PuzzleLeaderboardSubmitRecordInput, PuzzleMergedGroupRecord, - PuzzlePieceStateRecord, PuzzlePublishRecordInput, PuzzleRecommendedNextWorkRecord, - PuzzleResultDraftRecord, PuzzleResultPreviewBlockerRecord, PuzzleResultPreviewFindingRecord, - PuzzleResultPreviewRecord, PuzzleRunDragRecordInput, PuzzleRunNextLevelRecordInput, - PuzzleRunPauseRecordInput, PuzzleRunPropRecordInput, PuzzleRunRecord, - PuzzleRunStartRecordInput, PuzzleRunSwapRecordInput, PuzzleRuntimeLevelRecord, - PuzzleSelectCoverImageRecordInput, PuzzleUiBackgroundSaveRecordInput, + PuzzleLeaderboardEntryRecord, PuzzleLeaderboardSubmitRecordInput, + PuzzleLevelGenerationFailureRecordInput, PuzzleMergedGroupRecord, PuzzlePieceStateRecord, + PuzzlePublishRecordInput, PuzzleRecommendedNextWorkRecord, PuzzleResultDraftRecord, + PuzzleResultPreviewBlockerRecord, PuzzleResultPreviewFindingRecord, PuzzleResultPreviewRecord, + PuzzleRunDragRecordInput, PuzzleRunNextLevelRecordInput, PuzzleRunPauseRecordInput, + PuzzleRunPropRecordInput, PuzzleRunRecord, PuzzleRunStartRecordInput, PuzzleRunSwapRecordInput, + PuzzleRuntimeLevelRecord, PuzzleSelectCoverImageRecordInput, PuzzleUiBackgroundSaveRecordInput, PuzzleWorkLikeReportRecordInput, PuzzleWorkPointIncentiveClaimRecordInput, PuzzleWorkProfileRecord, PuzzleWorkRemixRecordInput, PuzzleWorkUpsertRecordInput, }; @@ -178,6 +185,10 @@ pub(crate) use self::custom_world::{ parse_rpg_agent_operation_status_record, parse_rpg_agent_operation_type_record, parse_rpg_agent_stage_record, }; +pub(crate) use self::external_generation::{ + map_external_generation_job_claim_result, map_external_generation_job_procedure_result, + map_external_generation_queue_stats_result, +}; pub(crate) use self::inventory::{ map_runtime_inventory_state_procedure_result, map_runtime_item_reward_item_snapshot, map_runtime_item_reward_item_snapshot_back, diff --git a/server-rs/crates/spacetime-client/src/mapper/external_generation.rs b/server-rs/crates/spacetime-client/src/mapper/external_generation.rs new file mode 100644 index 000000000..9e3729835 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/mapper/external_generation.rs @@ -0,0 +1,255 @@ +use super::*; + +impl From for ExternalGenerationJobEnqueueInput { + fn from(input: ExternalGenerationJobEnqueueRecordInput) -> Self { + Self { + job_id: input.job_id, + dedupe_key: input.dedupe_key, + job_kind: input.job_kind, + owner_user_id: input.owner_user_id, + source_module: input.source_module, + source_entity_id: input.source_entity_id, + request_label: input.request_label, + request_payload_json: input.request_payload_json, + max_attempts: input.max_attempts, + available_at_micros: input.available_at_micros, + created_at_micros: input.created_at_micros, + } + } +} + +impl From for ExternalGenerationJobClaimInput { + fn from(input: ExternalGenerationJobClaimRecordInput) -> Self { + Self { + worker_id: input.worker_id, + limit: input.limit, + lease_expires_at_micros: input.lease_expires_at_micros, + claimed_at_micros: input.claimed_at_micros, + } + } +} + +impl From for ExternalGenerationJobCompleteInput { + fn from(input: ExternalGenerationJobCompleteRecordInput) -> Self { + Self { + job_id: input.job_id, + worker_id: input.worker_id, + lease_token: input.lease_token, + result_payload_json: input.result_payload_json, + completed_at_micros: input.completed_at_micros, + } + } +} + +impl From for ExternalGenerationJobRenewLeaseInput { + fn from(input: ExternalGenerationJobRenewLeaseRecordInput) -> Self { + Self { + job_id: input.job_id, + worker_id: input.worker_id, + lease_token: input.lease_token, + lease_expires_at_micros: input.lease_expires_at_micros, + renewed_at_micros: input.renewed_at_micros, + } + } +} + +impl From for ExternalGenerationJobFailInput { + fn from(input: ExternalGenerationJobFailRecordInput) -> Self { + Self { + job_id: input.job_id, + worker_id: input.worker_id, + lease_token: input.lease_token, + error_message: input.error_message, + retry_after_micros: input.retry_after_micros, + failed_at_micros: input.failed_at_micros, + } + } +} + +impl From for ExternalGenerationJobGetInput { + fn from(input: ExternalGenerationJobGetRecordInput) -> Self { + Self { + job_id: input.job_id, + owner_user_id: input.owner_user_id, + } + } +} + +pub(crate) fn map_external_generation_job_procedure_result( + result: ExternalGenerationJobProcedureResult, +) -> Result { + if !result.ok { + return Err(SpacetimeClientError::procedure_failed(result.error_message)); + } + + let job = result + .job + .ok_or_else(|| SpacetimeClientError::missing_snapshot("external_generation_job 快照"))?; + + Ok(map_external_generation_job_snapshot(job)) +} + +pub(crate) fn map_external_generation_job_claim_result( + result: ExternalGenerationJobProcedureResult, +) -> Result, SpacetimeClientError> { + if !result.ok { + return Err(SpacetimeClientError::procedure_failed(result.error_message)); + } + + Ok(result + .jobs + .into_iter() + .map(map_external_generation_job_snapshot) + .collect()) +} + +pub(crate) fn map_external_generation_queue_stats_result( + result: ExternalGenerationQueueStatsProcedureResult, +) -> Result { + if !result.ok { + return Err(SpacetimeClientError::procedure_failed(result.error_message)); + } + + let stats = result.stats.ok_or_else(|| { + SpacetimeClientError::missing_snapshot("external_generation queue stats 快照") + })?; + + Ok(ExternalGenerationQueueStatsRecord { + pending_count: stats.pending_count, + delayed_pending_count: stats.delayed_pending_count, + claimable_pending_count: stats.claimable_pending_count, + running_active_count: stats.running_active_count, + expired_running_count: stats.expired_running_count, + terminal_count: stats.terminal_count, + claimable_count: stats.claimable_count, + oldest_claimable_age_micros: stats.oldest_claimable_age_micros, + now_micros: stats.now_micros, + }) +} + +fn map_external_generation_job_snapshot( + snapshot: ExternalGenerationJobSnapshot, +) -> ExternalGenerationJobRecord { + ExternalGenerationJobRecord { + job_id: snapshot.job_id, + dedupe_key: snapshot.dedupe_key, + job_kind: snapshot.job_kind, + owner_user_id: snapshot.owner_user_id, + source_module: snapshot.source_module, + source_entity_id: snapshot.source_entity_id, + request_label: snapshot.request_label, + request_payload_json: snapshot.request_payload_json, + status: snapshot.status, + attempt: snapshot.attempt, + max_attempts: snapshot.max_attempts, + last_error_message: snapshot.last_error_message, + worker_id: snapshot.worker_id, + lease_expires_at: snapshot + .lease_expires_at_micros + .map(format_timestamp_micros), + available_at: format_timestamp_micros(snapshot.available_at_micros), + result_payload_json: snapshot.result_payload_json, + created_at: format_timestamp_micros(snapshot.created_at_micros), + started_at: snapshot.started_at_micros.map(format_timestamp_micros), + completed_at: snapshot.completed_at_micros.map(format_timestamp_micros), + updated_at: format_timestamp_micros(snapshot.updated_at_micros), + updated_at_micros: snapshot.updated_at_micros, + lease_token: snapshot.lease_token, + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationJobEnqueueRecordInput { + pub job_id: String, + pub dedupe_key: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_payload_json: String, + pub max_attempts: u32, + pub available_at_micros: i64, + pub created_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationJobClaimRecordInput { + pub worker_id: String, + pub limit: u32, + pub lease_expires_at_micros: i64, + pub claimed_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationJobCompleteRecordInput { + pub job_id: String, + pub worker_id: String, + pub lease_token: String, + pub result_payload_json: Option, + pub completed_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationJobRenewLeaseRecordInput { + pub job_id: String, + pub worker_id: String, + pub lease_token: String, + pub lease_expires_at_micros: i64, + pub renewed_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationJobFailRecordInput { + pub job_id: String, + pub worker_id: String, + pub lease_token: String, + pub error_message: String, + pub retry_after_micros: i64, + pub failed_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationJobGetRecordInput { + pub job_id: String, + pub owner_user_id: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationJobRecord { + pub job_id: String, + pub dedupe_key: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_payload_json: String, + pub status: String, + pub attempt: u32, + pub max_attempts: u32, + pub last_error_message: Option, + pub worker_id: Option, + pub lease_expires_at: Option, + pub available_at: String, + pub result_payload_json: Option, + pub created_at: String, + pub started_at: Option, + pub completed_at: Option, + pub updated_at: String, + pub updated_at_micros: i64, + pub lease_token: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalGenerationQueueStatsRecord { + pub pending_count: u32, + pub delayed_pending_count: u32, + pub claimable_pending_count: u32, + pub running_active_count: u32, + pub expired_running_count: u32, + pub terminal_count: u32, + pub claimable_count: u32, + pub oldest_claimable_age_micros: Option, + pub now_micros: i64, +} diff --git a/server-rs/crates/spacetime-client/src/mapper/puzzle.rs b/server-rs/crates/spacetime-client/src/mapper/puzzle.rs index d11564de1..2ce8b6a91 100644 --- a/server-rs/crates/spacetime-client/src/mapper/puzzle.rs +++ b/server-rs/crates/spacetime-client/src/mapper/puzzle.rs @@ -669,6 +669,22 @@ pub struct PuzzleDraftCompileFailureRecordInput { pub owner_user_id: String, pub error_message: String, pub failed_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PuzzleLevelGenerationFailureRecordInput { + pub session_id: String, + pub owner_user_id: String, + pub level_id: Option, + pub levels_json: Option, + pub error_message: String, + pub failed_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -679,6 +695,9 @@ pub struct PuzzleGeneratedImagesSaveRecordInput { pub levels_json: Option, pub candidates_json: String, pub saved_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -691,6 +710,9 @@ pub struct PuzzleUiBackgroundSaveRecordInput { pub image_src: String, pub image_object_key: Option, pub saved_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/server-rs/crates/spacetime-client/src/module_bindings.rs b/server-rs/crates/spacetime-client/src/module_bindings.rs index 556771684..62e6d5068 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings.rs @@ -203,6 +203,7 @@ pub mod chapter_progression_snapshot_type; pub mod chapter_progression_table; pub mod chapter_progression_type; pub mod checkpoint_wooden_fish_run_procedure; +pub mod claim_external_generation_jobs_and_return_procedure; pub mod claim_profile_task_reward_and_return_procedure; pub mod claim_puzzle_background_compile_task_procedure; pub mod claim_puzzle_work_point_incentive_procedure; @@ -221,6 +222,7 @@ pub mod compile_visual_novel_work_profile_procedure; pub mod compile_wooden_fish_draft_procedure; pub mod complete_ai_stage_and_return_procedure; pub mod complete_ai_task_and_return_procedure; +pub mod complete_external_generation_job_and_return_procedure; pub mod confirm_asset_object_and_return_procedure; pub mod confirm_asset_object_reducer; pub mod consume_inventory_item_input_type; @@ -343,12 +345,26 @@ pub mod delete_visual_novel_work_procedure; pub mod delete_wooden_fish_work_procedure; pub mod drag_puzzle_piece_or_group_procedure; pub mod drop_square_hole_shape_procedure; +pub mod enqueue_external_generation_job_and_return_procedure; pub mod ensure_analytics_date_dimension_for_date_reducer; pub mod equip_inventory_item_input_type; pub mod execute_custom_world_agent_action_procedure; pub mod export_auth_store_snapshot_from_tables_procedure; pub mod export_database_migration_to_file_procedure; +pub mod external_generation_job_claim_input_type; +pub mod external_generation_job_complete_input_type; +pub mod external_generation_job_enqueue_input_type; +pub mod external_generation_job_fail_input_type; +pub mod external_generation_job_get_input_type; +pub mod external_generation_job_procedure_result_type; +pub mod external_generation_job_renew_lease_input_type; +pub mod external_generation_job_snapshot_type; +pub mod external_generation_job_table; +pub mod external_generation_job_type; +pub mod external_generation_queue_stats_procedure_result_type; +pub mod external_generation_queue_stats_snapshot_type; pub mod fail_ai_task_and_return_procedure; +pub mod fail_external_generation_job_and_return_procedure; pub mod finalize_big_fish_agent_message_turn_procedure; pub mod finalize_custom_world_agent_message_turn_procedure; pub mod finalize_match_3_d_agent_message_turn_procedure; @@ -373,6 +389,8 @@ pub mod get_custom_world_agent_session_procedure; pub mod get_custom_world_gallery_detail_by_code_procedure; pub mod get_custom_world_gallery_detail_procedure; pub mod get_custom_world_library_detail_procedure; +pub mod get_external_generation_job_and_return_procedure; +pub mod get_external_generation_queue_stats_and_return_procedure; pub mod get_jump_hop_agent_session_procedure; pub mod get_jump_hop_leaderboard_procedure; pub mod get_jump_hop_run_procedure; @@ -497,6 +515,7 @@ pub mod list_wooden_fish_works_procedure; pub mod mark_profile_recharge_order_paid_and_return_procedure; pub mod mark_puzzle_clear_level_time_up_procedure; pub mod mark_puzzle_draft_generation_failed_procedure; +pub mod mark_puzzle_level_generation_failed_procedure; pub mod match_3_d_agent_message_finalize_input_type; pub mod match_3_d_agent_message_row_type; pub mod match_3_d_agent_message_snapshot_type; @@ -692,6 +711,7 @@ pub mod puzzle_leaderboard_entry_row_type; pub mod puzzle_leaderboard_entry_table; pub mod puzzle_leaderboard_entry_type; pub mod puzzle_leaderboard_submit_input_type; +pub mod puzzle_level_generation_failure_input_type; pub mod puzzle_merged_group_state_type; pub mod puzzle_piece_state_type; pub mod puzzle_publication_status_type; @@ -776,6 +796,7 @@ pub mod release_puzzle_background_compile_task_procedure; pub mod remix_big_fish_work_procedure; pub mod remix_custom_world_profile_procedure; pub mod remix_puzzle_work_procedure; +pub mod renew_external_generation_job_lease_and_return_procedure; pub mod resolve_combat_action_and_return_procedure; pub mod resolve_combat_action_input_type; pub mod resolve_combat_action_procedure_result_type; @@ -1318,6 +1339,7 @@ pub use chapter_progression_snapshot_type::ChapterProgressionSnapshot; pub use chapter_progression_table::*; pub use chapter_progression_type::ChapterProgression; pub use checkpoint_wooden_fish_run_procedure::checkpoint_wooden_fish_run; +pub use claim_external_generation_jobs_and_return_procedure::claim_external_generation_jobs_and_return; pub use claim_profile_task_reward_and_return_procedure::claim_profile_task_reward_and_return; pub use claim_puzzle_background_compile_task_procedure::claim_puzzle_background_compile_task; pub use claim_puzzle_work_point_incentive_procedure::claim_puzzle_work_point_incentive; @@ -1336,6 +1358,7 @@ pub use compile_visual_novel_work_profile_procedure::compile_visual_novel_work_p pub use compile_wooden_fish_draft_procedure::compile_wooden_fish_draft; pub use complete_ai_stage_and_return_procedure::complete_ai_stage_and_return; pub use complete_ai_task_and_return_procedure::complete_ai_task_and_return; +pub use complete_external_generation_job_and_return_procedure::complete_external_generation_job_and_return; pub use confirm_asset_object_and_return_procedure::confirm_asset_object_and_return; pub use confirm_asset_object_reducer::confirm_asset_object; pub use consume_inventory_item_input_type::ConsumeInventoryItemInput; @@ -1458,12 +1481,26 @@ pub use delete_visual_novel_work_procedure::delete_visual_novel_work; pub use delete_wooden_fish_work_procedure::delete_wooden_fish_work; pub use drag_puzzle_piece_or_group_procedure::drag_puzzle_piece_or_group; pub use drop_square_hole_shape_procedure::drop_square_hole_shape; +pub use enqueue_external_generation_job_and_return_procedure::enqueue_external_generation_job_and_return; pub use ensure_analytics_date_dimension_for_date_reducer::ensure_analytics_date_dimension_for_date; pub use equip_inventory_item_input_type::EquipInventoryItemInput; pub use execute_custom_world_agent_action_procedure::execute_custom_world_agent_action; pub use export_auth_store_snapshot_from_tables_procedure::export_auth_store_snapshot_from_tables; pub use export_database_migration_to_file_procedure::export_database_migration_to_file; +pub use external_generation_job_claim_input_type::ExternalGenerationJobClaimInput; +pub use external_generation_job_complete_input_type::ExternalGenerationJobCompleteInput; +pub use external_generation_job_enqueue_input_type::ExternalGenerationJobEnqueueInput; +pub use external_generation_job_fail_input_type::ExternalGenerationJobFailInput; +pub use external_generation_job_get_input_type::ExternalGenerationJobGetInput; +pub use external_generation_job_procedure_result_type::ExternalGenerationJobProcedureResult; +pub use external_generation_job_renew_lease_input_type::ExternalGenerationJobRenewLeaseInput; +pub use external_generation_job_snapshot_type::ExternalGenerationJobSnapshot; +pub use external_generation_job_table::*; +pub use external_generation_job_type::ExternalGenerationJob; +pub use external_generation_queue_stats_procedure_result_type::ExternalGenerationQueueStatsProcedureResult; +pub use external_generation_queue_stats_snapshot_type::ExternalGenerationQueueStatsSnapshot; pub use fail_ai_task_and_return_procedure::fail_ai_task_and_return; +pub use fail_external_generation_job_and_return_procedure::fail_external_generation_job_and_return; pub use finalize_big_fish_agent_message_turn_procedure::finalize_big_fish_agent_message_turn; pub use finalize_custom_world_agent_message_turn_procedure::finalize_custom_world_agent_message_turn; pub use finalize_match_3_d_agent_message_turn_procedure::finalize_match_3_d_agent_message_turn; @@ -1488,6 +1525,8 @@ pub use get_custom_world_agent_session_procedure::get_custom_world_agent_session pub use get_custom_world_gallery_detail_by_code_procedure::get_custom_world_gallery_detail_by_code; pub use get_custom_world_gallery_detail_procedure::get_custom_world_gallery_detail; pub use get_custom_world_library_detail_procedure::get_custom_world_library_detail; +pub use get_external_generation_job_and_return_procedure::get_external_generation_job_and_return; +pub use get_external_generation_queue_stats_and_return_procedure::get_external_generation_queue_stats_and_return; pub use get_jump_hop_agent_session_procedure::get_jump_hop_agent_session; pub use get_jump_hop_leaderboard_procedure::get_jump_hop_leaderboard; pub use get_jump_hop_run_procedure::get_jump_hop_run; @@ -1612,6 +1651,7 @@ pub use list_wooden_fish_works_procedure::list_wooden_fish_works; pub use mark_profile_recharge_order_paid_and_return_procedure::mark_profile_recharge_order_paid_and_return; pub use mark_puzzle_clear_level_time_up_procedure::mark_puzzle_clear_level_time_up; pub use mark_puzzle_draft_generation_failed_procedure::mark_puzzle_draft_generation_failed; +pub use mark_puzzle_level_generation_failed_procedure::mark_puzzle_level_generation_failed; pub use match_3_d_agent_message_finalize_input_type::Match3DAgentMessageFinalizeInput; pub use match_3_d_agent_message_row_type::Match3DAgentMessageRow; pub use match_3_d_agent_message_snapshot_type::Match3DAgentMessageSnapshot; @@ -1807,6 +1847,7 @@ pub use puzzle_leaderboard_entry_row_type::PuzzleLeaderboardEntryRow; pub use puzzle_leaderboard_entry_table::*; pub use puzzle_leaderboard_entry_type::PuzzleLeaderboardEntry; pub use puzzle_leaderboard_submit_input_type::PuzzleLeaderboardSubmitInput; +pub use puzzle_level_generation_failure_input_type::PuzzleLevelGenerationFailureInput; pub use puzzle_merged_group_state_type::PuzzleMergedGroupState; pub use puzzle_piece_state_type::PuzzlePieceState; pub use puzzle_publication_status_type::PuzzlePublicationStatus; @@ -1891,6 +1932,7 @@ pub use release_puzzle_background_compile_task_procedure::release_puzzle_backgro pub use remix_big_fish_work_procedure::remix_big_fish_work; pub use remix_custom_world_profile_procedure::remix_custom_world_profile; pub use remix_puzzle_work_procedure::remix_puzzle_work; +pub use renew_external_generation_job_lease_and_return_procedure::renew_external_generation_job_lease_and_return; pub use resolve_combat_action_and_return_procedure::resolve_combat_action_and_return; pub use resolve_combat_action_input_type::ResolveCombatActionInput; pub use resolve_combat_action_procedure_result_type::ResolveCombatActionProcedureResult; @@ -2547,6 +2589,7 @@ pub struct DbUpdate { custom_world_session: __sdk::TableUpdate, database_migration_import_chunk: __sdk::TableUpdate, database_migration_operator: __sdk::TableUpdate, + external_generation_job: __sdk::TableUpdate, inventory_slot: __sdk::TableUpdate, jump_hop_agent_session: __sdk::TableUpdate, jump_hop_event: __sdk::TableUpdate, @@ -2759,6 +2802,9 @@ impl TryFrom<__ws::v2::TransactionUpdate> for DbUpdate { "database_migration_operator" => db_update.database_migration_operator.append( database_migration_operator_table::parse_table_update(table_update)?, ), + "external_generation_job" => db_update.external_generation_job.append( + external_generation_job_table::parse_table_update(table_update)?, + ), "inventory_slot" => db_update .inventory_slot .append(inventory_slot_table::parse_table_update(table_update)?), @@ -3219,6 +3265,12 @@ impl __sdk::DbUpdate for DbUpdate { &self.database_migration_operator, ) .with_updates_by_pk(|row| &row.operator_identity); + diff.external_generation_job = cache + .apply_diff_to_table::( + "external_generation_job", + &self.external_generation_job, + ) + .with_updates_by_pk(|row| &row.job_id); diff.inventory_slot = cache .apply_diff_to_table::("inventory_slot", &self.inventory_slot) .with_updates_by_pk(|row| &row.slot_id); @@ -3746,6 +3798,9 @@ impl __sdk::DbUpdate for DbUpdate { "database_migration_operator" => db_update .database_migration_operator .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), + "external_generation_job" => db_update + .external_generation_job + .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), "inventory_slot" => db_update .inventory_slot .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), @@ -4113,6 +4168,9 @@ impl __sdk::DbUpdate for DbUpdate { "database_migration_operator" => db_update .database_migration_operator .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), + "external_generation_job" => db_update + .external_generation_job + .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), "inventory_slot" => db_update .inventory_slot .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), @@ -4406,6 +4464,7 @@ pub struct AppliedDiff<'r> { custom_world_session: __sdk::TableAppliedDiff<'r, CustomWorldSession>, database_migration_import_chunk: __sdk::TableAppliedDiff<'r, DatabaseMigrationImportChunk>, database_migration_operator: __sdk::TableAppliedDiff<'r, DatabaseMigrationOperator>, + external_generation_job: __sdk::TableAppliedDiff<'r, ExternalGenerationJob>, inventory_slot: __sdk::TableAppliedDiff<'r, InventorySlot>, jump_hop_agent_session: __sdk::TableAppliedDiff<'r, JumpHopAgentSessionRow>, jump_hop_event: __sdk::TableAppliedDiff<'r, JumpHopEventRow>, @@ -4686,6 +4745,11 @@ impl<'r> __sdk::AppliedDiff<'r> for AppliedDiff<'r> { &self.database_migration_operator, event, ); + callbacks.invoke_table_row_callbacks::( + "external_generation_job", + &self.external_generation_job, + event, + ); callbacks.invoke_table_row_callbacks::( "inventory_slot", &self.inventory_slot, @@ -5768,6 +5832,7 @@ impl __sdk::SpacetimeModule for RemoteModule { custom_world_session_table::register_table(client_cache); database_migration_import_chunk_table::register_table(client_cache); database_migration_operator_table::register_table(client_cache); + external_generation_job_table::register_table(client_cache); inventory_slot_table::register_table(client_cache); jump_hop_agent_session_table::register_table(client_cache); jump_hop_event_table::register_table(client_cache); @@ -5888,6 +5953,7 @@ impl __sdk::SpacetimeModule for RemoteModule { "custom_world_session", "database_migration_import_chunk", "database_migration_operator", + "external_generation_job", "inventory_slot", "jump_hop_agent_session", "jump_hop_event", diff --git a/server-rs/crates/spacetime-client/src/module_bindings/claim_external_generation_jobs_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/claim_external_generation_jobs_and_return_procedure.rs new file mode 100644 index 000000000..6455c7b2e --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/claim_external_generation_jobs_and_return_procedure.rs @@ -0,0 +1,59 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_claim_input_type::ExternalGenerationJobClaimInput; +use super::external_generation_job_procedure_result_type::ExternalGenerationJobProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct ClaimExternalGenerationJobsAndReturnArgs { + pub input: ExternalGenerationJobClaimInput, +} + +impl __sdk::InModule for ClaimExternalGenerationJobsAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `claim_external_generation_jobs_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait claim_external_generation_jobs_and_return { + fn claim_external_generation_jobs_and_return(&self, input: ExternalGenerationJobClaimInput) { + self.claim_external_generation_jobs_and_return_then(input, |_, _| {}); + } + + fn claim_external_generation_jobs_and_return_then( + &self, + input: ExternalGenerationJobClaimInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl claim_external_generation_jobs_and_return for super::RemoteProcedures { + fn claim_external_generation_jobs_and_return_then( + &self, + input: ExternalGenerationJobClaimInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobProcedureResult>( + "claim_external_generation_jobs_and_return", + ClaimExternalGenerationJobsAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/complete_external_generation_job_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/complete_external_generation_job_and_return_procedure.rs new file mode 100644 index 000000000..9c923b968 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/complete_external_generation_job_and_return_procedure.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_complete_input_type::ExternalGenerationJobCompleteInput; +use super::external_generation_job_procedure_result_type::ExternalGenerationJobProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct CompleteExternalGenerationJobAndReturnArgs { + pub input: ExternalGenerationJobCompleteInput, +} + +impl __sdk::InModule for CompleteExternalGenerationJobAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `complete_external_generation_job_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait complete_external_generation_job_and_return { + fn complete_external_generation_job_and_return( + &self, + input: ExternalGenerationJobCompleteInput, + ) { + self.complete_external_generation_job_and_return_then(input, |_, _| {}); + } + + fn complete_external_generation_job_and_return_then( + &self, + input: ExternalGenerationJobCompleteInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl complete_external_generation_job_and_return for super::RemoteProcedures { + fn complete_external_generation_job_and_return_then( + &self, + input: ExternalGenerationJobCompleteInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobProcedureResult>( + "complete_external_generation_job_and_return", + CompleteExternalGenerationJobAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/enqueue_external_generation_job_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/enqueue_external_generation_job_and_return_procedure.rs new file mode 100644 index 000000000..cd14e1432 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/enqueue_external_generation_job_and_return_procedure.rs @@ -0,0 +1,59 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_enqueue_input_type::ExternalGenerationJobEnqueueInput; +use super::external_generation_job_procedure_result_type::ExternalGenerationJobProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct EnqueueExternalGenerationJobAndReturnArgs { + pub input: ExternalGenerationJobEnqueueInput, +} + +impl __sdk::InModule for EnqueueExternalGenerationJobAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `enqueue_external_generation_job_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait enqueue_external_generation_job_and_return { + fn enqueue_external_generation_job_and_return(&self, input: ExternalGenerationJobEnqueueInput) { + self.enqueue_external_generation_job_and_return_then(input, |_, _| {}); + } + + fn enqueue_external_generation_job_and_return_then( + &self, + input: ExternalGenerationJobEnqueueInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl enqueue_external_generation_job_and_return for super::RemoteProcedures { + fn enqueue_external_generation_job_and_return_then( + &self, + input: ExternalGenerationJobEnqueueInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobProcedureResult>( + "enqueue_external_generation_job_and_return", + EnqueueExternalGenerationJobAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_claim_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_claim_input_type.rs new file mode 100644 index 000000000..0a2ed3f14 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_claim_input_type.rs @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobClaimInput { + pub worker_id: String, + pub limit: u32, + pub lease_expires_at_micros: i64, + pub claimed_at_micros: i64, +} + +impl __sdk::InModule for ExternalGenerationJobClaimInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_complete_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_complete_input_type.rs new file mode 100644 index 000000000..e2a93bec8 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_complete_input_type.rs @@ -0,0 +1,19 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobCompleteInput { + pub job_id: String, + pub worker_id: String, + pub lease_token: String, + pub result_payload_json: Option, + pub completed_at_micros: i64, +} + +impl __sdk::InModule for ExternalGenerationJobCompleteInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_enqueue_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_enqueue_input_type.rs new file mode 100644 index 000000000..760a5c3e1 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_enqueue_input_type.rs @@ -0,0 +1,25 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobEnqueueInput { + pub job_id: String, + pub dedupe_key: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_payload_json: String, + pub max_attempts: u32, + pub available_at_micros: i64, + pub created_at_micros: i64, +} + +impl __sdk::InModule for ExternalGenerationJobEnqueueInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_fail_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_fail_input_type.rs new file mode 100644 index 000000000..54bbaf3c2 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_fail_input_type.rs @@ -0,0 +1,20 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobFailInput { + pub job_id: String, + pub worker_id: String, + pub lease_token: String, + pub error_message: String, + pub retry_after_micros: i64, + pub failed_at_micros: i64, +} + +impl __sdk::InModule for ExternalGenerationJobFailInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_get_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_get_input_type.rs new file mode 100644 index 000000000..9b4bd341e --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_get_input_type.rs @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobGetInput { + pub job_id: String, + pub owner_user_id: String, +} + +impl __sdk::InModule for ExternalGenerationJobGetInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_procedure_result_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_procedure_result_type.rs new file mode 100644 index 000000000..4f3c0f810 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_procedure_result_type.rs @@ -0,0 +1,20 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_snapshot_type::ExternalGenerationJobSnapshot; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobProcedureResult { + pub ok: bool, + pub job: Option, + pub jobs: Vec, + pub error_message: Option, +} + +impl __sdk::InModule for ExternalGenerationJobProcedureResult { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_renew_lease_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_renew_lease_input_type.rs new file mode 100644 index 000000000..4ec70e628 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_renew_lease_input_type.rs @@ -0,0 +1,19 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobRenewLeaseInput { + pub job_id: String, + pub worker_id: String, + pub lease_token: String, + pub lease_expires_at_micros: i64, + pub renewed_at_micros: i64, +} + +impl __sdk::InModule for ExternalGenerationJobRenewLeaseInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_snapshot_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_snapshot_type.rs new file mode 100644 index 000000000..8449e819c --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_snapshot_type.rs @@ -0,0 +1,35 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobSnapshot { + pub job_id: String, + pub dedupe_key: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_payload_json: String, + pub status: String, + pub attempt: u32, + pub max_attempts: u32, + pub last_error_message: Option, + pub worker_id: Option, + pub lease_expires_at_micros: Option, + pub available_at_micros: i64, + pub result_payload_json: Option, + pub created_at_micros: i64, + pub started_at_micros: Option, + pub completed_at_micros: Option, + pub updated_at_micros: i64, + pub lease_token: Option, +} + +impl __sdk::InModule for ExternalGenerationJobSnapshot { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_table.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_table.rs new file mode 100644 index 000000000..7318cd496 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_table.rs @@ -0,0 +1,192 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use super::external_generation_job_type::ExternalGenerationJob; +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +/// Table handle for the table `external_generation_job`. +/// +/// Obtain a handle from the [`ExternalGenerationJobTableAccess::external_generation_job`] method on [`super::RemoteTables`], +/// like `ctx.db.external_generation_job()`. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.external_generation_job().on_insert(...)`. +pub struct ExternalGenerationJobTableHandle<'ctx> { + imp: __sdk::TableHandle, + ctx: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the table `external_generation_job`. +/// +/// Implemented for [`super::RemoteTables`]. +pub trait ExternalGenerationJobTableAccess { + #[allow(non_snake_case)] + /// Obtain a [`ExternalGenerationJobTableHandle`], which mediates access to the table `external_generation_job`. + fn external_generation_job(&self) -> ExternalGenerationJobTableHandle<'_>; +} + +impl ExternalGenerationJobTableAccess for super::RemoteTables { + fn external_generation_job(&self) -> ExternalGenerationJobTableHandle<'_> { + ExternalGenerationJobTableHandle { + imp: self + .imp + .get_table::("external_generation_job"), + ctx: std::marker::PhantomData, + } + } +} + +pub struct ExternalGenerationJobInsertCallbackId(__sdk::CallbackId); +pub struct ExternalGenerationJobDeleteCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::Table for ExternalGenerationJobTableHandle<'ctx> { + type Row = ExternalGenerationJob; + type EventContext = super::EventContext; + + fn count(&self) -> u64 { + self.imp.count() + } + fn iter(&self) -> impl Iterator + '_ { + self.imp.iter() + } + + type InsertCallbackId = ExternalGenerationJobInsertCallbackId; + + fn on_insert( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> ExternalGenerationJobInsertCallbackId { + ExternalGenerationJobInsertCallbackId(self.imp.on_insert(Box::new(callback))) + } + + fn remove_on_insert(&self, callback: ExternalGenerationJobInsertCallbackId) { + self.imp.remove_on_insert(callback.0) + } + + type DeleteCallbackId = ExternalGenerationJobDeleteCallbackId; + + fn on_delete( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> ExternalGenerationJobDeleteCallbackId { + ExternalGenerationJobDeleteCallbackId(self.imp.on_delete(Box::new(callback))) + } + + fn remove_on_delete(&self, callback: ExternalGenerationJobDeleteCallbackId) { + self.imp.remove_on_delete(callback.0) + } +} + +pub struct ExternalGenerationJobUpdateCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::TableWithPrimaryKey for ExternalGenerationJobTableHandle<'ctx> { + type UpdateCallbackId = ExternalGenerationJobUpdateCallbackId; + + fn on_update( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row, &Self::Row) + Send + 'static, + ) -> ExternalGenerationJobUpdateCallbackId { + ExternalGenerationJobUpdateCallbackId(self.imp.on_update(Box::new(callback))) + } + + fn remove_on_update(&self, callback: ExternalGenerationJobUpdateCallbackId) { + self.imp.remove_on_update(callback.0) + } +} + +/// Access to the `job_id` unique index on the table `external_generation_job`, +/// which allows point queries on the field of the same name +/// via the [`ExternalGenerationJobJobIdUnique::find`] method. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.external_generation_job().job_id().find(...)`. +pub struct ExternalGenerationJobJobIdUnique<'ctx> { + imp: __sdk::UniqueConstraintHandle, + phantom: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +impl<'ctx> ExternalGenerationJobTableHandle<'ctx> { + /// Get a handle on the `job_id` unique index on the table `external_generation_job`. + pub fn job_id(&self) -> ExternalGenerationJobJobIdUnique<'ctx> { + ExternalGenerationJobJobIdUnique { + imp: self.imp.get_unique_constraint::("job_id"), + phantom: std::marker::PhantomData, + } + } +} + +impl<'ctx> ExternalGenerationJobJobIdUnique<'ctx> { + /// Find the subscribed row whose `job_id` column value is equal to `col_val`, + /// if such a row is present in the client cache. + pub fn find(&self, col_val: &String) -> Option { + self.imp.find(col_val) + } +} + +/// Access to the `dedupe_key` unique index on the table `external_generation_job`, +/// which allows point queries on the field of the same name +/// via the [`ExternalGenerationJobDedupeKeyUnique::find`] method. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.external_generation_job().dedupe_key().find(...)`. +pub struct ExternalGenerationJobDedupeKeyUnique<'ctx> { + imp: __sdk::UniqueConstraintHandle, + phantom: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +impl<'ctx> ExternalGenerationJobTableHandle<'ctx> { + /// Get a handle on the `dedupe_key` unique index on the table `external_generation_job`. + pub fn dedupe_key(&self) -> ExternalGenerationJobDedupeKeyUnique<'ctx> { + ExternalGenerationJobDedupeKeyUnique { + imp: self.imp.get_unique_constraint::("dedupe_key"), + phantom: std::marker::PhantomData, + } + } +} + +impl<'ctx> ExternalGenerationJobDedupeKeyUnique<'ctx> { + /// Find the subscribed row whose `dedupe_key` column value is equal to `col_val`, + /// if such a row is present in the client cache. + pub fn find(&self, col_val: &String) -> Option { + self.imp.find(col_val) + } +} + +#[doc(hidden)] +pub(super) fn register_table(client_cache: &mut __sdk::ClientCache) { + let _table = client_cache.get_or_make_table::("external_generation_job"); + _table.add_unique_constraint::("job_id", |row| &row.job_id); + _table.add_unique_constraint::("dedupe_key", |row| &row.dedupe_key); +} + +#[doc(hidden)] +pub(super) fn parse_table_update( + raw_updates: __ws::v2::TableUpdate, +) -> __sdk::Result<__sdk::TableUpdate> { + __sdk::TableUpdate::parse_table_update(raw_updates).map_err(|e| { + __sdk::InternalError::failed_parse("TableUpdate", "TableUpdate") + .with_cause(e) + .into() + }) +} + +#[allow(non_camel_case_types)] +/// Extension trait for query builder access to the table `ExternalGenerationJob`. +/// +/// Implemented for [`__sdk::QueryTableAccessor`]. +pub trait external_generation_jobQueryTableAccess { + #[allow(non_snake_case)] + /// Get a query builder for the table `ExternalGenerationJob`. + fn external_generation_job(&self) -> __sdk::__query_builder::Table; +} + +impl external_generation_jobQueryTableAccess for __sdk::QueryTableAccessor { + fn external_generation_job(&self) -> __sdk::__query_builder::Table { + __sdk::__query_builder::Table::new("external_generation_job") + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_type.rs new file mode 100644 index 000000000..4a4e0afab --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_type.rs @@ -0,0 +1,122 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJob { + pub job_id: String, + pub dedupe_key: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_payload_json: String, + pub status: String, + pub attempt: u32, + pub max_attempts: u32, + pub last_error_message: Option, + pub worker_id: Option, + pub lease_expires_at: Option<__sdk::Timestamp>, + pub available_at: __sdk::Timestamp, + pub result_payload_json: Option, + pub created_at: __sdk::Timestamp, + pub started_at: Option<__sdk::Timestamp>, + pub completed_at: Option<__sdk::Timestamp>, + pub updated_at: __sdk::Timestamp, + pub lease_token: Option, +} + +impl __sdk::InModule for ExternalGenerationJob { + type Module = super::RemoteModule; +} + +/// Column accessor struct for the table `ExternalGenerationJob`. +/// +/// Provides typed access to columns for query building. +pub struct ExternalGenerationJobCols { + pub job_id: __sdk::__query_builder::Col, + pub dedupe_key: __sdk::__query_builder::Col, + pub job_kind: __sdk::__query_builder::Col, + pub owner_user_id: __sdk::__query_builder::Col, + pub source_module: __sdk::__query_builder::Col, + pub source_entity_id: __sdk::__query_builder::Col, + pub request_label: __sdk::__query_builder::Col, + pub request_payload_json: __sdk::__query_builder::Col, + pub status: __sdk::__query_builder::Col, + pub attempt: __sdk::__query_builder::Col, + pub max_attempts: __sdk::__query_builder::Col, + pub last_error_message: __sdk::__query_builder::Col>, + pub worker_id: __sdk::__query_builder::Col>, + pub lease_expires_at: + __sdk::__query_builder::Col>, + pub available_at: __sdk::__query_builder::Col, + pub result_payload_json: __sdk::__query_builder::Col>, + pub created_at: __sdk::__query_builder::Col, + pub started_at: __sdk::__query_builder::Col>, + pub completed_at: __sdk::__query_builder::Col>, + pub updated_at: __sdk::__query_builder::Col, + pub lease_token: __sdk::__query_builder::Col>, +} + +impl __sdk::__query_builder::HasCols for ExternalGenerationJob { + type Cols = ExternalGenerationJobCols; + fn cols(table_name: &'static str) -> Self::Cols { + ExternalGenerationJobCols { + job_id: __sdk::__query_builder::Col::new(table_name, "job_id"), + dedupe_key: __sdk::__query_builder::Col::new(table_name, "dedupe_key"), + job_kind: __sdk::__query_builder::Col::new(table_name, "job_kind"), + owner_user_id: __sdk::__query_builder::Col::new(table_name, "owner_user_id"), + source_module: __sdk::__query_builder::Col::new(table_name, "source_module"), + source_entity_id: __sdk::__query_builder::Col::new(table_name, "source_entity_id"), + request_label: __sdk::__query_builder::Col::new(table_name, "request_label"), + request_payload_json: __sdk::__query_builder::Col::new( + table_name, + "request_payload_json", + ), + status: __sdk::__query_builder::Col::new(table_name, "status"), + attempt: __sdk::__query_builder::Col::new(table_name, "attempt"), + max_attempts: __sdk::__query_builder::Col::new(table_name, "max_attempts"), + last_error_message: __sdk::__query_builder::Col::new(table_name, "last_error_message"), + worker_id: __sdk::__query_builder::Col::new(table_name, "worker_id"), + lease_expires_at: __sdk::__query_builder::Col::new(table_name, "lease_expires_at"), + available_at: __sdk::__query_builder::Col::new(table_name, "available_at"), + result_payload_json: __sdk::__query_builder::Col::new( + table_name, + "result_payload_json", + ), + created_at: __sdk::__query_builder::Col::new(table_name, "created_at"), + started_at: __sdk::__query_builder::Col::new(table_name, "started_at"), + completed_at: __sdk::__query_builder::Col::new(table_name, "completed_at"), + updated_at: __sdk::__query_builder::Col::new(table_name, "updated_at"), + lease_token: __sdk::__query_builder::Col::new(table_name, "lease_token"), + } + } +} + +/// Indexed column accessor struct for the table `ExternalGenerationJob`. +/// +/// Provides typed access to indexed columns for query building. +pub struct ExternalGenerationJobIxCols { + pub dedupe_key: __sdk::__query_builder::IxCol, + pub job_id: __sdk::__query_builder::IxCol, + pub owner_user_id: __sdk::__query_builder::IxCol, + pub worker_id: __sdk::__query_builder::IxCol>, +} + +impl __sdk::__query_builder::HasIxCols for ExternalGenerationJob { + type IxCols = ExternalGenerationJobIxCols; + fn ix_cols(table_name: &'static str) -> Self::IxCols { + ExternalGenerationJobIxCols { + dedupe_key: __sdk::__query_builder::IxCol::new(table_name, "dedupe_key"), + job_id: __sdk::__query_builder::IxCol::new(table_name, "job_id"), + owner_user_id: __sdk::__query_builder::IxCol::new(table_name, "owner_user_id"), + worker_id: __sdk::__query_builder::IxCol::new(table_name, "worker_id"), + } + } +} + +impl __sdk::__query_builder::CanBeLookupTable for ExternalGenerationJob {} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_queue_stats_procedure_result_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_queue_stats_procedure_result_type.rs new file mode 100644 index 000000000..2061d33fb --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_queue_stats_procedure_result_type.rs @@ -0,0 +1,19 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_queue_stats_snapshot_type::ExternalGenerationQueueStatsSnapshot; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationQueueStatsProcedureResult { + pub ok: bool, + pub stats: Option, + pub error_message: Option, +} + +impl __sdk::InModule for ExternalGenerationQueueStatsProcedureResult { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_queue_stats_snapshot_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_queue_stats_snapshot_type.rs new file mode 100644 index 000000000..ae98e521a --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_queue_stats_snapshot_type.rs @@ -0,0 +1,23 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationQueueStatsSnapshot { + pub pending_count: u32, + pub delayed_pending_count: u32, + pub claimable_pending_count: u32, + pub running_active_count: u32, + pub expired_running_count: u32, + pub terminal_count: u32, + pub claimable_count: u32, + pub oldest_claimable_age_micros: Option, + pub now_micros: i64, +} + +impl __sdk::InModule for ExternalGenerationQueueStatsSnapshot { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/fail_external_generation_job_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/fail_external_generation_job_and_return_procedure.rs new file mode 100644 index 000000000..46c1f8846 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/fail_external_generation_job_and_return_procedure.rs @@ -0,0 +1,59 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_fail_input_type::ExternalGenerationJobFailInput; +use super::external_generation_job_procedure_result_type::ExternalGenerationJobProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct FailExternalGenerationJobAndReturnArgs { + pub input: ExternalGenerationJobFailInput, +} + +impl __sdk::InModule for FailExternalGenerationJobAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `fail_external_generation_job_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait fail_external_generation_job_and_return { + fn fail_external_generation_job_and_return(&self, input: ExternalGenerationJobFailInput) { + self.fail_external_generation_job_and_return_then(input, |_, _| {}); + } + + fn fail_external_generation_job_and_return_then( + &self, + input: ExternalGenerationJobFailInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl fail_external_generation_job_and_return for super::RemoteProcedures { + fn fail_external_generation_job_and_return_then( + &self, + input: ExternalGenerationJobFailInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobProcedureResult>( + "fail_external_generation_job_and_return", + FailExternalGenerationJobAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_job_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_job_and_return_procedure.rs new file mode 100644 index 000000000..e2bc98a33 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_job_and_return_procedure.rs @@ -0,0 +1,59 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_get_input_type::ExternalGenerationJobGetInput; +use super::external_generation_job_procedure_result_type::ExternalGenerationJobProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct GetExternalGenerationJobAndReturnArgs { + pub input: ExternalGenerationJobGetInput, +} + +impl __sdk::InModule for GetExternalGenerationJobAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `get_external_generation_job_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait get_external_generation_job_and_return { + fn get_external_generation_job_and_return(&self, input: ExternalGenerationJobGetInput) { + self.get_external_generation_job_and_return_then(input, |_, _| {}); + } + + fn get_external_generation_job_and_return_then( + &self, + input: ExternalGenerationJobGetInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl get_external_generation_job_and_return for super::RemoteProcedures { + fn get_external_generation_job_and_return_then( + &self, + input: ExternalGenerationJobGetInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobProcedureResult>( + "get_external_generation_job_and_return", + GetExternalGenerationJobAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_queue_stats_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_queue_stats_and_return_procedure.rs new file mode 100644 index 000000000..9d7a98a05 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/get_external_generation_queue_stats_and_return_procedure.rs @@ -0,0 +1,54 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_queue_stats_procedure_result_type::ExternalGenerationQueueStatsProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct GetExternalGenerationQueueStatsAndReturnArgs {} + +impl __sdk::InModule for GetExternalGenerationQueueStatsAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `get_external_generation_queue_stats_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait get_external_generation_queue_stats_and_return { + fn get_external_generation_queue_stats_and_return(&self) { + self.get_external_generation_queue_stats_and_return_then(|_, _| {}); + } + + fn get_external_generation_queue_stats_and_return_then( + &self, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl get_external_generation_queue_stats_and_return for super::RemoteProcedures { + fn get_external_generation_queue_stats_and_return_then( + &self, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationQueueStatsProcedureResult>( + "get_external_generation_queue_stats_and_return", + GetExternalGenerationQueueStatsAndReturnArgs {}, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/mark_puzzle_level_generation_failed_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/mark_puzzle_level_generation_failed_procedure.rs new file mode 100644 index 000000000..1906e8d89 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/mark_puzzle_level_generation_failed_procedure.rs @@ -0,0 +1,59 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::puzzle_agent_session_procedure_result_type::PuzzleAgentSessionProcedureResult; +use super::puzzle_level_generation_failure_input_type::PuzzleLevelGenerationFailureInput; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct MarkPuzzleLevelGenerationFailedArgs { + pub input: PuzzleLevelGenerationFailureInput, +} + +impl __sdk::InModule for MarkPuzzleLevelGenerationFailedArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `mark_puzzle_level_generation_failed`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait mark_puzzle_level_generation_failed { + fn mark_puzzle_level_generation_failed(&self, input: PuzzleLevelGenerationFailureInput) { + self.mark_puzzle_level_generation_failed_then(input, |_, _| {}); + } + + fn mark_puzzle_level_generation_failed_then( + &self, + input: PuzzleLevelGenerationFailureInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl mark_puzzle_level_generation_failed for super::RemoteProcedures { + fn mark_puzzle_level_generation_failed_then( + &self, + input: PuzzleLevelGenerationFailureInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, PuzzleAgentSessionProcedureResult>( + "mark_puzzle_level_generation_failed", + MarkPuzzleLevelGenerationFailedArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/puzzle_draft_compile_failure_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_draft_compile_failure_input_type.rs index ccda3ff5b..cfa6818b7 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings/puzzle_draft_compile_failure_input_type.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_draft_compile_failure_input_type.rs @@ -11,6 +11,9 @@ pub struct PuzzleDraftCompileFailureInput { pub owner_user_id: String, pub error_message: String, pub failed_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } impl __sdk::InModule for PuzzleDraftCompileFailureInput { diff --git a/server-rs/crates/spacetime-client/src/module_bindings/puzzle_draft_compile_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_draft_compile_input_type.rs index 3b5f565f8..138277251 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings/puzzle_draft_compile_input_type.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_draft_compile_input_type.rs @@ -10,6 +10,9 @@ pub struct PuzzleDraftCompileInput { pub session_id: String, pub owner_user_id: String, pub compiled_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } impl __sdk::InModule for PuzzleDraftCompileInput { diff --git a/server-rs/crates/spacetime-client/src/module_bindings/puzzle_generated_images_save_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_generated_images_save_input_type.rs index f5debfcd2..d246f74a9 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings/puzzle_generated_images_save_input_type.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_generated_images_save_input_type.rs @@ -13,6 +13,9 @@ pub struct PuzzleGeneratedImagesSaveInput { pub levels_json: Option, pub candidates_json: String, pub saved_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } impl __sdk::InModule for PuzzleGeneratedImagesSaveInput { diff --git a/server-rs/crates/spacetime-client/src/module_bindings/puzzle_level_generation_failure_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_level_generation_failure_input_type.rs new file mode 100644 index 000000000..6d96e4807 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_level_generation_failure_input_type.rs @@ -0,0 +1,23 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct PuzzleLevelGenerationFailureInput { + pub session_id: String, + pub owner_user_id: String, + pub level_id: Option, + pub levels_json: Option, + pub error_message: String, + pub failed_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, +} + +impl __sdk::InModule for PuzzleLevelGenerationFailureInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/puzzle_ui_background_save_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_ui_background_save_input_type.rs index 28862433f..ebdeea950 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings/puzzle_ui_background_save_input_type.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings/puzzle_ui_background_save_input_type.rs @@ -15,6 +15,9 @@ pub struct PuzzleUiBackgroundSaveInput { pub image_src: String, pub image_object_key: Option, pub saved_at_micros: i64, + pub external_generation_job_id: Option, + pub external_generation_worker_id: Option, + pub external_generation_lease_token: Option, } impl __sdk::InModule for PuzzleUiBackgroundSaveInput { diff --git a/server-rs/crates/spacetime-client/src/module_bindings/renew_external_generation_job_lease_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/renew_external_generation_job_lease_and_return_procedure.rs new file mode 100644 index 000000000..4cbd45fc1 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/renew_external_generation_job_lease_and_return_procedure.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_procedure_result_type::ExternalGenerationJobProcedureResult; +use super::external_generation_job_renew_lease_input_type::ExternalGenerationJobRenewLeaseInput; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct RenewExternalGenerationJobLeaseAndReturnArgs { + pub input: ExternalGenerationJobRenewLeaseInput, +} + +impl __sdk::InModule for RenewExternalGenerationJobLeaseAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `renew_external_generation_job_lease_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait renew_external_generation_job_lease_and_return { + fn renew_external_generation_job_lease_and_return( + &self, + input: ExternalGenerationJobRenewLeaseInput, + ) { + self.renew_external_generation_job_lease_and_return_then(input, |_, _| {}); + } + + fn renew_external_generation_job_lease_and_return_then( + &self, + input: ExternalGenerationJobRenewLeaseInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl renew_external_generation_job_lease_and_return for super::RemoteProcedures { + fn renew_external_generation_job_lease_and_return_then( + &self, + input: ExternalGenerationJobRenewLeaseInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobProcedureResult>( + "renew_external_generation_job_lease_and_return", + RenewExternalGenerationJobLeaseAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/puzzle.rs b/server-rs/crates/spacetime-client/src/puzzle.rs index 8916e853f..080e13534 100644 --- a/server-rs/crates/spacetime-client/src/puzzle.rs +++ b/server-rs/crates/spacetime-client/src/puzzle.rs @@ -149,10 +149,56 @@ impl SpacetimeClient { owner_user_id: String, compiled_at_micros: i64, ) -> Result { + self.compile_puzzle_agent_draft_inner( + session_id, + owner_user_id, + compiled_at_micros, + (None, None, None), + ) + .await + } + + pub async fn compile_puzzle_agent_draft_with_external_generation_guard( + &self, + session_id: String, + owner_user_id: String, + compiled_at_micros: i64, + external_generation_job_id: Option, + external_generation_worker_id: Option, + external_generation_lease_token: Option, + ) -> Result { + self.compile_puzzle_agent_draft_inner( + session_id, + owner_user_id, + compiled_at_micros, + ( + external_generation_job_id, + external_generation_worker_id, + external_generation_lease_token, + ), + ) + .await + } + + async fn compile_puzzle_agent_draft_inner( + &self, + session_id: String, + owner_user_id: String, + compiled_at_micros: i64, + external_generation_guard: (Option, Option, Option), + ) -> Result { + let ( + external_generation_job_id, + external_generation_worker_id, + external_generation_lease_token, + ) = external_generation_guard; let procedure_input = PuzzleDraftCompileInput { session_id, owner_user_id, compiled_at_micros, + external_generation_job_id, + external_generation_worker_id, + external_generation_lease_token, }; self.call_after_connect("compile_puzzle_agent_draft", move |connection, sender| { @@ -178,6 +224,9 @@ impl SpacetimeClient { owner_user_id: input.owner_user_id, error_message: input.error_message, failed_at_micros: input.failed_at_micros, + external_generation_job_id: input.external_generation_job_id, + external_generation_worker_id: input.external_generation_worker_id, + external_generation_lease_token: input.external_generation_lease_token, }; self.call_after_connect( @@ -196,6 +245,38 @@ impl SpacetimeClient { .await } + pub async fn mark_puzzle_level_generation_failed( + &self, + input: PuzzleLevelGenerationFailureRecordInput, + ) -> Result { + let procedure_input = PuzzleLevelGenerationFailureInput { + session_id: input.session_id, + owner_user_id: input.owner_user_id, + level_id: input.level_id, + levels_json: input.levels_json, + error_message: input.error_message, + failed_at_micros: input.failed_at_micros, + external_generation_job_id: input.external_generation_job_id, + external_generation_worker_id: input.external_generation_worker_id, + external_generation_lease_token: input.external_generation_lease_token, + }; + + self.call_after_connect( + "mark_puzzle_level_generation_failed", + move |connection, sender| { + connection + .procedures() + .mark_puzzle_level_generation_failed_then(procedure_input, move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_puzzle_agent_session_procedure_result); + send_once(&sender, mapped); + }); + }, + ) + .await + } + pub async fn claim_puzzle_background_compile_task( &self, input: PuzzleBackgroundCompileTaskClaimRecordInput, @@ -268,6 +349,9 @@ impl SpacetimeClient { levels_json: input.levels_json, candidates_json: input.candidates_json, saved_at_micros: input.saved_at_micros, + external_generation_job_id: input.external_generation_job_id, + external_generation_worker_id: input.external_generation_worker_id, + external_generation_lease_token: input.external_generation_lease_token, }; self.call_after_connect("save_puzzle_generated_images", move |connection, sender| { @@ -297,6 +381,9 @@ impl SpacetimeClient { image_src: input.image_src, image_object_key: input.image_object_key, saved_at_micros: input.saved_at_micros, + external_generation_job_id: input.external_generation_job_id, + external_generation_worker_id: input.external_generation_worker_id, + external_generation_lease_token: input.external_generation_lease_token, }; self.call_after_connect("save_puzzle_ui_background", move |connection, sender| { diff --git a/server-rs/crates/spacetime-client/src/puzzle_clear.rs b/server-rs/crates/spacetime-client/src/puzzle_clear.rs index 8d1dc0ed3..ba9876c76 100644 --- a/server-rs/crates/spacetime-client/src/puzzle_clear.rs +++ b/server-rs/crates/spacetime-client/src/puzzle_clear.rs @@ -124,6 +124,51 @@ impl SpacetimeClient { action_type: payload.action_type, session, work, + queue_state: None, + }) + } + + pub async fn mark_puzzle_clear_generation_queued( + &self, + session_id: String, + owner_user_id: String, + author_display_name: String, + payload: PuzzleClearActionRequest, + ) -> Result { + let current = self + .get_puzzle_clear_session(session_id.clone(), owner_user_id.clone()) + .await?; + let action_type = payload.action_type.clone(); + let scope = match action_type { + PuzzleClearActionType::CompileDraft => PuzzleClearDraftMergeScope::CompileDraft, + PuzzleClearActionType::RegenerateAtlas => PuzzleClearDraftMergeScope::RegenerateAtlas, + _ => { + return Err(SpacetimeClientError::validation_failed( + "puzzle-clear queued generation 只支持 compile-draft/regenerate-atlas", + )); + } + }; + let mut draft = merge_action_into_draft(current.draft.clone(), &payload, scope)?; + let profile_id = + resolve_puzzle_clear_profile_id(&draft, &action_type, payload.profile_id.as_deref())?; + draft.profile_id = Some(profile_id.clone()); + draft.generation_status = PuzzleClearGenerationStatus::Generating; + let session = self + .compile_puzzle_clear_draft(build_generating_compile_input( + ¤t, + &owner_user_id, + &author_display_name, + &profile_id, + &draft, + current_unix_micros(), + )?) + .await?; + + Ok(PuzzleClearActionResponse { + action_type, + session, + work: None, + queue_state: None, }) } @@ -647,6 +692,38 @@ fn build_compile_input( }) } +fn build_generating_compile_input( + current: &PuzzleClearSessionSnapshotResponse, + owner_user_id: &str, + author_display_name: &str, + profile_id: &str, + draft: &PuzzleClearDraftResponse, + now_micros: i64, +) -> Result { + Ok(PuzzleClearDraftCompileInput { + session_id: current.session_id.clone(), + owner_user_id: owner_user_id.to_string(), + profile_id: profile_id.to_string(), + author_display_name: non_empty_str(author_display_name) + .unwrap_or_else(|| "拼消消玩家".to_string()), + work_title: draft.work_title.clone(), + work_description: draft.work_description.clone(), + theme_prompt: draft.theme_prompt.clone(), + board_background_prompt: draft.board_background_prompt.clone(), + generate_board_background: draft.generate_board_background, + board_background_asset_json: draft + .board_background_asset + .as_ref() + .map(json_string) + .transpose()?, + atlas_asset_json: draft.atlas_asset.as_ref().map(json_string).transpose()?, + pattern_groups_json: Some(json_string(&draft.pattern_groups)?), + card_assets_json: Some(json_string(&draft.card_assets)?), + generation_status: Some("generating".to_string()), + compiled_at_micros: now_micros, + }) +} + fn build_failed_compile_input( current: &PuzzleClearSessionSnapshotResponse, owner_user_id: &str, diff --git a/server-rs/crates/spacetime-client/src/wooden_fish.rs b/server-rs/crates/spacetime-client/src/wooden_fish.rs index ddc7f8674..66ad6fc29 100644 --- a/server-rs/crates/spacetime-client/src/wooden_fish.rs +++ b/server-rs/crates/spacetime-client/src/wooden_fish.rs @@ -119,6 +119,53 @@ impl SpacetimeClient { action_type: payload.action_type, session, work, + queue_state: None, + }) + } + + pub async fn mark_wooden_fish_generation_queued( + &self, + session_id: String, + owner_user_id: String, + author_display_name: String, + payload: WoodenFishActionRequest, + ) -> Result { + let current = self + .get_wooden_fish_session(session_id.clone(), owner_user_id.clone()) + .await?; + let action_type = payload.action_type.clone(); + let scope = match action_type { + WoodenFishActionType::CompileDraft => WoodenFishDraftMergeScope::CompileDraft, + WoodenFishActionType::RegenerateHitObject => { + WoodenFishDraftMergeScope::RegenerateHitObject + } + _ => { + return Err(SpacetimeClientError::validation_failed( + "wooden-fish queued generation 只支持 compile-draft/regenerate-hit-object", + )); + } + }; + let mut draft = merge_action_into_draft(current.draft.clone(), &payload, scope)?; + let profile_id = + resolve_wooden_fish_profile_id(&draft, &action_type, payload.profile_id.as_deref())?; + draft.profile_id = Some(profile_id.clone()); + draft.generation_status = WoodenFishGenerationStatus::Generating; + let session = self + .compile_wooden_fish_draft(build_generating_compile_input( + ¤t, + &owner_user_id, + &author_display_name, + &profile_id, + &draft, + current_unix_micros(), + )?) + .await?; + + Ok(WoodenFishActionResponse { + action_type, + session, + work: None, + queue_state: None, }) } @@ -689,6 +736,52 @@ fn build_compile_input( }) } +fn build_generating_compile_input( + current: &WoodenFishSessionSnapshotResponse, + owner_user_id: &str, + author_display_name: &str, + profile_id: &str, + draft: &WoodenFishDraftResponse, + now_micros: i64, +) -> Result { + Ok(WoodenFishDraftCompileInput { + session_id: current.session_id.clone(), + owner_user_id: owner_user_id.to_string(), + profile_id: profile_id.to_string(), + author_display_name: author_display_name.trim().to_string(), + work_title: draft.work_title.clone(), + work_description: draft.work_description.clone(), + theme_tags_json: Some(json_string(&draft.theme_tags)?), + hit_object_prompt: draft.hit_object_prompt.clone(), + hit_object_reference_image_src: draft.hit_object_reference_image_src.clone(), + hit_sound_prompt: draft.hit_sound_prompt.clone(), + hit_object_asset_json: draft + .hit_object_asset + .as_ref() + .map(json_string) + .transpose()?, + background_asset_json: draft + .background_asset + .as_ref() + .map(json_string) + .transpose()?, + hit_sound_asset_json: draft + .hit_sound_asset + .as_ref() + .map(json_string) + .transpose()?, + back_button_asset_json: draft + .back_button_asset + .as_ref() + .map(json_string) + .transpose()?, + floating_words_json: Some(json_string(&draft.floating_words)?), + cover_image_src: draft.cover_image_src.clone(), + generation_status: Some("generating".to_string()), + compiled_at_micros: now_micros, + }) +} + fn build_failed_compile_input( current: &WoodenFishSessionSnapshotResponse, owner_user_id: &str, diff --git a/server-rs/crates/spacetime-module/src/external_generation.rs b/server-rs/crates/spacetime-module/src/external_generation.rs new file mode 100644 index 000000000..0de980e1b --- /dev/null +++ b/server-rs/crates/spacetime-module/src/external_generation.rs @@ -0,0 +1,936 @@ +use crate::*; + +const EXTERNAL_GENERATION_STATUS_PENDING: &str = "pending"; +const EXTERNAL_GENERATION_STATUS_RUNNING: &str = "running"; +const EXTERNAL_GENERATION_STATUS_COMPLETED: &str = "completed"; +const EXTERNAL_GENERATION_STATUS_FAILED: &str = "failed"; +const EXTERNAL_GENERATION_STATUS_CANCELLED: &str = "cancelled"; + +#[spacetimedb::table( + accessor = external_generation_job, + index( + accessor = by_external_generation_job_status_available, + btree(columns = [status, available_at]) + ), + index( + accessor = by_external_generation_job_worker_id, + btree(columns = [worker_id]) + ), + index( + accessor = by_external_generation_job_source, + btree(columns = [source_module, source_entity_id]) + ), + index( + accessor = by_external_generation_job_owner_user_id, + btree(columns = [owner_user_id]) + ) +)] +#[derive(Clone)] +pub struct ExternalGenerationJob { + #[primary_key] + pub(crate) job_id: String, + #[unique] + pub(crate) dedupe_key: String, + pub(crate) job_kind: String, + pub(crate) owner_user_id: String, + pub(crate) source_module: String, + pub(crate) source_entity_id: String, + pub(crate) request_label: String, + pub(crate) request_payload_json: String, + pub(crate) status: String, + pub(crate) attempt: u32, + pub(crate) max_attempts: u32, + pub(crate) last_error_message: Option, + pub(crate) worker_id: Option, + pub(crate) lease_expires_at: Option, + pub(crate) available_at: Timestamp, + pub(crate) result_payload_json: Option, + pub(crate) created_at: Timestamp, + pub(crate) started_at: Option, + pub(crate) completed_at: Option, + pub(crate) updated_at: Timestamp, + #[default(None::)] + pub(crate) lease_token: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobEnqueueInput { + pub job_id: String, + pub dedupe_key: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_payload_json: String, + pub max_attempts: u32, + pub available_at_micros: i64, + pub created_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobClaimInput { + pub worker_id: String, + pub limit: u32, + pub lease_expires_at_micros: i64, + pub claimed_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobRenewLeaseInput { + pub job_id: String, + pub worker_id: String, + pub lease_token: String, + pub lease_expires_at_micros: i64, + pub renewed_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobCompleteInput { + pub job_id: String, + pub worker_id: String, + pub lease_token: String, + pub result_payload_json: Option, + pub completed_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobFailInput { + pub job_id: String, + pub worker_id: String, + pub lease_token: String, + pub error_message: String, + pub retry_after_micros: i64, + pub failed_at_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobGetInput { + pub job_id: String, + pub owner_user_id: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobSnapshot { + pub job_id: String, + pub dedupe_key: String, + pub job_kind: String, + pub owner_user_id: String, + pub source_module: String, + pub source_entity_id: String, + pub request_label: String, + pub request_payload_json: String, + pub status: String, + pub attempt: u32, + pub max_attempts: u32, + pub last_error_message: Option, + pub worker_id: Option, + pub lease_expires_at_micros: Option, + pub available_at_micros: i64, + pub result_payload_json: Option, + pub created_at_micros: i64, + pub started_at_micros: Option, + pub completed_at_micros: Option, + pub updated_at_micros: i64, + pub lease_token: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobProcedureResult { + pub ok: bool, + pub job: Option, + pub jobs: Vec, + pub error_message: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationQueueStatsSnapshot { + pub pending_count: u32, + pub delayed_pending_count: u32, + pub claimable_pending_count: u32, + pub running_active_count: u32, + pub expired_running_count: u32, + // 中文注释：保留字段兼容已生成 bindings；controller 只按非终态队列压力扩缩容，不每轮扫描历史终态任务。 + pub terminal_count: u32, + pub claimable_count: u32, + pub oldest_claimable_age_micros: Option, + pub now_micros: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationQueueStatsProcedureResult { + pub ok: bool, + pub stats: Option, + pub error_message: Option, +} + +#[spacetimedb::procedure] +pub fn enqueue_external_generation_job_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobEnqueueInput, +) -> ExternalGenerationJobProcedureResult { + match ctx.try_with_tx(|tx| enqueue_external_generation_job_tx(tx, input.clone())) { + Ok(job) => single_external_generation_job_result(job), + Err(message) => failed_external_generation_job_result(message), + } +} + +#[spacetimedb::procedure] +pub fn claim_external_generation_jobs_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobClaimInput, +) -> ExternalGenerationJobProcedureResult { + match ctx.try_with_tx(|tx| claim_external_generation_jobs_tx(tx, input.clone())) { + Ok(jobs) => ExternalGenerationJobProcedureResult { + ok: true, + job: None, + jobs, + error_message: None, + }, + Err(message) => failed_external_generation_job_result(message), + } +} + +#[spacetimedb::procedure] +pub fn complete_external_generation_job_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobCompleteInput, +) -> ExternalGenerationJobProcedureResult { + match ctx.try_with_tx(|tx| complete_external_generation_job_tx(tx, input.clone())) { + Ok(job) => single_external_generation_job_result(job), + Err(message) => failed_external_generation_job_result(message), + } +} + +#[spacetimedb::procedure] +pub fn renew_external_generation_job_lease_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobRenewLeaseInput, +) -> ExternalGenerationJobProcedureResult { + match ctx.try_with_tx(|tx| renew_external_generation_job_lease_tx(tx, input.clone())) { + Ok(job) => single_external_generation_job_result(job), + Err(message) => failed_external_generation_job_result(message), + } +} + +#[spacetimedb::procedure] +pub fn fail_external_generation_job_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobFailInput, +) -> ExternalGenerationJobProcedureResult { + match ctx.try_with_tx(|tx| fail_external_generation_job_tx(tx, input.clone())) { + Ok(job) => single_external_generation_job_result(job), + Err(message) => failed_external_generation_job_result(message), + } +} + +#[spacetimedb::procedure] +pub fn get_external_generation_job_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobGetInput, +) -> ExternalGenerationJobProcedureResult { + match ctx.try_with_tx(|tx| get_external_generation_job_tx(tx, input.clone())) { + Ok(job) => single_external_generation_job_result(job), + Err(message) => failed_external_generation_job_result(message), + } +} + +#[spacetimedb::procedure] +pub fn get_external_generation_queue_stats_and_return( + ctx: &mut ProcedureContext, +) -> ExternalGenerationQueueStatsProcedureResult { + match ctx.try_with_tx(|tx| get_external_generation_queue_stats_tx(tx)) { + Ok(stats) => ExternalGenerationQueueStatsProcedureResult { + ok: true, + stats: Some(stats), + error_message: None, + }, + Err(message) => ExternalGenerationQueueStatsProcedureResult { + ok: false, + stats: None, + error_message: Some(message), + }, + } +} + +fn enqueue_external_generation_job_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobEnqueueInput, +) -> Result { + validate_required("external_generation_job.job_id", &input.job_id)?; + validate_required("external_generation_job.dedupe_key", &input.dedupe_key)?; + validate_required("external_generation_job.job_kind", &input.job_kind)?; + validate_required( + "external_generation_job.owner_user_id", + &input.owner_user_id, + )?; + validate_required( + "external_generation_job.source_module", + &input.source_module, + )?; + validate_required( + "external_generation_job.source_entity_id", + &input.source_entity_id, + )?; + validate_required( + "external_generation_job.request_label", + &input.request_label, + )?; + validate_required( + "external_generation_job.request_payload_json", + &input.request_payload_json, + )?; + + if let Some(row) = ctx + .db + .external_generation_job() + .dedupe_key() + .find(&input.dedupe_key) + { + return Ok(map_external_generation_job_row(row)); + } + if ctx + .db + .external_generation_job() + .job_id() + .find(&input.job_id) + .is_some() + { + return Err("external_generation_job.job_id 已存在".to_string()); + } + + let now = Timestamp::from_micros_since_unix_epoch(input.created_at_micros); + let available_at = Timestamp::from_micros_since_unix_epoch(input.available_at_micros); + let row = ExternalGenerationJob { + job_id: input.job_id.trim().to_string(), + dedupe_key: input.dedupe_key.trim().to_string(), + job_kind: input.job_kind.trim().to_string(), + owner_user_id: input.owner_user_id.trim().to_string(), + source_module: input.source_module.trim().to_string(), + source_entity_id: input.source_entity_id.trim().to_string(), + request_label: input.request_label.trim().to_string(), + request_payload_json: input.request_payload_json.trim().to_string(), + status: EXTERNAL_GENERATION_STATUS_PENDING.to_string(), + attempt: 0, + max_attempts: input.max_attempts.max(1), + last_error_message: None, + worker_id: None, + lease_expires_at: None, + available_at, + result_payload_json: None, + created_at: now, + started_at: None, + completed_at: None, + updated_at: now, + lease_token: None, + }; + ctx.db.external_generation_job().insert(row.clone()); + Ok(map_external_generation_job_row(row)) +} + +fn claim_external_generation_jobs_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobClaimInput, +) -> Result, String> { + validate_required("external_generation_job.worker_id", &input.worker_id)?; + if input.limit == 0 { + return Ok(Vec::new()); + } + + let claim_time = ctx.timestamp; + let lease_duration_micros = duration_between_micros( + input.lease_expires_at_micros, + input.claimed_at_micros, + "external_generation_job.lease_duration", + )?; + let lease_expires_at = timestamp_after_micros(claim_time, lease_duration_micros); + let worker_id = input.worker_id.trim().to_string(); + let limit = input.limit.min(64) as usize; + let mut candidates = Vec::new(); + + candidates.extend( + ctx.db + .external_generation_job() + .by_external_generation_job_status_available() + .filter(&EXTERNAL_GENERATION_STATUS_PENDING.to_string()) + .filter(|row| is_external_generation_job_claimable(row, claim_time)), + ); + candidates.extend( + ctx.db + .external_generation_job() + .by_external_generation_job_status_available() + .filter(&EXTERNAL_GENERATION_STATUS_RUNNING.to_string()) + .filter(|row| is_external_generation_job_claimable(row, claim_time)), + ); + + candidates.sort_by(|left, right| { + left.available_at + .to_micros_since_unix_epoch() + .cmp(&right.available_at.to_micros_since_unix_epoch()) + .then_with(|| { + left.created_at + .to_micros_since_unix_epoch() + .cmp(&right.created_at.to_micros_since_unix_epoch()) + }) + .then_with(|| left.job_id.cmp(&right.job_id)) + }); + + let mut claimed = Vec::new(); + for mut row in candidates.into_iter().take(limit) { + let next_attempt = row.attempt.saturating_add(1); + let lease_token = build_external_generation_lease_token( + &row.job_id, + &worker_id, + next_attempt, + claim_time, + ); + row.status = EXTERNAL_GENERATION_STATUS_RUNNING.to_string(); + row.worker_id = Some(worker_id.clone()); + row.lease_expires_at = Some(lease_expires_at); + row.lease_token = Some(lease_token); + row.attempt = next_attempt; + if row.started_at.is_none() { + row.started_at = Some(claim_time); + } + row.updated_at = claim_time; + persist_external_generation_job_row(ctx, row.clone()); + claimed.push(map_external_generation_job_row(row)); + } + + Ok(claimed) +} + +fn complete_external_generation_job_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobCompleteInput, +) -> Result { + let mut row = get_worker_owned_external_generation_job( + ctx, + &input.job_id, + &input.worker_id, + &input.lease_token, + )?; + let completed_at = ctx.timestamp; + row.status = EXTERNAL_GENERATION_STATUS_COMPLETED.to_string(); + row.result_payload_json = input + .result_payload_json + .and_then(|value| normalize_optional_text(value.as_str())); + row.lease_expires_at = None; + row.completed_at = Some(completed_at); + row.updated_at = completed_at; + persist_external_generation_job_row(ctx, row.clone()); + Ok(map_external_generation_job_row(row)) +} + +fn get_external_generation_job_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobGetInput, +) -> Result { + validate_required("external_generation_job.job_id", &input.job_id)?; + validate_required( + "external_generation_job.owner_user_id", + &input.owner_user_id, + )?; + let row = ctx + .db + .external_generation_job() + .job_id() + .find(&input.job_id.trim().to_string()) + .ok_or_else(|| "external_generation_job 不存在".to_string())?; + if row.owner_user_id.trim() != input.owner_user_id.trim() { + return Err("external_generation_job 不存在".to_string()); + } + + Ok(map_external_generation_job_row(row)) +} + +fn renew_external_generation_job_lease_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobRenewLeaseInput, +) -> Result { + let mut row = get_worker_owned_external_generation_job( + ctx, + &input.job_id, + &input.worker_id, + &input.lease_token, + )?; + let renewed_at = ctx.timestamp; + let lease_duration_micros = duration_between_micros( + input.lease_expires_at_micros, + input.renewed_at_micros, + "external_generation_job.lease_duration", + )?; + row.lease_expires_at = Some(timestamp_after_micros(renewed_at, lease_duration_micros)); + row.updated_at = renewed_at; + persist_external_generation_job_row(ctx, row.clone()); + Ok(map_external_generation_job_row(row)) +} + +fn fail_external_generation_job_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobFailInput, +) -> Result { + let error_message = input.error_message.trim(); + if error_message.is_empty() { + return Err("external_generation_job.error_message 不能为空".to_string()); + } + + let mut row = get_worker_owned_external_generation_job( + ctx, + &input.job_id, + &input.worker_id, + &input.lease_token, + )?; + let failed_at = ctx.timestamp; + let retry_delay_micros = duration_between_micros( + input.retry_after_micros, + input.failed_at_micros, + "external_generation_job.retry_delay", + )?; + row.last_error_message = Some(error_message.to_string()); + row.lease_expires_at = None; + row.worker_id = None; + row.lease_token = None; + row.updated_at = failed_at; + + if row.attempt < row.max_attempts { + row.status = EXTERNAL_GENERATION_STATUS_PENDING.to_string(); + row.available_at = timestamp_after_micros(failed_at, retry_delay_micros); + } else { + row.status = EXTERNAL_GENERATION_STATUS_FAILED.to_string(); + row.completed_at = Some(failed_at); + } + + persist_external_generation_job_row(ctx, row.clone()); + Ok(map_external_generation_job_row(row)) +} + +fn get_external_generation_queue_stats_tx( + ctx: &ReducerContext, +) -> Result { + let now = ctx.timestamp; + let now_micros = now.to_micros_since_unix_epoch(); + let mut stats = ExternalGenerationQueueStatsSnapshot { + pending_count: 0, + delayed_pending_count: 0, + claimable_pending_count: 0, + running_active_count: 0, + expired_running_count: 0, + terminal_count: 0, + claimable_count: 0, + oldest_claimable_age_micros: None, + now_micros, + }; + + for row in ctx + .db + .external_generation_job() + .by_external_generation_job_status_available() + .filter(&EXTERNAL_GENERATION_STATUS_PENDING.to_string()) + { + stats.pending_count = stats.pending_count.saturating_add(1); + if is_external_generation_job_claimable(&row, now) { + stats.claimable_pending_count = stats.claimable_pending_count.saturating_add(1); + record_external_generation_claimable_age(&mut stats, &row, now_micros); + } else { + stats.delayed_pending_count = stats.delayed_pending_count.saturating_add(1); + } + } + + for row in ctx + .db + .external_generation_job() + .by_external_generation_job_status_available() + .filter(&EXTERNAL_GENERATION_STATUS_RUNNING.to_string()) + { + if is_external_generation_job_claimable(&row, now) { + stats.expired_running_count = stats.expired_running_count.saturating_add(1); + record_external_generation_claimable_age(&mut stats, &row, now_micros); + } else { + stats.running_active_count = stats.running_active_count.saturating_add(1); + } + } + + stats.claimable_count = stats + .claimable_pending_count + .saturating_add(stats.expired_running_count); + Ok(stats) +} + +pub(crate) fn validate_external_generation_job_lease_for_tx( + ctx: &ReducerContext, + job_id: &str, + worker_id: &str, + lease_token: &str, + expected_job_kinds: &[&str], + expected_owner_user_id: &str, + expected_source_module: &str, + expected_source_entity_ids: &[String], +) -> Result<(), String> { + let row = get_worker_owned_external_generation_job(ctx, job_id, worker_id, lease_token)?; + if !expected_job_kinds.is_empty() + && !expected_job_kinds + .iter() + .any(|expected| row.job_kind.trim() == expected.trim()) + { + return Err("external_generation_job job_kind 与业务写回不匹配".to_string()); + } + if row.owner_user_id.trim() != expected_owner_user_id.trim() { + return Err("external_generation_job owner_user_id 与业务写回不匹配".to_string()); + } + if row.source_module.trim() != expected_source_module.trim() { + return Err("external_generation_job source_module 与业务写回不匹配".to_string()); + } + if !expected_source_entity_ids + .iter() + .any(|expected| row.source_entity_id.trim() == expected.trim()) + { + return Err("external_generation_job source_entity_id 与业务写回不匹配".to_string()); + } + Ok(()) +} + +fn get_worker_owned_external_generation_job( + ctx: &ReducerContext, + job_id: &str, + worker_id: &str, + lease_token: &str, +) -> Result { + validate_required("external_generation_job.job_id", job_id)?; + validate_required("external_generation_job.worker_id", worker_id)?; + validate_required("external_generation_job.lease_token", lease_token)?; + let row = ctx + .db + .external_generation_job() + .job_id() + .find(&job_id.trim().to_string()) + .ok_or_else(|| "external_generation_job 不存在".to_string())?; + if row.status != EXTERNAL_GENERATION_STATUS_RUNNING { + return Err("external_generation_job 当前不是 running 状态".to_string()); + } + if !is_external_generation_job_owned_by_worker(&row, worker_id) { + return Err("external_generation_job worker lease 不匹配".to_string()); + } + if !is_external_generation_job_owned_by_lease_token(&row, lease_token) { + return Err("external_generation_job lease token 不匹配".to_string()); + } + if !is_external_generation_job_lease_active(&row, ctx.timestamp) { + return Err("external_generation_job lease 已过期".to_string()); + } + Ok(row) +} + +fn is_external_generation_job_owned_by_worker( + row: &ExternalGenerationJob, + worker_id: &str, +) -> bool { + row.worker_id.as_deref() == Some(worker_id.trim()) +} + +fn is_external_generation_job_owned_by_lease_token( + row: &ExternalGenerationJob, + lease_token: &str, +) -> bool { + row.lease_token.as_deref() == Some(lease_token.trim()) +} + +fn is_external_generation_job_lease_active(row: &ExternalGenerationJob, now: Timestamp) -> bool { + row.lease_expires_at + .map(|lease_expires_at| lease_expires_at > now) + .unwrap_or(false) +} + +fn is_external_generation_job_claimable(row: &ExternalGenerationJob, now: Timestamp) -> bool { + match row.status.as_str() { + EXTERNAL_GENERATION_STATUS_PENDING => row.available_at <= now, + EXTERNAL_GENERATION_STATUS_RUNNING => row + .lease_expires_at + .map(|lease_expires_at| lease_expires_at <= now) + .unwrap_or(true), + EXTERNAL_GENERATION_STATUS_COMPLETED + | EXTERNAL_GENERATION_STATUS_FAILED + | EXTERNAL_GENERATION_STATUS_CANCELLED => false, + _ => false, + } +} + +fn record_external_generation_claimable_age( + stats: &mut ExternalGenerationQueueStatsSnapshot, + row: &ExternalGenerationJob, + now_micros: i64, +) { + let age = now_micros + .saturating_sub(row.available_at.to_micros_since_unix_epoch()) + .max(0); + stats.oldest_claimable_age_micros = Some( + stats + .oldest_claimable_age_micros + .map(|current| current.max(age)) + .unwrap_or(age), + ); +} + +fn persist_external_generation_job_row(ctx: &ReducerContext, row: ExternalGenerationJob) { + ctx.db + .external_generation_job() + .job_id() + .delete(&row.job_id); + ctx.db.external_generation_job().insert(row); +} + +fn map_external_generation_job_row(row: ExternalGenerationJob) -> ExternalGenerationJobSnapshot { + ExternalGenerationJobSnapshot { + job_id: row.job_id, + dedupe_key: row.dedupe_key, + job_kind: row.job_kind, + owner_user_id: row.owner_user_id, + source_module: row.source_module, + source_entity_id: row.source_entity_id, + request_label: row.request_label, + request_payload_json: row.request_payload_json, + status: row.status, + attempt: row.attempt, + max_attempts: row.max_attempts, + last_error_message: row.last_error_message, + worker_id: row.worker_id, + lease_expires_at_micros: row + .lease_expires_at + .map(|value| value.to_micros_since_unix_epoch()), + available_at_micros: row.available_at.to_micros_since_unix_epoch(), + result_payload_json: row.result_payload_json, + created_at_micros: row.created_at.to_micros_since_unix_epoch(), + started_at_micros: row + .started_at + .map(|value| value.to_micros_since_unix_epoch()), + completed_at_micros: row + .completed_at + .map(|value| value.to_micros_since_unix_epoch()), + updated_at_micros: row.updated_at.to_micros_since_unix_epoch(), + lease_token: row.lease_token, + } +} + +fn single_external_generation_job_result( + job: ExternalGenerationJobSnapshot, +) -> ExternalGenerationJobProcedureResult { + ExternalGenerationJobProcedureResult { + ok: true, + job: Some(job), + jobs: Vec::new(), + error_message: None, + } +} + +fn failed_external_generation_job_result(message: String) -> ExternalGenerationJobProcedureResult { + ExternalGenerationJobProcedureResult { + ok: false, + job: None, + jobs: Vec::new(), + error_message: Some(message), + } +} + +fn validate_required(field: &str, value: &str) -> Result<(), String> { + if value.trim().is_empty() { + return Err(format!("{field} 不能为空")); + } + Ok(()) +} + +fn duration_between_micros(later: i64, earlier: i64, field: &str) -> Result { + let duration = later.saturating_sub(earlier); + if duration <= 0 { + return Err(format!("{field} 必须大于 0")); + } + Ok(duration) +} + +fn timestamp_after_micros(timestamp: Timestamp, duration_micros: i64) -> Timestamp { + Timestamp::from_micros_since_unix_epoch( + timestamp + .to_micros_since_unix_epoch() + .saturating_add(duration_micros.max(0)), + ) +} + +fn build_external_generation_lease_token( + job_id: &str, + worker_id: &str, + attempt: u32, + claimed_at: Timestamp, +) -> String { + format!( + "{}:{}:{}:{}", + job_id.trim(), + worker_id.trim(), + attempt, + claimed_at.to_micros_since_unix_epoch() + ) +} + +fn normalize_optional_text(value: &str) -> Option { + let normalized = value.trim(); + if normalized.is_empty() { + None + } else { + Some(normalized.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn external_generation_job_result_failure_is_structured() { + let result = failed_external_generation_job_result("失败".to_string()); + assert!(!result.ok); + assert_eq!(result.error_message.as_deref(), Some("失败")); + assert!(result.jobs.is_empty()); + } + + #[test] + fn pending_job_is_claimable_only_after_available_time() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_PENDING); + row.available_at = micros(1_000); + + assert!(!is_external_generation_job_claimable(&row, micros(999))); + assert!(is_external_generation_job_claimable(&row, micros(1_000))); + } + + #[test] + fn running_job_is_claimable_only_after_lease_expires() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_RUNNING); + row.lease_expires_at = Some(micros(2_000)); + + assert!(!is_external_generation_job_claimable(&row, micros(1_999))); + assert!(is_external_generation_job_claimable(&row, micros(2_000))); + } + + #[test] + fn terminal_job_is_not_claimable() { + for status in [ + EXTERNAL_GENERATION_STATUS_COMPLETED, + EXTERNAL_GENERATION_STATUS_FAILED, + EXTERNAL_GENERATION_STATUS_CANCELLED, + ] { + let row = external_generation_job_fixture(status); + + assert!(!is_external_generation_job_claimable(&row, micros(10_000))); + } + } + + #[test] + fn worker_ownership_requires_matching_trimmed_worker_id() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_RUNNING); + row.worker_id = Some("worker-a".to_string()); + + assert!(is_external_generation_job_owned_by_worker( + &row, + " worker-a " + )); + assert!(!is_external_generation_job_owned_by_worker( + &row, "worker-b" + )); + } + + #[test] + fn worker_ownership_requires_matching_trimmed_lease_token() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_RUNNING); + row.lease_token = Some("job-1:worker-a:1:1000".to_string()); + + assert!(is_external_generation_job_owned_by_lease_token( + &row, + " job-1:worker-a:1:1000 " + )); + assert!(!is_external_generation_job_owned_by_lease_token( + &row, + "job-1:worker-a:2:2000" + )); + } + + #[test] + fn worker_lease_is_active_only_before_expiry() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_RUNNING); + row.lease_expires_at = Some(micros(2_000)); + + assert!(is_external_generation_job_lease_active(&row, micros(1_999))); + assert!(!is_external_generation_job_lease_active( + &row, + micros(2_000) + )); + } + + #[test] + fn lease_token_changes_with_claim_attempt() { + let first = + build_external_generation_lease_token("extgen-test", "worker-a", 1, micros(1_000)); + let second = + build_external_generation_lease_token("extgen-test", "worker-a", 2, micros(2_000)); + + assert_ne!(first, second); + } + + #[test] + fn claimable_age_keeps_oldest_available_job() { + let mut stats = ExternalGenerationQueueStatsSnapshot { + pending_count: 0, + delayed_pending_count: 0, + claimable_pending_count: 0, + running_active_count: 0, + expired_running_count: 0, + terminal_count: 0, + claimable_count: 0, + oldest_claimable_age_micros: None, + now_micros: 10_000, + }; + let mut old_job = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_PENDING); + old_job.available_at = micros(1_000); + let mut newer_job = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_RUNNING); + newer_job.available_at = micros(8_000); + + record_external_generation_claimable_age(&mut stats, &newer_job, 10_000); + record_external_generation_claimable_age(&mut stats, &old_job, 10_000); + + assert_eq!(stats.oldest_claimable_age_micros, Some(9_000)); + } + + #[test] + fn positive_duration_between_client_times_is_preserved() { + assert_eq!( + duration_between_micros(3_500, 1_000, "external_generation_job.lease_duration"), + Ok(2_500), + ); + assert!(duration_between_micros(1_000, 1_000, "duration").is_err()); + } + + fn external_generation_job_fixture(status: &str) -> ExternalGenerationJob { + ExternalGenerationJob { + job_id: "extgen-test".to_string(), + dedupe_key: "puzzle:compile:test".to_string(), + job_kind: "puzzle_compile_draft".to_string(), + owner_user_id: "user-1".to_string(), + source_module: "puzzle".to_string(), + source_entity_id: "session-1".to_string(), + request_label: "拼图首关草稿生成".to_string(), + request_payload_json: r#"{"sessionId":"session-1"}"#.to_string(), + status: status.to_string(), + attempt: 0, + max_attempts: 1, + last_error_message: None, + worker_id: None, + lease_expires_at: None, + available_at: micros(0), + result_payload_json: None, + created_at: micros(0), + started_at: None, + completed_at: None, + updated_at: micros(0), + lease_token: None, + } + } + + fn micros(value: i64) -> Timestamp { + Timestamp::from_micros_since_unix_epoch(value) + } +} diff --git a/server-rs/crates/spacetime-module/src/lib.rs b/server-rs/crates/spacetime-module/src/lib.rs index c54055cea..51cfd9eae 100644 --- a/server-rs/crates/spacetime-module/src/lib.rs +++ b/server-rs/crates/spacetime-module/src/lib.rs @@ -31,6 +31,7 @@ mod big_fish; mod custom_world; mod domain_types; mod entry; +mod external_generation; mod gameplay; mod jump_hop; mod match3d; @@ -51,6 +52,7 @@ pub use big_fish::*; pub use custom_world::*; pub use domain_types::*; pub use entry::*; +pub use external_generation::*; pub use gameplay::*; pub use jump_hop::*; pub use match3d::*; diff --git a/server-rs/crates/spacetime-module/src/migration.rs b/server-rs/crates/spacetime-module/src/migration.rs index 6de73df5c..8734e09c1 100644 --- a/server-rs/crates/spacetime-module/src/migration.rs +++ b/server-rs/crates/spacetime-module/src/migration.rs @@ -183,6 +183,7 @@ macro_rules! migration_tables { ai_text_chunk, ai_result_reference, ai_task_event, + external_generation_job, runtime_snapshot, runtime_setting, creation_entry_config, diff --git a/server-rs/crates/spacetime-module/src/puzzle.rs b/server-rs/crates/spacetime-module/src/puzzle.rs index 2106cccbe..6fded8c1b 100644 --- a/server-rs/crates/spacetime-module/src/puzzle.rs +++ b/server-rs/crates/spacetime-module/src/puzzle.rs @@ -14,12 +14,12 @@ use module_puzzle::{ PuzzleBackgroundCompileTaskProcedureResult, PuzzleBackgroundCompileTaskReleaseInput, PuzzleDraftCompileFailureInput, PuzzleDraftCompileInput, PuzzleFormDraftSaveInput, PuzzleGeneratedImageCandidate, PuzzleGeneratedImagesSaveInput, PuzzleLeaderboardEntry, - PuzzleLeaderboardSubmitInput, PuzzlePublicationStatus, PuzzlePublishInput, - PuzzleRecommendedNextWork, PuzzleResultDraft, PuzzleRunDragInput, PuzzleRunGetInput, - PuzzleRunNextLevelInput, PuzzleRunPauseInput, PuzzleRunProcedureResult, PuzzleRunPropInput, - PuzzleRunSnapshot, PuzzleRunStartInput, PuzzleRunSwapInput, PuzzleRuntimeLevelStatus, - PuzzleSelectCoverImageInput, PuzzleUiBackgroundSaveInput, PuzzleWorkDeleteInput, - PuzzleWorkGetInput, PuzzleWorkLikeRecordInput as PuzzleWorkLikeInput, + PuzzleLeaderboardSubmitInput, PuzzleLevelGenerationFailureInput, PuzzlePublicationStatus, + PuzzlePublishInput, PuzzleRecommendedNextWork, PuzzleResultDraft, PuzzleRunDragInput, + PuzzleRunGetInput, PuzzleRunNextLevelInput, PuzzleRunPauseInput, PuzzleRunProcedureResult, + PuzzleRunPropInput, PuzzleRunSnapshot, PuzzleRunStartInput, PuzzleRunSwapInput, + PuzzleRuntimeLevelStatus, PuzzleSelectCoverImageInput, PuzzleUiBackgroundSaveInput, + PuzzleWorkDeleteInput, PuzzleWorkGetInput, PuzzleWorkLikeRecordInput as PuzzleWorkLikeInput, PuzzleWorkPointIncentiveClaimInput, PuzzleWorkProcedureResult, PuzzleWorkProfile, PuzzleWorkRemixInput, PuzzleWorkUpsertInput, PuzzleWorksListInput, PuzzleWorksProcedureResult, apply_publish_overrides_to_draft, apply_selected_candidate, build_form_draft_from_seed, @@ -38,10 +38,15 @@ use spacetimedb::{ }; use crate::auth::user_account; +use crate::validate_external_generation_job_lease_for_tx; const PUZZLE_POINT_INCENTIVE_DEFAULT_U64: u64 = 0; const PUZZLE_BACKGROUND_COMPILE_TASK_LEASE_MICROS: i64 = 30 * 60 * 1_000_000; const WORK_VISIBLE_DEFAULT: bool = true; +const PUZZLE_EXTERNAL_GENERATION_SOURCE_MODULE: &str = "puzzle"; +const PUZZLE_COMPILE_DRAFT_JOB_KIND: &str = "puzzle_compile_draft"; +const PUZZLE_GENERATE_IMAGES_JOB_KIND: &str = "puzzle_generate_images"; +const PUZZLE_GENERATE_UI_BACKGROUND_JOB_KIND: &str = "puzzle_generate_ui_background"; /// 拼图 Agent session 真相表。 /// 当前只保存结构化字段与 JSON 草稿，不提前拆出更多编辑态子表。 @@ -407,6 +412,25 @@ pub fn mark_puzzle_draft_generation_failed( } } +#[spacetimedb::procedure] +pub fn mark_puzzle_level_generation_failed( + ctx: &mut ProcedureContext, + input: PuzzleLevelGenerationFailureInput, +) -> PuzzleAgentSessionProcedureResult { + match ctx.try_with_tx(|tx| mark_puzzle_level_generation_failed_tx(tx, input.clone())) { + Ok(session) => PuzzleAgentSessionProcedureResult { + ok: true, + session: Some(session), + error_message: None, + }, + Err(message) => PuzzleAgentSessionProcedureResult { + ok: false, + session: None, + error_message: Some(message), + }, + } +} + #[spacetimedb::procedure] pub fn claim_puzzle_background_compile_task( ctx: &mut ProcedureContext, @@ -444,7 +468,6 @@ pub fn release_puzzle_background_compile_task( }, } } - /// 保存拼图入口表单草稿。 /// 中文注释：该 procedure 只更新 session 与创作中心草稿卡，不触发图片生成或发布校验。 #[spacetimedb::procedure] @@ -1035,6 +1058,17 @@ fn compile_puzzle_agent_draft_tx( ctx: &TxContext, input: PuzzleDraftCompileInput, ) -> Result { + validate_optional_puzzle_external_generation_write_guard( + ctx, + input.external_generation_job_id.as_deref(), + input.external_generation_worker_id.as_deref(), + input.external_generation_lease_token.as_deref(), + &[PUZZLE_COMPILE_DRAFT_JOB_KIND], + &input.session_id, + &input.owner_user_id, + None, + "拼图草稿编译外部生成 guard 不完整", + )?; let row = get_owned_session_row(ctx, &input.session_id, &input.owner_user_id)?; if row.seed_text.trim().is_empty() { return Err("请先填写拼图作品信息".to_string()); @@ -1163,6 +1197,17 @@ fn mark_puzzle_draft_generation_failed_tx( ctx: &TxContext, input: PuzzleDraftCompileFailureInput, ) -> Result { + validate_optional_puzzle_external_generation_write_guard( + ctx, + input.external_generation_job_id.as_deref(), + input.external_generation_worker_id.as_deref(), + input.external_generation_lease_token.as_deref(), + &[PUZZLE_COMPILE_DRAFT_JOB_KIND], + &input.session_id, + &input.owner_user_id, + None, + "拼图草稿失败态外部生成 guard 不完整", + )?; let row = get_owned_session_row(ctx, &input.session_id, &input.owner_user_id)?; let updated_at = Timestamp::from_micros_since_unix_epoch(input.failed_at_micros); let draft = match deserialize_optional_draft(&row.draft_json)? { @@ -1214,6 +1259,89 @@ fn mark_puzzle_draft_generation_failed_tx( ) } +fn mark_puzzle_level_generation_failed_tx( + ctx: &TxContext, + input: PuzzleLevelGenerationFailureInput, +) -> Result { + validate_optional_puzzle_external_generation_write_guard( + ctx, + input.external_generation_job_id.as_deref(), + input.external_generation_worker_id.as_deref(), + input.external_generation_lease_token.as_deref(), + &[ + PUZZLE_GENERATE_IMAGES_JOB_KIND, + PUZZLE_GENERATE_UI_BACKGROUND_JOB_KIND, + ], + &input.session_id, + &input.owner_user_id, + input.level_id.as_deref(), + "拼图关卡失败态外部生成 guard 不完整", + )?; + let row = get_owned_session_row(ctx, &input.session_id, &input.owner_user_id)?; + let updated_at = Timestamp::from_micros_since_unix_epoch(input.failed_at_micros); + let mut draft = match deserialize_optional_draft(&row.draft_json)? { + Some(draft) => draft, + None => { + let anchor_pack = deserialize_anchor_pack(&row.anchor_pack_json)?; + let messages = list_session_messages(ctx, &row.session_id); + compile_result_draft_from_seed(&anchor_pack, &messages, Some(&row.seed_text)) + } + }; + if let Some(levels) = deserialize_optional_levels_input(input.levels_json.as_deref())? { + // 中文注释：新增关卡可能还没完成自动保存，失败回写必须以本次 action 快照作为目标集合。 + draft.levels = levels; + } + draft = mark_puzzle_level_generation_failed_draft(draft, input.level_id.as_deref())?; + let next_stage = resolve_failed_puzzle_agent_stage(row.stage, &draft); + upsert_puzzle_draft_work_profile( + ctx, + &row.session_id, + &row.owner_user_id, + &draft, + input.failed_at_micros, + )?; + + replace_puzzle_agent_session( + ctx, + &row, + PuzzleAgentSessionRow { + session_id: row.session_id.clone(), + owner_user_id: row.owner_user_id.clone(), + seed_text: row.seed_text.clone(), + current_turn: row.current_turn, + progress_percent: row.progress_percent.max(94), + stage: next_stage, + anchor_pack_json: row.anchor_pack_json.clone(), + draft_json: Some(serialize_json(&draft)), + last_assistant_reply: Some(input.error_message), + published_profile_id: row.published_profile_id.clone(), + created_at: row.created_at, + updated_at, + }, + ); + + get_puzzle_agent_session_tx( + ctx, + PuzzleAgentSessionGetInput { + session_id: input.session_id, + owner_user_id: input.owner_user_id, + }, + ) +} + +fn mark_puzzle_level_generation_failed_draft( + draft: PuzzleResultDraft, + level_id: Option<&str>, +) -> Result { + let target_level = + selected_puzzle_level(&draft, level_id).ok_or_else(|| "拼图关卡不存在".to_string())?; + let mut next_level = target_level; + next_level.generation_status = "failed".to_string(); + let mut draft = replace_puzzle_level(&draft, next_level).map_err(|error| error.to_string())?; + module_puzzle::sync_primary_level_fields(&mut draft); + Ok(draft) +} + fn resolve_failed_puzzle_agent_stage( current_stage: PuzzleAgentStage, draft: &PuzzleResultDraft, @@ -1229,6 +1357,63 @@ fn resolve_failed_puzzle_agent_stage( } } +fn validate_puzzle_external_generation_write_guard( + ctx: &TxContext, + job_id: &str, + worker_id: &str, + lease_token: &str, + expected_job_kinds: &[&str], + session_id: &str, + owner_user_id: &str, + level_id: Option<&str>, +) -> Result<(), String> { + let session_entity_id = session_id.trim().to_string(); + let mut source_entity_ids = vec![session_entity_id.clone()]; + if let Some(level_id) = level_id.map(str::trim).filter(|value| !value.is_empty()) { + source_entity_ids.push(format!("{session_entity_id}:{level_id}")); + } + + validate_external_generation_job_lease_for_tx( + ctx.as_ref(), + job_id, + worker_id, + lease_token, + expected_job_kinds, + owner_user_id, + PUZZLE_EXTERNAL_GENERATION_SOURCE_MODULE, + &source_entity_ids, + ) +} + +fn validate_optional_puzzle_external_generation_write_guard( + ctx: &TxContext, + job_id: Option<&str>, + worker_id: Option<&str>, + lease_token: Option<&str>, + expected_job_kinds: &[&str], + session_id: &str, + owner_user_id: &str, + level_id: Option<&str>, + incomplete_message: &str, +) -> Result<(), String> { + match (job_id, worker_id, lease_token) { + (Some(job_id), Some(worker_id), Some(lease_token)) => { + validate_puzzle_external_generation_write_guard( + ctx, + job_id, + worker_id, + lease_token, + expected_job_kinds, + session_id, + owner_user_id, + level_id, + ) + } + (None, None, None) => Ok(()), + _ => Err(incomplete_message.to_string()), + } +} + fn save_puzzle_form_draft_tx( ctx: &TxContext, input: PuzzleFormDraftSaveInput, @@ -1286,6 +1471,20 @@ fn save_puzzle_generated_images_tx( ctx: &TxContext, input: PuzzleGeneratedImagesSaveInput, ) -> Result { + validate_optional_puzzle_external_generation_write_guard( + ctx, + input.external_generation_job_id.as_deref(), + input.external_generation_worker_id.as_deref(), + input.external_generation_lease_token.as_deref(), + &[ + PUZZLE_COMPILE_DRAFT_JOB_KIND, + PUZZLE_GENERATE_IMAGES_JOB_KIND, + ], + &input.session_id, + &input.owner_user_id, + input.level_id.as_deref(), + "拼图图片保存外部生成 guard 不完整", + )?; let row = get_owned_session_row(ctx, &input.session_id, &input.owner_user_id)?; let mut draft = deserialize_draft_required(&row.draft_json)?; let previous_primary_level_name = draft.level_name.clone(); @@ -1370,6 +1569,17 @@ fn save_puzzle_ui_background_tx( ctx: &TxContext, input: PuzzleUiBackgroundSaveInput, ) -> Result { + validate_optional_puzzle_external_generation_write_guard( + ctx, + input.external_generation_job_id.as_deref(), + input.external_generation_worker_id.as_deref(), + input.external_generation_lease_token.as_deref(), + &[PUZZLE_GENERATE_UI_BACKGROUND_JOB_KIND], + &input.session_id, + &input.owner_user_id, + input.level_id.as_deref(), + "拼图 UI 背景保存外部生成 guard 不完整", + )?; let row = get_owned_session_row(ctx, &input.session_id, &input.owner_user_id)?; let mut draft = deserialize_draft_required(&row.draft_json)?; if let Some(levels) = deserialize_optional_levels_input(input.levels_json.as_deref())? { @@ -4040,6 +4250,39 @@ mod tests { ); } + #[test] + fn level_generation_failure_only_marks_target_level_failed() { + let anchor_pack = infer_anchor_pack("画面描述：一只猫在雨夜灯牌下回头。", None); + let mut draft = compile_result_draft_from_seed( + &anchor_pack, + &[], + Some("画面描述：一只猫在雨夜灯牌下回头。"), + ); + draft.levels[0].generation_status = "ready".to_string(); + draft.levels[0].cover_image_src = Some("/generated-puzzle-assets/first.png".to_string()); + let mut second_level = draft.levels[0].clone(); + second_level.level_id = "puzzle-level-2".to_string(); + second_level.level_name = "第二关".to_string(); + second_level.picture_description = "第二关画面".to_string(); + second_level.cover_image_src = None; + second_level.cover_asset_id = None; + second_level.candidates = Vec::new(); + second_level.selected_candidate_id = None; + second_level.generation_status = "generating".to_string(); + draft.levels.push(second_level); + + let failed = mark_puzzle_level_generation_failed_draft(draft, Some("puzzle-level-2")) + .expect("target level should be marked failed"); + + assert_eq!(failed.levels[0].generation_status, "ready"); + assert_eq!( + failed.levels[0].cover_image_src.as_deref(), + Some("/generated-puzzle-assets/first.png") + ); + assert_eq!(failed.levels[1].generation_status, "failed"); + assert_eq!(failed.generation_status, "ready"); + } + #[test] fn puzzle_recommendation_score_prefers_same_author_weight() { let left = PuzzleWorkProfile { diff --git a/server-rs/crates/spacetime-module/src/puzzle_clear.rs b/server-rs/crates/spacetime-module/src/puzzle_clear.rs index ce917767b..62b7a42a3 100644 --- a/server-rs/crates/spacetime-module/src/puzzle_clear.rs +++ b/server-rs/crates/spacetime-module/src/puzzle_clear.rs @@ -345,6 +345,9 @@ fn compile_puzzle_clear_draft_tx( if input.generation_status.as_deref() == Some(PUZZLE_CLEAR_GENERATION_FAILED) { return mark_puzzle_clear_generation_failed_tx(ctx, input, session); } + if input.generation_status.as_deref() == Some(PUZZLE_CLEAR_GENERATION_GENERATING) { + return mark_puzzle_clear_generation_generating_tx(ctx, input, session); + } let pattern_groups: Vec = input .pattern_groups_json .as_deref() @@ -457,6 +460,71 @@ fn compile_puzzle_clear_draft_tx( ) } +fn mark_puzzle_clear_generation_generating_tx( + ctx: &ReducerContext, + input: PuzzleClearDraftCompileInput, + session: PuzzleClearAgentSessionRow, +) -> Result { + let updated_at = Timestamp::from_micros_since_unix_epoch(input.compiled_at_micros); + let mut draft = if session.draft_json.trim().is_empty() { + None + } else { + parse_json::(&session.draft_json).ok() + } + .unwrap_or_else(|| PuzzleClearDraftSnapshot { + template_id: PUZZLE_CLEAR_TEMPLATE_ID.to_string(), + template_name: PUZZLE_CLEAR_TEMPLATE_NAME.to_string(), + profile_id: Some(input.profile_id.clone()), + work_title: clean_string(&input.work_title, PUZZLE_CLEAR_TEMPLATE_NAME), + work_description: input.work_description.trim().to_string(), + theme_prompt: clean_string(&input.theme_prompt, PUZZLE_CLEAR_TEMPLATE_NAME), + generate_board_background: input.generate_board_background, + board_background_asset: None, + board_background_prompt: clean_string(&input.board_background_prompt, &input.theme_prompt), + card_back_image_src: Some(PUZZLE_CLEAR_CARD_BACK_IMAGE_SRC.to_string()), + atlas_asset: None, + pattern_groups: Vec::new(), + card_assets: Vec::new(), + generation_status: PUZZLE_CLEAR_GENERATION_GENERATING.to_string(), + }); + draft.profile_id = Some(input.profile_id.clone()); + draft.work_title = clean_string(&input.work_title, PUZZLE_CLEAR_TEMPLATE_NAME); + draft.work_description = input.work_description.trim().to_string(); + draft.theme_prompt = clean_string(&input.theme_prompt, PUZZLE_CLEAR_TEMPLATE_NAME); + draft.generate_board_background = input.generate_board_background; + draft.board_background_prompt = + clean_string(&input.board_background_prompt, &input.theme_prompt); + if let Some(board_background_asset) = input + .board_background_asset_json + .as_deref() + .map(parse_json) + .transpose()? + { + draft.board_background_asset = Some(board_background_asset); + } + draft.generation_status = PUZZLE_CLEAR_GENERATION_GENERATING.to_string(); + + replace_session( + ctx, + &session, + PuzzleClearAgentSessionRow { + status: PUZZLE_CLEAR_GENERATION_GENERATING.to_string(), + draft_json: to_json_string(&draft), + published_profile_id: input.profile_id, + updated_at, + ..clone_session(&session) + }, + ); + + get_puzzle_clear_agent_session_tx( + ctx, + PuzzleClearAgentSessionGetInput { + session_id: input.session_id, + owner_user_id: input.owner_user_id, + }, + ) +} + fn mark_puzzle_clear_generation_failed_tx( ctx: &ReducerContext, input: PuzzleClearDraftCompileInput, diff --git a/src/components/CustomWorldGenerationView.tsx b/src/components/CustomWorldGenerationView.tsx index 9c786a9c0..886172b43 100644 --- a/src/components/CustomWorldGenerationView.tsx +++ b/src/components/CustomWorldGenerationView.tsx @@ -31,8 +31,16 @@ interface CustomWorldGenerationViewProps { idleBadgeLabel?: string; structuredEmptyText?: string; hideBatchModule?: boolean; + queueStatus?: ExternalGenerationQueueStatus | null; } +export type ExternalGenerationQueueStatus = { + currentStatus?: 'queued' | 'running' | 'completed' | 'failed' | null; + currentProgress?: number | null; + pendingCount?: number | null; + runningCount?: number | null; +}; + function formatDuration(ms: number) { const safeMs = Math.max(0, Math.round(ms)); const totalSeconds = Math.ceil(safeMs / 1000); @@ -85,6 +93,49 @@ function getStepStatusLabel(step: { status: string }) { return '待处理'; } +function resolveQueueStatusLabel( + status: ExternalGenerationQueueStatus['currentStatus'], +) { + if (status === 'queued') { + return '排队中'; + } + + if (status === 'running') { + return '生成中'; + } + + if (status === 'failed') { + return '生成失败'; + } + + if (status === 'completed') { + return '已完成'; + } + + return null; +} + +function hasQueueStatus(status: ExternalGenerationQueueStatus | null | undefined) { + return Boolean( + status && + (status.currentStatus || + typeof status.pendingCount === 'number' || + typeof status.runningCount === 'number'), + ); +} + +function formatQueueCount(value: number | null | undefined) { + return Math.max(0, Math.round(value ?? 0)).toString(); +} + +function formatQueueProgress(value: number | null | undefined) { + if (typeof value !== 'number' || !Number.isFinite(value)) { + return null; + } + + return `${Math.max(0, Math.min(100, Math.round(value)))}%`; +} + function resolveCurrentGenerationStep( progress: CustomWorldGenerationProgress | null, ) { @@ -111,6 +162,7 @@ export function CustomWorldGenerationView({ activeBadgeLabel = '世界建设中', idleBadgeLabel = '等待操作', hideBatchModule = false, + queueStatus = null, }: CustomWorldGenerationViewProps) { void hideBatchModule; const progressValue = getProgressPercentage(progress); @@ -131,6 +183,11 @@ export function CustomWorldGenerationView({ : '校准中'; const elapsedText = progress != null ? formatDuration(progress.elapsedMs) : '启动中'; + const queueStatusLabel = resolveQueueStatusLabel( + queueStatus?.currentStatus ?? null, + ); + const queueProgressText = formatQueueProgress(queueStatus?.currentProgress); + const shouldShowQueueStatus = hasQueueStatus(queueStatus); return (
@@ -167,6 +224,21 @@ export function CustomWorldGenerationView({ />
+ {shouldShowQueueStatus ? ( +
+ {queueStatusLabel ? ( + + {queueProgressText + ? `${queueStatusLabel} ${queueProgressText}` + : queueStatusLabel} + + ) : null} + 排队 {formatQueueCount(queueStatus?.pendingCount)} + + 生成 {formatQueueCount(queueStatus?.runningCount)} +
+ ) : null} +
{!isGenerating ? ( { @@ -57,3 +58,34 @@ describe('resolveMiniGameGenerationViewBusy', () => { ); }); }); + +describe('buildExternalGenerationQueueStatus', () => { + test('合并队列概览和当前任务状态', () => { + expect( + buildExternalGenerationQueueStatus( + { + pendingCount: 7, + runningCount: 3, + updatedAtMicros: 1_781_222_400_000_000, + }, + { + operationId: 'extgen-1', + status: 'running', + phaseLabel: '正在生成。', + phaseDetail: '正在生成。', + progress: 35, + updatedAtMicros: 1_781_222_400_000_000, + }, + ), + ).toEqual({ + currentStatus: 'running', + currentProgress: 35, + pendingCount: 7, + runningCount: 3, + }); + }); + + test('没有队列或任务信息时不显示状态条', () => { + expect(buildExternalGenerationQueueStatus(null, null)).toBeNull(); + }); +}); diff --git a/src/components/platform-entry/PlatformEntryFlowShellImpl.tsx b/src/components/platform-entry/PlatformEntryFlowShellImpl.tsx index 7191bc68d..875665e28 100644 --- a/src/components/platform-entry/PlatformEntryFlowShellImpl.tsx +++ b/src/components/platform-entry/PlatformEntryFlowShellImpl.tsx @@ -37,6 +37,10 @@ import type { BabyObjectMatchDraft, CreateBabyObjectMatchDraftRequest, } from '../../../packages/shared/src/contracts/edutainmentBabyObject'; +import type { + ExternalGenerationJobStatusRecord, + ExternalGenerationQueueOverview, +} from '../../../packages/shared/src/contracts/externalGeneration'; import type { JumpHopJumpRequest, JumpHopWorkSummaryResponse, @@ -172,6 +176,7 @@ import { streamCreativeAgentMessage, streamCreativeDraftEdit, } from '../../services/creative-agent'; +import { getExternalGenerationQueueOverview } from '../../services/external-generation'; import { readCustomWorldAgentUiState, shouldRestoreCustomWorldAgentUiState, @@ -454,6 +459,7 @@ import { resolveWoodenFishCreationUrlRestoreStage, } from './platformCreationUrlStateModel'; import { resolvePlatformCreationWorkDeleteConfirmationModel } from './platformCreationWorkDeleteFlow'; +import { buildExternalGenerationQueueStatus } from './platformExternalGenerationQueueStatusModel'; import { buildPlatformErrorDialogDismissKey, buildPlatformTaskCompletionDialogDismissKey, @@ -717,6 +723,20 @@ export function resolveMiniGameGenerationViewBusy( return isBusy || isMiniGameDraftGenerating(state ?? null); } +function isExternalGenerationQueueStage(selectionStage: SelectionStage) { + return ( + selectionStage === 'puzzle-generating' || + selectionStage === 'big-fish-generating' || + selectionStage === 'square-hole-generating' || + selectionStage === 'match3d-generating' || + selectionStage === 'baby-object-match-generating' || + selectionStage === 'jump-hop-generating' || + selectionStage === 'puzzle-clear-generating' || + selectionStage === 'wooden-fish-generating' || + selectionStage === 'visual-novel-generating' + ); +} + type PuzzleDetailReturnTarget = { tab: PlatformHomeTab; }; @@ -890,9 +910,143 @@ function pushPuzzleResultHistoryEntry( } const PUZZLE_DRAFT_GENERATION_POINT_COST = 2; +const PUZZLE_BACKGROUND_ACTION_POLL_INTERVAL_MS = 3000; +const PUZZLE_BACKGROUND_ACTION_MAX_POLL_ATTEMPTS = 160; const MATCH3D_DRAFT_GENERATION_POINT_COST = 10; const BARK_BATTLE_DRAFT_GENERATION_POINT_COST = 3; +function isPuzzleBackgroundAction(payload: PuzzleAgentActionRequest) { + return ( + payload.action === 'generate_puzzle_images' || + payload.action === 'generate_puzzle_ui_background' + ); +} + +function findPuzzleActionLevel( + session: PuzzleAgentSessionSnapshot | null | undefined, + payload: PuzzleAgentActionRequest, +) { + const levels = session?.draft?.levels ?? []; + const targetLevelId = + 'levelId' in payload ? payload.levelId?.trim() : undefined; + if (targetLevelId) { + return levels.find((level) => level.levelId === targetLevelId) ?? null; + } + + return levels[0] ?? null; +} + +function buildPuzzleGeneratedImageSignature( + level: PuzzleDraftLevel | null | undefined, +) { + if (!level) { + return ''; + } + + return [ + level.levelName, + level.selectedCandidateId ?? '', + level.coverImageSrc ?? '', + level.coverAssetId ?? '', + level.levelSceneImageSrc ?? '', + level.levelSceneImageObjectKey ?? '', + level.uiSpritesheetImageSrc ?? '', + level.uiSpritesheetImageObjectKey ?? '', + level.levelBackgroundImageSrc ?? '', + level.levelBackgroundImageObjectKey ?? '', + (level.candidates ?? []) + .map((candidate) => + [ + candidate.candidateId, + candidate.imageSrc, + candidate.assetId, + String(candidate.selected), + ].join(':'), + ) + .join('|'), + ].join('::'); +} + +function buildPuzzleUiBackgroundSignature( + level: PuzzleDraftLevel | null | undefined, +) { + if (!level) { + return ''; + } + + return [ + level.uiBackgroundPrompt ?? '', + level.uiBackgroundImageSrc ?? '', + level.uiBackgroundImageObjectKey ?? '', + ].join('::'); +} + +function buildPuzzleBackgroundActionSignature( + payload: PuzzleAgentActionRequest, + level: PuzzleDraftLevel | null | undefined, +) { + if (payload.action === 'generate_puzzle_ui_background') { + return buildPuzzleUiBackgroundSignature(level); + } + + return buildPuzzleGeneratedImageSignature(level); +} + +function hasPuzzleBackgroundActionAsset( + payload: PuzzleAgentActionRequest, + level: PuzzleDraftLevel | null | undefined, +) { + if (!level) { + return false; + } + + if (payload.action === 'generate_puzzle_ui_background') { + return Boolean(level.uiBackgroundImageSrc?.trim()); + } + + return Boolean( + level.coverImageSrc?.trim() || + level.candidates?.some((candidate) => candidate.imageSrc.trim()), + ); +} + +function isPuzzleBackgroundActionSettled( + payload: PuzzleAgentActionRequest, + baselineSession: PuzzleAgentSessionSnapshot, + latestSession: PuzzleAgentSessionSnapshot, +) { + const latestLevel = findPuzzleActionLevel(latestSession, payload); + if (!latestLevel) { + return false; + } + if (latestLevel.generationStatus === 'failed') { + return true; + } + if (latestLevel.generationStatus === 'generating') { + return false; + } + + const baselineSignature = buildPuzzleBackgroundActionSignature( + payload, + findPuzzleActionLevel(baselineSession, payload), + ); + const latestSignature = buildPuzzleBackgroundActionSignature( + payload, + latestLevel, + ); + + return ( + hasPuzzleBackgroundActionAsset(payload, latestLevel) && + latestSignature !== baselineSignature + ); +} + +function waitForPuzzleBackgroundActionPollTick() { + return new Promise((resolve) => { + window.setTimeout(resolve, PUZZLE_BACKGROUND_ACTION_POLL_INTERVAL_MS); + }); +} + /** 中文注释：首页公开数据降级时，入口关闭错误不弹窗；真实创作动作仍由对应工作台提示。 */ function isCreationEntryDisabledErrorMessage( message: string | null | undefined, @@ -1613,9 +1767,17 @@ export function PlatformEntryFlowShellImpl({ const [isStartingRecommendEntry, setIsStartingRecommendEntry] = useState(false); const recommendRuntimeStartRequestRef = useRef(0); - const [, setPuzzleOperation] = useState( + const [puzzleOperation, setPuzzleOperation] = useState( null, ); + const [externalGenerationQueueOverview, setExternalGenerationQueueOverview] = + useState(null); + const [jumpHopQueueState, setJumpHopQueueState] = + useState(null); + const [puzzleClearQueueState, setPuzzleClearQueueState] = + useState(null); + const [woodenFishQueueState, setWoodenFishQueueState] = + useState(null); const [puzzleWorks, setPuzzleWorks] = useState([]); const [puzzleGalleryEntries, setPuzzleGalleryEntries] = useState< PuzzleWorkSummary[] @@ -1682,6 +1844,8 @@ export function PlatformEntryFlowShellImpl({ useState(null); const [puzzleBackgroundCompileTasks, setPuzzleBackgroundCompileTasks] = useState>({}); + const puzzleRuntimeReturnSessionRef = + useRef(null); const [miniGameGenerationProgressNowMs, setMiniGameGenerationProgressNowMs] = useState(() => Date.now()); const [puzzleFormDraftPayload, setPuzzleFormDraftPayload] = @@ -4159,6 +4323,7 @@ export function PlatformEntryFlowShellImpl({ setPuzzleRun(run); setPuzzleRuntimeAuthMode('default'); setPuzzleRuntimeReturnStage('puzzle-result'); + puzzleRuntimeReturnSessionRef.current = response.session; pushPuzzleResultHistoryEntry(response.session); openPuzzleRuntimeStage( setSelectionStage, @@ -4524,6 +4689,7 @@ export function PlatformEntryFlowShellImpl({ setPuzzleRun(run); setPuzzleRuntimeAuthMode('default'); setPuzzleRuntimeReturnStage('puzzle-result'); + puzzleRuntimeReturnSessionRef.current = latestSession; pushPuzzleResultHistoryEntry(latestSession); openPuzzleRuntimeStage( setSelectionStage, @@ -4615,6 +4781,82 @@ export function PlatformEntryFlowShellImpl({ isWoodenFishBusy, woodenFishGenerationState, ); + const shouldShowExternalGenerationQueueStatus = + isExternalGenerationQueueStage(selectionStage); + useEffect(() => { + if (!shouldShowExternalGenerationQueueStatus) { + setExternalGenerationQueueOverview(null); + return; + } + + let disposed = false; + let controller: AbortController | null = null; + + const refreshQueueOverview = () => { + controller?.abort(); + controller = new AbortController(); + getExternalGenerationQueueOverview(controller.signal) + .then((response) => { + if (!disposed) { + setExternalGenerationQueueOverview(response.overview); + } + }) + .catch(() => { + if (!disposed) { + setExternalGenerationQueueOverview(null); + } + }); + }; + + refreshQueueOverview(); + const intervalId = window.setInterval(refreshQueueOverview, 4000); + + return () => { + disposed = true; + controller?.abort(); + window.clearInterval(intervalId); + }; + }, [shouldShowExternalGenerationQueueStatus]); + const puzzleExternalGenerationQueueStatus = useMemo( + () => + buildExternalGenerationQueueStatus( + externalGenerationQueueOverview, + puzzleOperation?.queueState ?? null, + ), + [externalGenerationQueueOverview, puzzleOperation], + ); + const jumpHopExternalGenerationQueueStatus = useMemo( + () => + buildExternalGenerationQueueStatus( + externalGenerationQueueOverview, + jumpHopQueueState, + ), + [externalGenerationQueueOverview, jumpHopQueueState], + ); + const puzzleClearExternalGenerationQueueStatus = useMemo( + () => + buildExternalGenerationQueueStatus( + externalGenerationQueueOverview, + puzzleClearQueueState, + ), + [externalGenerationQueueOverview, puzzleClearQueueState], + ); + const woodenFishExternalGenerationQueueStatus = useMemo( + () => + buildExternalGenerationQueueStatus( + externalGenerationQueueOverview, + woodenFishQueueState, + ), + [externalGenerationQueueOverview, woodenFishQueueState], + ); + const externalGenerationQueueStatus = useMemo( + () => + buildExternalGenerationQueueStatus( + externalGenerationQueueOverview, + null, + ), + [externalGenerationQueueOverview], + ); const platformBootstrapErrorForDisplay = isCreationEntryDisabledErrorMessage( platformBootstrap.platformError, ) @@ -5508,6 +5750,7 @@ export function PlatformEntryFlowShellImpl({ setPuzzleRun(run); setPuzzleRuntimeAuthMode('default'); setPuzzleRuntimeReturnStage('puzzle-result'); + puzzleRuntimeReturnSessionRef.current = response.session; pushPuzzleResultHistoryEntry(response.session); openPuzzleRuntimeStage( setSelectionStage, @@ -7330,6 +7573,7 @@ export function PlatformEntryFlowShellImpl({ setJumpHopRun(null); setJumpHopRuntimeRequestOptions(null); setJumpHopGenerationState(generationState); + setJumpHopQueueState(null); setIsJumpHopBusy(true); setSelectionStage('jump-hop-generating'); markDraftGenerating('jump-hop', [ @@ -7346,6 +7590,19 @@ export function PlatformEntryFlowShellImpl({ draft: created.session.draft, }), ); + if (response.queueState && response.session.status === 'generating') { + setJumpHopQueueState(response.queueState); + setJumpHopSession(response.session); + setJumpHopWork(response.work ?? null); + writeCreationUrlState( + buildJumpHopCreationUrlState({ + session: response.session, + work: response.work, + }), + ); + return; + } + setJumpHopQueueState(null); const readyState = createReadyJumpHopGenerationState(generationState); setJumpHopSession(response.session); setJumpHopWork(response.work ?? null); @@ -7382,6 +7639,7 @@ export function PlatformEntryFlowShellImpl({ '生成跳一跳草稿失败。', ); setJumpHopError(errorMessage); + setJumpHopQueueState(null); setJumpHopGenerationState( resolveFinishedMiniGameDraftGenerationState( generationState, @@ -7451,6 +7709,7 @@ export function PlatformEntryFlowShellImpl({ const generationState = createMiniGameDraftGenerationState('jump-hop'); setJumpHopError(null); setJumpHopGenerationState(generationState); + setJumpHopQueueState(null); setIsJumpHopBusy(true); setSelectionStage('jump-hop-generating'); try { @@ -7460,6 +7719,19 @@ export function PlatformEntryFlowShellImpl({ draft: jumpHopSession.draft, }), ); + if (response.queueState && response.session.status === 'generating') { + setJumpHopQueueState(response.queueState); + setJumpHopSession(response.session); + setJumpHopWork(response.work ?? jumpHopWork); + writeCreationUrlState( + buildJumpHopCreationUrlState({ + session: response.session, + work: response.work ?? jumpHopWork, + }), + ); + return; + } + setJumpHopQueueState(null); setJumpHopSession(response.session); setJumpHopWork(response.work ?? jumpHopWork); writeCreationUrlState( @@ -7478,6 +7750,7 @@ export function PlatformEntryFlowShellImpl({ '重新生成跳一跳地块失败。', ); setJumpHopError(errorMessage); + setJumpHopQueueState(null); setJumpHopGenerationState( resolveFinishedMiniGameDraftGenerationState( generationState, @@ -7779,6 +8052,7 @@ export function PlatformEntryFlowShellImpl({ setPuzzleClearWork(null); setPuzzleClearRun(null); setPuzzleClearGenerationState(generationState); + setPuzzleClearQueueState(null); setIsPuzzleClearBusy(true); markDraftGenerating('puzzle-clear', [created.session.sessionId]); markPendingDraftGenerating('puzzle-clear', created.session.sessionId); @@ -7807,6 +8081,19 @@ export function PlatformEntryFlowShellImpl({ created.session.draft?.boardBackgroundAsset, }, ); + if (response.queueState && response.session.status === 'generating') { + setPuzzleClearQueueState(response.queueState); + setPuzzleClearSession(response.session); + setPuzzleClearWork(response.work ?? null); + writeCreationUrlState( + buildPuzzleClearCreationUrlState({ + session: response.session, + work: response.work, + }), + ); + return; + } + setPuzzleClearQueueState(null); setPuzzleClearSession(response.session); setPuzzleClearWork(response.work ?? null); writeCreationUrlState( @@ -7851,6 +8138,7 @@ export function PlatformEntryFlowShellImpl({ '生成拼消消草稿失败。', ); setPuzzleClearError(errorMessage); + setPuzzleClearQueueState(null); setPuzzleClearGenerationState( resolveFinishedMiniGameDraftGenerationState( generationState, @@ -7932,6 +8220,7 @@ export function PlatformEntryFlowShellImpl({ const generationState = createMiniGameDraftGenerationState('puzzle-clear'); setPuzzleClearError(null); setPuzzleClearGenerationState(generationState); + setPuzzleClearQueueState(null); setIsPuzzleClearBusy(true); selectionStageRef.current = 'puzzle-clear-generating'; setSelectionStage('puzzle-clear-generating'); @@ -7953,6 +8242,19 @@ export function PlatformEntryFlowShellImpl({ boardBackgroundAsset: puzzleClearSession.draft?.boardBackgroundAsset, }, ); + if (response.queueState && response.session.status === 'generating') { + setPuzzleClearQueueState(response.queueState); + setPuzzleClearSession(response.session); + setPuzzleClearWork(response.work ?? puzzleClearWork); + writeCreationUrlState( + buildPuzzleClearCreationUrlState({ + session: response.session, + work: response.work ?? puzzleClearWork, + }), + ); + return; + } + setPuzzleClearQueueState(null); setPuzzleClearSession(response.session); setPuzzleClearWork(response.work ?? puzzleClearWork); writeCreationUrlState( @@ -7971,6 +8273,7 @@ export function PlatformEntryFlowShellImpl({ '重新生成拼消消图集失败。', ); setPuzzleClearError(errorMessage); + setPuzzleClearQueueState(null); setPuzzleClearGenerationState( resolveFinishedMiniGameDraftGenerationState(generationState, 'failed', { error: errorMessage, @@ -8281,6 +8584,7 @@ export function PlatformEntryFlowShellImpl({ setWoodenFishWork(null); setWoodenFishRun(null); setWoodenFishGenerationState(generationState); + setWoodenFishQueueState(null); setIsWoodenFishBusy(true); setSelectionStage('wooden-fish-generating'); markDraftGenerating('wooden-fish', [created.session.sessionId]); @@ -8300,6 +8604,19 @@ export function PlatformEntryFlowShellImpl({ draft: created.session.draft, }), ); + if (response.queueState && response.session.status === 'generating') { + setWoodenFishQueueState(response.queueState); + setWoodenFishSession(response.session); + setWoodenFishWork(response.work ?? null); + writeCreationUrlState( + buildWoodenFishCreationUrlState({ + session: response.session, + work: response.work, + }), + ); + return; + } + setWoodenFishQueueState(null); setWoodenFishSession(response.session); setWoodenFishWork(response.work ?? null); writeCreationUrlState( @@ -8344,6 +8661,7 @@ export function PlatformEntryFlowShellImpl({ '生成敲木鱼草稿失败。', ); setWoodenFishError(errorMessage); + setWoodenFishQueueState(null); setWoodenFishGenerationState( resolveFinishedMiniGameDraftGenerationState( generationState, @@ -8429,6 +8747,7 @@ export function PlatformEntryFlowShellImpl({ ); setWoodenFishError(null); setWoodenFishGenerationState(generationState); + setWoodenFishQueueState(null); setIsWoodenFishBusy(true); setSelectionStage('wooden-fish-generating'); try { @@ -8438,6 +8757,19 @@ export function PlatformEntryFlowShellImpl({ draft: woodenFishSession.draft, }), ); + if (response.queueState && response.session.status === 'generating') { + setWoodenFishQueueState(response.queueState); + setWoodenFishSession(response.session); + setWoodenFishWork(response.work ?? woodenFishWork); + writeCreationUrlState( + buildWoodenFishCreationUrlState({ + session: response.session, + work: response.work ?? woodenFishWork, + }), + ); + return; + } + setWoodenFishQueueState(null); setWoodenFishSession(response.session); setWoodenFishWork(response.work ?? woodenFishWork); writeCreationUrlState( @@ -8456,6 +8788,7 @@ export function PlatformEntryFlowShellImpl({ '重新生成敲击物图案失败。', ); setWoodenFishError(errorMessage); + setWoodenFishQueueState(null); setWoodenFishGenerationState( resolveFinishedMiniGameDraftGenerationState( generationState, @@ -8676,6 +9009,113 @@ export function PlatformEntryFlowShellImpl({ const executePuzzleAction = puzzleFlow.executeAction; + const pollPuzzleBackgroundActionUntilSettled = useCallback( + ( + payload: PuzzleAgentActionRequest, + baselineSession: PuzzleAgentSessionSnapshot, + ) => { + if (!isPuzzleBackgroundAction(payload)) { + return; + } + + void (async () => { + for ( + let attempt = 0; + attempt < PUZZLE_BACKGROUND_ACTION_MAX_POLL_ATTEMPTS; + attempt += 1 + ) { + await waitForPuzzleBackgroundActionPollTick(); + try { + const response = await getPuzzleAgentSession( + baselineSession.sessionId, + ); + puzzleFlow.setSession(response.session); + if ( + !isPuzzleBackgroundActionSettled( + payload, + baselineSession, + response.session, + ) + ) { + continue; + } + + const latestLevel = findPuzzleActionLevel(response.session, payload); + if (latestLevel?.generationStatus === 'failed') { + const errorMessage = '拼图图片生成失败，请调整描述后重试。'; + const failedGenerationState = + resolveFinishedMiniGameDraftGenerationState( + createPuzzleDraftGenerationStateFromPayload( + buildPuzzleFormPayloadFromSession(response.session), + response.session, + ), + 'failed', + { error: errorMessage }, + ); + setPuzzleBackgroundCompileTasks((current) => ({ + ...current, + [response.session.sessionId]: { + session: response.session, + payload: buildPuzzleFormPayloadFromSession(response.session), + generationState: failedGenerationState, + error: errorMessage, + }, + })); + markPendingDraftFailed('puzzle', response.session.sessionId); + markDraftFailed( + 'puzzle', + [ + response.session.sessionId, + buildPuzzleResultWorkId(response.session.sessionId), + response.session.publishedProfileId, + buildPuzzleResultProfileId(response.session.sessionId), + ], + errorMessage, + !isViewingPuzzleGeneration(response.session.sessionId), + ); + if (isViewingPuzzleGeneration(response.session.sessionId)) { + setPuzzleGenerationState(failedGenerationState); + } + setPuzzleError(errorMessage); + } else { + setPuzzleError(null); + } + + refreshPlatformDashboardSilently(); + await Promise.allSettled([ + refreshPuzzleShelf(), + refreshPuzzleGallery(), + ]); + return; + } catch (pollError) { + if (attempt >= 2) { + setPuzzleError( + resolvePuzzleErrorMessage( + pollError, + '刷新拼图图片生成结果失败。', + ), + ); + return; + } + } + } + + setPuzzleError('拼图图片仍在后台生成，请稍后刷新草稿查看结果。'); + })(); + }, + [ + isViewingPuzzleGeneration, + markDraftFailed, + markPendingDraftFailed, + puzzleFlow, + refreshPlatformDashboardSilently, + refreshPuzzleGallery, + refreshPuzzleShelf, + resolvePuzzleErrorMessage, + setPuzzleError, + ], + ); + const executePuzzleBackgroundAction = useCallback( async (payload: PuzzleAgentActionRequest) => { const targetSession = puzzleSession; @@ -8696,6 +9136,13 @@ export function PlatformEntryFlowShellImpl({ ); setPuzzleOperation(response.operation); puzzleFlow.setSession(response.session); + if ( + isPuzzleBackgroundAction(payload) && + (response.operation.status === 'queued' || + response.operation.status === 'running') + ) { + pollPuzzleBackgroundActionUntilSettled(payload, targetSession); + } } catch (error) { setPuzzleError(resolvePuzzleErrorMessage(error, '执行拼图操作失败。')); } finally { @@ -8710,6 +9157,7 @@ export function PlatformEntryFlowShellImpl({ [ puzzleFlow, puzzleSession, + pollPuzzleBackgroundActionUntilSettled, refreshPlatformDashboardSilently, resolvePuzzleErrorMessage, setPuzzleError, @@ -9374,6 +9822,10 @@ export function PlatformEntryFlowShellImpl({ setPuzzleRun(run); setPuzzleRuntimeAuthMode('default'); setPuzzleRuntimeReturnStage('puzzle-result'); + puzzleRuntimeReturnSessionRef.current = + puzzleSession?.draft && !isPuzzleFormOnlyDraft(puzzleSession) + ? puzzleSession + : null; pushPuzzleResultHistoryEntry(puzzleSession); openPuzzleRuntimeStage( setSelectionStage, @@ -9397,6 +9849,27 @@ export function PlatformEntryFlowShellImpl({ ], ); + const returnFromPuzzleRuntime = useCallback(() => { + const targetStage = puzzleRuntimeReturnStage; + let sessionForResultUrl = puzzleSession; + if ( + targetStage === 'puzzle-result' && + (!puzzleSession?.draft || isPuzzleFormOnlyDraft(puzzleSession)) && + puzzleRuntimeReturnSessionRef.current + ) { + sessionForResultUrl = puzzleRuntimeReturnSessionRef.current; + puzzleFlow.setSession(puzzleRuntimeReturnSessionRef.current); + } + + clearPuzzleRuntimeUrlState(); + pushAppHistoryPath(resolvePathForSelectionStage(targetStage)); + if (targetStage === 'puzzle-result') { + writeCreationUrlState(buildPuzzleCreationUrlState(sessionForResultUrl)); + } + selectionStageRef.current = targetStage; + setSelectionStage(targetStage); + }, [puzzleFlow, puzzleRuntimeReturnStage, puzzleSession, setSelectionStage]); + const submitBigFishInput = useCallback( async (payload: SubmitBigFishInputRequest) => { if ( @@ -15103,6 +15576,7 @@ export function PlatformEntryFlowShellImpl({ activeBadgeLabel="草稿生成中" pausedBadgeLabel="草稿生成已暂停" idleBadgeLabel="等待返回工作区" + queueStatus={jumpHopExternalGenerationQueueStatus} /> @@ -15247,6 +15721,7 @@ export function PlatformEntryFlowShellImpl({ setSelectionStage('match3d-agent-workspace'); }} onRetry={retryMatch3DDraftGeneration} + queueStatus={puzzleClearExternalGenerationQueueStatus} hideBatchModule /> @@ -15517,6 +15992,7 @@ export function PlatformEntryFlowShellImpl({ activeBadgeLabel="草稿生成中" pausedBadgeLabel="草稿生成已暂停" idleBadgeLabel="等待返回工作区" + queueStatus={woodenFishExternalGenerationQueueStatus} /> @@ -15717,6 +16193,7 @@ export function PlatformEntryFlowShellImpl({ activeBadgeLabel="图片生成中" pausedBadgeLabel="图片生成已暂停" idleBadgeLabel="等待返回结果页" + queueStatus={externalGenerationQueueStatus} /> @@ -15921,6 +16398,7 @@ export function PlatformEntryFlowShellImpl({ setSelectionStage('jump-hop-workspace'); }} onRetry={retryJumpHopDraftGeneration} + queueStatus={externalGenerationQueueStatus} /> @@ -16069,6 +16547,7 @@ export function PlatformEntryFlowShellImpl({ activeBadgeLabel="素材生成中" pausedBadgeLabel="素材生成已暂停" idleBadgeLabel="等待返回工作区" + queueStatus={externalGenerationQueueStatus} /> @@ -16198,6 +16677,7 @@ export function PlatformEntryFlowShellImpl({ setSelectionStage('wooden-fish-workspace'); }} onRetry={retryWoodenFishDraftGeneration} + queueStatus={externalGenerationQueueStatus} /> @@ -16391,6 +16871,7 @@ export function PlatformEntryFlowShellImpl({ setSelectionStage('puzzle-agent-workspace'); }} onRetry={retryPuzzleDraftGeneration} + queueStatus={puzzleExternalGenerationQueueStatus} hideBatchModule /> @@ -16517,6 +16998,7 @@ export function PlatformEntryFlowShellImpl({ activeBadgeLabel="草稿生成中" pausedBadgeLabel="草稿生成已暂停" idleBadgeLabel="等待返回工作区" + queueStatus={externalGenerationQueueStatus} /> @@ -16686,15 +17168,7 @@ export function PlatformEntryFlowShellImpl({ } error={puzzleError} hideBackButton={Boolean(puzzleOnboardingDraft)} - onBack={() => { - const returnStage = puzzleRuntimeReturnStage; - setSelectionStage(returnStage); - if (returnStage === 'puzzle-result') { - writeCreationUrlState( - buildPuzzleCreationUrlState(puzzleSession), - ); - } - }} + onBack={returnFromPuzzleRuntime} onRemodelWork={ selectedPuzzleDetail?.publicationStatus === 'published' ? remodelCurrentPuzzleRuntimeWork @@ -16768,6 +17242,7 @@ export function PlatformEntryFlowShellImpl({ activeBadgeLabel="草稿编译中" pausedBadgeLabel="草稿生成已暂停" idleBadgeLabel="等待返回工作区" + queueStatus={externalGenerationQueueStatus} /> diff --git a/src/components/platform-entry/platformExternalGenerationQueueStatusModel.ts b/src/components/platform-entry/platformExternalGenerationQueueStatusModel.ts new file mode 100644 index 000000000..7238377b3 --- /dev/null +++ b/src/components/platform-entry/platformExternalGenerationQueueStatusModel.ts @@ -0,0 +1,21 @@ +import type { + ExternalGenerationJobStatusRecord, + ExternalGenerationQueueOverview, +} from '../../../packages/shared/src/contracts/externalGeneration'; +import type { ExternalGenerationQueueStatus } from '../CustomWorldGenerationView'; + +export function buildExternalGenerationQueueStatus( + overview: ExternalGenerationQueueOverview | null, + job: ExternalGenerationJobStatusRecord | null, +): ExternalGenerationQueueStatus | null { + if (!overview && !job) { + return null; + } + + return { + currentStatus: job?.status ?? null, + currentProgress: job?.progress ?? null, + pendingCount: overview?.pendingCount ?? null, + runningCount: overview?.runningCount ?? null, + }; +} diff --git a/src/components/rpg-entry/RpgEntryFlowShell.agent.interaction.test.tsx b/src/components/rpg-entry/RpgEntryFlowShell.agent.interaction.test.tsx index 782c239ed..e91bc908f 100644 --- a/src/components/rpg-entry/RpgEntryFlowShell.agent.interaction.test.tsx +++ b/src/components/rpg-entry/RpgEntryFlowShell.agent.interaction.test.tsx @@ -4933,6 +4933,60 @@ test('running puzzle form generation creates a new puzzle draft on same template }); }); +test('queued puzzle form generation stays on generation progress', async () => { + const user = userEvent.setup(); + const queuedSession = buildMockPuzzleAgentSession({ + sessionId: 'puzzle-queued-session-1', + progressPercent: 5, + lastAssistantReply: '拼图生成任务已进入后台队列。', + }); + vi.mocked(createPuzzleAgentSession).mockResolvedValueOnce({ + session: queuedSession, + }); + vi.mocked(executePuzzleAgentAction).mockResolvedValueOnce({ + operation: { + operationId: 'compile-puzzle-queued-1', + type: 'compile_puzzle_draft', + status: 'queued', + phaseLabel: '已进入后台队列', + phaseDetail: '拼图草稿生成已进入后台队列。', + progress: 5, + error: null, + }, + session: queuedSession, + }); + vi.mocked(getPuzzleAgentSession).mockResolvedValue({ + session: { + ...queuedSession, + progressPercent: 12, + }, + }); + + render(); + + await openCreateTemplateHub(user); + await user.click(await findCreationTypeButton('拼图')); + await user.click(await screen.findByRole('button', { name: '生成草稿' })); + + const progressbar = await screen.findByRole('progressbar', { + name: '拼图图片生成进度', + }); + expect(progressbar).toBeTruthy(); + expect(updatePuzzleWork).not.toHaveBeenCalled(); + expect(startLocalPuzzleRun).not.toHaveBeenCalled(); + expect(screen.queryByText('拼图结果页')).toBeNull(); + expect(window.location.pathname).not.toBe('/runtime/puzzle'); + await user.click(screen.getByRole('button', { name: '返回创作中心' })); + await openDraftHub(user); + await waitFor(() => { + expect( + within(getPlatformTabPanel('saves')).getAllByRole('button', { + name: /继续创作《[^》]+》，生成中/u, + }).length, + ).toBeGreaterThanOrEqual(1); + }); +}); + test('failed parallel puzzle generations stay as separate non-generating drafts', async () => { const user = userEvent.setup(); const firstSession = buildMockPuzzleAgentSession({ diff --git a/src/components/unified-creation/UnifiedGenerationPage.test.tsx b/src/components/unified-creation/UnifiedGenerationPage.test.tsx index adbc94dc0..c41c656d2 100644 --- a/src/components/unified-creation/UnifiedGenerationPage.test.tsx +++ b/src/components/unified-creation/UnifiedGenerationPage.test.tsx @@ -70,4 +70,28 @@ describe('UnifiedGenerationPage', () => { expect(screen.queryByText('当前跳一跳信息')).toBeNull(); expect(screen.queryByText('云端糖果塔')).toBeNull(); }); + + test('显示外部生成队列状态', () => { + render( + {}} + onEditSetting={() => {}} + onRetry={() => {}} + />, + ); + + expect(screen.getByText('排队中 18%')).toBeTruthy(); + expect(screen.getByText('排队 6')).toBeTruthy(); + expect(screen.getByText('生成 2')).toBeTruthy(); + }); }); diff --git a/src/components/unified-creation/UnifiedGenerationPage.tsx b/src/components/unified-creation/UnifiedGenerationPage.tsx index b943f549a..04ddf4f54 100644 --- a/src/components/unified-creation/UnifiedGenerationPage.tsx +++ b/src/components/unified-creation/UnifiedGenerationPage.tsx @@ -1,6 +1,9 @@ import type { CustomWorldGenerationProgress } from '../../../packages/shared/src/contracts/runtime'; import type { CustomWorldStructuredAnchorEntry } from '../../services/customWorldAgentGenerationProgress'; -import { CustomWorldGenerationView } from '../CustomWorldGenerationView'; +import { + CustomWorldGenerationView, + type ExternalGenerationQueueStatus, +} from '../CustomWorldGenerationView'; import type { UnifiedGenerationPlayId } from './unifiedGenerationCopy'; import { getUnifiedGenerationCopy } from './unifiedGenerationCopy'; @@ -15,6 +18,7 @@ type UnifiedGenerationPageProps = { onEditSetting: () => void; onRetry: () => void; hideBatchModule?: boolean; + queueStatus?: ExternalGenerationQueueStatus | null; }; export function UnifiedGenerationPage({ @@ -28,6 +32,7 @@ export function UnifiedGenerationPage({ onEditSetting, onRetry, hideBatchModule = false, + queueStatus = null, }: UnifiedGenerationPageProps) { const copy = getUnifiedGenerationCopy(playId); @@ -51,6 +56,7 @@ export function UnifiedGenerationPage({ pausedBadgeLabel="素材生成已暂停" idleBadgeLabel="等待返回工作区" hideBatchModule={hideBatchModule} + queueStatus={queueStatus} /> ); } diff --git a/src/services/external-generation/externalGenerationClient.ts b/src/services/external-generation/externalGenerationClient.ts new file mode 100644 index 000000000..f91875489 --- /dev/null +++ b/src/services/external-generation/externalGenerationClient.ts @@ -0,0 +1,49 @@ +import type { + ExternalGenerationJobStatusResponse, + ExternalGenerationQueueOverviewResponse, +} from '../../../packages/shared/src/contracts/externalGeneration'; +import { BACKGROUND_AUTH_REQUEST_OPTIONS, requestJson } from '../apiClient'; + +const EXTERNAL_GENERATION_API_BASE = '/api/runtime/external-generation'; +const EXTERNAL_GENERATION_READ_OPTIONS = { + ...BACKGROUND_AUTH_REQUEST_OPTIONS, + retry: { + maxRetries: 1, + baseDelayMs: 200, + maxDelayMs: 600, + }, +} as const; + +export async function getExternalGenerationQueueOverview( + signal?: AbortSignal, +) { + return requestJson( + `${EXTERNAL_GENERATION_API_BASE}/queue-overview`, + { + method: 'GET', + signal, + }, + '读取生成队列状态失败', + EXTERNAL_GENERATION_READ_OPTIONS, + ); +} + +export async function getExternalGenerationJobStatus( + jobId: string, + signal?: AbortSignal, +) { + return requestJson( + `${EXTERNAL_GENERATION_API_BASE}/jobs/${encodeURIComponent(jobId)}`, + { + method: 'GET', + signal, + }, + '读取生成任务状态失败', + EXTERNAL_GENERATION_READ_OPTIONS, + ); +} + +export const externalGenerationClient = { + getQueueOverview: getExternalGenerationQueueOverview, + getJobStatus: getExternalGenerationJobStatus, +}; diff --git a/src/services/external-generation/index.ts b/src/services/external-generation/index.ts new file mode 100644 index 000000000..83c70bad9 --- /dev/null +++ b/src/services/external-generation/index.ts @@ -0,0 +1,5 @@ +export { + externalGenerationClient, + getExternalGenerationJobStatus, + getExternalGenerationQueueOverview, +} from './externalGenerationClient';