diff --git a/deploy/systemd/genarrative-database-backup.service b/deploy/systemd/genarrative-database-backup.service index 276b9f9ab..8a7d95535 100644 --- a/deploy/systemd/genarrative-database-backup.service +++ b/deploy/systemd/genarrative-database-backup.service @@ -11,6 +11,17 @@ WorkingDirectory=/opt/genarrative/current EnvironmentFile=/etc/genarrative/api-server.env ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service +# 备份脚本必须受独立内存上限保护，不能因目录扫描异常拖垮整台 release 主机。 +Environment=NODE_OPTIONS=--max-old-space-size=768 +Environment=GENARRATIVE_DATABASE_BACKUP_STOP_MARKER=/var/lib/genarrative/database-backups/.spacetimedb-stopped +MemoryHigh=768M +MemoryMax=1G +OOMPolicy=stop + +# 主进程可能在停库后被 MemoryMax/OOMPolicy 强制终止，JS finally 无法执行； +# 仅当备份脚本留下停库 marker 且本次 service 非正常成功时，由 systemd 兜底恢复全部依赖服务。 +ExecStopPost=/bin/sh -c 'if [ "${SERVICE_RESULT}" != "success" ] && [ -f "${GENARRATIVE_DATABASE_BACKUP_STOP_MARKER}" ]; then systemctl start spacetimedb.service; systemctl restart genarrative-api.service; systemctl restart genarrative-external-generation-worker@1.service; systemctl restart genarrative-external-generation-controller.service; if systemctl is-active --quiet spacetimedb.service && systemctl is-active --quiet genarrative-api.service && systemctl is-active --quiet genarrative-external-generation-worker@1.service && systemctl is-active --quiet genarrative-external-generation-controller.service; then rm -f "${GENARRATIVE_DATABASE_BACKUP_STOP_MARKER}"; fi; fi' + # 备份需要停止 / 启动 spacetimedb.service，并读取 /stdb、写入 /var/lib/genarrative/database-backups。 # 停止 SpacetimeDB 会连带停止 Requires 它的 API / worker / controller，冷备份后必须显式拉起。 PrivateTmp=true diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index dad81b194..82659cf5b 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -32,6 +32,14 @@ --- +## 2026-08-27 外部生成历史采用受控保留清理 + +- 背景：`external_generation_job`、`external_generation_job_summary` 与 `external_generation_job_event` 都是持久化表；摘要和 payload 边界收紧后，已确认的终态历史仍会继续占用 SpacetimeDB 常驻内存，且事件审计链会随任务数量增长。 +- 决策：新增仅 migration operator 可调用的 `prune_external_generation_job_history_and_return`。默认按 `source_module=editor-canvas`、30 天保留期和 `job_id` 游标分批运行；只删除主任务与摘要状态一致、属于 completed / failed / cancelled、摘要已有 `notification_acknowledged_at` 且终态时间达到 cutoff 的任务，并在同一事务内删除该任务的全部事件、摘要和主任务。默认 dry-run，必须固定 dry-run 返回的 cutoff 后再 apply；pending / running、未确认通知、摘要缺失或状态不一致的数据永不删除。其他 source module 必须显式指定并单独评估；资产对象和钱包流水不随任务历史删除；不新增自动定时器或 runtime 清理权限。 +- 影响范围：`server-rs/crates/spacetime-module/src/external_generation.rs`、外部生成事件 job_id 单列索引、SpacetimeDB 生成 bindings、`scripts/spacetime-maintain-external-generation-jobs.mjs`、架构与生产运维文档。 +- 验证方式：覆盖终态 / 活跃态 / 已确认与未确认摘要、状态或身份不一致、cutoff 边界测试；运行 SpacetimeDB module tests/check、bindings 生成、schema/encoding/diff 门禁，并在维护窗口先 dry-run 再 apply。 +- 关联文档：`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`、PR #203。 + ## 2026-08-27 SpacetimeDB 工具链统一升级到 2.8.3 - 背景：SpacetimeDB 2.8.0 引入 TypeScript submodule 与调度延迟观测，2.8.1 修复 v1 WebSocket 订阅移除死锁、TypeScript SDK `array` 读缓存别名和 Rust string 默认值支持，2.8.2 修复 table accessor 改名自动迁移，2.8.3 修复 scheduled function 从实际执行时间重排导致的长期漂移。仓库若继续锁定 2.7.0，会保留这些已知运行时与 SDK 问题。 @@ -1333,8 +1341,8 @@ CI 上 `background_agent_runtime_recovers_stale_running_before_pending_task` 在 - OSS 固定恢复入口为 `//latest.json`。CAS 文件和 full/history catalog 保持不可变；latest pointer 只保存最新 full catalog 与已发布 history catalog 的 object key、长度和 SHA，不包含主机绝对路径或文件内容。每次 state 变化先验真全部引用 catalog，再覆盖上传并 HEAD 验真 latest pointer，成功后才落本地 state；history 还必须在 pointer 成功后才允许删除源文件。全新机器可仅凭 bucket、database、prefix 与 OSS 凭据自动下载 pointer 和 full catalog。 - dev 带宽不足时，允许把已冻结的 dev 基线经 `10.2.0.10 -> 10.2.4.16` 内网 rsync 到 release 独立 staging，再用 release 出口上传 dev bucket；staging 不得指向 release `/stdb`，不得停止或修改 release 服务，传输凭据必须临时创建并在演练后移除。catalog 不记录 staging 绝对路径，files state 可回传 dev 继续 history。 - 恢复边界：恢复时默认从 OSS `latest.json` 自动定位 full catalog，创建目录并按相对路径下载每个对象、逐文件校验长度与 SHA；本地 state 只用于备份续跑，不再是异机恢复前置条件。远程 dev 已完成真实 OSS、清理、重启和异机隔离恢复演练；release timer 与 publish 前备份继续保持原行为。 -- systemd 接线：主 service 保持 `archive-full`。Server-Provision 新增默认值为 `archive-full` 的 `DATABASE_BACKUP_PROFILE`；dev 或 release 显式选择 `files-history` 时，必须为各自主机指定独立 work-dir，并先用 current release 脚本执行 history dry-run，确认已有 full state 后才安装仓库托管 drop-in，并删除现场手写旧 drop-in。切回默认 profile 必须删除所有 history 覆盖。 -- 影响范围：`scripts/database-backup-to-oss.mjs`、备份门禁、生产 env 示例、systemd 模板、Server-Provision、SpacetimeDB 运维与恢复流程；release timer 可在独立 baseline 验证后显式选择 profile，publish 前备份是否切换仍需单独决策。 +- systemd 接线：主 service 保持 `archive-full`。Server-Provision 新增默认值为 `archive-full` 的 `DATABASE_BACKUP_PROFILE`；development 可显式选择 `files-history`，必须指定独立 work-dir 并先用 current release 脚本执行 history dry-run，确认已有 full state 后才安装仓库托管 drop-in；release 拒绝 `files-history`，直到流式 catalog 改造完成，以免大目录扫描再次触发 Node 内存峰值。切回默认 profile 必须删除所有 history 覆盖；备份 unit 同时设置 Node heap 与 systemd memory 上限，避免备份异常拖垮业务主机。 +- 影响范围：`scripts/database-backup-to-oss.mjs`、备份门禁、生产 env 示例、systemd 模板、Server-Provision、SpacetimeDB 运维与恢复流程；release timer 固定使用 archive-full，publish 前备份是否切换仍需单独决策。 - 验证方式：`npm run check:database-backup`、`npm run check:production-ops`、`npm run check:encoding`、`git diff --check`；dev 现场必须完成逐文件 full catalog、重复 full 零 PUT、history dry-run、上传后清理、STDB 重启和按 catalog 隔离恢复 roundtrip。 - 关联：。 @@ -7785,3 +7793,11 @@ CI 上 `background_agent_runtime_recovers_stale_running_before_pending_task` 在 - 决策：`profile_wallet_refund_outbox` 是跨 API 节点退款的正式持久化队列。扣费失败、外部生成 attempt 失败或最终 lease 过期时，在同一个 SpacetimeDB 事务内按 `refund_ledger_id` 幂等写入 pending 行；worker 从库内 pending 行批量处理，退款账本写入与 outbox 成功删除保持在同一事务内，失败由 `available_at` / `attempts` 驱动重试。`asset_operation_wallet_settlement` 继续负责退款先于 consume 可见时的取消 intent，阻止迟到扣费。只有 SpacetimeDB 完全不可达时，api-server 才写本机 `wallet-refund-outbox` emergency spool；本机文件不能替代库内队列，必须持久挂载、告警、恢复演练并支持人工补偿。 - 影响范围：`profile_wallet_refund_outbox` 表及 bindings、runtime enqueue/process procedure、外部生成失败事务、inline 资产退款、api-server 跨节点 worker 和 emergency spool、后端架构与开发运维文档。 - 验证方式：运行 `npm run spacetime:generate`、`npm run check:spacetime-schema`、`npm run check:server-rs-ddd`、`cargo check -p spacetime-module -p spacetime-client -p api-server --manifest-path server-rs/Cargo.toml`、退款 outbox / asset billing / external generation 定向测试、`npm run check:encoding` 和 `git diff --check`。 + +## 2026-08-27 release 内存增长修复与备份 OOM 恢复兜底 + +- 决策：外部生成 worker 每轮主动 `try_join_next` 回收已完成 `JoinHandle`，避免持续有队列任务时只归还 semaphore permit 却让 `JoinSet` 句柄集合无界增长；超时脱管任务在 abort 后等待句柄结束，执行许可保持到 work 真正结束或被取消。 +- 决策：`module-ai` 的阶段终态写入与流式增量统一受文本、结构化 JSON、warning 和全局 retained 工作集上限约束；认证投影恢复对过滤后的 refresh session 重新计数，超过 8192 条直接拒绝启动恢复，避免超限快照灌入内存。 +- 复审补充：`spacetime-module` 的 AI procedure 复用同一组任务元数据 / payload / 输出 / 结果引用上限，流式文本聚合超限时回滚事务，terminal task 收口后删除 `ai_text_chunk` 明细，避免真实持久化链路绕过内存边界。 +- 决策：备份脚本停库前写入受保护 `.spacetimedb-stopped` marker，正常恢复完成后清理；systemd 备份 service 通过 `MemoryHigh/MemoryMax/OOMPolicy` 和 `ExecStopPost` 在 Node OOM kill、无法执行 JS finally 时兜底拉起 SpacetimeDB、API、worker、controller，恢复不完整则保留 marker。 +- 验证：worker/module-auth/module-ai 定向 Rust 测试、database-backup/production-ops/encoding 门禁和 `git diff --check` 必须在提交前通过；release 现场需按 archive-full 重新 provision 并核验旧 files-history drop-in 已删除、四个服务 active。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 189cb3cf9..dee419cf0 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -329,6 +329,8 @@ Responses 的终态载荷既是工具调用的恢复源，也是正文的恢复 - Rust 结构体：`AiTask` - 源码：`server-rs/crates/spacetime-module/src/ai/tasks.rs` +- `module-ai` 的进程内热状态不是持久化真相：文本增量按阶段有序聚合并受单阶段 512 KiB 上限约束；terminal task 立即释放增量明细，内存工作集最多保留 1024 个任务。需要长期查询时必须读取 SpacetimeDB 的 `ai_task` / `ai_task_stage` 投影，不得依赖进程重启后仍存在的内存快照。 +- SpacetimeDB 的 AI 写入 procedure 必须复用同一组任务元数据、payload、文本、结构化输出、warning、失败消息和结果引用上限；流式聚合超过 512 KiB 时在事务内拒绝，terminal task 收口后删除 `ai_text_chunk` 明细，只保留阶段最终快照和结果引用。 ### `ai_task_event` @@ -360,13 +362,14 @@ Responses 的终态载荷既是工具调用的恢复源，也是正文的恢复 - 源码：`server-rs/crates/spacetime-module/src/external_generation.rs` - 用途：外部生成正式任务列表的轻量投影，按 `job_id` 保存 owner、来源、状态、可选 `phase`、价格、有界错误摘要、通知确认时间、各阶段时间和入队时提取的 `request_prompt`，不包含 request/result payload、worker lease 或 dedupe 内部字段。错误摘要统一拒绝内联媒体并限制为 2048 字符；列表在单次 owner 扫描中同时计数并只保留请求 limit 的固定大小 top-N，不得先收集全量历史再截断。enqueue、claim、renew、phase update、complete、fail 事务同步投影；acknowledge 只更新该轻量表并写审计事件，后续主任务同步必须保留已有确认时间，禁止为了写确认时间加载 / 重写大 payload 行。BFF 的列表、状态和确认只调用 summary procedure；`running + processing` 映射为“正在处理”，其它 running（含旧行 `phase=None`）映射为“正在生成”。历史终态任务由迁移操作员的游标分批 maintenance procedure 在压缩 payload 时同步回填摘要，正式列表不得为兼容旧数据回扫完整主表。 - 非阻断告警：摘要字段 `warning_message` 是展示投影，由完成任务的轻量 `result_payload_json.warning.reason` 原样提取，不等同于公开 inline / external v1 的原始结构化诊断字段。complete 和历史 backfill 共用同一构建路径；历史任务按其结果载荷中已写入的 `reason` 快照投影，不为格式升级重写或补前缀。单 job 状态和任务列表 BFF 以 `warning: string` 返回该可直接展示的完整文案，不再返回结构化 code，Web 不得再次补前缀或按字符串推断告警类型。错误与告警摘要都不复制内联媒体并限制为 2048 字符。`phase` 与 `warning_message` 分别表示当前执行阶段和成功降级提示，不得混用；worker / BFF / Web 必须同版本协调发布，不保证滚动混部或旧 Web 缓存下的字符串语义兼容。 -- 正式读取 procedure 为 `get_external_generation_job_summary_and_return`、`list_external_generation_job_summaries_and_return` 和 `acknowledge_external_generation_job_summaries_and_return`。历史维护 procedure 为 `compact_external_generation_job_payloads_and_return` 与 `backfill_external_generation_job_summaries_and_return`，仅 migration operator 可调用；运维入口统一使用 `npm run spacetime:external-generation:maintain -- ...`，默认 dry-run、单批最多 25 条。B-tree cursor 选择阶段最多反序列化 `limit + 1` 行，apply 再按主键逐条读取选中行；怀疑存在单行异常巨型 JSON 时必须先使用 `--limit 1`。payload 压缩额外固定使用 `source_module = editor-canvas` 的复合 cursor 索引，不得静默改写其它玩法历史任务。 +- 正式读取 procedure 为 `get_external_generation_job_summary_and_return`、`list_external_generation_job_summaries_and_return` 和 `acknowledge_external_generation_job_summaries_and_return`。历史维护 procedure 为 `compact_external_generation_job_payloads_and_return`、`backfill_external_generation_job_summaries_and_return` 与 `prune_external_generation_job_history_and_return`，仅 migration operator 可调用；运维入口统一使用 `npm run spacetime:external-generation:maintain -- ...`，默认 dry-run、单批最多 25 条。B-tree cursor 选择阶段最多反序列化 `limit + 1` 行，apply 再按主键逐条读取选中行；怀疑存在单行异常巨型 JSON 时必须先使用 `--limit 1`。payload 压缩额外固定使用 `source_module = editor-canvas` 的复合 cursor 索引，不得静默改写其它玩法历史任务。历史清理默认使用 `--prune-history`、`source_module = editor-canvas` 和 30 天保留期；只有主任务与摘要状态一致且属于 completed / failed / cancelled、摘要已有 `notification_acknowledged_at`、终态时间不晚于 cutoff 的记录才是候选。apply 在同一事务内按事件 → 摘要 → 主任务顺序删除，事件不得独立清理；pending / running、未确认通知、摘要缺失或状态不一致的记录永不删除。清理不触碰资产对象或钱包流水，其他 source module 必须显式指定并单独评估。 ### `external_generation_job_event` - Rust 结构体：`ExternalGenerationJobEvent` - 源码：`server-rs/crates/spacetime-module/src/external_generation.rs` - 用途：外部生成任务审计事件表，按 `job_id` 和 `owner_user_id` 记录 `enqueued`、`claimed`、`lease_renewed`、`completed`、`failed`、`acknowledged` 等状态转换事实。状态转换只能由 SpacetimeDB procedure 写入，不由前端或 worker 直接改表；该表用于追溯任务生命周期和排障，不替代 `external_generation_job` 当前状态。 +- 保留策略：事件只会随已确认通知的终态任务由 `prune_external_generation_job_history_and_return` 原子删除，不支持按事件单独清理，以保持任务、摘要和审计链一致。 ### `ai_text_chunk` @@ -410,11 +413,14 @@ Responses 的终态载荷既是工具调用的恢复源，也是正文的恢复 ### `auth_store_projection_meta` +启动投影恢复会对过滤后的 retained refresh session 重新计数；超过 8192 条时直接失败关闭并继续重试，不得把超限快照一次性灌入内存。 + - Rust 结构体：`AuthStoreProjectionMeta` - 源码：`server-rs/crates/spacetime-module/src/auth/tables.rs` - 职责：保存 typed 认证投影的单调版本，以及短期手机号验证码和微信 OAuth state 的序列化投影；`phone_codes_json` / `wechat_states_json` 只承载短期认证状态，不替代 `user_account`、`auth_identity` 或 `refresh_session` 的正式表语义。 认证恢复策略：`api-server` 启动时从 SpacetimeDB 正式认证表（`user_account` / `auth_identity` / `refresh_session`）以及 `auth_store_projection_meta` 中的短期状态投影导出 typed `AuthStoreProjectionView`，再恢复 `module-auth` 的进程内认证工作集；生产 Bearer 中间件不再从 `InMemoryAuthStore` 读取用户或会话，而是每次通过 typed `validate_auth_session` procedure 在 SpacetimeDB 事务内校验 `token_version`、会话归属、撤销时间和过期时间，SpacetimeDB 不可用时 fail closed 返回服务错误。`validate_auth_session`、投影导出和投影同步均从 `ctx.sender()` 派生调用方，并复用现役 runtime service identity 白名单；启动恢复先完成该服务身份初始化，普通 SpacetimeDB identity 不能读取或改写私有认证表。测试构建仍可使用显式的内存测试夹具。所有会读取或变更本机认证工作集的认证主链路（登录、刷新、`/me`、会话管理、密码、绑定和微信 state）在领域操作前先从正式投影做一次受 CAS 保护的只读刷新，刷新失败时 fail closed；refresh cookie 仍只按正式 `refresh_session` 校验，其他认证数据也不得绕过正式同步。`module-auth` 只保留内存工作集和 projection 导入 / 导出能力，不再保留 JSON 快照导入 / 导出能力，也不写本地持久化文件；`auth-store.json` / `GENARRATIVE_AUTH_STORE_PATH` 不再是兼容恢复源。认证创建、登录会话、刷新、退出、改密、重置密码、绑定和资料变更等写操作仍必须在返回客户端前通过 `sync_auth_store_projection` 成功同步 SpacetimeDB 正式认证表；同步失败时接口返回错误，不允许把只存在于当前进程内存的账号、会话、短信验证码或微信 state 当成成功结果。每个 API 工作集绑定启动恢复或上次成功同步得到的 `auth_store_projection_meta.updated_at` 版本作为 `base_updated_at_micros`，SpacetimeDB 在同一事务内执行基线 CAS，并要求新的 `updated_at_micros` 严格递增；基线不一致或版本不晚于当前值时整包写入失败，冲突节点只有在确认本次同步尝试期间没有新的本地认证变更后，才可丢弃失败工作集并从正式表恢复，不能用陈旧工作集删除、恢复或覆盖另一节点的新状态；若同期仍有本地变更则保留 pending revision，并由后续认证请求先重试同步，不把临时数据库故障变成永久卡死；同步成功但期间又出现新本地变更时最多连续补同步三轮，仍未稳定则失败关闭。这只是迁移期并发保护，不改变正式认证表的权威地位。新用户注册奖励、邀请码绑定和登录埋点必须排在认证同步成功之后，避免认证没落库时先写出钱包或邀请关系。若启动恢复阶段 SpacetimeDB 不可连接或超时，`api-server` 会按固定间隔持续重试认证工作集恢复，恢复成功后才开始监听 HTTP，避免一次短超时让进程永久停留在依赖不可用状态。 +认证工作集容量限制：refresh session 最多保留 8192 条，短信验证码最多保留 4096 条；写入前清理过期项，达到上限时拒绝新增而不继续膨胀。 `auth_store_snapshot` 表和旧 `import_auth_store_snapshot_json` / `export_auth_store_snapshot_from_tables` procedure 已删除。认证投影同步只读写 `user_account`、`auth_identity`、`refresh_session` 和 `auth_store_projection_meta`；`auth_identity` 不再写 `phone_e164`、`display_name`、`avatar_url`，这些账号资料只以 `user_account` 为准。`api-server` 多节点必须使用相同的部署级验证码哈希盐（当前复用 `GENARRATIVE_JWT_SECRET`）；轮换该 secret 会使尚未消费的短信验证码失效，但不会改变已持久化账号或 session。 @@ -1242,6 +1248,7 @@ RPG 创作入口的配置 ID 是 `rpg`，当前 `visible=true`、`open=true`； - Rust 结构体：`RefreshSession` - 源码：`server-rs/crates/spacetime-module/src/auth/tables.rs` +- 认证工作集只保留 active 会话以及最近 24 小时内的 revoked / expired 会话；超过宽限期的失效会话在 refresh session 写路径和 projection 导出前从内存索引移除，并随下一次 typed projection 同步从正式表清理。该清理不改变 active 多端登录、单端登出或全端登出语义。 ### `runtime_setting` diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 062547649..064bae248 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -99,7 +99,7 @@ HTTP 角色的 `GENARRATIVE_SPACETIME_POOL_SIZE` 只表示 procedure / reducer 生产拆分角色时，`external-generation-worker` 和 `external-generation-controller` 的专属 env 示例会把 `GENARRATIVE_SPACETIME_POOL_SIZE` 覆盖为 `1`；非 HTTP 角色不创建 API 缓存读连接，只保留 `external_generation_job` 队列窄订阅作为响应式唤醒信号，实际抢占和扩缩容判断仍走 SpacetimeDB procedure。worker / controller 不执行模型定价 seed，启动时先调用受 runtime writer 鉴权的 queue-stats procedure 做只读预检，身份不匹配时 fail-fast；当前正式 systemd unit 通过共同加载 API env 继承同一 `GENARRATIVE_SPACETIME_TOKEN`，默认路径为 `/etc/genarrative/api-server.env`，自定义部署由 provision 和 API deploy 按实际参数渲染，专属角色 env 示例不重复配置该 token。`GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS` 与 controller poll interval 只作为订阅失效、漏事件和 lease 过期这类时间条件的兜底，不作为正常领取任务的主路径。 -生产 worker 默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=600`，只覆盖 worker 心跳抖动和短暂断连窗口，不再把 lease 当成完整任务时长；默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS=900`。`editor_image_generation`、`editor_image_edit`、`editor_icon_spritesheet_generation`、`editor_ui_design_asset_extraction` 四类 VectorEngine 图片任务与角色动画 / 视频类长任务使用 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS=1800`，手动去背景、音效和背景音乐继续使用普通预算。worker 在单次尝试超过执行预算后会停止续租并释放 worker 槽位，但不会取消已启动的业务 future 或主动写入失败 / 重试状态；在途执行由 lease fencing 仲裁，有效租约内写回仍可完成，租约过期后任务才可重新领取，attempt 耗尽时由认领事务标记失败并结算退款。生产部署和 provision 脚本会给 `/etc/genarrative/api-server.env` 与 `/etc/genarrative/external-generation-worker.env` 补齐这些变量；已有自定义值不覆盖，只会把历史旧默认 `3600` 迁移为 `600`。 +生产 worker 默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=600`，只覆盖 worker 心跳抖动和短暂断连窗口，不再把 lease 当成完整任务时长；默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS=900`。`editor_image_generation`、`editor_image_edit`、`editor_icon_spritesheet_generation`、`editor_ui_design_asset_extraction` 四类 VectorEngine 图片任务与角色动画 / 视频类长任务使用 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS=1800`，手动去背景、音效和背景音乐继续使用普通预算。worker 在单次尝试超过执行预算后会停止续租，但不会取消已启动的业务 future 或主动写入失败 / 重试状态；执行许可会一直绑定到 active 或 detached work 真正结束（或超过租约仲裁窗口被取消），避免超时任务脱管后立即补进新的高内存任务。在途执行由 lease fencing 仲裁，有效租约内写回仍可完成，租约过期后任务才可重新领取，attempt 耗尽时由认领事务标记失败并结算退款。生产部署和 provision 脚本会给 `/etc/genarrative/api-server.env` 与 `/etc/genarrative/external-generation-worker.env` 补齐这些变量；已有自定义值不覆盖，只会把历史旧默认 `3600` 迁移为 `600`。 lease 过期后不代表任务一定再次执行：claim transaction 只有在 `attempt < max_attempts` 时才会递增 attempt 并返回 worker；如果过期的是最终 attempt，则直接把 job 收口为 `failed`、清理 lease，并按入队冻结价格为当前 attempt 原子退款或写 cancellation intent。该终态任务不会再次进入 provider executor，迟到 consume 会被 settlement intent 拒绝。 @@ -115,7 +115,7 @@ BgFilter 对已经落入私有 OSS 的生成原图、动作抽取帧和手动去 图片编辑器任务侧栏与生成提交工作流只读取 BFF 队列接口：`GET /api/runtime/external-generation/jobs` 列出当前用户任务，`GET /api/runtime/external-generation/jobs/{jobId}` 查看单 job 状态，概览场景可使用 `GET /api/runtime/external-generation/queue-overview`。前端不直接查询 `external_generation_job` private table，也不展示 worker 内部 payload；完成态以编辑器项目和资源接口返回的正式数据为准。 -外部生成任务摘要投影与历史 payload 维护使用 `npm run spacetime:external-generation:maintain -- ...`，且只能由已授权 migration operator 的 SpacetimeDB CLI 登录态执行。脚本默认 dry-run、每次只处理一批，绝不自动循环全表；`--apply` 才写入。先发布包含 `external_generation_job_summary` 与 cursor 索引的 SpacetimeDB 模块，在维护模式内对事故时间以前的编辑器终态任务执行小批 dry-run，例如 `npm run spacetime:external-generation:maintain -- --database --server-url --limit 5 --completed-before-micros `；核对 `matched_count`、`before_bytes`、`after_bytes` 和 `inline_media_count` 后，保持本批输入 cursor 不变并追加 `--apply` 重跑同一批，即使最后一批 `has_more = false`，只要 dry-run 仍有 `matched_count` / `selected_count` 也必须 apply；只有 apply 成功后才使用它返回的 `next_cursor_job_id` 继续。B-tree cursor 的选择阶段最多反序列化 `limit + 1` 行，apply 会再按主键逐条读取选中行但不会同时保留整批 payload；如怀疑存在单行异常巨型历史 JSON，先用 `--limit 1`。payload 压缩硬限制 `source_module = editor-canvas`；终态压缩完成后，用 `--backfill-summaries` 先 dry-run、再 `--apply` 分批补齐仍缺失的活动任务或无内联媒体历史任务摘要，直到 `has_more = false`，最后再切换使用 summary procedure 的 api-server。Stdb 构建 artifact 和完整 release 包都必须包含 `scripts/spacetime-maintain-external-generation-jobs.mjs` 与 `scripts/spacetime-migration-common.mjs`。首次上线不得让 Full Build 从 Stdb 自动直落 API：`STDB_API_ROLLOUT_MODE` 默认 fail-closed 为 `pause-after-stdb`，必须填写受限的 `STDB_API_ROLLOUT_APPROVERS`；Stdb Publish 通过 `KEEP_MAINTENANCE_MODE` 保持维护文件并停止旧 API/controller/worker，暂停点最多等待 4 小时，完成上述维护并确认无后续批次后才由指定审批人放行 API。定时构建缺少审批人时必须在发布前失败，不能静默退回 `normal`；也可分开运行 Stdb publish、维护、API deploy 三个受控 Job。任一批次都不得处理 pending / running payload；不要用 runtime writer、bootstrap secret 或匿名 identity 代替 migration operator，也不要在未核对 dry-run 时直接 apply。 +外部生成任务摘要投影与历史 payload / history 维护使用 `npm run spacetime:external-generation:maintain -- ...`，且只能由已授权 migration operator 的 SpacetimeDB CLI 登录态执行。脚本默认 dry-run、每次只处理一批，绝不自动循环全表；`--apply` 才写入。先发布包含 `external_generation_job_summary` 与 cursor 索引的 SpacetimeDB 模块，在维护模式内对事故时间以前的编辑器终态任务执行小批 dry-run，例如 `npm run spacetime:external-generation:maintain -- --database --server-url --limit 5 --completed-before-micros `；核对 `matched_count`、`before_bytes`、`after_bytes` 和 `inline_media_count` 后，保持本批输入 cursor 不变并追加 `--apply` 重跑同一批，即使最后一批 `has_more = false`，只要 dry-run 仍有 `matched_count` / `selected_count` 也必须 apply；只有 apply 成功后才使用它返回的 `next_cursor_job_id` 继续。B-tree cursor 的选择阶段最多反序列化 `limit + 1` 行，apply 会再按主键逐条读取选中行但不会同时保留整批 payload；如怀疑存在单行异常巨型历史 JSON，先用 `--limit 1`。payload 压缩硬限制 `source_module = editor-canvas`；终态压缩完成后，用 `--backfill-summaries` 先 dry-run、再 `--apply` 分批补齐仍缺失的活动任务或无内联媒体历史任务摘要，直到 `has_more = false`，最后再切换使用 summary procedure 的 api-server。历史清理使用 `--prune-history`，默认 `source_module=editor-canvas`、30 天保留期；候选必须是 completed / failed / cancelled 终态、主任务与摘要状态一致、摘要存在 `notification_acknowledged_at` 且终态时间不晚于 `completed_before_micros`，否则永不删除。先 dry-run，记下输出的 `completed_before_micros`，再保持相同 `--cursor-job-id` 与 cutoff 追加 `--apply`；apply 在一个事务中删除该 job 的所有 event、summary 和主任务，资产对象与钱包流水保留。需要清理其它 source module 时必须显式 `--source-module` 并先完成业务评估；这不是自动 systemd 任务，不得授予 runtime writer 清理权限。Stdb 构建 artifact 和完整 release 包都必须包含 `scripts/spacetime-maintain-external-generation-jobs.mjs` 与 `scripts/spacetime-migration-common.mjs`。首次上线不得让 Full Build 从 Stdb 自动直落 API：`STDB_API_ROLLOUT_MODE` 默认 fail-closed 为 `pause-after-stdb`，必须填写受限的 `STDB_API_ROLLOUT_APPROVERS`；Stdb Publish 通过 `KEEP_MAINTENANCE_MODE` 保持维护文件并停止旧 API/controller/worker，暂停点最多等待 4 小时，完成上述维护并确认无后续批次后才由指定审批人放行 API。定时构建缺少审批人时必须在发布前失败，不能静默退回 `normal`；也可分开运行 Stdb publish、维护、API deploy 三个受控 Job。任一批次都不得处理 pending / running payload；不要用 runtime writer、bootstrap secret 或匿名 identity 代替 migration operator，也不要在未核对 dry-run 时直接 apply。 角色动作正式字段收口使用 `node scripts/spacetime-normalize-editor-character-actions.mjs --database --server-url `，且同样只能由已授权 migration operator 执行。必须先发布包含 normalization cursor 索引和 `normalize_editor_character_animation_metadata_and_return` 的 SpacetimeDB 模块，在 API / worker 仍处于维护模式时先运行默认全量 dry-run；脚本固定按 `asset → project-resource → showcase → canvas` 扫描，普通 scope 每批最多 25 行，canvas 每批最多 5 行。全量 dry-run 会在不写库的情况下把 asset 计划结果投影给同 owner / task / 首帧对象精确匹配的 project-resource，再把前置 scope 的计划结果投影给 canvas 检查；因此同 task 的误标预览 MP4 会先按权威视频对象排除，最终图片序列会逐帧核对并补齐精确 `asset_object` 身份。canvas 中仍引用误标 preview resource 的普通 video layer 会按 project-resource 计划态 `video` 跳过，只有 layout 明确声明动作却指向视频，或资源规划本身失败时才形成 blocker。apply 时仍要求前置 scope 已按顺序物理完成，不能跳过 asset 直接让 project-resource 借未落库结果。历史 canvas 复制的 `sourceResourceId` 不是迁移证据，不要因它仍指向原角色而手工改库，补建资源会采用最终账号素材的 DB 血缘。出现 blocker 时脚本会打印 ID、原因、owner、project、task、对象身份和来源资源；先据此区分最终候选为零 / 多个、正式与旧版冲突、帧对象不匹配或缺失资源，不得跳过 scope。确认 dry-run 后追加 `--apply`，脚本会对每批重新 dry-run、携带该批 SHA-256 apply，并在最后从头要求四个 scope 均为零匹配、零 blocker。只有该复核通过后才发布移除 action fallback 的 API / Web。Stdb build artifact 和完整 release 包必须同时包含 `scripts/spacetime-normalize-editor-character-actions.mjs` 与 `scripts/spacetime-migration-common.mjs`。本地切换分支时若要避免 dev publish 因 schema 冲突使用 `-c=on-conflict` 清库，启动命令必须追加 `--preserve-database`，让冲突直接失败。动作视频抽帧临时目录固定使用 `/var/lib/genarrative/character-animation-tmp`，该路径已由生产 API / worker unit 放行；不要让动作抽帧重新依赖 `PrivateTmp` 下的 `/tmp`。 @@ -401,7 +401,9 @@ UI 相关修改要重点验证： ### SpacetimeDB 数据目录 OSS 备份 -数据库备份不放进 `spacetime-module` reducer / procedure：备份属于文件系统与 OSS 外部副作用，必须由运维脚本在 SpacetimeDB 宿主外执行。当前统一脚本为 `scripts/database-backup-to-oss.mjs`（npm 命令 `npm run database:backup:oss`）。默认 `--storage-format archive --mode full` 保持原有全量压缩包冷备行为；`--storage-format files` 不生成 tar.gz，而是把目录树映射成逐文件 CAS 对象与 catalog，full 重跑只上传新增或内容变化的文件，history 只处理已被最新 snapshot 完全覆盖的历史 commitlog 与旧 snapshot。`Genarrative-Server-Provision` 的 `DATABASE_BACKUP_PROFILE` 默认是 `archive-full`，继续安装每天 `03:20` 左右执行的全量冷备主 service；development 和 release 都可以显式选择 `files-history`，但指定 work-dir 必须已经有与本机 database/bucket 匹配且已发布的 full baseline state： +脚本停库前会在固定 work-dir 写入 `.spacetimedb-stopped` marker；正常 finally 恢复 SpacetimeDB 及 `--restart-service-after` 指定的 API / worker / controller 后才清理 marker。若 Node 因 `MemoryMax` / OOM 被强制终止，systemd `ExecStopPost` 会根据仍存在的 marker 兜底恢复这些服务；恢复未全部成功时保留 marker 供后续重试。 + +数据库备份不放进 `spacetime-module` reducer / procedure：备份属于文件系统与 OSS 外部副作用，必须由运维脚本在 SpacetimeDB 宿主外执行。当前统一脚本为 `scripts/database-backup-to-oss.mjs`（npm 命令 `npm run database:backup:oss`）。默认 `--storage-format archive --mode full` 保持原有全量压缩包冷备行为；`--storage-format files` 不生成 tar.gz，而是把目录树映射成逐文件 CAS 对象与 catalog，full 重跑只上传新增或内容变化的文件，history 只处理已被最新 snapshot 完全覆盖的历史 commitlog 与旧 snapshot。`Genarrative-Server-Provision` 的 `DATABASE_BACKUP_PROFILE` 默认是 `archive-full`，继续安装每天 `03:20` 左右执行的全量冷备主 service；当前 release 只允许 `archive-full`，避免 `files-history` 在大目录上构造全量 catalog 导致 Node 内存峰值；development 才可以显式选择 `files-history`，且指定 work-dir 必须已经有与本机 database/bucket 匹配且已发布的 full baseline state： ```bash npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service @@ -437,7 +439,7 @@ GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET= `GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET` 为空时会回退 `ALIYUN_OSS_BUCKET`；AccessKey 默认复用 `ALIYUN_OSS_ACCESS_KEY_ID` / `ALIYUN_OSS_ACCESS_KEY_SECRET`，也可用 `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID` / `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET` 为备份 bucket 单独配置最小权限账号。冷备脚本会在停止 SpacetimeDB 前检查 `GENARRATIVE_DATABASE_BACKUP_WORK_DIR` 所在文件系统剩余空间；未设置 `GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES` 时，按数据目录大小加安全余量估算，空间不足会在停库前失败，避免写满根分区。即使打包或上传前步骤失败，只要脚本已经停过 SpacetimeDB，也会先恢复 SpacetimeDB 并执行 `--restart-service-after` 指定的 API / worker / controller，再带着原始备份错误退出。`Genarrative-Server-Provision` 会创建 `/var/lib/genarrative/database-backups` 并归属 `genarrative:genarrative`，同时安装并启用 `genarrative-database-backup.timer`。手动检查定时器：`systemctl list-timers genarrative-database-backup.timer`；手动触发一次：`systemctl start genarrative-database-backup.service`。如果 timer 显示 `enabled` 但 `inactive/dead` 且 `NEXT` / `Trigger` 为空，先写入当前 stamp 避免 `Persistent=true` 在白天立刻补跑冷备份：`touch /var/lib/systemd/timers/stamp-genarrative-database-backup.timer && systemctl daemon-reload && systemctl start genarrative-database-backup.timer`，随后确认下一次触发时间约为次日 `03:20`。 -`files-history` 使用仓库模板 `deploy/systemd/genarrative-database-backup-files-history.conf` 覆盖主 service 的 `ExecStart`，从 `/etc/genarrative/api-server.env` 读取 data-dir、database、bucket、prefix 与 OSS 凭据，不在 unit 写死环境目标，也不传 `--stop-service`。Server-Provision 在改动 drop-in 前，先用 current release 的同一脚本、同一 env 和 `DATABASE_BACKUP_FILES_HISTORY_WORK_DIR` 执行一次 history `--dry-run`；缺少已发布 full catalog 的 files state、current 脚本过旧或配置不匹配都会在安装 drop-in 和 `daemon-reload` 前失败。选择 `archive-full` 会主动删除仓库托管的 `10-files-history.conf` 与 dev 试点遗留的 `10-dev-files.conf`，防止 systemd 继续合并旧覆盖。dev 可继续指定已有 `/var/lib/genarrative/database-backups/dev-files`，release 建议先在 `/var/lib/genarrative/database-backups/release-files` 建立自己的 full baseline；两台机器不得复用或互传本地 state 目录冒充本机基线。启用时通过 Server-Provision Job 选择目标、`DATABASE_BACKUP_PROFILE=files-history` 和对应 work-dir，先保持 `DRY_RUN=true` 核对，再以同参数正式 provision。不要直接在 `/etc/systemd/system` 手写第二份 drop-in。 +`files-history` 使用仓库模板 `deploy/systemd/genarrative-database-backup-files-history.conf` 覆盖主 service 的 `ExecStart`，从 `/etc/genarrative/api-server.env` 读取 data-dir、database、bucket、prefix 与 OSS 凭据，不在 unit 写死环境目标，也不传 `--stop-service`。Server-Provision 在 development 改动 drop-in 前，先用 current release 的同一脚本、同一 env 和 `DATABASE_BACKUP_FILES_HISTORY_WORK_DIR` 执行一次 history `--dry-run`；缺少已发布 full catalog 的 files state、current 脚本过旧或配置不匹配都会在安装 drop-in 和 `daemon-reload` 前失败。选择 `archive-full` 会主动删除仓库托管的 `10-files-history.conf` 与 dev 试点遗留的 `10-dev-files.conf`，防止 systemd 继续合并旧覆盖。`genarrative-database-backup.service` 还通过 `NODE_OPTIONS=--max-old-space-size=768`、`MemoryHigh=768M`、`MemoryMax=1G` 和 `OOMPolicy=stop` 给备份进程设置独立护栏；release 若现场残留 files-history drop-in，必须先按 archive-full 重新 provision 并确认 drop-in 已删除，再恢复定时器。dev 可继续指定已有 `/var/lib/genarrative/database-backups/dev-files`；两台机器不得复用或互传本地 state 目录冒充本机基线。启用时通过 Server-Provision Job 选择目标、`DATABASE_BACKUP_PROFILE=files-history` 和对应 work-dir，先保持 `DRY_RUN=true` 核对，再以同参数正式 provision。不要直接在 `/etc/systemd/system` 手写第二份 drop-in。 files full 会递归扫描 data-dir，保留空目录、每个普通文件的相对路径，以及目标仍位于 data-dir 内部的相对符号链接；绝对链接或解析后越界的链接直接拒绝。文件按 SHA-256 上传到不可变对象 key，catalog 记录目录、路径、长度、SHA、对象 key 和相对链接目标，不写 staging 主机的绝对路径。相同 catalog 重跑不重复 PUT；新增或变化文件先 HEAD CAS 对象，存在且长度/SHA 元数据一致就复用，否则上传。16 MiB 及以下对象使用单次 PUT 后 HEAD 验真，大对象继续使用 multipart；对象操作默认以 16 路并行执行，可用 `GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=1..64` 调整。需要给线上入口留带宽时设置 `GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND=`，该共享限速器只包裹备份上传流，空值或 `0` 表示不限速，不修改主机全局 qdisc。并发、限速和单次 PUT 都不改变“全部对象、catalog 与 latest pointer 成功后才推进 state/清理”的顺序。full 基线必须来自停库后的 data-dir 或已通过恢复验证的冻结副本；源文件上传前后 stat 虽会复核，但在线扫描不能保证大量文件属于同一跨文件一致时点。catalog 验真后，脚本把最新 full/history 引用发布到固定 `//latest.json`，全新机器不需要本地 state 即可自动发现恢复入口。 @@ -478,7 +480,7 @@ node -- scripts/database-backup-to-oss.mjs \ dev 出口过慢时，可以把冻结基线经内网 rsync 到 release 独立 staging，再由 release 上传 dev bucket。staging 必须位于 `/var/lib/genarrative/dev-database-backup-staging/` 一类隔离目录，命令显式传 staging `--data-dir`、独立 `--work-dir`、dev `--bucket`，且不得传 `--stop-service`；禁止指向或修改 release `/stdb`。中转 key 只为本次传输临时授权，结束后从 dev 私钥和 release `authorized_keys` 同时移除。上传完成后把整个 files work-dir/state 回传 dev，history 才能延续同一 baseline catalog。 -完整恢复默认从 OSS 固定 `latest.json` 读取最新 full catalog：先创建 `directories`，再把每个 `files[].objectKey` 下载到 `/` 并逐项核对 `sizeBytes` / `sha256`；history catalog 用于证明已清理历史仍有 OSS 对象，不需要把已被 full baseline 覆盖的旧文件叠回当前恢复目录。本地 state 仍可作为兼容入口，并同时支持旧 v1 JSON 与 v2 gzip，但不再是异机恢复的前置条件。随后用隔离 data-dir 启动同版本 standalone，验证 `/v1/ping`、日志中的 snapshot restore / commitlog replay / module launch、代表性 SQL 和 reducer。dev 已完成这轮 OSS-only 异机恢复与重启演练；release 已使用独立 `/var/lib/genarrative/database-backups/release-files` full baseline 和 `files-history` profile，现场最终 `ExecStart`、timer 状态与最近备份结果仍须在变更时重新核对。 +完整恢复默认从 OSS 固定 `latest.json` 读取最新 full catalog：先创建 `directories`，再把每个 `files[].objectKey` 下载到 `/` 并逐项核对 `sizeBytes` / `sha256`；history catalog 用于证明已清理历史仍有 OSS 对象，不需要把已被 full baseline 覆盖的旧文件叠回当前恢复目录。本地 state 仍可作为兼容入口，并同时支持旧 v1 JSON 与 v2 gzip，但不再是异机恢复的前置条件。随后用隔离 data-dir 启动同版本 standalone，验证 `/v1/ping`、日志中的 snapshot restore / commitlog replay / module launch、代表性 SQL 和 reducer。dev 已完成这轮 OSS-only 异机恢复与重启演练；release 使用 archive-full 时，现场最终 `ExecStart`、timer 状态与最近备份结果仍须在变更时重新核对。 ```bash node -- scripts/database-backup-to-oss.mjs \ diff --git a/jenkins/Jenkinsfile.production-server-provision b/jenkins/Jenkinsfile.production-server-provision index f0233318b..d2104e68c 100644 --- a/jenkins/Jenkinsfile.production-server-provision +++ b/jenkins/Jenkinsfile.production-server-provision @@ -34,8 +34,8 @@ pipeline { string(name: 'WEB_LINK', defaultValue: '/srv/genarrative/web', description: 'Nginx 静态站点目录或软链接') string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'api-server 环境文件') string(name: 'API_PORT', defaultValue: '8082', description: 'api-server 本机监听端口') - choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profile；默认 archive-full，files-history 仅在指定 work-dir 已有完整 full baseline 后启用') - string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'files-history 的本地 state/catalog 目录；dev/release 必须使用各自已建立 full baseline 的独立目录') + choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profile；release 仅允许 archive-full，files-history 仅供 development 在指定 work-dir 已有完整 full baseline 后启用') + string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'development files-history 的本地 state/catalog 目录；必须使用已建立 full baseline 的独立目录') choice(name: 'NGINX_CONFIG_MODE', choices: ['none', 'production-https', 'development-http'], description: 'Nginx 配置模式；开发服无域名时选 development-http，release 正式入口选 production-https') booleanParam(name: 'ENABLE_SERVICES', defaultValue: true, description: '启用并启动 spacetimedb 与 api-server systemd 服务') booleanParam(name: 'ENABLE_OTELCOL', defaultValue: true, description: '安装并启用本机 OpenTelemetry Collector；api-server 模板默认开启 OTLP，如需关闭请在 API_ENV_FILE 中将 GENARRATIVE_OTEL_ENABLED 改为 false') @@ -113,6 +113,9 @@ pipeline { if (!(databaseBackupProfile in ['archive-full', 'files-history'])) { error("DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${params.DATABASE_BACKUP_PROFILE}") } + if (params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history') { + error('release 仅允许 archive-full；files-history 会把整棵历史目录加载到 Node 内存，需先完成流式 catalog 改造后才能重新启用。') + } def databaseBackupFilesHistoryWorkDir = params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR?.trim() if (!(databaseBackupFilesHistoryWorkDir ==~ /^\/var\/lib\/genarrative\/database-backups\/[A-Za-z0-9._\/-]+$/) || databaseBackupFilesHistoryWorkDir.contains('..')) { error("DATABASE_BACKUP_FILES_HISTORY_WORK_DIR 必须是 /var/lib/genarrative/database-backups/ 下不含连续点号的绝对路径，当前值: ${params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}") diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index 57bc09368..689295ca6 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -50,6 +50,7 @@ async function main() { assertDeferredArchiveDiscoveryIsBoundedAndDeterministic(); assertCanonicalQueryAndAuthorizationIncludeMultipartParameters(); assertInsufficientSpaceStopsBeforeServiceChanges(); + assertStopFailureRetainsRecoveryMarker(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); await assertUploadBandwidthLimiterSharesBudgetAndPropagatesErrors(); @@ -748,6 +749,27 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() { assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。'); } +function assertStopFailureRetainsRecoveryMarker() { + const fixture = createFixture('stop-failure-marker'); + writeExecutable( + path.join(fixture.binDir, 'systemctl'), + `#!/usr/bin/env bash +printf 'systemctl %s\\n' "$*" >> "${fixture.systemctlLog}" +if [ "$1" = stop ]; then + exit 9 +fi +exit 0 +`, + ); + const result = runBackup(fixture, ['--stop-service', 'spacetimedb.service']); + + assertStatus(result, 1, '停止服务失败时备份必须失败。'); + assertTrue( + existsSync(path.join(fixture.workDir, '.spacetimedb-stopped')), + '停止服务命令失败时必须保留 marker，供 systemd ExecStopPost 兜底恢复。', + ); +} + function assertArchiveFailureStillRestoresDependentServices() { const fixture = createFixture('tar-failure'); const result = runBackup(fixture, [ @@ -776,6 +798,10 @@ function assertArchiveFailureStillRestoresDependentServices() { for (const command of expectedCommands) { assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`); } + assertFileMissing( + path.join(fixture.workDir, '.spacetimedb-stopped'), + '正常执行 finally 恢复全部服务后必须清理停库 marker。', + ); } async function assertMultipartUploadRetriesAndVerifiesRemoteLength() { diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index c9463ace6..bacd4b949 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -821,6 +821,31 @@ const checks = [ reason: '生产冷备份 service 必须用 node -- 分隔脚本参数，避免 Node 22 抢占业务 --env-file。', }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: 'Environment=NODE_OPTIONS=--max-old-space-size=768', + reason: + '备份 Node 进程必须设置独立 heap 上限，避免目录扫描异常拖垮 release 主机。', + }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: + 'Environment=GENARRATIVE_DATABASE_BACKUP_STOP_MARKER=/var/lib/genarrative/database-backups/.spacetimedb-stopped', + reason: + '备份停库 marker 必须固定在受保护的 release work-dir，供 OOM 后 systemd 兜底恢复服务。', + }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: 'MemoryMax=1G', + reason: + '备份 service 必须设置 systemd 内存硬上限，避免异常进程消耗整机内存。', + }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: 'ExecStopPost=/bin/sh -c', + reason: + '备份主进程被 OOM kill 后必须由 systemd 兜底恢复停掉的 SpacetimeDB、API、worker 和 controller。', + }, { file: 'deploy/systemd/genarrative-database-backup.service', excludes: '--storage-format files', @@ -941,10 +966,9 @@ const checks = [ }, { file: 'jenkins/Jenkinsfile.production-server-provision', - excludes: - "params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history'", + includes: 'release 仅允许 archive-full；files-history', reason: - 'release 必须能在显式选择 profile 且 baseline 预检通过后启用 files-history。', + 'release 必须拒绝 files-history，避免逐文件 catalog 扫描再次触发生产内存峰值。', }, { file: 'scripts/database-backup-to-oss.mjs', @@ -954,7 +978,7 @@ const checks = [ { file: 'scripts/database-backup-to-oss.mjs', includes: - 'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter})', + 'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath})', reason: '生产冷备份打包失败时也必须恢复 SpacetimeDB 及依赖服务。', }, { diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index d9a1ff9eb..f3788301a 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -35,6 +35,7 @@ const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/ const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups'); const DEFAULT_PRODUCTION_DATA_DIR = '/stdb'; const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups'; +const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join(DEFAULT_PRODUCTION_WORK_DIR, '.spacetimedb-stopped'); const DEFAULT_SPACE_SAFETY_RATIO = 1.1; const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024; const OSS_ALGORITHM = 'OSS4-HMAC-SHA256'; @@ -555,7 +556,11 @@ function assertSafeRelativePath(dataDir, absolutePath) { } function statFingerprint(absolutePath, rootPath = absolutePath) { - const entries = []; + // 候选 snapshot 可能包含数十万条目录项；增量更新摘要，避免把每条 + // fingerprint 字符串同时保存在 entries[] 后再 join，造成一次性内存峰值。 + const fingerprintHash = createHash('sha256'); + let isFirstEntry = true; + let entryCount = 0; let totalSize = 0n; const visit = (currentPath) => { const stat = lstatSync(currentPath, {bigint: true}); @@ -567,7 +572,7 @@ function statFingerprint(absolutePath, rootPath = absolutePath) { if (kind === 'other') { throw new Error(`history 候选只允许普通文件或目录: ${currentPath}`); } - entries.push([ + const entry = [ entryPath, kind, stat.dev.toString(), @@ -575,7 +580,13 @@ function statFingerprint(absolutePath, rootPath = absolutePath) { stat.mode.toString(), stat.size.toString(), stat.mtimeNs.toString(), - ].join('\0')); + ].join('\0'); + if (!isFirstEntry) { + fingerprintHash.update('\n'); + } + fingerprintHash.update(entry); + isFirstEntry = false; + entryCount += 1; if (stat.isFile()) { totalSize += stat.size; } else { @@ -586,9 +597,9 @@ function statFingerprint(absolutePath, rootPath = absolutePath) { }; visit(rootPath); return { - fingerprint: sha256Hex(entries.join('\n')), + fingerprint: fingerprintHash.digest('hex'), sizeBytes: totalSize.toString(), - entryCount: entries.length, + entryCount, }; } @@ -880,11 +891,37 @@ function collectRestartServicesAfterBackup({args, env}) { return [...new Set(serviceNames.filter(Boolean))]; } -function stopServiceIfNeeded(serviceName) { +function databaseBackupStopMarkerPath(workDir) { + return resolvePath(firstNonEmpty( + process.env.GENARRATIVE_DATABASE_BACKUP_STOP_MARKER, + workDir === DEFAULT_PRODUCTION_WORK_DIR + ? DEFAULT_DATABASE_BACKUP_STOP_MARKER + : join(workDir, '.spacetimedb-stopped'), + )); +} + +function writeDatabaseBackupStopMarker(markerPath, serviceName) { + atomicWriteJson(markerPath, { + serviceName, + pid: process.pid, + stoppedAt: new Date().toISOString(), + }); +} + +function clearDatabaseBackupStopMarker(markerPath) { + if (markerPath) { + rmSync(markerPath, {force: true}); + } +} + +function stopServiceIfNeeded(serviceName, stopMarkerPath) { if (!serviceName) { return false; } console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`); + writeDatabaseBackupStopMarker(stopMarkerPath, serviceName); + // stop 命令失败时仍保留 marker：systemd 的 ExecStopPost 需要它判断是否要 + // 兜底恢复，不能因为当前进程还能捕获异常就抹掉上一次停库证据。 runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'}); return true; } @@ -915,7 +952,7 @@ function restartServicesAfterBackup(serviceNames) { } } -function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}) { +function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath}) { const errors = []; try { startServiceIfNeeded(stopService, serviceStopped); @@ -930,6 +967,7 @@ function restoreServicesAfterBackup({stopService, serviceStopped, restartService if (errors.length > 0) { throw new AggregateError(errors, `恢复冷备份相关服务失败: ${errors.map((error) => error.message).join('; ')}`); } + clearDatabaseBackupStopMarker(stopMarkerPath); } function createArchive({dataDir, workDir, fileName}) { @@ -1285,14 +1323,17 @@ export async function collectDirectFileEntries({dataDir, candidates = null, obje throw new Error(`files 扫描期间源文件发生变化: ${relativePath}`); } const basePrefix = normalizeObjectPrefix(objectPrefix, database); - files.set(relativePath, { + const file = { path: relativePath, sizeBytes: Number(after.size), sha256, mode: after.mode, objectKey: `${basePrefix}/files/sha256/${sha256.slice(0, 2)}/${sha256}`, - sourceStat: after, - }); + }; + // 上传前后的 inode/stat 仍用于防止在线扫描漂移，但设为不可枚举，避免 + // 把仅供本地校验的副本再次写入 catalog 或 result JSON。 + Object.defineProperty(file, 'sourceStat', {value: after, enumerable: false}); + files.set(relativePath, file); }; for (const root of roots.sort((left, right) => left.relativePath.localeCompare(right.relativePath))) { @@ -1309,14 +1350,40 @@ export async function collectDirectFileEntries({dataDir, candidates = null, obje } function directCatalogIdentity({mode, baselineCatalogId, rootName, directories, files, symlinks}) { - return sha256Hex(JSON.stringify({ - mode, - baselineCatalogId: baselineCatalogId || '', - rootName, - directories, - files: files.map(({path, sizeBytes, sha256, mode, objectKey}) => ({path, sizeBytes, sha256, mode, objectKey})), - symlinks, + // 不把数十万条文件元数据先拼成一个巨型 JSON 字符串；分段写入 hash + // 保持与 JSON.stringify 同样的字段顺序和转义结果，同时把峰值降到单条记录。 + const hash = createHash('sha256'); + hash.update('{"mode":'); + hash.update(JSON.stringify(mode)); + hash.update(',"baselineCatalogId":'); + hash.update(JSON.stringify(baselineCatalogId || '')); + hash.update(',"rootName":'); + hash.update(JSON.stringify(rootName)); + hash.update(',"directories":'); + updateJsonArrayHash(hash, directories, (directory) => JSON.stringify(directory)); + hash.update(',"files":'); + updateJsonArrayHash(hash, files, (file) => JSON.stringify({ + path: file.path, + sizeBytes: file.sizeBytes, + sha256: file.sha256, + mode: file.mode, + objectKey: file.objectKey, })); + hash.update(',"symlinks":'); + updateJsonArrayHash(hash, symlinks, (symlink) => JSON.stringify(symlink)); + hash.update('}'); + return hash.digest('hex'); +} + +function updateJsonArrayHash(hash, values, serialize) { + hash.update('['); + values.forEach((value, index) => { + if (index > 0) { + hash.update(','); + } + hash.update(serialize(value)); + }); + hash.update(']'); } function readDirectFilesState(statePath, {database, bucket}) { @@ -1621,7 +1688,7 @@ export async function runDirectFilesBackup({ baselineCatalogId, rootName, directories: collected.directories, - files: collected.files.map(({sourceStat: _sourceStat, ...file}) => file), + files: collected.files, symlinks: collected.symlinks, }; writeManifest({manifestPath: catalogPath, payload: catalog}); @@ -3302,12 +3369,13 @@ async function main() { } const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE); const restartServicesAfter = collectRestartServicesAfterBackup({args, env}); + const stopMarkerPath = databaseBackupStopMarkerPath(workDir); let serviceStopped = false; let backupError = null; let restoreError = null; try { if (args.mode === 'full' && !args.dryRun) { - serviceStopped = stopServiceIfNeeded(stopService); + serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); } await runDirectFilesBackup({ mode: args.mode, @@ -3326,7 +3394,7 @@ async function main() { } finally { try { if (serviceStopped) { - restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}); + restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath}); } else if (!backupError && args.mode === 'full' && !args.dryRun) { restartServicesAfterBackup(restartServicesAfter); } @@ -3380,16 +3448,17 @@ async function main() { let restoreError = null; const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE); const restartServicesAfter = collectRestartServicesAfterBackup({args, env}); + const stopMarkerPath = databaseBackupStopMarkerPath(workDir); try { assertSufficientWorkDirSpace({dataDir, workDir, args, env}); - serviceStopped = stopServiceIfNeeded(stopService); + serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); archivePath = createArchive({dataDir, workDir, fileName}); } catch (error) { backupError = error; } finally { try { if (serviceStopped) { - restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}); + restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath}); } else if (!backupError) { restartServicesAfterBackup(restartServicesAfter); } diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 894458fd2..259b6f6c8 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -78,6 +78,10 @@ validate_database_backup_profile() { exit 1 ;; esac + if [[ "${DEPLOY_TARGET}" == "release" && "${DATABASE_BACKUP_PROFILE}" == "files-history" ]]; then + echo "[server-provision] release 仅允许 archive-full；files-history 会把整棵历史目录加载到 Node 内存，需先完成流式 catalog 改造后才能重新启用。" >&2 + exit 1 + fi if [[ ! "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" =~ ^/var/lib/genarrative/database-backups/[A-Za-z0-9._/-]+$ || "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" == *..* ]]; then echo "[server-provision] DATABASE_BACKUP_FILES_HISTORY_WORK_DIR 必须是 /var/lib/genarrative/database-backups/ 下不含连续点号的绝对路径，当前值: ${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" >&2 exit 1 diff --git a/scripts/spacetime-maintain-external-generation-jobs.mjs b/scripts/spacetime-maintain-external-generation-jobs.mjs index 69ead2f3c..49a6c8625 100644 --- a/scripts/spacetime-maintain-external-generation-jobs.mjs +++ b/scripts/spacetime-maintain-external-generation-jobs.mjs @@ -8,23 +8,29 @@ import { } from './spacetime-migration-common.mjs'; const MAX_BATCH_SIZE = 25; +const DEFAULT_RETENTION_DAYS = 30; +const MICROS_PER_DAY = 86_400_000_000; function usage() { return `用法： node scripts/spacetime-maintain-external-generation-jobs.mjs --database [选项] 默认只 dry-run 一批历史终态任务 payload 压缩，不修改数据库。 +使用 --prune-history 时改为清理已确认通知且超过保留期的历史任务、摘要与事件。 公共选项： --database 目标数据库（必填，也可用 GENARRATIVE_SPACETIME_DATABASE） --server spacetime CLI server 名或 URL --server-url 显式 server URL - --limit <1-${MAX_BATCH_SIZE}> 单批任务数，默认 10 + --limit <1-${MAX_BATCH_SIZE}> 单批任务数，默认 10 --cursor-job-id 从上一批 next_cursor_job_id 继续 --apply 执行写入；省略时始终 dry-run --backfill-summaries 改为回填轻量摘要投影 + --prune-history 改为清理已确认通知的终态历史 --owner-user-id 仅摘要回填可选，限定 owner - --completed-before-micros 仅 payload 压缩可选，限定终态完成时间 + --source-module 仅历史清理可选，默认 editor-canvas + --retention-days 仅历史清理可选，默认 ${DEFAULT_RETENTION_DAYS} 天 + --completed-before-micros 限定终态完成时间；历史清理默认按 retention-days 计算 --help 显示帮助 必须使用已授权 migration operator 的 spacetime CLI 登录态。脚本每次只处理一批； @@ -40,6 +46,9 @@ function parseOptions(argv) { database: process.env.GENARRATIVE_SPACETIME_DATABASE || '', limit: 10, ownerUserId: '', + pruneHistory: false, + retentionDays: DEFAULT_RETENTION_DAYS, + sourceModule: 'editor-canvas', passthrough: [], server: process.env.GENARRATIVE_SPACETIME_SERVER || '', serverUrl: process.env.GENARRATIVE_SPACETIME_SERVER_URL || '', @@ -82,6 +91,15 @@ function parseOptions(argv) { options.apply = true; } else if (arg === '--backfill-summaries') { options.backfillSummaries = true; + } else if (arg === '--prune-history') { + options.pruneHistory = true; + } else if (arg === '--source-module') { + options.sourceModule = readValue(arg).trim(); + if (!options.sourceModule) { + throw new Error('--source-module 不能为空。'); + } + } else if (arg === '--retention-days') { + options.retentionDays = parsePositiveInteger(readValue(arg), arg); } else if (arg === '--help' || arg === '-h') { options.help = true; } else { @@ -95,12 +113,49 @@ function parseOptions(argv) { if (options.ownerUserId && !options.backfillSummaries) { throw new Error('--owner-user-id 只能与 --backfill-summaries 一起使用。'); } + if (options.backfillSummaries && options.pruneHistory) { + throw new Error('--backfill-summaries 与 --prune-history 不能同时使用。'); + } + if (options.sourceModule !== 'editor-canvas' && !options.pruneHistory) { + throw new Error('--source-module 只能与 --prune-history 一起使用。'); + } + if ( + options.retentionDays !== DEFAULT_RETENTION_DAYS && + !options.pruneHistory + ) { + throw new Error('--retention-days 只能与 --prune-history 一起使用。'); + } if (options.completedBeforeMicros !== null && options.backfillSummaries) { throw new Error('--completed-before-micros 不能用于摘要回填。'); } + if ( + options.completedBeforeMicros !== null && + options.pruneHistory && + options.retentionDays !== DEFAULT_RETENTION_DAYS + ) { + throw new Error( + '--completed-before-micros 与 --retention-days 不能同时使用。', + ); + } return options; } +function resolveRetentionCutoffMicros(options) { + if (!options.pruneHistory) { + return options.completedBeforeMicros; + } + if (options.completedBeforeMicros !== null) { + return options.completedBeforeMicros; + } + const cutoff = Date.now() * 1000 - options.retentionDays * MICROS_PER_DAY; + if (!Number.isSafeInteger(cutoff)) { + throw new Error( + '--retention-days 计算出的 completed_before_micros 超出安全整数范围。', + ); + } + return cutoff; +} + try { const options = parseOptions(process.argv.slice(2)); if (options.help) { @@ -113,24 +168,36 @@ try { ); } - const procedureName = options.backfillSummaries - ? 'backfill_external_generation_job_summaries_and_return' - : 'compact_external_generation_job_payloads_and_return'; - const input = options.backfillSummaries + const completedBeforeMicros = resolveRetentionCutoffMicros(options); + + const procedureName = options.pruneHistory + ? 'prune_external_generation_job_history_and_return' + : options.backfillSummaries + ? 'backfill_external_generation_job_summaries_and_return' + : 'compact_external_generation_job_payloads_and_return'; + const input = options.pruneHistory ? { - owner_user_id: encodeSpacetimeCliOption(options.ownerUserId || null), + source_module: options.sourceModule, limit: options.limit, cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), + completed_before_micros: completedBeforeMicros, dry_run: !options.apply, } - : { - dry_run: !options.apply, - limit: options.limit, - cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), - completed_before_micros: encodeSpacetimeCliOption( - options.completedBeforeMicros, - ), - }; + : options.backfillSummaries + ? { + owner_user_id: encodeSpacetimeCliOption(options.ownerUserId || null), + limit: options.limit, + cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), + dry_run: !options.apply, + } + : { + dry_run: !options.apply, + limit: options.limit, + cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), + completed_before_micros: encodeSpacetimeCliOption( + completedBeforeMicros, + ), + }; const result = await callSpacetimeProcedureViaCli( options, procedureName, @@ -138,10 +205,29 @@ try { ); ensureProcedureOk(result); - console.log(JSON.stringify({ procedure: procedureName, ...result }, null, 2)); - const pendingApplyCount = options.backfillSummaries - ? Number(result.selected_count ?? 0) - : Number(result.matched_count ?? 0); + console.log( + JSON.stringify( + { + procedure: procedureName, + ...(options.pruneHistory + ? { + source_module: options.sourceModule, + completed_before_micros: completedBeforeMicros, + ...(options.completedBeforeMicros === null + ? { retention_days: options.retentionDays } + : {}), + } + : {}), + ...result, + }, + null, + 2, + ), + ); + const pendingApplyCount = + options.pruneHistory || options.backfillSummaries + ? Number(result.selected_count ?? 0) + : Number(result.matched_count ?? 0); if (result.has_more && options.apply) { console.log( `仍有后续批次；下一次追加 --cursor-job-id ${result.next_cursor_job_id ?? ''}。`, @@ -150,8 +236,11 @@ try { const currentCursor = options.cursorJobId ? `保留 --cursor-job-id ${options.cursorJobId}` : '仍从首批开始'; + const cutoffHint = options.pruneHistory + ? `并固定 --completed-before-micros ${completedBeforeMicros}` + : ''; console.log( - `当前仅 dry-run；请${currentCursor}并追加 --apply 重跑同一批。apply 成功后再使用其 next_cursor_job_id 进入下一批。`, + `当前仅 dry-run；请${currentCursor}${cutoffHint}并追加 --apply 重跑同一批。apply 成功后再使用其 next_cursor_job_id 进入下一批。`, ); } } catch (error) { diff --git a/server-rs/crates/api-server/src/external_generation_worker.rs b/server-rs/crates/api-server/src/external_generation_worker.rs index b27804a9d..fa80d9b32 100644 --- a/server-rs/crates/api-server/src/external_generation_worker.rs +++ b/server-rs/crates/api-server/src/external_generation_worker.rs @@ -14,6 +14,7 @@ use spacetime_client::{ ExternalGenerationJobRenewLeaseRecordInput, ExternalGenerationQueueWakeSubscription, }; use tokio::{ + sync::{OwnedSemaphorePermit, Semaphore}, task::{JoinHandle, JoinSet}, time::sleep, }; @@ -92,6 +93,10 @@ pub(crate) async fn run_external_generation_worker(state: AppState) -> Result<() let concurrency = state.config.external_generation_worker_concurrency.max(1); let poll_interval = state.config.external_generation_worker_poll_interval; let lease = state.config.external_generation_worker_lease; + // 超时任务不能立即取消（在途 procedure 仍可能写回），因此执行容量必须同时 + // 约束 active 与 detached work；否则每次超时都会释放 tasks 槽位，实际内存占用 + // 会超过配置并发。 + let work_slots = std::sync::Arc::new(Semaphore::new(concurrency)); let mut tasks = JoinSet::new(); let mut shutdown = external_generation_worker_shutdown_signal(); let mut queue_wake = None; @@ -113,16 +118,29 @@ pub(crate) async fn run_external_generation_worker(state: AppState) -> Result<() ); loop { + // 持续有队列任务时不会进入等待分支，因此必须在每轮主动回收已完成的 + // JoinHandle；否则 permit 虽已归还，JoinSet 仍会保留每个历史任务的句柄。 + reap_finished_external_generation_worker_tasks(&mut tasks); ensure_external_generation_queue_wake_subscription(&state, &mut queue_wake).await; - while tasks.len() >= concurrency { - if await_worker_task_or_shutdown(&mut tasks, &mut shutdown).await { - drain_external_generation_worker_tasks(&mut tasks).await; - return Ok(()); + while work_slots.available_permits() == 0 { + tokio::select! { + _ = shutdown.as_mut() => { + drain_external_generation_worker_tasks(&mut tasks).await; + return Ok(()); + } + permit = work_slots.clone().acquire_owned() => { + if permit.is_err() { + drain_external_generation_worker_tasks(&mut tasks).await; + return Ok(()); + } + // 只用 acquire 作为容量变化唤醒信号，许可立即归还；真正领取任务 + // 时在下方按返回的 job 数量逐个 try_acquire。 + } } } - let available = concurrency.saturating_sub(tasks.len()).max(1); + let available = work_slots.available_permits().max(1); let now_micros = current_utc_micros(); let lease_expires_at_micros = now_micros.saturating_add(duration_micros_i64(lease)); @@ -178,9 +196,13 @@ pub(crate) async fn run_external_generation_worker(state: AppState) -> Result<() for job in jobs { let state = state.clone(); let worker_id = worker_id.clone(); + let permit = work_slots + .clone() + .try_acquire_owned() + .expect("claimed job must have an execution capacity permit"); tasks.spawn(async move { if let Err(error) = - process_external_generation_job(state, worker_id, lease, job).await + process_external_generation_job(state, worker_id, lease, job, permit).await { error!(error = %error, "external generation worker 执行任务失败"); } @@ -255,13 +277,11 @@ async fn await_worker_task(tasks: &mut JoinSet<()>) { } } -async fn await_worker_task_or_shutdown( - tasks: &mut JoinSet<()>, - shutdown: &mut ExternalGenerationShutdownSignal, -) -> bool { - tokio::select! { - _ = shutdown.as_mut() => true, - _ = await_worker_task(tasks) => false, +fn reap_finished_external_generation_worker_tasks(tasks: &mut JoinSet<()>) { + while let Some(result) = tasks.try_join_next() { + if let Err(error) = result { + error!(error = %error, "external generation worker 子任务 panic"); + } } } @@ -326,6 +346,7 @@ async fn process_external_generation_job( worker_id: String, lease: Duration, job: ExternalGenerationJobRecord, + permit: OwnedSemaphorePermit, ) -> Result<(), String> { let heartbeat_interval = external_generation_worker_heartbeat_interval(lease); let job_timeout = external_generation_worker_job_timeout(&state.config, job.job_kind.as_str()); @@ -377,13 +398,14 @@ async fn process_external_generation_job( job_id = %job.job_id, job_kind = %job.job_kind, timeout_seconds = job_timeout.as_secs(), - "external generation worker 任务超过执行预算，停止续租并释放 worker 槽位，在途执行交由租约仲裁" + "external generation worker 任务超过执行预算，停止续租并保留 worker 槽位，在途执行交由租约仲裁" ); detach_external_generation_work_until_lease_expiry( work_handle, &job, lease, "任务超过执行预算", + Some(permit), ); Err(message) } @@ -393,6 +415,7 @@ async fn process_external_generation_job( &job, lease, "任务租约续期失败", + Some(permit), ); Err(error) } @@ -417,6 +440,7 @@ fn detach_external_generation_work_until_lease_expiry( job: &ExternalGenerationJobRecord, lease: Duration, reason: &'static str, + permit: Option, ) { let job_id = job.job_id.clone(); let job_kind = job.job_kind.clone(); @@ -445,6 +469,9 @@ fn detach_external_generation_work_until_lease_expiry( ), Err(_) => { work_handle.abort(); + // 仅调用 abort 不会从 JoinHandle/JoinSet 中消费完成结果；等待被取消 + // 的 handle，确保 permit 与任务句柄在同一生命周期内一起释放。 + let _ = work_handle.await; warn!( job_id = %job_id, job_kind = %job_kind, @@ -454,6 +481,9 @@ fn detach_external_generation_work_until_lease_expiry( ); } } + // 保持执行许可直到 work 真正结束或被取消，避免超时任务脱管后继续 + // 累积图片/音频响应占用。 + drop(permit); }); } @@ -1972,6 +2002,7 @@ mod tests { &job, Duration::from_millis(200), "任务超过执行预算", + None, ); tokio::time::sleep(Duration::from_millis(100)).await; @@ -1981,6 +2012,61 @@ mod tests { ); } + #[tokio::test] + async fn worker_detached_work_keeps_execution_slot_until_finished() { + let slots = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let permit = slots + .clone() + .acquire_owned() + .await + .expect("the only execution slot should be available"); + let work_handle = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(20)).await; + Ok(()) + }); + let job = external_generation_job_record_fixture(Some("lease-1")); + + detach_external_generation_work_until_lease_expiry( + work_handle, + &job, + Duration::from_millis(200), + "任务超过执行预算", + Some(permit), + ); + + assert!( + slots.try_acquire().is_err(), + "脱管 work 完成前不得重新领取执行容量" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + slots.try_acquire().is_ok(), + "脱管 work 完成后应归还执行容量" + ); + } + + #[tokio::test] + async fn worker_reaps_completed_tasks_while_queue_remains_busy() { + let mut tasks = JoinSet::new(); + let completed = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + const TASK_COUNT: usize = 128; + for _ in 0..TASK_COUNT { + let completed = completed.clone(); + tasks.spawn(async move { + completed.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }); + } + + while completed.load(std::sync::atomic::Ordering::SeqCst) < TASK_COUNT { + tokio::task::yield_now().await; + } + assert_eq!(tasks.len(), TASK_COUNT); + + reap_finished_external_generation_worker_tasks(&mut tasks); + + assert!(tasks.is_empty(), "已完成任务的 JoinHandle 应在每轮被回收"); + } + #[tokio::test] async fn worker_detached_work_is_aborted_after_lease_arbitration_window() { let connection = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); @@ -1999,6 +2085,7 @@ mod tests { &job, Duration::from_millis(10), "任务超过执行预算", + None, ); let reacquired = diff --git a/server-rs/crates/module-ai/src/application/service.rs b/server-rs/crates/module-ai/src/application/service.rs index 713f3cd33..6f42f1d69 100644 --- a/server-rs/crates/module-ai/src/application/service.rs +++ b/server-rs/crates/module-ai/src/application/service.rs @@ -28,7 +28,7 @@ impl AiTaskService { validate_task_create_input(&input).map_err(AiTaskServiceError::Field)?; let snapshot = AiTaskSnapshot { - task_id: input.task_id.clone(), + task_id: normalize_required_string(input.task_id).unwrap_or_default(), task_kind: input.task_kind, owner_user_id: normalize_required_string(input.owner_user_id).unwrap_or_default(), request_label: normalize_required_string(input.request_label).unwrap_or_default(), diff --git a/server-rs/crates/module-ai/src/application/store.rs b/server-rs/crates/module-ai/src/application/store.rs index f2d0c1175..6150dc397 100644 --- a/server-rs/crates/module-ai/src/application/store.rs +++ b/server-rs/crates/module-ai/src/application/store.rs @@ -1,10 +1,12 @@ use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, sync::{Arc, Mutex}, }; use crate::{ AiTaskServiceError, AiTaskSnapshot, AiTaskStageStatus, AiTaskStatus, AiTextChunkSnapshot, + MAX_AI_TASK_RETAINED_OUTPUT_BYTES, MAX_AI_TASK_RETAINED_TASKS, MAX_AI_TASK_TEXT_OUTPUT_BYTES, + validate_ai_task_snapshot_memory_limits, }; use super::ensure_task_is_not_terminal; @@ -17,7 +19,9 @@ pub struct InMemoryAiTaskStore { #[derive(Debug, Default)] struct InMemoryAiTaskStoreState { tasks: HashMap, - text_chunks: HashMap>, + // Keep only the ordered deltas needed to handle an out-of-order chunk. + // Completed tasks drop this map immediately; it is not a second durable log. + text_chunks: HashMap>>, } impl InMemoryAiTaskStore { @@ -34,7 +38,48 @@ impl InMemoryAiTaskStore { return Err(AiTaskServiceError::TaskAlreadyExists); } - state.text_chunks.insert(task.task_id.clone(), Vec::new()); + validate_task_memory_limits(&task)?; + + let oldest_terminal = if state.tasks.len() >= MAX_AI_TASK_RETAINED_TASKS { + let oldest_terminal = state + .tasks + .values() + .filter(|value| value.status.is_terminal()) + .min_by_key(|value| value.completed_at_micros.or(Some(value.updated_at_micros))) + .map(|value| value.task_id.clone()); + if oldest_terminal.is_none() { + return Err(AiTaskServiceError::Store( + "AI 任务仓储已达到内存容量上限".to_string(), + )); + } + oldest_terminal + } else { + None + }; + + let retained_output_bytes = retained_output_bytes(&state) + .saturating_sub( + oldest_terminal + .as_deref() + .and_then(|task_id| state.tasks.get(task_id)) + .map(task_output_bytes) + .unwrap_or_default(), + ) + .saturating_add(task_output_bytes(&task)); + if retained_output_bytes > MAX_AI_TASK_RETAINED_OUTPUT_BYTES { + return Err(AiTaskServiceError::Store( + "AI 任务仓储输出工作集超过内存上限".to_string(), + )); + } + + if let Some(task_id) = oldest_terminal { + state.tasks.remove(&task_id); + state.text_chunks.remove(&task_id); + } + + state + .text_chunks + .insert(task.task_id.clone(), HashMap::new()); state.tasks.insert(task.task_id.clone(), task.clone()); Ok(task) } @@ -51,12 +96,37 @@ impl InMemoryAiTaskStore { .inner .lock() .map_err(|_| AiTaskServiceError::Store("AI 任务仓储锁已中毒".to_string()))?; - let task = state - .tasks - .get_mut(task_id.trim()) - .ok_or(AiTaskServiceError::TaskNotFound)?; - apply(task)?; - Ok(task.clone()) + let (previous_task, snapshot) = { + let task = state + .tasks + .get_mut(task_id.trim()) + .ok_or(AiTaskServiceError::TaskNotFound)?; + let previous_task = task.clone(); + if let Err(error) = apply(task) { + *task = previous_task; + return Err(error); + } + (previous_task, task.clone()) + }; + if let Err(error) = validate_task_memory_limits(&snapshot) { + state + .tasks + .insert(task_id.trim().to_string(), previous_task); + return Err(error); + } + let retained_output_bytes = retained_output_bytes(&state); + if retained_output_bytes > MAX_AI_TASK_RETAINED_OUTPUT_BYTES { + state + .tasks + .insert(task_id.trim().to_string(), previous_task); + return Err(AiTaskServiceError::Store( + "AI 任务仓储输出工作集超过内存上限".to_string(), + )); + } + if snapshot.status.is_terminal() { + state.text_chunks.remove(task_id.trim()); + } + Ok(snapshot) } pub(super) fn append_text_chunk( @@ -67,13 +137,75 @@ impl InMemoryAiTaskStore { .inner .lock() .map_err(|_| AiTaskServiceError::Store("AI 任务仓储锁已中毒".to_string()))?; - { + if chunk.delta_text.len() > MAX_AI_TASK_TEXT_OUTPUT_BYTES { + return Err(AiTaskServiceError::Store( + "AI 任务文本输出超过内存上限".to_string(), + )); + } + let (previous_stage_output_bytes, previous_latest_output_bytes, previous_task) = { + let task = state + .tasks + .get(&chunk.task_id) + .ok_or(AiTaskServiceError::TaskNotFound)?; + ensure_task_is_not_terminal(task.status)?; + let stage = task + .stages + .iter() + .find(|stage| stage.stage_kind == chunk.stage_kind) + .ok_or(AiTaskServiceError::StageNotFound)?; + ( + stage.text_output.as_ref().map_or(0, String::len), + task.latest_text_output.as_ref().map_or(0, String::len), + task.clone(), + ) + }; + + let (previous_chunk, aggregated_bytes, aggregated_text) = { + let chunks = state + .text_chunks + .get_mut(&chunk.task_id) + .ok_or(AiTaskServiceError::TaskNotFound)?; + let stage_chunks = chunks.entry(chunk.stage_kind).or_default(); + let previous_chunk = stage_chunks.insert(chunk.sequence, chunk.delta_text.clone()); + let aggregated_bytes = stage_chunks + .values() + .fold(0_usize, |total, delta| total.saturating_add(delta.len())); + let mut aggregated_text = String::with_capacity(aggregated_bytes); + for delta in stage_chunks.values() { + aggregated_text.push_str(delta); + } + (previous_chunk, aggregated_bytes, aggregated_text) + }; + if aggregated_bytes > MAX_AI_TASK_TEXT_OUTPUT_BYTES { + rollback_text_chunk(&mut state, &chunk, previous_chunk); + return Err(AiTaskServiceError::Store( + "AI 任务文本输出超过内存上限".to_string(), + )); + } + + let projected_retained_output_bytes = retained_output_bytes(&state) + .saturating_sub(previous_stage_output_bytes) + .saturating_sub(previous_latest_output_bytes) + .saturating_add(aggregated_bytes.saturating_mul(2)); + if projected_retained_output_bytes > MAX_AI_TASK_RETAINED_OUTPUT_BYTES { + rollback_text_chunk(&mut state, &chunk, previous_chunk); + return Err(AiTaskServiceError::Store( + "AI 任务仓储输出工作集超过内存上限".to_string(), + )); + } + + let normalized_output = if aggregated_text.trim().is_empty() { + None + } else { + Some(aggregated_text) + }; + + let snapshot = { let task = state .tasks .get_mut(&chunk.task_id) .ok_or(AiTaskServiceError::TaskNotFound)?; ensure_task_is_not_terminal(task.status)?; - let stage = task .stages .iter_mut() @@ -83,45 +215,23 @@ impl InMemoryAiTaskStore { stage.status = AiTaskStageStatus::Running; stage.started_at_micros = Some(chunk.created_at_micros); } - task.status = AiTaskStatus::Running; task.started_at_micros .get_or_insert(chunk.created_at_micros); - } - - let chunks = state - .text_chunks - .get_mut(&chunk.task_id) - .ok_or(AiTaskServiceError::TaskNotFound)?; - chunks.push(chunk.clone()); - chunks.sort_by_key(|value| value.sequence); - - let aggregated_text = chunks - .iter() - .filter(|value| value.stage_kind == chunk.stage_kind) - .map(|value| value.delta_text.as_str()) - .collect::>() - .join(""); - let normalized_output = if aggregated_text.trim().is_empty() { - None - } else { - Some(aggregated_text) + stage.text_output = normalized_output.clone(); + task.latest_text_output = normalized_output; + task.updated_at_micros = chunk.created_at_micros; + task.version += 1; + task.clone() }; - - let task = state - .tasks - .get_mut(&chunk.task_id) - .ok_or(AiTaskServiceError::TaskNotFound)?; - let stage = task - .stages - .iter_mut() - .find(|stage| stage.stage_kind == chunk.stage_kind) - .ok_or(AiTaskServiceError::StageNotFound)?; - stage.text_output = normalized_output.clone(); - task.latest_text_output = normalized_output; - task.updated_at_micros = chunk.created_at_micros; - task.version += 1; - Ok(task.clone()) + if let Err(error) = validate_task_memory_limits(&snapshot) + .and_then(|_| validate_retained_output_bytes(&state)) + { + state.tasks.insert(chunk.task_id.clone(), previous_task); + rollback_text_chunk(&mut state, &chunk, previous_chunk); + return Err(error); + } + Ok(snapshot) } pub(super) fn get_task(&self, task_id: &str) -> Result { @@ -136,3 +246,109 @@ impl InMemoryAiTaskStore { .ok_or(AiTaskServiceError::TaskNotFound) } } + +fn rollback_text_chunk( + state: &mut InMemoryAiTaskStoreState, + chunk: &AiTextChunkSnapshot, + previous_chunk: Option, +) { + if let Some(stage_chunks) = state + .text_chunks + .get_mut(&chunk.task_id) + .and_then(|chunks| chunks.get_mut(&chunk.stage_kind)) + { + if let Some(previous_chunk) = previous_chunk { + stage_chunks.insert(chunk.sequence, previous_chunk); + } else { + stage_chunks.remove(&chunk.sequence); + } + } +} + +fn retained_output_bytes(state: &InMemoryAiTaskStoreState) -> usize { + let snapshot_bytes = state.tasks.values().fold(0_usize, |total, task| { + total.saturating_add(task_output_bytes(task)) + }); + state + .text_chunks + .values() + .fold(snapshot_bytes, |total, stages| { + stages.values().fold(total, |stage_total, chunks| { + chunks.values().fold(stage_total, |chunk_total, delta| { + chunk_total.saturating_add(delta.len()) + }) + }) + }) +} + +fn validate_retained_output_bytes( + state: &InMemoryAiTaskStoreState, +) -> Result<(), AiTaskServiceError> { + if retained_output_bytes(state) > MAX_AI_TASK_RETAINED_OUTPUT_BYTES { + return Err(AiTaskServiceError::Store( + "AI 任务仓储输出工作集超过内存上限".to_string(), + )); + } + Ok(()) +} + +fn task_output_bytes(task: &AiTaskSnapshot) -> usize { + let task_metadata = task + .task_id + .len() + .saturating_add(task.owner_user_id.len()) + .saturating_add(task.request_label.len()) + .saturating_add(task.source_module.len()) + .saturating_add(task.source_entity_id.as_ref().map_or(0, String::len)) + .saturating_add(task.stages.iter().fold(0_usize, |total, stage| { + total + .saturating_add(stage.label.len()) + .saturating_add(stage.detail.len()) + })); + let request_payload = task.request_payload_json.as_ref().map_or(0, String::len); + let failure_message = task.failure_message.as_ref().map_or(0, String::len); + let result_references = task + .result_references + .iter() + .fold(0_usize, |total, reference| { + total + .saturating_add(reference.result_ref_id.len()) + .saturating_add(reference.task_id.len()) + .saturating_add(reference.reference_id.len()) + .saturating_add(reference.label.as_ref().map_or(0, String::len)) + }); + let latest_text = task.latest_text_output.as_ref().map_or(0, String::len); + let latest_structured = task + .latest_structured_payload_json + .as_ref() + .map_or(0, String::len); + let stage_bytes = task.stages.iter().fold(0_usize, |total, stage| { + let text = stage.text_output.as_ref().map_or(0, String::len); + let structured = stage + .structured_payload_json + .as_ref() + .map_or(0, String::len); + let warnings = stage + .warning_messages + .iter() + .fold(0_usize, |warning_total, warning| { + warning_total.saturating_add(warning.len()) + }); + total + .saturating_add(text) + .saturating_add(structured) + .saturating_add(warnings) + }); + task_metadata + .saturating_add(request_payload) + .saturating_add(failure_message) + .saturating_add(result_references) + .saturating_add(latest_text) + .saturating_add(latest_structured) + .saturating_add(stage_bytes) +} + +fn validate_task_memory_limits(task: &AiTaskSnapshot) -> Result<(), AiTaskServiceError> { + validate_ai_task_snapshot_memory_limits(task) + .map_err(|message| AiTaskServiceError::Store(message.to_string())) +} diff --git a/server-rs/crates/module-ai/src/domain.rs b/server-rs/crates/module-ai/src/domain.rs index a9931048f..df225816e 100644 --- a/server-rs/crates/module-ai/src/domain.rs +++ b/server-rs/crates/module-ai/src/domain.rs @@ -1,4 +1,5 @@ mod ids; +mod limits; mod stages; mod types; @@ -8,6 +9,16 @@ pub use ids::{ generate_ai_task_stage_id, generate_ai_text_chunk_id, normalize_optional_text, normalize_string_list, }; +pub use limits::{ + MAX_AI_TASK_FAILURE_MESSAGE_BYTES, MAX_AI_TASK_ID_BYTES, MAX_AI_TASK_OWNER_USER_ID_BYTES, + MAX_AI_TASK_REFERENCE_ID_BYTES, MAX_AI_TASK_REFERENCE_LABEL_BYTES, + MAX_AI_TASK_REQUEST_LABEL_BYTES, MAX_AI_TASK_REQUEST_PAYLOAD_BYTES, + MAX_AI_TASK_RESULT_REFERENCES, MAX_AI_TASK_RETAINED_OUTPUT_BYTES, MAX_AI_TASK_RETAINED_TASKS, + MAX_AI_TASK_SOURCE_ENTITY_ID_BYTES, MAX_AI_TASK_SOURCE_MODULE_BYTES, + MAX_AI_TASK_STAGE_DETAIL_BYTES, MAX_AI_TASK_STAGE_LABEL_BYTES, + MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES, MAX_AI_TASK_TEXT_OUTPUT_BYTES, MAX_AI_TASK_WARNING_BYTES, + validate_ai_task_snapshot_memory_limits, +}; pub use types::{ AiResultReferenceKind, AiResultReferenceSnapshot, AiTaskKind, AiTaskSnapshot, AiTaskStageBlueprint, AiTaskStageKind, AiTaskStageSnapshot, AiTaskStageStatus, AiTaskStatus, diff --git a/server-rs/crates/module-ai/src/domain/limits.rs b/server-rs/crates/module-ai/src/domain/limits.rs new file mode 100644 index 000000000..759b4ac6a --- /dev/null +++ b/server-rs/crates/module-ai/src/domain/limits.rs @@ -0,0 +1,115 @@ +use super::types::AiTaskSnapshot; + +pub const MAX_AI_TASK_RETAINED_TASKS: usize = 1024; +pub const MAX_AI_TASK_ID_BYTES: usize = 256; +pub const MAX_AI_TASK_OWNER_USER_ID_BYTES: usize = 256; +pub const MAX_AI_TASK_REQUEST_LABEL_BYTES: usize = 4 * 1024; +pub const MAX_AI_TASK_SOURCE_MODULE_BYTES: usize = 256; +pub const MAX_AI_TASK_SOURCE_ENTITY_ID_BYTES: usize = 512; +pub const MAX_AI_TASK_STAGE_LABEL_BYTES: usize = 4 * 1024; +pub const MAX_AI_TASK_STAGE_DETAIL_BYTES: usize = 8 * 1024; +pub const MAX_AI_TASK_TEXT_OUTPUT_BYTES: usize = 512 * 1024; +pub const MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES: usize = 512 * 1024; +pub const MAX_AI_TASK_WARNING_BYTES: usize = 64 * 1024; +pub const MAX_AI_TASK_REQUEST_PAYLOAD_BYTES: usize = 512 * 1024; +pub const MAX_AI_TASK_FAILURE_MESSAGE_BYTES: usize = 64 * 1024; +pub const MAX_AI_TASK_RESULT_REFERENCES: usize = 64; +pub const MAX_AI_TASK_REFERENCE_ID_BYTES: usize = 512; +pub const MAX_AI_TASK_REFERENCE_LABEL_BYTES: usize = 2 * 1024; +pub const MAX_AI_TASK_RETAINED_OUTPUT_BYTES: usize = 64 * 1024 * 1024; + +pub fn validate_ai_task_snapshot_memory_limits(task: &AiTaskSnapshot) -> Result<(), &'static str> { + if task.task_id.len() > MAX_AI_TASK_ID_BYTES { + return Err("AI 任务 ID 超过内存上限"); + } + if task.owner_user_id.len() > MAX_AI_TASK_OWNER_USER_ID_BYTES { + return Err("AI 任务用户 ID 超过内存上限"); + } + if task.request_label.len() > MAX_AI_TASK_REQUEST_LABEL_BYTES { + return Err("AI 任务请求标签超过内存上限"); + } + if task.source_module.len() > MAX_AI_TASK_SOURCE_MODULE_BYTES { + return Err("AI 任务来源模块超过内存上限"); + } + if task + .source_entity_id + .as_ref() + .is_some_and(|entity_id| entity_id.len() > MAX_AI_TASK_SOURCE_ENTITY_ID_BYTES) + { + return Err("AI 任务来源实体 ID 超过内存上限"); + } + if task.stages.iter().any(|stage| { + stage.label.len() > MAX_AI_TASK_STAGE_LABEL_BYTES + || stage.detail.len() > MAX_AI_TASK_STAGE_DETAIL_BYTES + }) { + return Err("AI 任务阶段元数据超过内存上限"); + } + if task + .request_payload_json + .as_ref() + .is_some_and(|payload| payload.len() > MAX_AI_TASK_REQUEST_PAYLOAD_BYTES) + { + return Err("AI 任务请求 payload 超过内存上限"); + } + if task + .failure_message + .as_ref() + .is_some_and(|message| message.len() > MAX_AI_TASK_FAILURE_MESSAGE_BYTES) + { + return Err("AI 任务失败消息超过内存上限"); + } + if task.stages.iter().any(|stage| { + stage + .text_output + .as_ref() + .is_some_and(|text| text.len() > MAX_AI_TASK_TEXT_OUTPUT_BYTES) + }) || task + .latest_text_output + .as_ref() + .is_some_and(|text| text.len() > MAX_AI_TASK_TEXT_OUTPUT_BYTES) + { + return Err("AI 任务文本输出超过内存上限"); + } + if task.stages.iter().any(|stage| { + stage + .structured_payload_json + .as_ref() + .is_some_and(|payload| payload.len() > MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES) + }) || task + .latest_structured_payload_json + .as_ref() + .is_some_and(|payload| payload.len() > MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES) + { + return Err("AI 任务结构化输出超过内存上限"); + } + if task.stages.iter().any(|stage| { + stage + .warning_messages + .iter() + .fold(0_usize, |total, warning| { + total.saturating_add(warning.len()) + }) + > MAX_AI_TASK_WARNING_BYTES + }) { + return Err("AI 任务 warning 输出超过内存上限"); + } + if task.result_references.len() > MAX_AI_TASK_RESULT_REFERENCES { + return Err("AI 任务结果引用数量超过内存上限"); + } + if task + .result_references + .iter() + .any(|reference| reference.reference_id.len() > MAX_AI_TASK_REFERENCE_ID_BYTES) + { + return Err("AI 任务结果引用 ID 超过内存上限"); + } + if task.result_references.iter().any(|reference| { + reference + .label + .as_ref() + .is_some_and(|label| label.len() > MAX_AI_TASK_REFERENCE_LABEL_BYTES) + }) { + return Err("AI 任务结果引用标签超过内存上限"); + } + Ok(()) +} diff --git a/server-rs/crates/module-ai/src/lib.rs b/server-rs/crates/module-ai/src/lib.rs index e69b5609f..2eba88eb0 100644 --- a/server-rs/crates/module-ai/src/lib.rs +++ b/server-rs/crates/module-ai/src/lib.rs @@ -14,9 +14,17 @@ pub use domain::{ AI_RESULT_REF_ID_PREFIX, AI_TASK_ID_PREFIX, AI_TASK_STAGE_ID_PREFIX, AI_TEXT_CHUNK_ID_PREFIX, AiResultReferenceKind, AiResultReferenceSnapshot, AiTaskKind, AiTaskSnapshot, AiTaskStageBlueprint, AiTaskStageKind, AiTaskStageSnapshot, AiTaskStageStatus, AiTaskStatus, - AiTextChunkSnapshot, INITIAL_AI_TASK_VERSION, generate_ai_result_ref_id, generate_ai_task_id, - generate_ai_task_stage_id, generate_ai_text_chunk_id, normalize_optional_text, - normalize_string_list, + AiTextChunkSnapshot, INITIAL_AI_TASK_VERSION, MAX_AI_TASK_FAILURE_MESSAGE_BYTES, + MAX_AI_TASK_ID_BYTES, MAX_AI_TASK_OWNER_USER_ID_BYTES, MAX_AI_TASK_REFERENCE_ID_BYTES, + MAX_AI_TASK_REFERENCE_LABEL_BYTES, MAX_AI_TASK_REQUEST_LABEL_BYTES, + MAX_AI_TASK_REQUEST_PAYLOAD_BYTES, MAX_AI_TASK_RESULT_REFERENCES, + MAX_AI_TASK_RETAINED_OUTPUT_BYTES, MAX_AI_TASK_RETAINED_TASKS, + MAX_AI_TASK_SOURCE_ENTITY_ID_BYTES, MAX_AI_TASK_SOURCE_MODULE_BYTES, + MAX_AI_TASK_STAGE_DETAIL_BYTES, MAX_AI_TASK_STAGE_LABEL_BYTES, + MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES, MAX_AI_TASK_TEXT_OUTPUT_BYTES, MAX_AI_TASK_WARNING_BYTES, + generate_ai_result_ref_id, generate_ai_task_id, generate_ai_task_stage_id, + generate_ai_text_chunk_id, normalize_optional_text, normalize_string_list, + validate_ai_task_snapshot_memory_limits, }; pub use errors::{AiTaskFieldError, AiTaskServiceError}; pub use events::AiTaskDomainEvent; diff --git a/server-rs/crates/module-ai/src/tests.rs b/server-rs/crates/module-ai/src/tests.rs index 766320035..a1ba8f6cb 100644 --- a/server-rs/crates/module-ai/src/tests.rs +++ b/server-rs/crates/module-ai/src/tests.rs @@ -43,6 +43,32 @@ fn create_task_rejects_duplicate_stage_blueprints() { assert_eq!(error, AiTaskFieldError::DuplicateStageBlueprint); } +#[test] +fn create_task_rejects_oversized_request_payload() { + let service = build_service(); + let mut input = build_create_input(AiTaskKind::StoryGeneration); + input.request_payload_json = Some("x".repeat(MAX_AI_TASK_REQUEST_PAYLOAD_BYTES + 1)); + + let error = service + .create_task(input) + .expect_err("request payload over the memory cap should fail"); + assert!( + matches!(error, AiTaskServiceError::Store(message) if message.contains("请求 payload")) + ); +} + +#[test] +fn create_task_rejects_oversized_request_metadata() { + let service = build_service(); + let mut input = build_create_input(AiTaskKind::StoryGeneration); + input.request_label = "x".repeat(MAX_AI_TASK_REQUEST_LABEL_BYTES + 1); + + let error = service + .create_task(input) + .expect_err("request metadata over the memory cap should fail"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("请求标签"))); +} + #[test] fn generate_ai_task_stage_id_contains_task_and_stage_slug() { let stage_id = generate_ai_task_stage_id("aitask_demo", AiTaskStageKind::NormalizeResult); @@ -112,6 +138,47 @@ fn append_text_chunk_aggregates_stream_output_by_stage() { assert_eq!(second_chunk.sequence, 2); } +#[test] +fn append_text_chunk_rejects_output_over_stage_memory_limit_without_mutating_task() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::CharacterChat)) + .expect("task should create"); + let max_output = "a".repeat(512 * 1024); + + let (updated, _) = service + .append_text_chunk( + &task.task_id, + AiTaskStageKind::RequestModel, + 1, + max_output.clone(), + task.created_at_micros + 1, + ) + .expect("the stage limit itself should be accepted"); + assert_eq!( + updated.latest_text_output.as_deref().map(str::len), + Some(max_output.len()) + ); + + let error = service + .append_text_chunk( + &task.task_id, + AiTaskStageKind::RequestModel, + 2, + "b".to_string(), + task.created_at_micros + 2, + ) + .expect_err("output beyond the stage limit should fail"); + assert!(matches!(error, AiTaskServiceError::Store(_))); + let after_rejection = service + .get_task(&task.task_id) + .expect("task should remain readable"); + assert_eq!( + after_rejection.latest_text_output.as_deref().map(str::len), + Some(max_output.len()) + ); +} + #[test] fn complete_stage_updates_latest_outputs() { let service = build_service(); @@ -147,6 +214,150 @@ fn complete_stage_updates_latest_outputs() { assert_eq!(stage.warning_messages, vec!["使用了 fallback 选项池"]); } +#[test] +fn complete_stage_rejects_oversized_text_output_without_mutating_task() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::StoryGeneration)) + .expect("task should create"); + let oversized = "x".repeat(512 * 1024 + 1); + + let error = service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id.clone(), + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: Some(oversized), + structured_payload_json: None, + warning_messages: Vec::new(), + completed_at_micros: task.created_at_micros + 1, + }) + .expect_err("text output over the per-stage cap should fail"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("文本输出"))); + + let unchanged = service + .get_task(&task.task_id) + .expect("task should remain readable"); + let stage = unchanged + .stages + .iter() + .find(|stage| stage.stage_kind == AiTaskStageKind::NormalizeResult) + .expect("normalize stage should exist"); + assert_eq!(stage.status, AiTaskStageStatus::Pending); + assert!(stage.text_output.is_none()); + assert!(unchanged.latest_text_output.is_none()); +} + +#[test] +fn complete_stage_rejects_oversized_structured_output_without_mutating_task() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::StoryGeneration)) + .expect("task should create"); + let oversized = "x".repeat(512 * 1024 + 1); + + let error = service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id.clone(), + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: None, + structured_payload_json: Some(oversized), + warning_messages: Vec::new(), + completed_at_micros: task.created_at_micros + 1, + }) + .expect_err("structured output over the per-stage cap should fail"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("结构化输出"))); + + let unchanged = service + .get_task(&task.task_id) + .expect("task should remain readable"); + let stage = unchanged + .stages + .iter() + .find(|stage| stage.stage_kind == AiTaskStageKind::NormalizeResult) + .expect("normalize stage should exist"); + assert_eq!(stage.status, AiTaskStageStatus::Pending); + assert!(stage.structured_payload_json.is_none()); + assert!(unchanged.latest_structured_payload_json.is_none()); +} + +#[test] +fn complete_stage_rejects_oversized_warning_output_without_mutating_task() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::StoryGeneration)) + .expect("task should create"); + let oversized_warning = "w".repeat(64 * 1024 + 1); + + let error = service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id.clone(), + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: None, + structured_payload_json: None, + warning_messages: vec![oversized_warning], + completed_at_micros: task.created_at_micros + 1, + }) + .expect_err("warning output over the per-stage cap should fail"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("warning"))); + + let unchanged = service + .get_task(&task.task_id) + .expect("task should remain readable"); + let stage = unchanged + .stages + .iter() + .find(|stage| stage.stage_kind == AiTaskStageKind::NormalizeResult) + .expect("normalize stage should exist"); + assert_eq!(stage.status, AiTaskStageStatus::Pending); + assert!(stage.warning_messages.is_empty()); +} + +#[test] +fn complete_stage_enforces_global_retained_output_cap() { + let service = build_service(); + let structured_payload = "x".repeat(512 * 1024); + for index in 0..63 { + let task = service + .create_task(AiTaskCreateInput { + task_id: format!("task-structured-cap-{index}"), + ..build_create_input(AiTaskKind::StoryGeneration) + }) + .expect("task should create"); + service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id, + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: None, + structured_payload_json: Some(structured_payload.clone()), + warning_messages: Vec::new(), + completed_at_micros: task.created_at_micros + 1, + }) + .expect("63 MiB retained output should remain within the global cap"); + } + + let task = service + .create_task(AiTaskCreateInput { + task_id: "task-structured-cap-overflow".to_string(), + ..build_create_input(AiTaskKind::StoryGeneration) + }) + .expect("the overflow candidate task itself should create"); + let error = service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id.clone(), + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: None, + structured_payload_json: Some(structured_payload), + warning_messages: Vec::new(), + completed_at_micros: task.created_at_micros + 1, + }) + .expect_err("global retained output cap should reject the overflow"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("工作集"))); + let unchanged = service + .get_task(&task.task_id) + .expect("overflow task should remain readable"); + assert!(unchanged.latest_structured_payload_json.is_none()); +} + #[test] fn attach_result_reference_appends_binding() { let service = build_service(); @@ -172,6 +383,47 @@ fn attach_result_reference_appends_binding() { assert_eq!(updated.result_references[0].reference_id, "profile_001"); } +#[test] +fn attach_result_reference_rejects_unbounded_reference_growth() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::CustomWorldGeneration)) + .expect("task should create"); + + for index in 0..MAX_AI_TASK_RESULT_REFERENCES { + service + .attach_result_reference( + &task.task_id, + AiResultReferenceKind::CustomWorldProfile, + format!("profile_{index}"), + None, + task.created_at_micros + index as i64 + 1, + ) + .expect("references within the cap should attach"); + } + + let error = service + .attach_result_reference( + &task.task_id, + AiResultReferenceKind::CustomWorldProfile, + "profile_overflow".to_string(), + None, + task.created_at_micros + MAX_AI_TASK_RESULT_REFERENCES as i64 + 1, + ) + .expect_err("references over the cap should fail"); + assert!( + matches!(error, AiTaskServiceError::Store(message) if message.contains("结果引用数量")) + ); + + let unchanged = service + .get_task(&task.task_id) + .expect("task should remain readable"); + assert_eq!( + unchanged.result_references.len(), + MAX_AI_TASK_RESULT_REFERENCES + ); +} + #[test] fn fail_and_cancel_task_move_into_terminal_states() { let service = build_service(); diff --git a/server-rs/crates/module-auth/src/lib.rs b/server-rs/crates/module-auth/src/lib.rs index 92d254e25..c69c9cc7d 100644 --- a/server-rs/crates/module-auth/src/lib.rs +++ b/server-rs/crates/module-auth/src/lib.rs @@ -34,6 +34,9 @@ use tracing::{info, warn}; const DEFAULT_PHONE_VERIFY_CODE_SALT: &str = "genarrative-phone-verify-code-v1"; const PHONE_CODE_RESERVATION_MARKER: &str = "__genarrative_phone_code_reservation__"; const MAX_ACTIVE_WECHAT_AUTH_STATES: usize = 1024; +const REFRESH_SESSION_STALE_RETENTION: Duration = Duration::days(1); +const MAX_REFRESH_SESSIONS: usize = 8_192; +const MAX_PHONE_CODES: usize = 4_096; #[derive(Clone, Debug)] pub struct InMemoryAuthStore { @@ -390,6 +393,7 @@ impl RefreshSessionService { input: CreateRefreshSessionInput, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; self.store .find_by_user_id(&input.user_id) .map_err(map_password_store_error)? @@ -426,6 +430,7 @@ impl RefreshSessionService { input: RotateRefreshSessionInput, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; let Some(refresh_token_hash) = normalize_required_string(&input.refresh_token_hash) else { return Err(RefreshSessionError::MissingToken); }; @@ -480,6 +485,7 @@ impl RefreshSessionService { user_id: &str, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; self.store .find_by_user_id(user_id) .map_err(map_password_store_error)? @@ -494,6 +500,7 @@ impl RefreshSessionService { input: RevokeRefreshSessionByUserInput, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; self.store .find_by_user_id(&input.user_id) .map_err(map_password_store_error)? @@ -518,6 +525,7 @@ impl RefreshSessionService { session_id: &str, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; self.store .is_session_active_for_user(user_id, session_id.trim(), now) } @@ -556,6 +564,7 @@ impl PhoneAuthService { input: &SendPhoneCodeInput, now: OffsetDateTime, ) -> Result<(), PhoneAuthError> { + self.store.prune_expired_phone_codes(now)?; let scene = input.scene.clone(); validate_mainland_china_country_code(input.country_code.as_deref())?; let normalized_phone = normalize_mainland_china_phone_number(&input.pure_phone_number)?; @@ -605,6 +614,7 @@ impl PhoneAuthService { now: OffsetDateTime, check_local_cooldown: bool, ) -> Result { + self.store.prune_expired_phone_codes(now)?; let scene = input.scene.clone(); validate_mainland_china_country_code(input.country_code.as_deref())?; let normalized_phone = normalize_mainland_china_phone_number(&input.pure_phone_number)?; @@ -621,6 +631,8 @@ impl PhoneAuthService { self.store .ensure_phone_code_not_cooling_down(&normalized_phone.e164, &scene, now)?; } + self.store + .ensure_phone_code_capacity(&normalized_phone.e164, &scene)?; let expires_at = now .checked_add(Duration::minutes(SMS_CODE_TTL_MINUTES)) .ok_or_else(|| PhoneAuthError::Store("短信验证码过期时间计算溢出".to_string()))?; @@ -883,6 +895,7 @@ impl WechatAuthStateService { input: CreateWechatAuthStateInput, now: OffsetDateTime, ) -> Result { + self.store.prune_wechat_states(now)?; let created_at = format_rfc3339(now).map_err(|message| { WechatAuthError::Store(format!("微信 state 时间格式化失败：{message}")) })?; @@ -1162,10 +1175,25 @@ impl InMemoryAuthStoreState { } } + let now = OffsetDateTime::now_utc(); + let mut retained_refresh_session_count = 0_usize; for session in view.refresh_sessions { if !existing_user_ids.contains(&session.user_id) { continue; } + if should_prune_refresh_session_fields( + &session.expires_at, + session.revoked_at.as_deref(), + now, + ) { + continue; + } + retained_refresh_session_count += 1; + if retained_refresh_session_count > MAX_REFRESH_SESSIONS { + return Err(format!( + "认证投影中的 refresh session 数量超过内存上限（最多 {MAX_REFRESH_SESSIONS} 条）" + )); + } let client_info = serde_json::from_str::(&session.client_info_json) .map_err(|error| format!("解析 refresh session 客户端信息失败：{error}"))?; @@ -1371,6 +1399,8 @@ impl InMemoryAuthStore { &self, updated_at_micros: i64, ) -> Result { + self.prune_stale_sessions(OffsetDateTime::now_utc()) + .map_err(|error| error.to_string())?; let mut state = self .inner .lock() @@ -1499,6 +1529,38 @@ impl InMemoryAuthStore { Err("认证工作集在导出期间持续发生变化".to_string()) } + fn prune_stale_sessions(&self, now: OffsetDateTime) -> Result<(), RefreshSessionError> { + let mut state = self + .inner + .lock() + .map_err(|_| RefreshSessionError::Store("会话仓储锁已中毒".to_string()))?; + let stale_session_ids = state + .sessions_by_id + .iter() + .filter(|(_, stored)| should_prune_refresh_session(&stored.session, now)) + .map(|(session_id, _)| session_id.clone()) + .collect::>(); + if stale_session_ids.is_empty() { + return Ok(()); + } + + for session_id in stale_session_ids { + let Some(stored) = state.sessions_by_id.remove(&session_id) else { + continue; + }; + if state + .session_id_by_refresh_token_hash + .get(&stored.session.refresh_token_hash) + .is_some_and(|mapped_id| mapped_id == &session_id) + { + state + .session_id_by_refresh_token_hash + .remove(&stored.session.refresh_token_hash); + } + } + self.persist_refresh_state(&state) + } + fn persist_state(&self, state: &InMemoryAuthStoreState) -> Result<(), String> { let _ = state; self.revision.fetch_add(1, Ordering::Release); @@ -2132,6 +2194,11 @@ impl InMemoryAuthStore { "refresh token hash 已存在，无法重复创建会话".to_string(), )); } + if state.sessions_by_id.len() >= MAX_REFRESH_SESSIONS { + return Err(RefreshSessionError::Store( + "refresh session 内存容量已达到上限".to_string(), + )); + } state.session_id_by_refresh_token_hash.insert( session.refresh_token_hash.clone(), @@ -2156,11 +2223,42 @@ impl InMemoryAuthStore { .map_err(|_| PhoneAuthError::Store("短信验证码仓储锁已中毒".to_string()))?; // 手机号和业务场景共同决定同一份验证码快照，重复发送时直接覆盖旧值。 let key = build_phone_code_key(&code.phone_number, &code.scene); + if !state.phone_codes_by_key.contains_key(&key) + && state.phone_codes_by_key.len() >= MAX_PHONE_CODES + { + return Err(PhoneAuthError::Store( + "短信验证码内存容量已达到上限，请稍后重试".to_string(), + )); + } state.phone_codes_by_key.insert(key, code); self.persist_phone_state(&state)?; Ok(()) } + fn prune_expired_phone_codes(&self, now: OffsetDateTime) -> Result<(), PhoneAuthError> { + let mut state = self + .inner + .lock() + .map_err(|_| PhoneAuthError::Store("短信验证码仓储锁已中毒".to_string()))?; + let expired_keys = state + .phone_codes_by_key + .iter() + .filter_map(|(key, stored)| { + OffsetDateTime::parse( + &stored.expires_at, + &time::format_description::well_known::Rfc3339, + ) + .ok() + .filter(|expires_at| *expires_at <= now) + .map(|_| key.clone()) + }) + .collect::>(); + for key in expired_keys { + state.phone_codes_by_key.remove(&key); + } + Ok(()) + } + fn ensure_phone_code_not_cooling_down( &self, phone_number: &str, @@ -2200,6 +2298,26 @@ impl InMemoryAuthStore { }) } + fn ensure_phone_code_capacity( + &self, + phone_number: &str, + scene: &PhoneAuthScene, + ) -> Result<(), PhoneAuthError> { + let state = self + .inner + .lock() + .map_err(|_| PhoneAuthError::Store("短信验证码仓储锁已中毒".to_string()))?; + let key = build_phone_code_key(phone_number, scene); + if state.phone_codes_by_key.contains_key(&key) + || state.phone_codes_by_key.len() < MAX_PHONE_CODES + { + return Ok(()); + } + Err(PhoneAuthError::Store( + "短信验证码内存容量已达到上限，请稍后重试".to_string(), + )) + } + fn get_active_phone_code( &self, phone_number: &str, @@ -2658,6 +2776,33 @@ impl InMemoryAuthStore { Ok(()) } + fn prune_wechat_states(&self, now: OffsetDateTime) -> Result<(), WechatAuthError> { + let mut state = self + .inner + .lock() + .map_err(|_| WechatAuthError::Store("微信 state 仓储锁已中毒".to_string()))?; + let stale_tokens = state + .wechat_states_by_token + .iter() + .filter_map(|(token, stored)| { + if stored.state.consumed_at.is_some() { + return Some(token.clone()); + } + OffsetDateTime::parse( + &stored.state.expires_at, + &time::format_description::well_known::Rfc3339, + ) + .ok() + .filter(|expires_at| *expires_at <= now) + .map(|_| token.clone()) + }) + .collect::>(); + for token in stale_tokens { + state.wechat_states_by_token.remove(&token); + } + Ok(()) + } + fn revoke_session_by_user_and_session_id( &self, user_id: &str, @@ -2821,6 +2966,25 @@ impl InMemoryAuthStore { } } +fn should_prune_refresh_session(session: &RefreshSessionRecord, now: OffsetDateTime) -> bool { + should_prune_refresh_session_fields(&session.expires_at, session.revoked_at.as_deref(), now) +} + +fn should_prune_refresh_session_fields( + expires_at: &str, + revoked_at: Option<&str>, + now: OffsetDateTime, +) -> bool { + let stale_before = now.saturating_sub(REFRESH_SESSION_STALE_RETENTION); + if let Some(revoked_at) = revoked_at { + return OffsetDateTime::parse(revoked_at, &time::format_description::well_known::Rfc3339) + .is_ok_and(|timestamp| timestamp <= stale_before); + } + + OffsetDateTime::parse(expires_at, &time::format_description::well_known::Rfc3339) + .is_ok_and(|timestamp| timestamp <= stale_before) +} + fn map_sms_provider_error_to_phone_error(error: SmsProviderError) -> PhoneAuthError { match error { SmsProviderError::InvalidVerifyCode => PhoneAuthError::InvalidVerifyCode, @@ -4413,6 +4577,108 @@ mod tests { ); } + #[tokio::test] + async fn stale_refresh_sessions_are_pruned_from_both_indexes() { + let store = build_store(); + let refresh_service = build_refresh_service(store.clone()); + let user = create_phone_login_user(store.clone(), "13800138008").await; + let now = OffsetDateTime::now_utc(); + + refresh_service + .create_session( + CreateRefreshSessionInput { + user_id: user.id.clone(), + refresh_token_hash: hash_refresh_session_token("stale-revoked"), + issued_by_provider: AuthLoginMethod::Password, + client_info: build_client_info(), + }, + now - Duration::days(2), + ) + .expect("stale session should create"); + store + .revoke_session_by_refresh_token_hash( + &hash_refresh_session_token("stale-revoked"), + now - Duration::days(2), + ) + .expect("stale session should revoke"); + + refresh_service + .create_session( + CreateRefreshSessionInput { + user_id: user.id.clone(), + refresh_token_hash: hash_refresh_session_token("recent-revoked"), + issued_by_provider: AuthLoginMethod::Password, + client_info: build_client_info(), + }, + now, + ) + .expect("recent session should create"); + store + .revoke_session_by_refresh_token_hash( + &hash_refresh_session_token("recent-revoked"), + now, + ) + .expect("recent session should revoke"); + + let projection = store + .export_projection_view(now.unix_timestamp()) + .expect("projection export should prune stale sessions"); + assert_eq!(projection.refresh_sessions.len(), 1); + assert_eq!( + projection.refresh_sessions[0].refresh_token_hash, + hash_refresh_session_token("recent-revoked") + ); + + let stale_error = refresh_service + .rotate_session( + RotateRefreshSessionInput { + refresh_token_hash: hash_refresh_session_token("stale-revoked"), + next_refresh_token_hash: hash_refresh_session_token("stale-next"), + }, + now, + ) + .expect_err("pruned session should no longer be indexed"); + assert_eq!(stale_error, RefreshSessionError::SessionNotFound); + } + + #[test] + fn projection_restore_rejects_too_many_retained_refresh_sessions() { + let client_info_json = + serde_json::to_string(&build_client_info()).expect("client info should serialize"); + let refresh_sessions = (0..=MAX_REFRESH_SESSIONS) + .map(|index| AuthStoreProjectionRefreshSession { + session_id: format!("session-{index}"), + user_id: "user_projection_cap".to_string(), + refresh_token_hash: format!("hash-{index}"), + issued_by_provider: "password".to_string(), + client_info_json: client_info_json.clone(), + expires_at: "2999-01-01T00:00:00Z".to_string(), + revoked_at: None, + created_at: "2026-01-01T00:00:00Z".to_string(), + updated_at: "2026-01-01T00:00:00Z".to_string(), + last_seen_at: "2026-01-01T00:00:00Z".to_string(), + }) + .collect(); + + let error = InMemoryAuthStore::from_projection_view(AuthStoreProjectionView { + base_updated_at_micros: 0, + updated_at_micros: 1, + users: vec![projection_user( + "user_projection_cap", + "projection_cap", + None, + )], + identities: vec![], + refresh_sessions, + phone_codes: vec![], + wechat_states: vec![], + }) + .expect_err("projection restore must enforce the refresh session cap"); + + assert!(error.contains("refresh session")); + assert!(error.contains(&MAX_REFRESH_SESSIONS.to_string())); + } + #[tokio::test] async fn wechat_login_hits_existing_user_by_union_id_before_openid() { let store = build_store(); diff --git a/server-rs/crates/spacetime-client/src/module_bindings.rs b/server-rs/crates/spacetime-client/src/module_bindings.rs index b1fc29d7b..f6c9c3b9d 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings.rs @@ -414,6 +414,8 @@ pub mod external_generation_job_procedure_result_type; pub mod external_generation_job_renew_lease_input_type; pub mod external_generation_job_result_procedure_result_type; pub mod external_generation_job_result_snapshot_type; +pub mod external_generation_job_retention_input_type; +pub mod external_generation_job_retention_procedure_result_type; pub mod external_generation_job_snapshot_type; pub mod external_generation_job_summary_backfill_input_type; pub mod external_generation_job_summary_backfill_procedure_result_type; @@ -578,6 +580,7 @@ pub mod profile_wallet_manual_restriction_table; pub mod profile_wallet_manual_restriction_type; pub mod profile_wallet_refund_outbox_table; pub mod profile_wallet_refund_outbox_type; +pub mod prune_external_generation_job_history_and_return_procedure; pub mod public_work_like_table; pub mod public_work_like_type; pub mod public_work_play_daily_stat_table; @@ -1285,6 +1288,8 @@ pub use external_generation_job_procedure_result_type::ExternalGenerationJobProc pub use external_generation_job_renew_lease_input_type::ExternalGenerationJobRenewLeaseInput; pub use external_generation_job_result_procedure_result_type::ExternalGenerationJobResultProcedureResult; pub use external_generation_job_result_snapshot_type::ExternalGenerationJobResultSnapshot; +pub use external_generation_job_retention_input_type::ExternalGenerationJobRetentionInput; +pub use external_generation_job_retention_procedure_result_type::ExternalGenerationJobRetentionProcedureResult; pub use external_generation_job_snapshot_type::ExternalGenerationJobSnapshot; pub use external_generation_job_summary_backfill_input_type::ExternalGenerationJobSummaryBackfillInput; pub use external_generation_job_summary_backfill_procedure_result_type::ExternalGenerationJobSummaryBackfillProcedureResult; @@ -1449,6 +1454,7 @@ pub use profile_wallet_manual_restriction_table::*; pub use profile_wallet_manual_restriction_type::ProfileWalletManualRestriction; pub use profile_wallet_refund_outbox_table::*; pub use profile_wallet_refund_outbox_type::ProfileWalletRefundOutbox; +pub use prune_external_generation_job_history_and_return_procedure::prune_external_generation_job_history_and_return; pub use public_work_like_table::*; pub use public_work_like_type::PublicWorkLike; pub use public_work_play_daily_stat_table::*; diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_event_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_event_type.rs index 32c820175..4658052db 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_event_type.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_event_type.rs @@ -56,6 +56,7 @@ impl __sdk::__query_builder::HasCols for ExternalGenerationJobEvent { /// Provides typed access to indexed columns for query building. pub struct ExternalGenerationJobEventIxCols { pub event_id: __sdk::__query_builder::IxCol, + pub job_id: __sdk::__query_builder::IxCol, } impl __sdk::__query_builder::HasIxCols for ExternalGenerationJobEvent { @@ -63,6 +64,7 @@ impl __sdk::__query_builder::HasIxCols for ExternalGenerationJobEvent { fn ix_cols(table_name: &'static str) -> Self::IxCols { ExternalGenerationJobEventIxCols { event_id: __sdk::__query_builder::IxCol::new(table_name, "event_id"), + job_id: __sdk::__query_builder::IxCol::new(table_name, "job_id"), } } } diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_input_type.rs new file mode 100644 index 000000000..cffb3527a --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_input_type.rs @@ -0,0 +1,19 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobRetentionInput { + pub source_module: String, + pub limit: u32, + pub cursor_job_id: Option, + pub completed_before_micros: i64, + pub dry_run: bool, +} + +impl __sdk::InModule for ExternalGenerationJobRetentionInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_procedure_result_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_procedure_result_type.rs new file mode 100644 index 000000000..1f3883749 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_procedure_result_type.rs @@ -0,0 +1,24 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobRetentionProcedureResult { + pub ok: bool, + pub dry_run: bool, + pub scanned_count: u64, + pub selected_count: u32, + pub deleted_job_count: u32, + pub deleted_summary_count: u32, + pub deleted_event_count: u32, + pub next_cursor_job_id: Option, + pub has_more: bool, + pub error_message: Option, +} + +impl __sdk::InModule for ExternalGenerationJobRetentionProcedureResult { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/prune_external_generation_job_history_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/prune_external_generation_job_history_and_return_procedure.rs new file mode 100644 index 000000000..877d06380 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/prune_external_generation_job_history_and_return_procedure.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_retention_input_type::ExternalGenerationJobRetentionInput; +use super::external_generation_job_retention_procedure_result_type::ExternalGenerationJobRetentionProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct PruneExternalGenerationJobHistoryAndReturnArgs { + pub input: ExternalGenerationJobRetentionInput, +} + +impl __sdk::InModule for PruneExternalGenerationJobHistoryAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `prune_external_generation_job_history_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait prune_external_generation_job_history_and_return { + fn prune_external_generation_job_history_and_return( + &self, + input: ExternalGenerationJobRetentionInput, + ) { + self.prune_external_generation_job_history_and_return_then(input, |_, _| {}); + } + + fn prune_external_generation_job_history_and_return_then( + &self, + input: ExternalGenerationJobRetentionInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl prune_external_generation_job_history_and_return for super::RemoteProcedures { + fn prune_external_generation_job_history_and_return_then( + &self, + input: ExternalGenerationJobRetentionInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobRetentionProcedureResult>( + "prune_external_generation_job_history_and_return", + PruneExternalGenerationJobHistoryAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-module/src/ai/snapshots.rs b/server-rs/crates/spacetime-module/src/ai/snapshots.rs index 8ee4c0bec..f76d209e7 100644 --- a/server-rs/crates/spacetime-module/src/ai/snapshots.rs +++ b/server-rs/crates/spacetime-module/src/ai/snapshots.rs @@ -139,17 +139,6 @@ pub(crate) fn build_ai_text_chunk_row_id(snapshot: &AiTextChunkSnapshot) -> Stri ) } -pub(crate) fn build_ai_text_chunk_snapshot_from_row(row: &AiTextChunk) -> AiTextChunkSnapshot { - AiTextChunkSnapshot { - chunk_id: row.chunk_id.clone(), - task_id: row.task_id.clone(), - stage_kind: row.stage_kind, - sequence: row.sequence, - delta_text: row.delta_text.clone(), - created_at_micros: row.created_at.to_micros_since_unix_epoch(), - } -} - pub(crate) fn build_ai_result_reference_row( snapshot: &AiResultReferenceSnapshot, ) -> AiResultReference { diff --git a/server-rs/crates/spacetime-module/src/ai/stages.rs b/server-rs/crates/spacetime-module/src/ai/stages.rs index 8ffea6f22..1fa2cab3f 100644 --- a/server-rs/crates/spacetime-module/src/ai/stages.rs +++ b/server-rs/crates/spacetime-module/src/ai/stages.rs @@ -1,7 +1,7 @@ use crate::*; use module_ai::{ - generate_ai_result_ref_id, generate_ai_text_chunk_id, normalize_optional_text, - normalize_string_list, + MAX_AI_TASK_TEXT_OUTPUT_BYTES, generate_ai_result_ref_id, generate_ai_text_chunk_id, + normalize_optional_text, normalize_string_list, validate_ai_task_snapshot_memory_limits, }; #[spacetimedb::table( @@ -178,6 +178,9 @@ pub(crate) fn append_ai_text_chunk_tx( if input.sequence == 0 { return Err("ai_text_chunk.sequence 必须大于 0".to_string()); } + if input.delta_text.trim().len() > MAX_AI_TASK_TEXT_OUTPUT_BYTES { + return Err("AI 任务文本输出超过内存上限".to_string()); + } let mut snapshot = get_ai_task_snapshot_tx(ctx, &input.task_id)?; ensure_ai_task_can_transition(snapshot.status)?; @@ -200,7 +203,7 @@ pub(crate) fn append_ai_text_chunk_tx( .ai_text_chunk() .insert(build_ai_text_chunk_row(&chunk)); - let aggregated_text = collect_ai_stage_text_output(ctx, &chunk.task_id, chunk.stage_kind); + let aggregated_text = collect_ai_stage_text_output(ctx, &chunk.task_id, chunk.stage_kind)?; snapshot.status = AiTaskStatus::Running; if snapshot.started_at_micros.is_none() { @@ -215,6 +218,7 @@ pub(crate) fn append_ai_text_chunk_tx( snapshot.updated_at_micros = input.created_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; emit_ai_task_event( ctx, @@ -252,6 +256,7 @@ pub(crate) fn complete_ai_stage_tx( snapshot.updated_at_micros = input.completed_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; emit_ai_task_event( ctx, @@ -285,26 +290,27 @@ pub(crate) fn attach_ai_result_reference_tx( label: normalize_optional_text(input.label), created_at_micros: input.created_at_micros, }; - ctx.db - .ai_result_reference() - .insert(build_ai_result_reference_row(&reference)); - snapshot.result_references.push(reference); snapshot.updated_at_micros = input.created_at_micros; snapshot.version += 1; - persist_ai_task_snapshot(ctx, &snapshot)?; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; let reference = snapshot .result_references .last() + .cloned() .ok_or_else(|| "ai_result_reference 写入后缺少快照".to_string())?; + ctx.db + .ai_result_reference() + .insert(build_ai_result_reference_row(&reference)); + persist_ai_task_snapshot(ctx, &snapshot)?; emit_ai_task_event( ctx, &snapshot, AiTaskEventKind::ResultReferenceAttached, None, None, - Some(build_ai_result_reference_row_id(reference)), + Some(build_ai_result_reference_row_id(&reference)), input.created_at_micros, ); Ok(snapshot) @@ -333,29 +339,48 @@ pub(crate) fn replace_ai_task_stages( } } +pub(crate) fn delete_ai_text_chunks_for_task(ctx: &ReducerContext, task_id: &str) { + let chunk_row_ids = ctx + .db + .ai_text_chunk() + .by_ai_text_chunk_task_id() + .filter(task_id) + .map(|row| row.text_chunk_row_id.clone()) + .collect::>(); + for row_id in chunk_row_ids { + ctx.db.ai_text_chunk().text_chunk_row_id().delete(&row_id); + } +} + pub(crate) fn collect_ai_stage_text_output( ctx: &ReducerContext, task_id: &str, stage_kind: AiTaskStageKind, -) -> Option { - let mut chunks = ctx +) -> Result, String> { + let mut chunks = Vec::new(); + let mut aggregated_bytes = 0_usize; + for row in ctx .db .ai_text_chunk() .by_ai_text_chunk_task_id() .filter(task_id) .filter(|row| row.task_id == task_id && row.stage_kind == stage_kind) - .map(|row| build_ai_text_chunk_snapshot_from_row(&row)) - .collect::>(); - chunks.sort_by_key(|chunk| chunk.sequence); + { + aggregated_bytes = aggregated_bytes.saturating_add(row.delta_text.len()); + if aggregated_bytes > MAX_AI_TASK_TEXT_OUTPUT_BYTES { + return Err("AI 任务文本输出超过内存上限".to_string()); + } + chunks.push((row.sequence, row.delta_text.clone())); + } + chunks.sort_by_key(|(sequence, _)| *sequence); - let aggregated = chunks - .into_iter() - .map(|chunk| chunk.delta_text) - .collect::>() - .join(""); + let mut aggregated = String::with_capacity(aggregated_bytes); + for (_, delta) in chunks { + aggregated.push_str(&delta); + } if aggregated.trim().is_empty() { - None + Ok(None) } else { - Some(aggregated) + Ok(Some(aggregated)) } } diff --git a/server-rs/crates/spacetime-module/src/ai/tasks.rs b/server-rs/crates/spacetime-module/src/ai/tasks.rs index 66c0909d1..b6c646789 100644 --- a/server-rs/crates/spacetime-module/src/ai/tasks.rs +++ b/server-rs/crates/spacetime-module/src/ai/tasks.rs @@ -1,5 +1,8 @@ use crate::*; -use module_ai::{INITIAL_AI_TASK_VERSION, normalize_optional_text, validate_task_create_input}; +use module_ai::{ + INITIAL_AI_TASK_VERSION, normalize_optional_text, validate_ai_task_snapshot_memory_limits, + validate_task_create_input, +}; #[spacetimedb::table( accessor = ai_task, @@ -133,6 +136,7 @@ fn create_ai_task_tx( } let task_snapshot = build_ai_task_snapshot_from_create_input(&input); + validate_ai_task_snapshot_memory_limits(&task_snapshot).map_err(str::to_string)?; ctx.db.ai_task().insert(build_ai_task_row(&task_snapshot)); replace_ai_task_stages(ctx, &task_snapshot.task_id, &task_snapshot.stages); emit_ai_task_event( @@ -187,7 +191,9 @@ fn complete_ai_task_tx( snapshot.updated_at_micros = input.completed_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; + delete_ai_text_chunks_for_task(ctx, &snapshot.task_id); emit_ai_task_event( ctx, &snapshot, @@ -218,7 +224,9 @@ fn fail_ai_task_tx( snapshot.updated_at_micros = input.completed_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; + delete_ai_text_chunks_for_task(ctx, &snapshot.task_id); emit_ai_task_event( ctx, &snapshot, @@ -243,7 +251,9 @@ fn cancel_ai_task_tx( snapshot.updated_at_micros = input.completed_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; + delete_ai_text_chunks_for_task(ctx, &snapshot.task_id); emit_ai_task_event( ctx, &snapshot, diff --git a/server-rs/crates/spacetime-module/src/external_generation.rs b/server-rs/crates/spacetime-module/src/external_generation.rs index 547c4282c..2978f3cef 100644 --- a/server-rs/crates/spacetime-module/src/external_generation.rs +++ b/server-rs/crates/spacetime-module/src/external_generation.rs @@ -97,6 +97,10 @@ pub struct ExternalGenerationJob { accessor = by_external_generation_job_event_job_id, btree(columns = [job_id, created_at]) ), + index( + accessor = by_external_generation_job_event_job_id_only, + btree(columns = [job_id]) + ), index( accessor = by_external_generation_job_event_owner, btree(columns = [owner_user_id, created_at]) @@ -378,6 +382,29 @@ pub struct ExternalGenerationJobPayloadCompactionProcedureResult { pub error_message: Option, } +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobRetentionInput { + pub source_module: String, + pub limit: u32, + pub cursor_job_id: Option, + pub completed_before_micros: i64, + pub dry_run: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobRetentionProcedureResult { + pub ok: bool, + pub dry_run: bool, + pub scanned_count: u64, + pub selected_count: u32, + pub deleted_job_count: u32, + pub deleted_summary_count: u32, + pub deleted_event_count: u32, + pub next_cursor_job_id: Option, + pub has_more: bool, + pub error_message: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] pub struct ExternalGenerationQueueStatsSnapshot { pub pending_count: u32, @@ -676,6 +703,21 @@ pub fn compact_external_generation_job_payloads_and_return( } } +#[spacetimedb::procedure] +pub fn prune_external_generation_job_history_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobRetentionInput, +) -> ExternalGenerationJobRetentionProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + crate::migration::require_migration_operator(tx, caller)?; + prune_external_generation_job_history_tx(tx, input.clone()) + }) { + Ok(result) => result, + Err(message) => failed_external_generation_job_retention_result(input.dry_run, message), + } +} + #[spacetimedb::procedure] pub fn get_external_generation_queue_stats_and_return( ctx: &mut ProcedureContext, @@ -1369,6 +1411,105 @@ fn compact_external_generation_job_payloads_tx( }) } +fn prune_external_generation_job_history_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobRetentionInput, +) -> Result { + let source_module = input.source_module.trim().to_string(); + validate_required("external_generation_job.source_module", &source_module)?; + let now_micros = ctx.timestamp.to_micros_since_unix_epoch(); + if input.completed_before_micros > now_micros { + return Err( + "external_generation_job.completed_before_micros 不能晚于数据库当前时间".to_string(), + ); + } + + let cursor_job_id = input + .cursor_job_id + .as_deref() + .and_then(normalize_optional_text); + let limit = input + .limit + .clamp(1, MAX_EXTERNAL_GENERATION_MAINTENANCE_BATCH_SIZE) as usize; + let cursor_range = external_generation_job_maintenance_cursor_range(cursor_job_id.as_deref()); + let cursor_to_skip = cursor_job_id.clone(); + let rows = ctx + .db + .external_generation_job() + .by_external_generation_job_source_cursor() + .filter((source_module.as_str(), cursor_range)) + .filter(move |row| { + cursor_to_skip + .as_deref() + .is_none_or(|cursor| row.job_id != cursor) + }); + let (job_ids, next_cursor_job_id, has_more, scanned_count) = + select_external_generation_job_ids_for_maintenance(rows, limit, |row| { + ctx.db + .external_generation_job_summary() + .job_id() + .find(&row.job_id) + .is_some_and(|summary| { + is_external_generation_job_retention_candidate( + row, + &summary, + &source_module, + input.completed_before_micros, + ) + }) + }); + + let mut deleted_job_count = 0u32; + let mut deleted_summary_count = 0u32; + let mut deleted_event_count = 0u32; + if !input.dry_run { + for job_id in &job_ids { + let Some(row) = ctx.db.external_generation_job().job_id().find(job_id) else { + continue; + }; + let Some(summary) = ctx + .db + .external_generation_job_summary() + .job_id() + .find(job_id) + else { + continue; + }; + if !is_external_generation_job_retention_candidate( + &row, + &summary, + &source_module, + input.completed_before_micros, + ) { + continue; + } + + deleted_event_count = deleted_event_count + .saturating_add(delete_external_generation_job_events_for_job(ctx, job_id)); + ctx.db + .external_generation_job_summary() + .job_id() + .delete(job_id); + deleted_summary_count = deleted_summary_count.saturating_add(1); + ctx.db.external_generation_job().job_id().delete(job_id); + deleted_job_count = deleted_job_count.saturating_add(1); + } + } + + Ok(ExternalGenerationJobRetentionProcedureResult { + ok: true, + dry_run: input.dry_run, + scanned_count, + selected_count: job_ids.len() as u32, + deleted_job_count, + deleted_summary_count, + deleted_event_count, + next_cursor_job_id, + has_more, + error_message: None, + }) +} + fn renew_external_generation_job_lease_tx( ctx: &ReducerContext, input: ExternalGenerationJobRenewLeaseInput, @@ -1801,6 +1942,25 @@ fn should_compact_external_generation_job_payloads( }) } +fn is_external_generation_job_retention_candidate( + row: &ExternalGenerationJob, + summary: &ExternalGenerationJobSummary, + source_module: &str, + completed_before_micros: i64, +) -> bool { + row.source_module.trim() == source_module.trim() + && summary.job_id == row.job_id + && summary.status == row.status + && is_external_generation_job_terminal(row) + && is_external_generation_job_summary_terminal(summary) + && summary.notification_acknowledged_at.is_some() + && row + .completed_at + .unwrap_or(row.updated_at) + .to_micros_since_unix_epoch() + <= completed_before_micros +} + fn external_generation_job_maintenance_cursor_range( cursor_job_id: Option<&str>, ) -> RangeFrom<&str> { @@ -1836,6 +1996,24 @@ fn select_external_generation_job_ids_for_maintenance( ) } +fn delete_external_generation_job_events_for_job(ctx: &ReducerContext, job_id: &str) -> u32 { + let event_ids = ctx + .db + .external_generation_job_event() + .by_external_generation_job_event_job_id_only() + .filter(job_id) + .map(|event| event.event_id.clone()) + .collect::>(); + let deleted_count = event_ids.len() as u32; + for event_id in event_ids { + ctx.db + .external_generation_job_event() + .event_id() + .delete(&event_id); + } + deleted_count +} + fn count_external_generation_job_summaries_for_owner( ctx: &ReducerContext, owner_user_id: &str, @@ -2548,6 +2726,24 @@ fn failed_external_generation_job_payload_compaction_result( } } +fn failed_external_generation_job_retention_result( + dry_run: bool, + message: String, +) -> ExternalGenerationJobRetentionProcedureResult { + ExternalGenerationJobRetentionProcedureResult { + ok: false, + dry_run, + scanned_count: 0, + selected_count: 0, + deleted_job_count: 0, + deleted_summary_count: 0, + deleted_event_count: 0, + next_cursor_job_id: None, + has_more: false, + error_message: Some(message), + } +} + fn validate_required(field: &str, value: &str) -> Result<(), String> { if value.trim().is_empty() { return Err(format!("{field} 不能为空")); @@ -3438,6 +3634,86 @@ mod tests { )); } + #[test] + fn retention_only_selects_acknowledged_terminal_rows_before_cutoff() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_COMPLETED); + row.source_module = EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE.to_string(); + row.completed_at = Some(micros(1_000)); + row.updated_at = micros(1_000); + let mut summary = build_external_generation_job_summary_row(&row, None); + summary.notification_acknowledged_at = Some(micros(2_000)); + + assert!(is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + summary.notification_acknowledged_at = None; + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + summary.notification_acknowledged_at = Some(micros(2_000)); + row.status = EXTERNAL_GENERATION_STATUS_RUNNING.to_string(); + summary.status = EXTERNAL_GENERATION_STATUS_RUNNING.to_string(); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + row.status = EXTERNAL_GENERATION_STATUS_COMPLETED.to_string(); + summary.status = EXTERNAL_GENERATION_STATUS_COMPLETED.to_string(); + row.completed_at = Some(micros(1_001)); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + row.completed_at = Some(micros(1_000)); + row.source_module = "puzzle".to_string(); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + } + + #[test] + fn retention_rejects_mismatched_summary_identity_or_status() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_FAILED); + row.source_module = EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE.to_string(); + row.completed_at = Some(micros(1_000)); + let mut summary = build_external_generation_job_summary_row(&row, None); + summary.notification_acknowledged_at = Some(micros(2_000)); + + summary.job_id = "different-job".to_string(); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + summary.job_id = row.job_id.clone(); + summary.status = EXTERNAL_GENERATION_STATUS_CANCELLED.to_string(); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + } + #[test] fn maintenance_selector_bounds_scanned_rows_and_advances_by_last_scanned_job() { let rows = (1..=4).map(|index| {