diff --git a/deploy/container/docker-compose.loadtest.yml b/deploy/container/docker-compose.loadtest.yml index c2b78c73b..cdb7aed22 100644 --- a/deploy/container/docker-compose.loadtest.yml +++ b/deploy/container/docker-compose.loadtest.yml @@ -53,6 +53,7 @@ services: - "host.docker.internal:host-gateway" volumes: - api-tracking-outbox:/var/lib/genarrative/tracking-outbox + - api-wallet-refund-outbox:/var/lib/genarrative/wallet-refund-outbox ulimits: nofile: soft: 4096 @@ -85,6 +86,9 @@ services: OTEL_SERVICE_NAME: genarrative-external-generation-worker extra_hosts: - "host.docker.internal:host-gateway" + volumes: + - external-generation-tracking-outbox:/var/lib/genarrative/tracking-outbox-worker + - external-generation-wallet-refund-outbox:/var/lib/genarrative/wallet-refund-outbox ulimits: nofile: soft: 4096 @@ -142,4 +146,7 @@ services: volumes: spacetime-data: api-tracking-outbox: + api-wallet-refund-outbox: + external-generation-tracking-outbox: + external-generation-wallet-refund-outbox: nginx-logs: diff --git a/deploy/systemd/genarrative-database-backup.service b/deploy/systemd/genarrative-database-backup.service index 276b9f9ab..8a7d95535 100644 --- a/deploy/systemd/genarrative-database-backup.service +++ b/deploy/systemd/genarrative-database-backup.service @@ -11,6 +11,17 @@ WorkingDirectory=/opt/genarrative/current EnvironmentFile=/etc/genarrative/api-server.env ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service +# 备份脚本必须受独立内存上限保护，不能因目录扫描异常拖垮整台 release 主机。 +Environment=NODE_OPTIONS=--max-old-space-size=768 +Environment=GENARRATIVE_DATABASE_BACKUP_STOP_MARKER=/var/lib/genarrative/database-backups/.spacetimedb-stopped +MemoryHigh=768M +MemoryMax=1G +OOMPolicy=stop + +# 主进程可能在停库后被 MemoryMax/OOMPolicy 强制终止，JS finally 无法执行； +# 仅当备份脚本留下停库 marker 且本次 service 非正常成功时，由 systemd 兜底恢复全部依赖服务。 +ExecStopPost=/bin/sh -c 'if [ "${SERVICE_RESULT}" != "success" ] && [ -f "${GENARRATIVE_DATABASE_BACKUP_STOP_MARKER}" ]; then systemctl start spacetimedb.service; systemctl restart genarrative-api.service; systemctl restart genarrative-external-generation-worker@1.service; systemctl restart genarrative-external-generation-controller.service; if systemctl is-active --quiet spacetimedb.service && systemctl is-active --quiet genarrative-api.service && systemctl is-active --quiet genarrative-external-generation-worker@1.service && systemctl is-active --quiet genarrative-external-generation-controller.service; then rm -f "${GENARRATIVE_DATABASE_BACKUP_STOP_MARKER}"; fi; fi' + # 备份需要停止 / 启动 spacetimedb.service，并读取 /stdb、写入 /var/lib/genarrative/database-backups。 # 停止 SpacetimeDB 会连带停止 Requires 它的 API / worker / controller，冷备份后必须显式拉起。 PrivateTmp=true diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 16609f13b..95d15253c 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -24,6 +24,31 @@ - 验证方式：首次 collecting 带 invented-confirmation 必须拒绝且不落 GDD；reject continuation 再交 `user_revision` 的 v2 仍成功。 - 关联文档：`docs/technical/【技术方案】立项策划Agent（Fast GDD）-2026-08-10.md`。 +## 2026-08-27 退款 emergency spool 容量溢出保持可恢复 +## 2026-08-27 退款 emergency spool 容量溢出保持可恢复 + +- 背景：本机 emergency spool 仅作为 SpacetimeDB 完全不可达时的最后恢复路径，原有 `MAX_BYTES` 分支会直接返回 `Dropped`，导致扣费已经完成但没有可重放记录。 +- 决策：达到普通 outbox `MAX_BYTES` 时，将退款记录写入同一持久目录的 `refund-overflow-*` 文件；该文件与普通 pending 文件一样由启动恢复和后台 worker 重放到 SpacetimeDB，且按 refund ledger id 保持幂等。溢出文件不计入普通阈值，但必须触发容量告警；底层磁盘写入失败仍进入关键退款人工补偿流程。 +- 影响范围：api-server wallet refund emergency spool、资产失败退款日志、loadtest / 预览 Compose 持久卷、后端架构与开发运维文档。 +- 验证方式：运行 api-server `wallet_refund_outbox` 定向测试，确认超限写入并保留 overflow 文件；运行 SpacetimeDB profile 测试、Compose 配置校验、编码和 diff 门禁。 + +## 2026-08-27 短期认证状态进入共享 typed projection + +- 背景：短信验证码和微信 OAuth state 仍只存在 API 进程内 HashMap，多节点请求或 API 重启会直接丢失，无法满足无粘性会话的鉴权恢复要求。 +- 决策：`AuthStoreProjectionView` 增加 `phone_codes` 与 `wechat_states` typed 字段，由 `auth_store_projection_meta` 以 JSON 投影持久化；启动恢复、CAS 同步和失败后的权威刷新都覆盖这两类短期状态。验证码哈希使用部署级稳定盐（当前复用 `GENARRATIVE_JWT_SECRET`），各 API 节点必须一致；发码前先刷新权威投影并用占位验证码记录做一次 projection CAS，只有占用成功才调用短信 provider，避免跨节点冷却竞态；认证 handler 在发码、消费验证码、创建/消费微信 state 后都要完成 projection sync，失败即返回服务错误；所有会读取或变更本机认证工作集的认证主链路（登录、刷新、`/me`、会话管理、密码、绑定和微信 state）在领域操作前先从正式投影做一次受 CAS 保护的只读刷新，受保护 Bearer 中间件也会在进入业务 handler 前执行同样的刷新，刷新失败时 fail closed，不能依赖粘性会话；同步遇到 CAS 冲突时，若本次尝试期间没有新的本地变更则恢复正式快照，若仍有待同步 revision 则由后续认证请求重试，避免节点永久卡在 pending。微信 OAuth state 设置有界活动数量，避免单个 JSON 投影无界膨胀。短期状态仍由 `module-auth` 内存工作集执行领域校验，但不再把本机 HashMap 当作持久化或跨节点真相。 +- 影响范围：`module-auth` projection、`spacetime-module` auth schema/procedure、`spacetime-client` bindings/facade、api-server 手机号 / 微信 handler、认证架构与运维文档。 +- 验证方式：运行 module-auth projection roundtrip（验证码可跨恢复校验、微信 state 可跨恢复消费）、SpacetimeDB schema/runtime/DDD 门禁、api-server 定向测试、编码和 diff 检查。 + +--- + +## 2026-08-27 外部生成历史采用受控保留清理 + +- 背景：`external_generation_job`、`external_generation_job_summary` 与 `external_generation_job_event` 都是持久化表；摘要和 payload 边界收紧后，已确认的终态历史仍会继续占用 SpacetimeDB 常驻内存，且事件审计链会随任务数量增长。 +- 决策：新增仅 migration operator 可调用的 `prune_external_generation_job_history_and_return`。默认按 `source_module=editor-canvas`、30 天保留期和 `job_id` 游标分批运行；只删除主任务与摘要状态一致、属于 completed / failed / cancelled、摘要已有 `notification_acknowledged_at` 且终态时间达到 cutoff 的任务。事件、摘要和主任务仍按同一事务顺序删除，但每次事务最多删除 256 条事件；事件未删完时保留任务与摘要并返回同一个 job cursor，维护脚本下一次继续，避免单个任务形成无界事务写集。默认 dry-run，必须固定 dry-run 返回的 cutoff 后再 apply；pending / running、未确认通知、摘要缺失或状态不一致的数据永不删除。其他 source module 必须显式指定并单独评估；资产对象和钱包流水不随任务历史删除；不新增自动定时器或 runtime 清理权限。 +- 影响范围：`server-rs/crates/spacetime-module/src/external_generation.rs`、外部生成事件 job_id 单列索引、SpacetimeDB 生成 bindings、`scripts/spacetime-maintain-external-generation-jobs.mjs`、架构与生产运维文档。 +- 验证方式：覆盖终态 / 活跃态 / 已确认与未确认摘要、状态或身份不一致、cutoff 边界测试；运行 SpacetimeDB module tests/check、bindings 生成、schema/encoding/diff 门禁，并在维护窗口先 dry-run 再 apply。 +- 关联文档：`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`、PR #203。 + ## 2026-08-27 SpacetimeDB 工具链统一升级到 2.8.3 - 背景：SpacetimeDB 2.8.0 引入 TypeScript submodule 与调度延迟观测，2.8.1 修复 v1 WebSocket 订阅移除死锁、TypeScript SDK `array` 读缓存别名和 Rust string 默认值支持，2.8.2 修复 table accessor 改名自动迁移，2.8.3 修复 scheduled function 从实际执行时间重排导致的长期漂移。仓库若继续锁定 2.7.0，会保留这些已知运行时与 SDK 问题。 @@ -1334,8 +1359,8 @@ CI 上 `background_agent_runtime_recovers_stale_running_before_pending_task` 在 - OSS 固定恢复入口为 `//latest.json`。CAS 文件和 full/history catalog 保持不可变；latest pointer 只保存最新 full catalog 与已发布 history catalog 的 object key、长度和 SHA，不包含主机绝对路径或文件内容。每次 state 变化先验真全部引用 catalog，再覆盖上传并 HEAD 验真 latest pointer，成功后才落本地 state；history 还必须在 pointer 成功后才允许删除源文件。全新机器可仅凭 bucket、database、prefix 与 OSS 凭据自动下载 pointer 和 full catalog。 - dev 带宽不足时，允许把已冻结的 dev 基线经 `10.2.0.10 -> 10.2.4.16` 内网 rsync 到 release 独立 staging，再用 release 出口上传 dev bucket；staging 不得指向 release `/stdb`，不得停止或修改 release 服务，传输凭据必须临时创建并在演练后移除。catalog 不记录 staging 绝对路径，files state 可回传 dev 继续 history。 - 恢复边界：恢复时默认从 OSS `latest.json` 自动定位 full catalog，创建目录并按相对路径下载每个对象、逐文件校验长度与 SHA；本地 state 只用于备份续跑，不再是异机恢复前置条件。远程 dev 已完成真实 OSS、清理、重启和异机隔离恢复演练；release timer 与 publish 前备份继续保持原行为。 -- systemd 接线：主 service 保持 `archive-full`。Server-Provision 新增默认值为 `archive-full` 的 `DATABASE_BACKUP_PROFILE`；dev 或 release 显式选择 `files-history` 时，必须为各自主机指定独立 work-dir，并先用 current release 脚本执行 history dry-run，确认已有 full state 后才安装仓库托管 drop-in，并删除现场手写旧 drop-in。切回默认 profile 必须删除所有 history 覆盖。 -- 影响范围：`scripts/database-backup-to-oss.mjs`、备份门禁、生产 env 示例、systemd 模板、Server-Provision、SpacetimeDB 运维与恢复流程；release timer 可在独立 baseline 验证后显式选择 profile，publish 前备份是否切换仍需单独决策。 +- systemd 接线：主 service 保持 `archive-full`。Server-Provision 新增默认值为 `archive-full` 的 `DATABASE_BACKUP_PROFILE`；development 可显式选择 `files-history`，必须指定独立 work-dir 并先用 current release 脚本执行 history dry-run，确认已有 full state 后才安装仓库托管 drop-in；release 拒绝 `files-history`，直到流式 catalog 改造完成，以免大目录扫描再次触发 Node 内存峰值。切回默认 profile 必须删除所有 history 覆盖；备份 unit 同时设置 Node heap 与 systemd memory 上限，避免备份异常拖垮业务主机。 +- 影响范围：`scripts/database-backup-to-oss.mjs`、备份门禁、生产 env 示例、systemd 模板、Server-Provision、SpacetimeDB 运维与恢复流程；release timer 固定使用 archive-full，publish 前备份是否切换仍需单独决策。 - 验证方式：`npm run check:database-backup`、`npm run check:production-ops`、`npm run check:encoding`、`git diff --check`；dev 现场必须完成逐文件 full catalog、重复 full 零 PUT、history dry-run、上传后清理、STDB 重启和按 catalog 隔离恢复 roundtrip。 - 关联：。 @@ -7793,3 +7818,17 @@ CI 上 `background_agent_runtime_recovers_stale_running_before_pending_task` 在 - 决策：`.codex/skills/` 下的 SpacetimeDB 指导收敛为单一 `.codex/skills/genarrative-spacetimedb/SKILL.md`。官方 `spacetimedb` 插件负责通用 concepts、Rust server、CLI、TypeScript client 和 MCP 知识；项目 skill 只保留 Genarrative 的架构边界、schema/migration 门禁、目标 server 安全规则、运行时排障和验证路径。 - 路由：涉及 SpacetimeDB 的任务统一先读取项目适配 skill，再按需读取 `spacetimedb:concepts`、`spacetimedb:rust-server`、`spacetimedb:cli`、`spacetimedb:typescript-client` 或 `spacetimedb:mcp`。插件通用示例不得覆盖项目禁止 `maincloud`、禁止人工 `spacetime --root-dir`、显式 server 和后端分层等约束。 - 安装：团队环境缺少插件时使用 `codex plugin marketplace add clockworklabs/SpacetimeDB --sparse .agents --sparse codex-plugin` 和 `codex plugin add spacetimedb\@spacetimedb-plugins`；个人配置、缓存和凭据不进入仓库。 + +## 2026-08-27 退款 outbox 主路径迁入 SpacetimeDB + +- 决策：`profile_wallet_refund_outbox` 是跨 API 节点退款的正式持久化队列。扣费失败、外部生成 attempt 失败或最终 lease 过期时，在同一个 SpacetimeDB 事务内按 `refund_ledger_id` 幂等写入 pending 行；worker 从库内 pending 行批量处理，退款账本写入与 outbox 成功删除保持在同一事务内，失败由 `available_at` / `attempts` 驱动重试。`asset_operation_wallet_settlement` 继续负责退款先于 consume 可见时的取消 intent，阻止迟到扣费。只有 SpacetimeDB 完全不可达时，api-server 才写本机 `wallet-refund-outbox` emergency spool；本机文件不能替代库内队列，必须持久挂载、告警、恢复演练并支持人工补偿。 +- 影响范围：`profile_wallet_refund_outbox` 表及 bindings、runtime enqueue/process procedure、外部生成失败事务、inline 资产退款、api-server 跨节点 worker 和 emergency spool、后端架构与开发运维文档。 +- 验证方式：运行 `npm run spacetime:generate`、`npm run check:spacetime-schema`、`npm run check:server-rs-ddd`、`cargo check -p spacetime-module -p spacetime-client -p api-server --manifest-path server-rs/Cargo.toml`、退款 outbox / asset billing / external generation 定向测试、`npm run check:encoding` 和 `git diff --check`。 + +## 2026-08-27 release 内存增长修复与备份 OOM 恢复兜底 + +- 决策：外部生成 worker 每轮主动 `try_join_next` 回收已完成 `JoinHandle`，避免持续有队列任务时只归还 semaphore permit 却让 `JoinSet` 句柄集合无界增长；超时脱管任务在 abort 后等待句柄结束，执行许可保持到 work 真正结束或被取消。 +- 决策：`module-ai` 的阶段终态写入与流式增量统一受文本、结构化 JSON、warning 和全局 retained 工作集上限约束；认证投影恢复对过滤后的 refresh session 重新计数，超过 8192 条直接拒绝启动恢复，避免超限快照灌入内存。 +- 复审补充：`spacetime-module` 的 AI procedure 复用同一组任务元数据 / payload / 输出 / 结果引用上限，流式文本聚合超过 512 KiB 或每阶段 8192 个 chunk 时回滚事务，terminal task 收口后按有界批次删除 `ai_text_chunk` 明细，避免真实持久化链路绕过内存边界。 +- 决策：备份脚本停库前写入受保护 `.spacetimedb-stopped` marker，正常恢复完成后清理；systemd 备份 service 通过 `MemoryHigh/MemoryMax/OOMPolicy` 和 `ExecStopPost` 在 Node OOM kill、无法执行 JS finally 时兜底拉起 SpacetimeDB、API、worker、controller，恢复不完整则保留 marker。 +- 验证：worker/module-auth/module-ai 定向 Rust 测试、database-backup/production-ops/encoding 门禁和 `git diff --check` 必须在提交前通过；release 现场需按 archive-full 重新 provision 并核验旧 files-history drop-in 已删除、四个服务 active。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 7c3560ac5..672d6c999 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -232,7 +232,7 @@ npm run check:server-rs-ddd 3. 结果页单图重生成、发布、道具使用和其它独立资产操作仍按各自业务操作成本执行；不要把初始草稿成本误套到这些单次操作上。 4. 资产操作的预扣费必须 fail-closed：钱包或 SpacetimeDB 预扣费不可达、超时或返回业务错误时，`api-server` 直接返回错误，不允许继续调用图片、音频、GLB 等外部生成 provider。 5. 需要支持 HTTP retry 的计费 ledger id 必须包含当前请求的 `request_id`；前端 `fetchWithApiAuth` 同一次业务请求的静默刷新重试复用同一个 `x-request-id`，后端不得再使用 prompt 指纹或随机 asset id 作为扣费幂等键。 -6. 外部生成已预扣费但后续失败时必须先同步调用钱包退款；若 SpacetimeDB 暂不可用，退款请求写入 `wallet-refund-outbox` 本地文件并由后台 worker 重放。默认启用，配置项为 `GENARRATIVE_WALLET_REFUND_OUTBOX_ENABLED`、`GENARRATIVE_WALLET_REFUND_OUTBOX_DIR`、`GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE`、`GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS` 和 `GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES`。outbox 文件按 refund ledger id 幂等落盘；成功重放后删除，坏文件隔离为 `corrupt-*`。外部生成任务触发的扣费和退款必须在 `profile_wallet_ledger.metadata_json` 中写入 `externalGenerationJobId`，outbox 重放也必须保留同一任务 ID，便于从退款记录追溯到正式生成任务。 +6. 外部生成已预扣费但后续失败时，失败/任务状态变更事务必须在 SpacetimeDB 内按 refund ledger id 幂等写入 `profile_wallet_refund_outbox` pending 行；跨节点 worker 从库内 pending 行批量处理并在库内事务执行退款，成功后删除 outbox 行，失败按 `available_at` 和 `attempts` 重试。当前 attempt 若 consume 尚不可见，事务仍必须先写 `asset_operation_wallet_settlement` 取消 intent，阻止迟到扣费。普通 inline 资产失败也先调用同一 DB outbox procedure；只有 SpacetimeDB 完全不可达时才写 `wallet-refund-outbox` 本机 emergency spool。默认启用，配置项为 `GENARRATIVE_WALLET_REFUND_OUTBOX_ENABLED`、`GENARRATIVE_WALLET_REFUND_OUTBOX_DIR`、`GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE`、`GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS` 和 `GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES`。本机文件按 refund ledger id 幂等落盘；成功重放后删除，坏文件隔离为 `corrupt-*`，不能替代库内 outbox。外部生成任务触发的扣费和退款必须在 `profile_wallet_ledger.metadata_json` 与两类 outbox 中保留 `externalGenerationJobId` 和 `externalGenerationClaimAttempt`，便于从退款记录追溯到具体 attempt。 7. 拼图首图后台生成的跨实例互斥锁必须落在 SpacetimeDB `puzzle_background_compile_task` 表，claim id 由 `task_id + request_id` 构成，释放时必须校验 claim id，避免旧后台任务释放新请求抢到的租约。 ## 用户钱包与编辑器生成扣费契约 @@ -329,6 +329,8 @@ Responses 的终态载荷既是工具调用的恢复源，也是正文的恢复 - Rust 结构体：`AiTask` - 源码：`server-rs/crates/spacetime-module/src/ai/tasks.rs` +- `module-ai` 的进程内热状态不是持久化真相：文本增量按阶段有序聚合并受单阶段 512 KiB、每阶段 8192 个 chunk 上限约束；terminal task 立即释放增量明细，内存工作集最多保留 1024 个任务。需要长期查询时必须读取 SpacetimeDB 的 `ai_task` / `ai_task_stage` 投影，不得依赖进程重启后仍存在的内存快照。 +- SpacetimeDB 的 AI 写入 procedure 必须复用同一组任务元数据、payload、文本、结构化输出、warning、失败消息和结果引用上限；流式聚合超过 512 KiB 或 8192 个 chunk 时在事务内拒绝，terminal task 收口后分批删除 `ai_text_chunk` 明细，只保留阶段最终快照和结果引用。 ### `ai_task_event` @@ -360,18 +362,20 @@ Responses 的终态载荷既是工具调用的恢复源，也是正文的恢复 - 源码：`server-rs/crates/spacetime-module/src/external_generation.rs` - 用途：外部生成正式任务列表的轻量投影，按 `job_id` 保存 owner、来源、状态、可选 `phase`、价格、有界错误摘要、通知确认时间、各阶段时间和入队时提取的 `request_prompt`，不包含 request/result payload、worker lease 或 dedupe 内部字段。错误摘要统一拒绝内联媒体并限制为 2048 字符；列表在单次 owner 扫描中同时计数并只保留请求 limit 的固定大小 top-N，不得先收集全量历史再截断。enqueue、claim、renew、phase update、complete、fail 事务同步投影；acknowledge 只更新该轻量表并写审计事件，后续主任务同步必须保留已有确认时间，禁止为了写确认时间加载 / 重写大 payload 行。BFF 的列表、状态和确认只调用 summary procedure；`running + processing` 映射为“正在处理”，其它 running（含旧行 `phase=None`）映射为“正在生成”。历史终态任务由迁移操作员的游标分批 maintenance procedure 在压缩 payload 时同步回填摘要，正式列表不得为兼容旧数据回扫完整主表。 - 非阻断告警：摘要字段 `warning_message` 是展示投影，由完成任务的轻量 `result_payload_json.warning.reason` 原样提取，不等同于公开 inline / external v1 的原始结构化诊断字段。complete 和历史 backfill 共用同一构建路径；历史任务按其结果载荷中已写入的 `reason` 快照投影，不为格式升级重写或补前缀。单 job 状态和任务列表 BFF 以 `warning: string` 返回该可直接展示的完整文案，不再返回结构化 code，Web 不得再次补前缀或按字符串推断告警类型。错误与告警摘要都不复制内联媒体并限制为 2048 字符。`phase` 与 `warning_message` 分别表示当前执行阶段和成功降级提示，不得混用；worker / BFF / Web 必须同版本协调发布，不保证滚动混部或旧 Web 缓存下的字符串语义兼容。 -- 正式读取 procedure 为 `get_external_generation_job_summary_and_return`、`list_external_generation_job_summaries_and_return` 和 `acknowledge_external_generation_job_summaries_and_return`。历史维护 procedure 为 `compact_external_generation_job_payloads_and_return` 与 `backfill_external_generation_job_summaries_and_return`，仅 migration operator 可调用；运维入口统一使用 `npm run spacetime:external-generation:maintain -- ...`，默认 dry-run、单批最多 25 条。B-tree cursor 选择阶段最多反序列化 `limit + 1` 行，apply 再按主键逐条读取选中行；怀疑存在单行异常巨型 JSON 时必须先使用 `--limit 1`。payload 压缩额外固定使用 `source_module = editor-canvas` 的复合 cursor 索引，不得静默改写其它玩法历史任务。 +- 正式读取 procedure 为 `get_external_generation_job_summary_and_return`、`list_external_generation_job_summaries_and_return` 和 `acknowledge_external_generation_job_summaries_and_return`。历史维护 procedure 为 `compact_external_generation_job_payloads_and_return`、`backfill_external_generation_job_summaries_and_return` 与 `prune_external_generation_job_history_and_return`，仅 migration operator 可调用；运维入口统一使用 `npm run spacetime:external-generation:maintain -- ...`，默认 dry-run、单批最多 25 条。B-tree cursor 选择阶段最多反序列化 `limit + 1` 行，apply 再按主键逐条读取选中行；怀疑存在单行异常巨型 JSON 时必须先使用 `--limit 1`。payload 压缩额外固定使用 `source_module = editor-canvas` 的复合 cursor 索引，不得静默改写其它玩法历史任务。历史清理默认使用 `--prune-history`、`source_module = editor-canvas` 和 30 天保留期；只有主任务与摘要状态一致且属于 completed / failed / cancelled、摘要已有 `notification_acknowledged_at`、终态时间不晚于 cutoff 的记录才是候选。apply 在同一事务内按事件 → 摘要 → 主任务顺序删除，事件不得独立清理；每次事务最多删除 256 条事件，若同一任务仍有事件则保留任务与摘要并返回同一个 `next_cursor_job_id`，下一次继续该任务，避免单个任务形成无界事务写集；pending / running、未确认通知、摘要缺失或状态不一致的记录永不删除。清理不触碰资产对象或钱包流水，其他 source module 必须显式指定并单独评估。 ### `external_generation_job_event` - Rust 结构体：`ExternalGenerationJobEvent` - 源码：`server-rs/crates/spacetime-module/src/external_generation.rs` - 用途：外部生成任务审计事件表，按 `job_id` 和 `owner_user_id` 记录 `enqueued`、`claimed`、`lease_renewed`、`completed`、`failed`、`acknowledged` 等状态转换事实。状态转换只能由 SpacetimeDB procedure 写入，不由前端或 worker 直接改表；该表用于追溯任务生命周期和排障，不替代 `external_generation_job` 当前状态。 +- 保留策略：事件只会随已确认通知的终态任务由 `prune_external_generation_job_history_and_return` 原子删除，不支持按事件单独清理，以保持任务、摘要和审计链一致。单次事务最多删除 256 条事件；若事件未删完，任务和摘要暂不删除，维护脚本用同一个 job cursor 重试剩余事件。 ### `ai_text_chunk` - Rust 结构体：`AiTextChunk` - 源码：`server-rs/crates/spacetime-module/src/ai/stages.rs` +- 单阶段最多保留 8192 个 chunk；聚合和终态清理均按有界批次处理，避免小 delta 堆积为无界行数或一次性 ID 列表。 ### `analytics_date_dimension` @@ -410,12 +414,18 @@ Responses 的终态载荷既是工具调用的恢复源，也是正文的恢复 ### `auth_store_projection_meta` +启动投影恢复会对过滤后的 retained refresh session 重新计数；超过 8192 条时直接失败关闭并继续重试，不得把超限快照一次性灌入内存。 + - Rust 结构体：`AuthStoreProjectionMeta` - 源码：`server-rs/crates/spacetime-module/src/auth/tables.rs` +- 职责：保存 typed 认证投影的单调版本，以及短期手机号验证码和微信 OAuth state 的序列化投影；`phone_codes_json` / `wechat_states_json` 只承载短期认证状态，不替代 `user_account`、`auth_identity` 或 `refresh_session` 的正式表语义。 -认证恢复策略：`api-server` 启动时只从 SpacetimeDB 正式认证表（`user_account` / `auth_identity` / `refresh_session`）导出 typed `AuthStoreProjectionView`，再恢复 `module-auth` 的进程内认证工作集；运行中 Bearer `sid` 或 refresh cookie 在本进程工作集内未命中时直接按失效处理，不再从 SpacetimeDB 导出整包认证状态刷新内存，避免旧投影把重复手机号或旧会话重新灌回进程。`module-auth` 只保留内存工作集和 projection 导入 / 导出能力，不再保留 JSON 快照导入 / 导出能力，也不写本地持久化文件；`auth-store.json` / `GENARRATIVE_AUTH_STORE_PATH` 不再是兼容恢复源。认证创建、登录会话、刷新、退出、改密、重置密码、绑定和资料变更等写操作必须在返回客户端前通过 `sync_auth_store_projection` 成功同步 SpacetimeDB 正式认证表；同步失败时接口返回错误，不允许把只存在于当前进程内存的账号或会话当成成功结果。新用户注册奖励、邀请码绑定和登录埋点必须排在认证同步成功之后，避免认证没落库时先写出钱包或邀请关系。若启动恢复阶段 SpacetimeDB 不可连接或超时，`api-server` 会按固定间隔持续重试认证工作集恢复，恢复成功后才开始监听 HTTP，避免一次短超时让进程永久停留在依赖不可用状态。 +认证恢复策略：`api-server` 启动时从 SpacetimeDB 正式认证表（`user_account` / `auth_identity` / `refresh_session`）以及 `auth_store_projection_meta` 中的短期状态投影导出 typed `AuthStoreProjectionView`，再恢复 `module-auth` 的进程内认证工作集；生产 Bearer 中间件不再从 `InMemoryAuthStore` 读取用户或会话，而是每次通过 typed `validate_auth_session` procedure 在 SpacetimeDB 事务内校验 `token_version`、会话归属、撤销时间和过期时间，SpacetimeDB 不可用时 fail closed 返回服务错误。`validate_auth_session`、投影导出和投影同步均从 `ctx.sender()` 派生调用方，并复用现役 runtime service identity 白名单；启动恢复先完成该服务身份初始化，普通 SpacetimeDB identity 不能读取或改写私有认证表。测试构建仍可使用显式的内存测试夹具。所有会读取或变更本机认证工作集的认证主链路（登录、刷新、`/me`、会话管理、密码、绑定和微信 state）在领域操作前先从正式投影做一次受 CAS 保护的只读刷新，刷新失败时 fail closed；refresh cookie 仍只按正式 `refresh_session` 校验，其他认证数据也不得绕过正式同步。`module-auth` 只保留内存工作集和 projection 导入 / 导出能力，不再保留 JSON 快照导入 / 导出能力，也不写本地持久化文件；`auth-store.json` / `GENARRATIVE_AUTH_STORE_PATH` 不再是兼容恢复源。认证创建、登录会话、刷新、退出、改密、重置密码、绑定和资料变更等写操作仍必须在返回客户端前通过 `sync_auth_store_projection` 成功同步 SpacetimeDB 正式认证表；同步失败时接口返回错误，不允许把只存在于当前进程内存的账号、会话、短信验证码或微信 state 当成成功结果。每个 API 工作集绑定启动恢复或上次成功同步得到的 `auth_store_projection_meta.updated_at` 版本作为 `base_updated_at_micros`，SpacetimeDB 在同一事务内执行基线 CAS，并要求新的 `updated_at_micros` 严格递增；基线不一致或版本不晚于当前值时整包写入失败，冲突节点只有在确认本次同步尝试期间没有新的本地认证变更后，才可丢弃失败工作集并从正式表恢复，不能用陈旧工作集删除、恢复或覆盖另一节点的新状态；若同期仍有本地变更则保留 pending revision，并由后续认证请求先重试同步，不把临时数据库故障变成永久卡死；同步成功但期间又出现新本地变更时最多连续补同步三轮，仍未稳定则失败关闭。这只是迁移期并发保护，不改变正式认证表的权威地位。新用户注册奖励、邀请码绑定和登录埋点必须排在认证同步成功之后，避免认证没落库时先写出钱包或邀请关系。若启动恢复阶段 SpacetimeDB 不可连接或超时，`api-server` 会按固定间隔持续重试认证工作集恢复，恢复成功后才开始监听 HTTP，避免一次短超时让进程永久停留在依赖不可用状态。 +认证工作集容量限制：refresh session 最多保留 8192 条，短信验证码最多保留 4096 条；写入前清理过期项，达到上限时拒绝新增而不继续膨胀。 -`auth_store_snapshot` 表和旧 `import_auth_store_snapshot_json` / `export_auth_store_snapshot_from_tables` procedure 已删除。认证投影同步只读写 `user_account`、`auth_identity`、`refresh_session` 和 `auth_store_projection_meta`；`auth_identity` 不再写 `phone_e164`、`display_name`、`avatar_url`，这些账号资料只以 `user_account` 为准。 +`auth_store_snapshot` 表和旧 `import_auth_store_snapshot_json` / `export_auth_store_snapshot_from_tables` procedure 已删除。认证投影同步只读写 `user_account`、`auth_identity`、`refresh_session` 和 `auth_store_projection_meta`；`auth_identity` 不再写 `phone_e164`、`display_name`、`avatar_url`，这些账号资料只以 `user_account` 为准。`api-server` 多节点必须使用相同的部署级验证码哈希盐（当前复用 `GENARRATIVE_JWT_SECRET`）；轮换该 secret 会使尚未消费的短信验证码失效，但不会改变已持久化账号或 session。 + +短期状态的并发保护：发短信前先从正式投影刷新工作集，再写入不可消费的占位验证码并通过 `sync_auth_store_projection` 的基线 CAS 占用手机号 / 场景冷却窗口；只有占用成功后才调用外部短信 provider，provider 成功后再同步真实验证码哈希。微信 OAuth state 在 `module-auth` 工作集内限制活动数量，超过上限直接拒绝创建，避免单行 JSON 投影无界增长；过期 state 仍由投影导出时清理。 ### `bark_battle_draft_config` @@ -1057,6 +1067,13 @@ Responses 的终态载荷既是工具调用的恢复源，也是正文的恢复 - 说明：资产操作 consume/refund 配对结算事实表，主键为 consume ledger ID，并保存配对 refund ledger、用户、金额和结算时间。退款先到且 consume 尚不可见时，该表作为持久化取消 intent；迟到 consume 必须检测该行并拒绝扣费，避免 worker 崩溃重领期间双扣。 - 索引：主键 `consume_ledger_id`。 +### `profile_wallet_refund_outbox` + +- Rust 结构体：`ProfileWalletRefundOutbox` +- 源码：`server-rs/crates/spacetime-module/src/runtime/active/profile.rs` +- 说明：跨节点资产退款的正式 pending 队列。主键为 refund ledger ID，保存 consume/refund 配对、用户、金额、资源、生成任务 attempt、失败原因和重试时间；失败事务先写入该表，worker 在 SpacetimeDB 事务内幂等执行钱包退款并删除成功行。只有数据库不可达时，api-server 才使用本机 `wallet-refund-outbox` emergency spool；本机 `MAX_BYTES` 达到阈值时改写入同目录 `refund-overflow-*` 溢出文件，保持可恢复而不静默丢弃。 +- 索引：`(status, available_at)`。 + ### `profile_wallet_config` - Rust 结构体：`ProfileWalletConfig` @@ -1232,6 +1249,7 @@ RPG 创作入口的配置 ID 是 `rpg`，当前 `visible=true`、`open=true`； - Rust 结构体：`RefreshSession` - 源码：`server-rs/crates/spacetime-module/src/auth/tables.rs` +- 认证工作集只保留 active 会话以及最近 24 小时内的 revoked / expired 会话；超过宽限期的失效会话在 refresh session 写路径和 projection 导出前从内存索引移除，并随下一次 typed projection 同步从正式表清理。该清理不改变 active 多端登录、单端登出或全端登出语义。 ### `runtime_setting` diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 73043e067..064bae248 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -99,7 +99,7 @@ HTTP 角色的 `GENARRATIVE_SPACETIME_POOL_SIZE` 只表示 procedure / reducer 生产拆分角色时，`external-generation-worker` 和 `external-generation-controller` 的专属 env 示例会把 `GENARRATIVE_SPACETIME_POOL_SIZE` 覆盖为 `1`；非 HTTP 角色不创建 API 缓存读连接，只保留 `external_generation_job` 队列窄订阅作为响应式唤醒信号，实际抢占和扩缩容判断仍走 SpacetimeDB procedure。worker / controller 不执行模型定价 seed，启动时先调用受 runtime writer 鉴权的 queue-stats procedure 做只读预检，身份不匹配时 fail-fast；当前正式 systemd unit 通过共同加载 API env 继承同一 `GENARRATIVE_SPACETIME_TOKEN`，默认路径为 `/etc/genarrative/api-server.env`，自定义部署由 provision 和 API deploy 按实际参数渲染，专属角色 env 示例不重复配置该 token。`GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS` 与 controller poll interval 只作为订阅失效、漏事件和 lease 过期这类时间条件的兜底，不作为正常领取任务的主路径。 -生产 worker 默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=600`，只覆盖 worker 心跳抖动和短暂断连窗口，不再把 lease 当成完整任务时长；默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS=900`。`editor_image_generation`、`editor_image_edit`、`editor_icon_spritesheet_generation`、`editor_ui_design_asset_extraction` 四类 VectorEngine 图片任务与角色动画 / 视频类长任务使用 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS=1800`，手动去背景、音效和背景音乐继续使用普通预算。worker 在单次尝试超过执行预算后会停止续租并释放 worker 槽位，但不会取消已启动的业务 future 或主动写入失败 / 重试状态；在途执行由 lease fencing 仲裁，有效租约内写回仍可完成，租约过期后任务才可重新领取，attempt 耗尽时由认领事务标记失败并结算退款。生产部署和 provision 脚本会给 `/etc/genarrative/api-server.env` 与 `/etc/genarrative/external-generation-worker.env` 补齐这些变量；已有自定义值不覆盖，只会把历史旧默认 `3600` 迁移为 `600`。 +生产 worker 默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=600`，只覆盖 worker 心跳抖动和短暂断连窗口，不再把 lease 当成完整任务时长；默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS=900`。`editor_image_generation`、`editor_image_edit`、`editor_icon_spritesheet_generation`、`editor_ui_design_asset_extraction` 四类 VectorEngine 图片任务与角色动画 / 视频类长任务使用 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS=1800`，手动去背景、音效和背景音乐继续使用普通预算。worker 在单次尝试超过执行预算后会停止续租，但不会取消已启动的业务 future 或主动写入失败 / 重试状态；执行许可会一直绑定到 active 或 detached work 真正结束（或超过租约仲裁窗口被取消），避免超时任务脱管后立即补进新的高内存任务。在途执行由 lease fencing 仲裁，有效租约内写回仍可完成，租约过期后任务才可重新领取，attempt 耗尽时由认领事务标记失败并结算退款。生产部署和 provision 脚本会给 `/etc/genarrative/api-server.env` 与 `/etc/genarrative/external-generation-worker.env` 补齐这些变量；已有自定义值不覆盖，只会把历史旧默认 `3600` 迁移为 `600`。 lease 过期后不代表任务一定再次执行：claim transaction 只有在 `attempt < max_attempts` 时才会递增 attempt 并返回 worker；如果过期的是最终 attempt，则直接把 job 收口为 `failed`、清理 lease，并按入队冻结价格为当前 attempt 原子退款或写 cancellation intent。该终态任务不会再次进入 provider executor，迟到 consume 会被 settlement intent 拒绝。 @@ -115,7 +115,7 @@ BgFilter 对已经落入私有 OSS 的生成原图、动作抽取帧和手动去 图片编辑器任务侧栏与生成提交工作流只读取 BFF 队列接口：`GET /api/runtime/external-generation/jobs` 列出当前用户任务，`GET /api/runtime/external-generation/jobs/{jobId}` 查看单 job 状态，概览场景可使用 `GET /api/runtime/external-generation/queue-overview`。前端不直接查询 `external_generation_job` private table，也不展示 worker 内部 payload；完成态以编辑器项目和资源接口返回的正式数据为准。 -外部生成任务摘要投影与历史 payload 维护使用 `npm run spacetime:external-generation:maintain -- ...`，且只能由已授权 migration operator 的 SpacetimeDB CLI 登录态执行。脚本默认 dry-run、每次只处理一批，绝不自动循环全表；`--apply` 才写入。先发布包含 `external_generation_job_summary` 与 cursor 索引的 SpacetimeDB 模块，在维护模式内对事故时间以前的编辑器终态任务执行小批 dry-run，例如 `npm run spacetime:external-generation:maintain -- --database --server-url --limit 5 --completed-before-micros `；核对 `matched_count`、`before_bytes`、`after_bytes` 和 `inline_media_count` 后，保持本批输入 cursor 不变并追加 `--apply` 重跑同一批，即使最后一批 `has_more = false`，只要 dry-run 仍有 `matched_count` / `selected_count` 也必须 apply；只有 apply 成功后才使用它返回的 `next_cursor_job_id` 继续。B-tree cursor 的选择阶段最多反序列化 `limit + 1` 行，apply 会再按主键逐条读取选中行但不会同时保留整批 payload；如怀疑存在单行异常巨型历史 JSON，先用 `--limit 1`。payload 压缩硬限制 `source_module = editor-canvas`；终态压缩完成后，用 `--backfill-summaries` 先 dry-run、再 `--apply` 分批补齐仍缺失的活动任务或无内联媒体历史任务摘要，直到 `has_more = false`，最后再切换使用 summary procedure 的 api-server。Stdb 构建 artifact 和完整 release 包都必须包含 `scripts/spacetime-maintain-external-generation-jobs.mjs` 与 `scripts/spacetime-migration-common.mjs`。首次上线不得让 Full Build 从 Stdb 自动直落 API：`STDB_API_ROLLOUT_MODE` 默认 fail-closed 为 `pause-after-stdb`，必须填写受限的 `STDB_API_ROLLOUT_APPROVERS`；Stdb Publish 通过 `KEEP_MAINTENANCE_MODE` 保持维护文件并停止旧 API/controller/worker，暂停点最多等待 4 小时，完成上述维护并确认无后续批次后才由指定审批人放行 API。定时构建缺少审批人时必须在发布前失败，不能静默退回 `normal`；也可分开运行 Stdb publish、维护、API deploy 三个受控 Job。任一批次都不得处理 pending / running payload；不要用 runtime writer、bootstrap secret 或匿名 identity 代替 migration operator，也不要在未核对 dry-run 时直接 apply。 +外部生成任务摘要投影与历史 payload / history 维护使用 `npm run spacetime:external-generation:maintain -- ...`，且只能由已授权 migration operator 的 SpacetimeDB CLI 登录态执行。脚本默认 dry-run、每次只处理一批，绝不自动循环全表；`--apply` 才写入。先发布包含 `external_generation_job_summary` 与 cursor 索引的 SpacetimeDB 模块，在维护模式内对事故时间以前的编辑器终态任务执行小批 dry-run，例如 `npm run spacetime:external-generation:maintain -- --database --server-url --limit 5 --completed-before-micros `；核对 `matched_count`、`before_bytes`、`after_bytes` 和 `inline_media_count` 后，保持本批输入 cursor 不变并追加 `--apply` 重跑同一批，即使最后一批 `has_more = false`，只要 dry-run 仍有 `matched_count` / `selected_count` 也必须 apply；只有 apply 成功后才使用它返回的 `next_cursor_job_id` 继续。B-tree cursor 的选择阶段最多反序列化 `limit + 1` 行，apply 会再按主键逐条读取选中行但不会同时保留整批 payload；如怀疑存在单行异常巨型历史 JSON，先用 `--limit 1`。payload 压缩硬限制 `source_module = editor-canvas`；终态压缩完成后，用 `--backfill-summaries` 先 dry-run、再 `--apply` 分批补齐仍缺失的活动任务或无内联媒体历史任务摘要，直到 `has_more = false`，最后再切换使用 summary procedure 的 api-server。历史清理使用 `--prune-history`，默认 `source_module=editor-canvas`、30 天保留期；候选必须是 completed / failed / cancelled 终态、主任务与摘要状态一致、摘要存在 `notification_acknowledged_at` 且终态时间不晚于 `completed_before_micros`，否则永不删除。先 dry-run，记下输出的 `completed_before_micros`，再保持相同 `--cursor-job-id` 与 cutoff 追加 `--apply`；apply 在一个事务中删除该 job 的所有 event、summary 和主任务，资产对象与钱包流水保留。需要清理其它 source module 时必须显式 `--source-module` 并先完成业务评估；这不是自动 systemd 任务，不得授予 runtime writer 清理权限。Stdb 构建 artifact 和完整 release 包都必须包含 `scripts/spacetime-maintain-external-generation-jobs.mjs` 与 `scripts/spacetime-migration-common.mjs`。首次上线不得让 Full Build 从 Stdb 自动直落 API：`STDB_API_ROLLOUT_MODE` 默认 fail-closed 为 `pause-after-stdb`，必须填写受限的 `STDB_API_ROLLOUT_APPROVERS`；Stdb Publish 通过 `KEEP_MAINTENANCE_MODE` 保持维护文件并停止旧 API/controller/worker，暂停点最多等待 4 小时，完成上述维护并确认无后续批次后才由指定审批人放行 API。定时构建缺少审批人时必须在发布前失败，不能静默退回 `normal`；也可分开运行 Stdb publish、维护、API deploy 三个受控 Job。任一批次都不得处理 pending / running payload；不要用 runtime writer、bootstrap secret 或匿名 identity 代替 migration operator，也不要在未核对 dry-run 时直接 apply。 角色动作正式字段收口使用 `node scripts/spacetime-normalize-editor-character-actions.mjs --database --server-url `，且同样只能由已授权 migration operator 执行。必须先发布包含 normalization cursor 索引和 `normalize_editor_character_animation_metadata_and_return` 的 SpacetimeDB 模块，在 API / worker 仍处于维护模式时先运行默认全量 dry-run；脚本固定按 `asset → project-resource → showcase → canvas` 扫描，普通 scope 每批最多 25 行，canvas 每批最多 5 行。全量 dry-run 会在不写库的情况下把 asset 计划结果投影给同 owner / task / 首帧对象精确匹配的 project-resource，再把前置 scope 的计划结果投影给 canvas 检查；因此同 task 的误标预览 MP4 会先按权威视频对象排除，最终图片序列会逐帧核对并补齐精确 `asset_object` 身份。canvas 中仍引用误标 preview resource 的普通 video layer 会按 project-resource 计划态 `video` 跳过，只有 layout 明确声明动作却指向视频，或资源规划本身失败时才形成 blocker。apply 时仍要求前置 scope 已按顺序物理完成，不能跳过 asset 直接让 project-resource 借未落库结果。历史 canvas 复制的 `sourceResourceId` 不是迁移证据，不要因它仍指向原角色而手工改库，补建资源会采用最终账号素材的 DB 血缘。出现 blocker 时脚本会打印 ID、原因、owner、project、task、对象身份和来源资源；先据此区分最终候选为零 / 多个、正式与旧版冲突、帧对象不匹配或缺失资源，不得跳过 scope。确认 dry-run 后追加 `--apply`，脚本会对每批重新 dry-run、携带该批 SHA-256 apply，并在最后从头要求四个 scope 均为零匹配、零 blocker。只有该复核通过后才发布移除 action fallback 的 API / Web。Stdb build artifact 和完整 release 包必须同时包含 `scripts/spacetime-normalize-editor-character-actions.mjs` 与 `scripts/spacetime-migration-common.mjs`。本地切换分支时若要避免 dev publish 因 schema 冲突使用 `-c=on-conflict` 清库，启动命令必须追加 `--preserve-database`，让冲突直接失败。动作视频抽帧临时目录固定使用 `/var/lib/genarrative/character-animation-tmp`，该路径已由生产 API / worker unit 放行；不要让动作抽帧重新依赖 `PrivateTmp` 下的 `/tmp`。 @@ -401,7 +401,9 @@ UI 相关修改要重点验证： ### SpacetimeDB 数据目录 OSS 备份 -数据库备份不放进 `spacetime-module` reducer / procedure：备份属于文件系统与 OSS 外部副作用，必须由运维脚本在 SpacetimeDB 宿主外执行。当前统一脚本为 `scripts/database-backup-to-oss.mjs`（npm 命令 `npm run database:backup:oss`）。默认 `--storage-format archive --mode full` 保持原有全量压缩包冷备行为；`--storage-format files` 不生成 tar.gz，而是把目录树映射成逐文件 CAS 对象与 catalog，full 重跑只上传新增或内容变化的文件，history 只处理已被最新 snapshot 完全覆盖的历史 commitlog 与旧 snapshot。`Genarrative-Server-Provision` 的 `DATABASE_BACKUP_PROFILE` 默认是 `archive-full`，继续安装每天 `03:20` 左右执行的全量冷备主 service；development 和 release 都可以显式选择 `files-history`，但指定 work-dir 必须已经有与本机 database/bucket 匹配且已发布的 full baseline state： +脚本停库前会在固定 work-dir 写入 `.spacetimedb-stopped` marker；正常 finally 恢复 SpacetimeDB 及 `--restart-service-after` 指定的 API / worker / controller 后才清理 marker。若 Node 因 `MemoryMax` / OOM 被强制终止，systemd `ExecStopPost` 会根据仍存在的 marker 兜底恢复这些服务；恢复未全部成功时保留 marker 供后续重试。 + +数据库备份不放进 `spacetime-module` reducer / procedure：备份属于文件系统与 OSS 外部副作用，必须由运维脚本在 SpacetimeDB 宿主外执行。当前统一脚本为 `scripts/database-backup-to-oss.mjs`（npm 命令 `npm run database:backup:oss`）。默认 `--storage-format archive --mode full` 保持原有全量压缩包冷备行为；`--storage-format files` 不生成 tar.gz，而是把目录树映射成逐文件 CAS 对象与 catalog，full 重跑只上传新增或内容变化的文件，history 只处理已被最新 snapshot 完全覆盖的历史 commitlog 与旧 snapshot。`Genarrative-Server-Provision` 的 `DATABASE_BACKUP_PROFILE` 默认是 `archive-full`，继续安装每天 `03:20` 左右执行的全量冷备主 service；当前 release 只允许 `archive-full`，避免 `files-history` 在大目录上构造全量 catalog 导致 Node 内存峰值；development 才可以显式选择 `files-history`，且指定 work-dir 必须已经有与本机 database/bucket 匹配且已发布的 full baseline state： ```bash npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service @@ -437,7 +439,7 @@ GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET= `GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET` 为空时会回退 `ALIYUN_OSS_BUCKET`；AccessKey 默认复用 `ALIYUN_OSS_ACCESS_KEY_ID` / `ALIYUN_OSS_ACCESS_KEY_SECRET`，也可用 `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID` / `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET` 为备份 bucket 单独配置最小权限账号。冷备脚本会在停止 SpacetimeDB 前检查 `GENARRATIVE_DATABASE_BACKUP_WORK_DIR` 所在文件系统剩余空间；未设置 `GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES` 时，按数据目录大小加安全余量估算，空间不足会在停库前失败，避免写满根分区。即使打包或上传前步骤失败，只要脚本已经停过 SpacetimeDB，也会先恢复 SpacetimeDB 并执行 `--restart-service-after` 指定的 API / worker / controller，再带着原始备份错误退出。`Genarrative-Server-Provision` 会创建 `/var/lib/genarrative/database-backups` 并归属 `genarrative:genarrative`，同时安装并启用 `genarrative-database-backup.timer`。手动检查定时器：`systemctl list-timers genarrative-database-backup.timer`；手动触发一次：`systemctl start genarrative-database-backup.service`。如果 timer 显示 `enabled` 但 `inactive/dead` 且 `NEXT` / `Trigger` 为空，先写入当前 stamp 避免 `Persistent=true` 在白天立刻补跑冷备份：`touch /var/lib/systemd/timers/stamp-genarrative-database-backup.timer && systemctl daemon-reload && systemctl start genarrative-database-backup.timer`，随后确认下一次触发时间约为次日 `03:20`。 -`files-history` 使用仓库模板 `deploy/systemd/genarrative-database-backup-files-history.conf` 覆盖主 service 的 `ExecStart`，从 `/etc/genarrative/api-server.env` 读取 data-dir、database、bucket、prefix 与 OSS 凭据，不在 unit 写死环境目标，也不传 `--stop-service`。Server-Provision 在改动 drop-in 前，先用 current release 的同一脚本、同一 env 和 `DATABASE_BACKUP_FILES_HISTORY_WORK_DIR` 执行一次 history `--dry-run`；缺少已发布 full catalog 的 files state、current 脚本过旧或配置不匹配都会在安装 drop-in 和 `daemon-reload` 前失败。选择 `archive-full` 会主动删除仓库托管的 `10-files-history.conf` 与 dev 试点遗留的 `10-dev-files.conf`，防止 systemd 继续合并旧覆盖。dev 可继续指定已有 `/var/lib/genarrative/database-backups/dev-files`，release 建议先在 `/var/lib/genarrative/database-backups/release-files` 建立自己的 full baseline；两台机器不得复用或互传本地 state 目录冒充本机基线。启用时通过 Server-Provision Job 选择目标、`DATABASE_BACKUP_PROFILE=files-history` 和对应 work-dir，先保持 `DRY_RUN=true` 核对，再以同参数正式 provision。不要直接在 `/etc/systemd/system` 手写第二份 drop-in。 +`files-history` 使用仓库模板 `deploy/systemd/genarrative-database-backup-files-history.conf` 覆盖主 service 的 `ExecStart`，从 `/etc/genarrative/api-server.env` 读取 data-dir、database、bucket、prefix 与 OSS 凭据，不在 unit 写死环境目标，也不传 `--stop-service`。Server-Provision 在 development 改动 drop-in 前，先用 current release 的同一脚本、同一 env 和 `DATABASE_BACKUP_FILES_HISTORY_WORK_DIR` 执行一次 history `--dry-run`；缺少已发布 full catalog 的 files state、current 脚本过旧或配置不匹配都会在安装 drop-in 和 `daemon-reload` 前失败。选择 `archive-full` 会主动删除仓库托管的 `10-files-history.conf` 与 dev 试点遗留的 `10-dev-files.conf`，防止 systemd 继续合并旧覆盖。`genarrative-database-backup.service` 还通过 `NODE_OPTIONS=--max-old-space-size=768`、`MemoryHigh=768M`、`MemoryMax=1G` 和 `OOMPolicy=stop` 给备份进程设置独立护栏；release 若现场残留 files-history drop-in，必须先按 archive-full 重新 provision 并确认 drop-in 已删除，再恢复定时器。dev 可继续指定已有 `/var/lib/genarrative/database-backups/dev-files`；两台机器不得复用或互传本地 state 目录冒充本机基线。启用时通过 Server-Provision Job 选择目标、`DATABASE_BACKUP_PROFILE=files-history` 和对应 work-dir，先保持 `DRY_RUN=true` 核对，再以同参数正式 provision。不要直接在 `/etc/systemd/system` 手写第二份 drop-in。 files full 会递归扫描 data-dir，保留空目录、每个普通文件的相对路径，以及目标仍位于 data-dir 内部的相对符号链接；绝对链接或解析后越界的链接直接拒绝。文件按 SHA-256 上传到不可变对象 key，catalog 记录目录、路径、长度、SHA、对象 key 和相对链接目标，不写 staging 主机的绝对路径。相同 catalog 重跑不重复 PUT；新增或变化文件先 HEAD CAS 对象，存在且长度/SHA 元数据一致就复用，否则上传。16 MiB 及以下对象使用单次 PUT 后 HEAD 验真，大对象继续使用 multipart；对象操作默认以 16 路并行执行，可用 `GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=1..64` 调整。需要给线上入口留带宽时设置 `GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND=`，该共享限速器只包裹备份上传流，空值或 `0` 表示不限速，不修改主机全局 qdisc。并发、限速和单次 PUT 都不改变“全部对象、catalog 与 latest pointer 成功后才推进 state/清理”的顺序。full 基线必须来自停库后的 data-dir 或已通过恢复验证的冻结副本；源文件上传前后 stat 虽会复核，但在线扫描不能保证大量文件属于同一跨文件一致时点。catalog 验真后，脚本把最新 full/history 引用发布到固定 `//latest.json`，全新机器不需要本地 state 即可自动发现恢复入口。 @@ -478,7 +480,7 @@ node -- scripts/database-backup-to-oss.mjs \ dev 出口过慢时，可以把冻结基线经内网 rsync 到 release 独立 staging，再由 release 上传 dev bucket。staging 必须位于 `/var/lib/genarrative/dev-database-backup-staging/` 一类隔离目录，命令显式传 staging `--data-dir`、独立 `--work-dir`、dev `--bucket`，且不得传 `--stop-service`；禁止指向或修改 release `/stdb`。中转 key 只为本次传输临时授权，结束后从 dev 私钥和 release `authorized_keys` 同时移除。上传完成后把整个 files work-dir/state 回传 dev，history 才能延续同一 baseline catalog。 -完整恢复默认从 OSS 固定 `latest.json` 读取最新 full catalog：先创建 `directories`，再把每个 `files[].objectKey` 下载到 `/` 并逐项核对 `sizeBytes` / `sha256`；history catalog 用于证明已清理历史仍有 OSS 对象，不需要把已被 full baseline 覆盖的旧文件叠回当前恢复目录。本地 state 仍可作为兼容入口，并同时支持旧 v1 JSON 与 v2 gzip，但不再是异机恢复的前置条件。随后用隔离 data-dir 启动同版本 standalone，验证 `/v1/ping`、日志中的 snapshot restore / commitlog replay / module launch、代表性 SQL 和 reducer。dev 已完成这轮 OSS-only 异机恢复与重启演练；release 已使用独立 `/var/lib/genarrative/database-backups/release-files` full baseline 和 `files-history` profile，现场最终 `ExecStart`、timer 状态与最近备份结果仍须在变更时重新核对。 +完整恢复默认从 OSS 固定 `latest.json` 读取最新 full catalog：先创建 `directories`，再把每个 `files[].objectKey` 下载到 `/` 并逐项核对 `sizeBytes` / `sha256`；history catalog 用于证明已清理历史仍有 OSS 对象，不需要把已被 full baseline 覆盖的旧文件叠回当前恢复目录。本地 state 仍可作为兼容入口，并同时支持旧 v1 JSON 与 v2 gzip，但不再是异机恢复的前置条件。随后用隔离 data-dir 启动同版本 standalone，验证 `/v1/ping`、日志中的 snapshot restore / commitlog replay / module launch、代表性 SQL 和 reducer。dev 已完成这轮 OSS-only 异机恢复与重启演练；release 使用 archive-full 时，现场最终 `ExecStart`、timer 状态与最近备份结果仍须在变更时重新核对。 ```bash node -- scripts/database-backup-to-oss.mjs \ @@ -777,7 +779,7 @@ node scripts/test-ve-llm.mjs ### 手机验证码短信 -手机验证码发送走阿里云普通短信 `SendSms`，验证码由 `module-auth` 在当前 `api-server` 进程内生成、哈希存储和校验，不再调用阿里云托管验证码的 `SendSmsVerifyCode` / `CheckSmsVerifyCode`。因此 `api-server` 重启后，已发送但未校验的验证码会失效。 +手机验证码发送走阿里云普通短信 `SendSms`，验证码由 `module-auth` 在当前 `api-server` 进程内生成并哈希，短期验证码投影随 `auth_store_projection_meta` 同步到 SpacetimeDB 后由任一 API 节点恢复和校验；不再调用阿里云托管验证码的 `SendSmsVerifyCode` / `CheckSmsVerifyCode`。因此只要 SpacetimeDB 正常，`api-server` 重启不会使已发送但未过期的验证码失效。 生产默认短信配置： @@ -853,7 +855,9 @@ GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES=268435456 GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS=5000 ``` -outbox 采用 NDJSON 文件保存原始事件。达到 `BATCH_SIZE` 时会立刻把当前 active 文件原子封存为 sealed 文件，并马上切到新的 active 继续写入；后台 worker 异步 flush sealed 文件，HTTP 请求线程不等待 SpacetimeDB。`FLUSH_INTERVAL_MS` 只负责兜底封存长时间未满批的 active 文件。SpacetimeDB 批量 procedure 返回成功后删除 sealed 文件，失败则保留文件并重试。`MAX_BYTES` 是每个 outbox 实例的磁盘保护阈值，不是 flush 阈值；超过后低价值 route tracking 和 BgFilter provider 失败审计可以被丢弃并记录日志 / 指标，关键同步事件不进入该丢弃路径。api-server 使用配置目录本身，BgFilter worker 固定使用其 `bgfilter-worker/` 子目录，两个进程不得操作同一个 active 文件。sealed 文件若出现无法解析的坏行，会重命名为 `corrupt-*` 隔离并记录 `genarrative.tracking_outbox.files.corrupt` 指标，避免一个坏文件阻塞后续批量入库。进程收到退出信号后会在 `GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS` 窗口内封存各自 active 文件并尽力 flush sealed 文件，超时或 SpacetimeDB 暂不可用时保留本地文件给下次同角色启动继续投递。该机制对已 enqueue 记录提供至少一次投递语义，依赖 `tracking_event.event_id` 幂等跳过重复事件；BgFilter 尚未 enqueue 或因硬上限 / 保护阈值被丢弃的审计不在该保证内。 +outbox 采用 NDJSON 文件保存原始事件。达到 `BATCH_SIZE` 时会立刻把当前 active 文件原子封存为 sealed 文件，并马上切到新的 active 继续写入；后台 worker 异步 flush sealed 文件，HTTP 请求线程不等待 SpacetimeDB。worker 启动时会先封存并 flush 已存在的 active / sealed 文件，恢复窗口内 SpacetimeDB 暂不可用则保留文件并按后续周期重试；`FLUSH_INTERVAL_MS` 只负责兜底封存长时间未满批的 active 文件。SpacetimeDB 批量 procedure 返回成功后删除 sealed 文件，失败则保留文件并重试。`MAX_BYTES` 是每个 outbox 实例的磁盘保护阈值，不是 flush 阈值；超过后低价值 route tracking 和 BgFilter provider 失败审计可以被丢弃并记录日志 / 指标，关键同步事件不进入该丢弃路径。api-server 使用配置目录本身，BgFilter worker 固定使用其 `bgfilter-worker/` 子目录，两个进程不得操作同一个 active 文件。sealed 文件若出现无法解析的坏行，会重命名为 `corrupt-*` 隔离并记录 `genarrative.tracking_outbox.files.corrupt` 指标，避免一个坏文件阻塞后续批量入库。进程收到退出信号后会在 `GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS` 窗口内封存各自 active 文件并尽力 flush sealed 文件，超时或 SpacetimeDB 暂不可用时保留本地文件给下次同角色启动继续投递。该机制对已 enqueue 记录提供至少一次投递语义，依赖 `tracking_event.event_id` 幂等跳过重复事件；BgFilter 尚未 enqueue 或因硬上限 / 保护阈值被丢弃的审计不在该保证内。 + +钱包退款正式 pending 队列在 SpacetimeDB 的 `profile_wallet_refund_outbox` 表中，由每个 API 节点的 worker 共同处理；worker 启动即扫描库内 pending 行，成功在同一事务内写钱包账本并删除 outbox 行，失败按库内 `available_at` / `attempts` 重试。只有 SpacetimeDB 完全不可达时才写本机 `wallet-refund-outbox` emergency spool；如果进程在“临时文件写完但尚未改名”阶段崩溃，启动恢复会校验 `tmp-*` 内容并原子提升为按 ledger id 命名的 pending 文件，损坏或冲突文件移入 `corrupt-*` 隔离目录。达到 `MAX_BYTES` 时不再静默丢弃退款，而是写入同一持久目录下的 `refund-overflow-*` 溢出文件并继续重放；溢出文件不计入普通容量阈值，但必须接入容量告警和人工补偿预案，底层磁盘写入失败仍按关键退款告警处理。worker 连接失败、库内 retry、emergency spool 写入 / 容量失败和 `corrupt-*` 出现都必须接入告警；人工补偿先按 refund ledger id 对账 `profile_wallet_ledger`、`asset_operation_wallet_settlement` 与两类 outbox，再通过受控退款 procedure 幂等重放，禁止直接手写钱包表。该目录不能替代库内 outbox；发布和主机替换必须保留 `/var/lib/genarrative/wallet-refund-outbox` 并纳入节点恢复 / 备份演练。容器 loadtest / 预览环境必须分别为 `api-server` 与 `external-generation-worker` 挂载各自的 tracking 与 wallet refund 命名卷，不能让节点重建清空本机恢复队列。 release 机器如果日志每秒刷 `tracking outbox ... Permission denied (os error 13)`，先检查 `/etc/genarrative/api-server.env` 是否缺少 `GENARRATIVE_TRACKING_OUTBOX_DIR`。缺少时 `api-server` 会回退到本地开发默认相对路径 `server-rs/.data/tracking-outbox`，而 systemd 的工作目录是只读发布目录 `/opt/genarrative/releases/`，`genarrative` 用户无法在其中创建 `server-rs`。修复顺序： @@ -864,7 +868,9 @@ systemctl restart genarrative-api.service journalctl -u genarrative-api.service --since '30 seconds ago' --no-pager | grep -E 'tracking outbox|Permission denied|os error 13' ``` -`Genarrative-Server-Provision` 和 `Genarrative-Api-Deploy` 会在保留旧 `/etc/genarrative/api-server.env` 的前提下补齐缺失的 tracking outbox 运行态路径，并确保 `/var/lib/genarrative/tracking-outbox` 归属 `genarrative:genarrative`。用户认证真相源只允许在 SpacetimeDB 正式认证表（`user_account` / `auth_identity` / `refresh_session`）恢复；不要再配置或依赖 `GENARRATIVE_AUTH_STORE_PATH` / `auth-store.json`，`module-auth` 也不再维护本地文件持久化；`auth_store_snapshot` 不再作为备查或运行期恢复源，只在正式认证表为空时一次性转移最新旧快照并清空，且旧 `get_auth_store_snapshot` / `upsert_auth_store_snapshot` / `import_auth_store_snapshot` 入口已经删除。如果 `api-server` 启动时连不上 SpacetimeDB，会持续重试启动恢复，直到认证工作集从 SpacetimeDB 正式表恢复成功后才开始监听 HTTP，以避免用空本地状态或旧快照覆盖认证表。 +`Genarrative-Server-Provision` 和 `Genarrative-Api-Deploy` 会在保留旧 `/etc/genarrative/api-server.env` 的前提下补齐缺失的 tracking outbox 运行态路径，并确保 `/var/lib/genarrative/tracking-outbox` 归属 `genarrative:genarrative`。用户认证真相源只允许从 SpacetimeDB 正式认证表（`user_account` / `auth_identity` / `refresh_session`）和 `auth_store_projection_meta` 中的短期验证码 / 微信 state 投影恢复；所有会读取或变更本机认证工作集的认证主链路在领域操作前都会从正式投影做一次只读刷新，刷新失败即 fail closed，不依赖粘性会话。不要再配置或依赖 `GENARRATIVE_AUTH_STORE_PATH` / `auth-store.json`，`module-auth` 也不再维护本地文件持久化；`auth_store_snapshot` 不再作为备查或运行期恢复源，只在正式认证表为空时一次性转移最新旧快照并清空，且旧 `get_auth_store_snapshot` / `upsert_auth_store_snapshot` / `import_auth_store_snapshot` 入口已经删除。所有 API 节点必须使用相同的 `GENARRATIVE_JWT_SECRET`，它也作为验证码哈希盐；轮换后尚未消费的验证码会失效。如果 `api-server` 启动时连不上 SpacetimeDB，会持续重试启动恢复，直到认证工作集从 SpacetimeDB 正式表和短期投影恢复成功后才开始监听 HTTP，以避免用空本地状态或旧快照覆盖认证表。 + +发短信运维门禁：handler 会先刷新正式认证投影，再通过 projection CAS 写入不可消费的占位验证码来占用跨节点冷却窗口；占用失败时不得调用短信 provider。微信 OAuth state 活动数量有上限，命中上限应返回服务错误并触发限流 / 入口告警；不要通过调大单个 `auth_store_projection_meta` JSON 字段来绕过该保护。 前端登录态恢复只把 `/api/auth/refresh` 的 `401` / `403` 当成权威失效信号；服务器重启窗口里的 `502` / `503` / `504`、浏览器 `Failed to fetch` 或 refresh 响应契约异常都必须保留已有本地 access token，不触发全局 auth 变化。refresh 成功响应以共享契约 `RefreshSessionResponse { token }` 为准，前端不要额外要求业务 `ok` 字段。排查“重启后用户都掉线”时，先区分前端是否被暂时不可用清掉本地 token，再检查 SpacetimeDB 正式认证表是否缺 `user_account` / `refresh_session` 数据。 diff --git a/jenkins/Jenkinsfile.production-server-provision b/jenkins/Jenkinsfile.production-server-provision index f0233318b..d2104e68c 100644 --- a/jenkins/Jenkinsfile.production-server-provision +++ b/jenkins/Jenkinsfile.production-server-provision @@ -34,8 +34,8 @@ pipeline { string(name: 'WEB_LINK', defaultValue: '/srv/genarrative/web', description: 'Nginx 静态站点目录或软链接') string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'api-server 环境文件') string(name: 'API_PORT', defaultValue: '8082', description: 'api-server 本机监听端口') - choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profile；默认 archive-full，files-history 仅在指定 work-dir 已有完整 full baseline 后启用') - string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'files-history 的本地 state/catalog 目录；dev/release 必须使用各自已建立 full baseline 的独立目录') + choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profile；release 仅允许 archive-full，files-history 仅供 development 在指定 work-dir 已有完整 full baseline 后启用') + string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'development files-history 的本地 state/catalog 目录；必须使用已建立 full baseline 的独立目录') choice(name: 'NGINX_CONFIG_MODE', choices: ['none', 'production-https', 'development-http'], description: 'Nginx 配置模式；开发服无域名时选 development-http，release 正式入口选 production-https') booleanParam(name: 'ENABLE_SERVICES', defaultValue: true, description: '启用并启动 spacetimedb 与 api-server systemd 服务') booleanParam(name: 'ENABLE_OTELCOL', defaultValue: true, description: '安装并启用本机 OpenTelemetry Collector；api-server 模板默认开启 OTLP，如需关闭请在 API_ENV_FILE 中将 GENARRATIVE_OTEL_ENABLED 改为 false') @@ -113,6 +113,9 @@ pipeline { if (!(databaseBackupProfile in ['archive-full', 'files-history'])) { error("DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${params.DATABASE_BACKUP_PROFILE}") } + if (params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history') { + error('release 仅允许 archive-full；files-history 会把整棵历史目录加载到 Node 内存，需先完成流式 catalog 改造后才能重新启用。') + } def databaseBackupFilesHistoryWorkDir = params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR?.trim() if (!(databaseBackupFilesHistoryWorkDir ==~ /^\/var\/lib\/genarrative\/database-backups\/[A-Za-z0-9._\/-]+$/) || databaseBackupFilesHistoryWorkDir.contains('..')) { error("DATABASE_BACKUP_FILES_HISTORY_WORK_DIR 必须是 /var/lib/genarrative/database-backups/ 下不含连续点号的绝对路径，当前值: ${params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}") diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index 57bc09368..b42f5dbfc 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -1,12 +1,24 @@ #!/usr/bin/env node -import {spawnSync} from 'node:child_process'; -import {createHash} from 'node:crypto'; -import {chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, statSync, symlinkSync, writeFileSync} from 'node:fs'; -import {tmpdir} from 'node:os'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readlinkSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; import path from 'node:path'; -import {Readable} from 'node:stream'; -import {gunzipSync, gzipSync} from 'node:zlib'; +import { Readable } from 'node:stream'; +import { gunzipSync, gzipSync } from 'node:zlib'; import { buildAuthorization, @@ -27,13 +39,15 @@ import { } from './database-backup-to-oss.mjs'; const BACKUP_SCRIPT = path.resolve('scripts/database-backup-to-oss.mjs'); -const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-database-backup-check-')); +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-database-backup-check-'), +); const failures = []; try { await main(); } finally { - rmSync(tmpRoot, {recursive: true, force: true}); + rmSync(tmpRoot, { recursive: true, force: true }); } if (failures.length > 0) { @@ -50,6 +64,7 @@ async function main() { assertDeferredArchiveDiscoveryIsBoundedAndDeterministic(); assertCanonicalQueryAndAuthorizationIncludeMultipartParameters(); assertInsufficientSpaceStopsBeforeServiceChanges(); + assertStopFailureRetainsRecoveryMarker(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); await assertUploadBandwidthLimiterSharesBudgetAndPropagatesErrors(); @@ -77,80 +92,149 @@ async function main() { function assertDeferredArchiveDiscoveryIsBoundedAndDeterministic() { const root = path.join(tmpRoot, 'deferred-archive-discovery'); - mkdirSync(root, {recursive: true}); - const createCandidate = ({name, status, database = 'test-db', withArchive = true}) => { + mkdirSync(root, { recursive: true }); + const createCandidate = ({ + name, + status, + database = 'test-db', + withArchive = true, + }) => { const archivePath = path.join(root, `${name}.tar.gz`); const manifestPath = `${archivePath}.manifest.json`; if (withArchive) { writeFileSync(archivePath, name); } - writeFileSync(manifestPath, `${JSON.stringify({ - backupKind: 'spacetimedb-data-dir', - database, - archivePath, - uploadStatus: status, - })}\n`); - return {archivePath, manifestPath}; + writeFileSync( + manifestPath, + `${JSON.stringify({ + backupKind: 'spacetimedb-data-dir', + database, + archivePath, + uploadStatus: status, + })}\n`, + ); + return { archivePath, manifestPath }; }; - const later = createCandidate({name: 'test-db-20260731T020000Z', status: 'pending'}); - const earlier = createCandidate({name: 'test-db-20260731T010000Z', status: 'deferred'}); - const uploaded = createCandidate({name: 'test-db-20260731T000000Z', status: 'uploaded'}); - createCandidate({name: 'other-db-20260731T000000Z', status: 'deferred', database: 'other-db'}); - const missing = createCandidate({name: 'test-db-20260730T230000Z', status: 'deferred', withArchive: false}); + const later = createCandidate({ + name: 'test-db-20260731T020000Z', + status: 'pending', + }); + const earlier = createCandidate({ + name: 'test-db-20260731T010000Z', + status: 'deferred', + }); + const uploaded = createCandidate({ + name: 'test-db-20260731T000000Z', + status: 'uploaded', + }); + createCandidate({ + name: 'other-db-20260731T000000Z', + status: 'deferred', + database: 'other-db', + }); + const missing = createCandidate({ + name: 'test-db-20260730T230000Z', + status: 'deferred', + withArchive: false, + }); - const result = discoverDeferredArchiveUploads({workDir: root, database: 'test-db'}); + const result = discoverDeferredArchiveUploads({ + workDir: root, + database: 'test-db', + }); assertEqual( - result.archives.map(({archivePath}) => archivePath).join(','), + result.archives.map(({ archivePath }) => archivePath).join(','), [earlier.archivePath, later.archivePath].join(','), 'deferred/pending 扫描必须只返回同库现存归档，并按文件名稳定排序。', ); - assertEqual(result.missingArchives.length, 1, '缺失归档的 deferred 清单必须单独报告。'); - assertEqual(result.missingArchives[0].manifestPath, missing.manifestPath, '缺失归档报告必须保留精确 manifest。'); - const cleanupResult = discoverDeferredArchiveUploads({workDir: root, database: 'test-db', includeUploaded: true}); assertEqual( - cleanupResult.archives.map(({archivePath}) => archivePath).join(','), + result.missingArchives.length, + 1, + '缺失归档的 deferred 清单必须单独报告。', + ); + assertEqual( + result.missingArchives[0].manifestPath, + missing.manifestPath, + '缺失归档报告必须保留精确 manifest。', + ); + const cleanupResult = discoverDeferredArchiveUploads({ + workDir: root, + database: 'test-db', + includeUploaded: true, + }); + assertEqual( + cleanupResult.archives.map(({ archivePath }) => archivePath).join(','), [uploaded.archivePath, earlier.archivePath, later.archivePath].join(','), '未要求保留本地归档时，补偿扫描必须同时收敛上传后未清理的本地归档。', ); - const cliDryRun = spawnSync(process.execPath, [ - BACKUP_SCRIPT, - '--upload-deferred-dir', root, - '--database', 'test-db', - '--bucket', 'test-bucket', - '--endpoint', 'oss-cn-shanghai.aliyuncs.com', - '--access-key-id', 'test-id', - '--access-key-secret', 'test-secret', - '--keep-local', - '--dry-run', - ], {encoding: 'utf8'}); - assertStatus(cliDryRun, 0, 'deferred 补偿扫描 dry-run 必须可通过统一 CLI 入口执行。'); - assertIncludes(cliDryRun.stdout, 'count=2', 'deferred 补偿扫描 CLI 必须报告待处理归档数量。'); - assertTrue(existsSync(earlier.archivePath) && existsSync(later.archivePath), 'dry-run 不得删除 deferred 本地归档。'); + const cliDryRun = spawnSync( + process.execPath, + [ + BACKUP_SCRIPT, + '--upload-deferred-dir', + root, + '--database', + 'test-db', + '--bucket', + 'test-bucket', + '--endpoint', + 'oss-cn-shanghai.aliyuncs.com', + '--access-key-id', + 'test-id', + '--access-key-secret', + 'test-secret', + '--keep-local', + '--dry-run', + ], + { encoding: 'utf8' }, + ); + assertStatus( + cliDryRun, + 0, + 'deferred 补偿扫描 dry-run 必须可通过统一 CLI 入口执行。', + ); + assertIncludes( + cliDryRun.stdout, + 'count=2', + 'deferred 补偿扫描 CLI 必须报告待处理归档数量。', + ); + assertTrue( + existsSync(earlier.archivePath) && existsSync(later.archivePath), + 'dry-run 不得删除 deferred 本地归档。', + ); const unsafeRoot = path.join(tmpRoot, 'deferred-archive-unsafe'); - mkdirSync(unsafeRoot, {recursive: true}); + mkdirSync(unsafeRoot, { recursive: true }); const escapedArchive = path.join(tmpRoot, 'outside.tar.gz'); writeFileSync(escapedArchive, 'outside'); writeFileSync( path.join(unsafeRoot, 'test-db-unsafe.tar.gz.manifest.json'), - `${JSON.stringify({database: 'test-db', archivePath: escapedArchive, uploadStatus: 'deferred'})}\n`, + `${JSON.stringify({ database: 'test-db', archivePath: escapedArchive, uploadStatus: 'deferred' })}\n`, ); assertThrows( - () => discoverDeferredArchiveUploads({workDir: unsafeRoot, database: 'test-db'}), + () => + discoverDeferredArchiveUploads({ + workDir: unsafeRoot, + database: 'test-db', + }), '路径与清单不匹配', 'deferred 扫描必须拒绝目录外归档或 manifest 名不匹配。', ); const symlinkRoot = path.join(tmpRoot, 'deferred-archive-symlink'); - mkdirSync(symlinkRoot, {recursive: true}); + mkdirSync(symlinkRoot, { recursive: true }); const symlinkArchive = path.join(symlinkRoot, 'test-db-symlink.tar.gz'); symlinkSync(escapedArchive, symlinkArchive); writeFileSync( `${symlinkArchive}.manifest.json`, - `${JSON.stringify({database: 'test-db', archivePath: symlinkArchive, uploadStatus: 'deferred'})}\n`, + `${JSON.stringify({ database: 'test-db', archivePath: symlinkArchive, uploadStatus: 'deferred' })}\n`, ); assertThrows( - () => discoverDeferredArchiveUploads({workDir: symlinkRoot, database: 'test-db'}), + () => + discoverDeferredArchiveUploads({ + workDir: symlinkRoot, + database: 'test-db', + }), '非符号链接的普通文件', 'deferred 扫描必须拒绝符号链接归档。', ); @@ -164,22 +248,36 @@ function createDirectOssHarness() { const objects = new Map(); const uploadedKeys = []; const verifiedKeys = []; - const uploadFn = async ({archivePath, objectKey, archiveSha256}) => { + const uploadFn = async ({ archivePath, objectKey, archiveSha256 }) => { const body = readFileSync(archivePath); const sha256 = createHash('sha256').update(body).digest('hex'); - assertEqual(sha256, archiveSha256, `direct file ${objectKey} 的上传 SHA 必须来自实际内容。`); - objects.set(objectKey, {body, contentLength: body.length, sha256}); + assertEqual( + sha256, + archiveSha256, + `direct file ${objectKey} 的上传 SHA 必须来自实际内容。`, + ); + objects.set(objectKey, { body, contentLength: body.length, sha256 }); uploadedKeys.push(objectKey); - return {objectKey, contentLength: body.length, archiveSha256: sha256, verifiedAt: '2026-07-16T01:00:00.000Z'}; + return { + objectKey, + contentLength: body.length, + archiveSha256: sha256, + verifiedAt: '2026-07-16T01:00:00.000Z', + }; }; - const uploadManifestFn = async ({manifestPath, objectKey}) => { + const uploadManifestFn = async ({ manifestPath, objectKey }) => { const body = readFileSync(manifestPath); const sha256 = createHash('sha256').update(body).digest('hex'); - objects.set(objectKey, {body, contentLength: body.length, sha256}); + objects.set(objectKey, { body, contentLength: body.length, sha256 }); uploadedKeys.push(objectKey); - return {objectKey, contentLength: body.length, archiveSha256: sha256, verifiedAt: '2026-07-16T01:00:01.000Z'}; + return { + objectKey, + contentLength: body.length, + archiveSha256: sha256, + verifiedAt: '2026-07-16T01:00:01.000Z', + }; }; - const verifyFn = async ({objectKey, contentLength, archiveSha256}) => { + const verifyFn = async ({ objectKey, contentLength, archiveSha256 }) => { verifiedKeys.push(objectKey); const object = objects.get(objectKey); if (!object) { @@ -187,12 +285,22 @@ function createDirectOssHarness() { error.status = 404; throw error; } - if (object.contentLength !== contentLength || object.sha256 !== archiveSha256) { + if ( + object.contentLength !== contentLength || + object.sha256 !== archiveSha256 + ) { throw new Error(`mismatch ${objectKey}`); } - return {verifiedAt: '2026-07-16T01:00:02.000Z'}; + return { verifiedAt: '2026-07-16T01:00:02.000Z' }; + }; + return { + objects, + uploadedKeys, + verifiedKeys, + uploadFn, + uploadManifestFn, + verifyFn, }; - return {objects, uploadedKeys, verifiedKeys, uploadFn, uploadManifestFn, verifyFn}; } async function assertDirectSmallFileUsesSinglePut() { @@ -208,13 +316,19 @@ async function assertDirectSmallFileUsesSinglePut() { for await (const chunk of options.body) { uploadedBytes += chunk.length; } - return new Response('', {status: 200, headers: {etag: '"single-etag"'}}); + return new Response('', { + status: 200, + headers: { etag: '"single-etag"' }, + }); } if (options.method === 'HEAD') { - return new Response(null, {status: 200, headers: { - 'content-length': String(body.length), - 'x-oss-meta-file-sha256': sha256, - }}); + return new Response(null, { + status: 200, + headers: { + 'content-length': String(body.length), + 'x-oss-meta-file-sha256': sha256, + }, + }); } throw new Error(`unexpected method ${options.method}`); }; @@ -230,12 +344,20 @@ async function assertDirectSmallFileUsesSinglePut() { bandwidthLimiter: createUploadBandwidthLimiter(64 * 1024), }); assertEqual(result.uploadMode, 'single', '小型逐文件对象必须使用单次 PUT。'); - assertEqual(methods.join(','), 'PUT,HEAD', '小型逐文件对象只能执行 PUT 后 HEAD 验真，不得进入 multipart。'); + assertEqual( + methods.join(','), + 'PUT,HEAD', + '小型逐文件对象只能执行 PUT 后 HEAD 验真，不得进入 multipart。', + ); assertEqual(uploadedBytes, body.length, '逐文件带宽限制流不得丢失上传内容。'); } async function assertUploadBandwidthLimiterSharesBudgetAndPropagatesErrors() { - assertEqual(createUploadBandwidthLimiter('0'), null, '上传带宽限制为 0 时必须关闭。'); + assertEqual( + createUploadBandwidthLimiter('0'), + null, + '上传带宽限制为 0 时必须关闭。', + ); assertThrows( () => createUploadBandwidthLimiter('1023'), '必须为空、0 或 >= 1024 的整数', @@ -256,120 +378,265 @@ async function assertUploadBandwidthLimiterSharesBudgetAndPropagatesErrors() { return totalBytes; }; const [firstBytes, secondBytes] = await Promise.all([ - consume(limiter.wrap(Readable.from([Buffer.alloc(1024), Buffer.alloc(1024)], {objectMode: false}))), - consume(limiter.wrap(Readable.from([Buffer.alloc(1024), Buffer.alloc(1024)], {objectMode: false}))), + consume( + limiter.wrap( + Readable.from([Buffer.alloc(1024), Buffer.alloc(1024)], { + objectMode: false, + }), + ), + ), + consume( + limiter.wrap( + Readable.from([Buffer.alloc(1024), Buffer.alloc(1024)], { + objectMode: false, + }), + ), + ), ]); - assertEqual(firstBytes + secondBytes, 4096, '共享上传限速器不得丢失并发流内容。'); - assertEqual(delays.join(','), '1000,2000,3000,4000', '两个并发上传流必须共享同一个累计带宽预算。'); + assertEqual( + firstBytes + secondBytes, + 4096, + '共享上传限速器不得丢失并发流内容。', + ); + assertEqual( + delays.join(','), + '1000,2000,3000,4000', + '两个并发上传流必须共享同一个累计带宽预算。', + ); let sourceError = null; try { - await consume(limiter.wrap(Readable.from((async function* failingSource() { - yield Buffer.alloc(1); - throw new Error('source-read-failed'); - })(), {objectMode: false}))); + await consume( + limiter.wrap( + Readable.from( + (async function* failingSource() { + yield Buffer.alloc(1); + throw new Error('source-read-failed'); + })(), + { objectMode: false }, + ), + ), + ); } catch (error) { sourceError = error; } - assertIncludes(sourceError?.message, 'source-read-failed', '限速流必须向上传请求透传源读取错误。'); + assertIncludes( + sourceError?.message, + 'source-read-failed', + '限速流必须向上传请求透传源读取错误。', + ); } async function assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload() { const root = path.join(tmpRoot, 'direct-files-incremental'); const dataDir = path.join(root, 'stdb'); const workDir = path.join(root, 'work'); - mkdirSync(path.join(dataDir, 'replicas', '1', 'snapshots', '00000000000000000010.snapshot_dir', 'objects'), {recursive: true}); - mkdirSync(path.join(dataDir, 'empty-directory'), {recursive: true}); - mkdirSync(path.join(dataDir, 'bin', '2.6.0'), {recursive: true}); + mkdirSync( + path.join( + dataDir, + 'replicas', + '1', + 'snapshots', + '00000000000000000010.snapshot_dir', + 'objects', + ), + { recursive: true }, + ); + mkdirSync(path.join(dataDir, 'empty-directory'), { recursive: true }); + mkdirSync(path.join(dataDir, 'bin', '2.6.0'), { recursive: true }); symlinkSync('2.6.0', path.join(dataDir, 'bin', 'current')); writeFileSync(path.join(dataDir, 'control-db'), 'control'); writeFileSync( - path.join(dataDir, 'replicas', '1', 'snapshots', '00000000000000000010.snapshot_dir', 'objects', 'object.bin'), + path.join( + dataDir, + 'replicas', + '1', + 'snapshots', + '00000000000000000010.snapshot_dir', + 'objects', + 'object.bin', + ), 'snapshot object', ); const harness = createDirectOssHarness(); const options = { - mode: 'full', dataDir, workDir, database: 'test-db', bucket: 'backup-bucket', objectPrefix: 'database-backups', - uploadOptions: {}, uploadFn: harness.uploadFn, uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn, + mode: 'full', + dataDir, + workDir, + database: 'test-db', + bucket: 'backup-bucket', + objectPrefix: 'database-backups', + uploadOptions: {}, + uploadFn: harness.uploadFn, + uploadManifestFn: harness.uploadManifestFn, + verifyFn: harness.verifyFn, }; - const collected = await collectDirectFileEntries({dataDir, database: 'test-db', objectPrefix: 'database-backups'}); + const collected = await collectDirectFileEntries({ + dataDir, + database: 'test-db', + objectPrefix: 'database-backups', + }); assertTrue( - collected.files.some(({path: filePath}) => filePath === 'replicas/1/snapshots/00000000000000000010.snapshot_dir/objects/object.bin'), + collected.files.some( + ({ path: filePath }) => + filePath === + 'replicas/1/snapshots/00000000000000000010.snapshot_dir/objects/object.bin', + ), 'files catalog 必须原样保留 snapshot 内文件的相对路径。', ); - assertTrue(collected.directories.includes('empty-directory'), 'files catalog 必须保留空目录。'); assertTrue( - collected.symlinks.some(({path: symlinkPath, target}) => symlinkPath === 'bin/current' && target === '2.6.0'), + collected.directories.includes('empty-directory'), + 'files catalog 必须保留空目录。', + ); + assertTrue( + collected.symlinks.some( + ({ path: symlinkPath, target }) => + symlinkPath === 'bin/current' && target === '2.6.0', + ), 'files catalog 必须保留指向 data-dir 内部的相对符号链接。', ); const first = await runDirectFilesBackup(options); assertEqual(first.uploadedCount, 2, '首次 files full 应上传全部普通文件。'); - assertTrue(!Object.hasOwn(first.catalog, 'dataDir'), '远端 files catalog 不得绑定 staging 主机的绝对 data-dir。'); - assertTrue(first.statePath.endsWith('.json.gz'), 'files state 必须使用 gzip 压缩文件。'); + assertTrue( + !Object.hasOwn(first.catalog, 'dataDir'), + '远端 files catalog 不得绑定 staging 主机的绝对 data-dir。', + ); + assertTrue( + first.statePath.endsWith('.json.gz'), + 'files state 必须使用 gzip 压缩文件。', + ); const compactState = readGzipJson(first.statePath); - assertEqual(compactState.schemaVersion, 2, 'files state 必须使用去重后的 v2 契约。'); - assertTrue(!Object.hasOwn(compactState.baselineCatalog, 'files'), 'baseline ref 不得重复嵌入 files。'); - assertTrue(!Object.hasOwn(compactState.latestCatalog, 'files'), 'latest ref 不得重复嵌入 files。'); - assertTrue(!existsSync(first.catalogPath), '本地 full catalog 原始 JSON 应在成功后压缩。'); - assertTrue(existsSync(`${first.catalogPath}.gz`), '本地应保留压缩后的 latest full catalog 供增量复用。'); + assertEqual( + compactState.schemaVersion, + 2, + 'files state 必须使用去重后的 v2 契约。', + ); + assertTrue( + !Object.hasOwn(compactState.baselineCatalog, 'files'), + 'baseline ref 不得重复嵌入 files。', + ); + assertTrue( + !Object.hasOwn(compactState.latestCatalog, 'files'), + 'latest ref 不得重复嵌入 files。', + ); + assertTrue( + !existsSync(first.catalogPath), + '本地 full catalog 原始 JSON 应在成功后压缩。', + ); + assertTrue( + existsSync(`${first.catalogPath}.gz`), + '本地应保留压缩后的 latest full catalog 供增量复用。', + ); const latestObjectKey = 'database-backups/test-db/latest.json'; - const latest = JSON.parse(harness.objects.get(latestObjectKey).body.toString('utf8')); - assertEqual(latest.latestFullCatalog.catalogId, first.catalogId, 'latest pointer 必须指向已验真的最新 full catalog。'); - assertTrue(!Object.hasOwn(latest.latestFullCatalog, 'files'), 'latest full ref 不得嵌入 files 数组。'); - assertTrue(latest.historyCatalogs.every((catalog) => !Object.hasOwn(catalog, 'files')), 'latest history ref 不得嵌入 files 数组。'); - const immutableUploadsAfterFirst = harness.uploadedKeys.filter((objectKey) => objectKey !== latestObjectKey).length; + const latest = JSON.parse( + harness.objects.get(latestObjectKey).body.toString('utf8'), + ); + assertEqual( + latest.latestFullCatalog.catalogId, + first.catalogId, + 'latest pointer 必须指向已验真的最新 full catalog。', + ); + assertTrue( + !Object.hasOwn(latest.latestFullCatalog, 'files'), + 'latest full ref 不得嵌入 files 数组。', + ); + assertTrue( + latest.historyCatalogs.every((catalog) => !Object.hasOwn(catalog, 'files')), + 'latest history ref 不得嵌入 files 数组。', + ); + const immutableUploadsAfterFirst = harness.uploadedKeys.filter( + (objectKey) => objectKey !== latestObjectKey, + ).length; const repeated = await runDirectFilesBackup(options); assertEqual(repeated.uploadedCount, 0, '相同目录重复运行不得重复上传文件。'); assertEqual( - harness.uploadedKeys.filter((objectKey) => objectKey !== latestObjectKey).length, + harness.uploadedKeys.filter((objectKey) => objectKey !== latestObjectKey) + .length, immutableUploadsAfterFirst, '相同 catalog 重跑不得重复 PUT 文件或 catalog，但应覆盖验真 latest pointer。', ); - rmSync(`${first.catalogPath}.gz`, {force: false}); + rmSync(`${first.catalogPath}.gz`, { force: false }); writeFileSync(path.join(dataDir, 'control-db'), 'control changed'); writeFileSync(path.join(dataDir, 'new-program.bin'), 'new program'); const incremental = await runDirectFilesBackup(options); - assertEqual(incremental.uploadedCount, 2, '增量 files full 只应上传新增和变化文件。'); - assertEqual(incremental.reusedCount, 1, '本地 full catalog 缓存缺失时仍应通过 OSS HEAD 复用未变化文件。'); + assertEqual( + incremental.uploadedCount, + 2, + '增量 files full 只应上传新增和变化文件。', + ); + assertEqual( + incremental.reusedCount, + 1, + '本地 full catalog 缓存缺失时仍应通过 OSS HEAD 复用未变化文件。', + ); } async function assertDirectFilesMigratesLegacyStateAndPrunesEmbeddedCatalogs() { const root = path.join(tmpRoot, 'direct-files-state-migration'); const dataDir = path.join(root, 'stdb'); const workDir = path.join(root, 'work'); - mkdirSync(dataDir, {recursive: true}); + mkdirSync(dataDir, { recursive: true }); writeFileSync(path.join(dataDir, 'control-db'), 'control'); const harness = createDirectOssHarness(); const options = { - mode: 'full', dataDir, workDir, database: 'test-db', bucket: 'backup-bucket', objectPrefix: 'database-backups', - uploadOptions: {}, uploadFn: harness.uploadFn, uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn, + mode: 'full', + dataDir, + workDir, + database: 'test-db', + bucket: 'backup-bucket', + objectPrefix: 'database-backups', + uploadOptions: {}, + uploadFn: harness.uploadFn, + uploadManifestFn: harness.uploadManifestFn, + verifyFn: harness.verifyFn, }; const first = await runDirectFilesBackup(options); const compactState = readGzipJson(first.statePath); - const catalog = JSON.parse(gunzipSync(readFileSync(`${first.catalogPath}.gz`)).toString('utf8')); + const catalog = JSON.parse( + gunzipSync(readFileSync(`${first.catalogPath}.gz`)).toString('utf8'), + ); const legacyCatalogRef = { ...compactState.latestCatalog, files: catalog.files, symlinks: catalog.symlinks, }; const legacyStatePath = first.statePath.slice(0, -3); - writeFileSync(legacyStatePath, `${JSON.stringify({ - ...compactState, - schemaVersion: 1, - baselineCatalog: legacyCatalogRef, - latestCatalog: legacyCatalogRef, - }, null, 2)}\n`); - rmSync(first.statePath, {force: false}); + writeFileSync( + legacyStatePath, + `${JSON.stringify( + { + ...compactState, + schemaVersion: 1, + baselineCatalog: legacyCatalogRef, + latestCatalog: legacyCatalogRef, + }, + null, + 2, + )}\n`, + ); + rmSync(first.statePath, { force: false }); const migrated = await runDirectFilesBackup(options); - assertTrue(migrated.unchanged, '旧 state 迁移不得改变相同 full catalog 的零上传语义。'); - assertTrue(existsSync(migrated.statePath), '旧 state 成功运行后必须生成压缩 state。'); - assertTrue(!existsSync(legacyStatePath), '压缩 state 原子落盘后应删除旧未压缩 state。'); + assertTrue( + migrated.unchanged, + '旧 state 迁移不得改变相同 full catalog 的零上传语义。', + ); + assertTrue( + existsSync(migrated.statePath), + '旧 state 成功运行后必须生成压缩 state。', + ); + assertTrue( + !existsSync(legacyStatePath), + '压缩 state 原子落盘后应删除旧未压缩 state。', + ); const migratedState = readGzipJson(migrated.statePath); assertEqual(migratedState.schemaVersion, 2, '旧 state 必须迁移到 v2。'); - assertTrue(!Object.hasOwn(migratedState.latestCatalog, 'files'), '迁移后 state 不得保留重复 files 清单。'); + assertTrue( + !Object.hasOwn(migratedState.latestCatalog, 'files'), + '迁移后 state 不得保留重复 files 清单。', + ); const latestCatalogPath = `${migrated.catalogPath}.gz`; const validCatalogBody = readFileSync(latestCatalogPath); @@ -388,7 +655,10 @@ async function assertDirectFilesMigratesLegacyStateAndPrunesEmbeddedCatalogs() { ); writeFileSync(latestCatalogPath, validCatalogBody); - writeFileSync(legacyStatePath, `${JSON.stringify({...migratedState, schemaVersion: 1})}\n`); + writeFileSync( + legacyStatePath, + `${JSON.stringify({ ...migratedState, schemaVersion: 1 })}\n`, + ); writeFileSync(migrated.statePath, 'not-a-gzip-state'); let corruptStateFailure = null; try { @@ -396,15 +666,21 @@ async function assertDirectFilesMigratesLegacyStateAndPrunesEmbeddedCatalogs() { } catch (error) { corruptStateFailure = error; } - assertTrue(corruptStateFailure instanceof Error, '压缩 state 损坏时必须失败。'); - assertTrue(existsSync(legacyStatePath), '压缩 state 损坏时不得静默回退并删除旧 state。'); + assertTrue( + corruptStateFailure instanceof Error, + '压缩 state 损坏时必须失败。', + ); + assertTrue( + existsSync(legacyStatePath), + '压缩 state 损坏时不得静默回退并删除旧 state。', + ); } async function assertDirectFilesConcurrencyIsBounded() { const root = path.join(tmpRoot, 'direct-files-concurrency'); const dataDir = path.join(root, 'stdb'); const workDir = path.join(root, 'work'); - mkdirSync(dataDir, {recursive: true}); + mkdirSync(dataDir, { recursive: true }); for (let index = 0; index < 12; index += 1) { writeFileSync(path.join(dataDir, `file-${index}.bin`), `content-${index}`); } @@ -439,8 +715,13 @@ async function assertDirectFilesConcurrencyIsBounded() { } async function assertDirectHistoryPublishesCatalogBeforeCleanup() { - const fixture = createHistoryFixture('direct-files-history-cleanup', {nestedData: false}); - createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const fixture = createHistoryFixture('direct-files-history-cleanup', { + nestedData: false, + }); + createReplicaHistory(fixture.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); const harness = createDirectOssHarness(); const common = { dataDir: fixture.dataDir, @@ -452,15 +733,29 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() { uploadFn: harness.uploadFn, verifyFn: harness.verifyFn, }; - const baseline = await runDirectFilesBackup({...common, mode: 'full', uploadManifestFn: harness.uploadManifestFn}); - const legacyResultFile = path.join(fixture.workDir, 'legacy-full-result.json'); - const legacyCatalogWithoutSymlinks = {...baseline.catalog}; + const baseline = await runDirectFilesBackup({ + ...common, + mode: 'full', + uploadManifestFn: harness.uploadManifestFn, + }); + const legacyResultFile = path.join( + fixture.workDir, + 'legacy-full-result.json', + ); + const legacyCatalogWithoutSymlinks = { ...baseline.catalog }; delete legacyCatalogWithoutSymlinks.symlinks; - writeFileSync(legacyResultFile, `${JSON.stringify({ - ...baseline, - catalog: legacyCatalogWithoutSymlinks, - }, null, 2)}\n`); - const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); + writeFileSync( + legacyResultFile, + `${JSON.stringify( + { + ...baseline, + catalog: legacyCatalogWithoutSymlinks, + }, + null, + 2, + )}\n`, + ); + const plan = discoverHistoryPlan({ dataDir: fixture.dataDir }); let failure = null; try { await runDirectFilesBackup({ @@ -473,9 +768,16 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() { } catch (error) { failure = error; } - assertIncludes(failure?.message ?? '', 'synthetic direct catalog failure', 'direct history catalog 发布失败必须向上返回。'); + assertIncludes( + failure?.message ?? '', + 'synthetic direct catalog failure', + 'direct history catalog 发布失败必须向上返回。', + ); for (const candidate of plan.candidates) { - assertTrue(existsSync(path.join(fixture.dataDir, candidate.path)), `direct history catalog 发布失败不得删除: ${candidate.path}`); + assertTrue( + existsSync(path.join(fixture.dataDir, candidate.path)), + `direct history catalog 发布失败不得删除: ${candidate.path}`, + ); } let pointerFailure = null; @@ -494,9 +796,16 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() { } catch (error) { pointerFailure = error; } - assertIncludes(pointerFailure?.message ?? '', 'synthetic latest pointer HEAD failure', 'latest pointer HEAD 验真失败必须向上返回。'); + assertIncludes( + pointerFailure?.message ?? '', + 'synthetic latest pointer HEAD failure', + 'latest pointer HEAD 验真失败必须向上返回。', + ); for (const candidate of plan.candidates) { - assertTrue(existsSync(path.join(fixture.dataDir, candidate.path)), `latest pointer 发布失败不得删除: ${candidate.path}`); + assertTrue( + existsSync(path.join(fixture.dataDir, candidate.path)), + `latest pointer 发布失败不得删除: ${candidate.path}`, + ); } const resultFile = path.join(fixture.workDir, 'history-result.json'); @@ -506,33 +815,80 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() { resultFile, uploadManifestFn: harness.uploadManifestFn, }); - assertEqual(success.uploadedCount, 0, 'history 文件已在 full CAS baseline 时不应重复上传内容。'); + assertEqual( + success.uploadedCount, + 0, + 'history 文件已在 full CAS baseline 时不应重复上传内容。', + ); for (const file of success.catalog.files) { assertTrue( harness.verifiedKeys.includes(file.objectKey), `history 清理前必须逐个验真 baseline 复用对象: ${file.path}`, ); } - assertEqual(success.cleanup?.deletedCount, plan.candidates.length, 'catalog 和 baseline 验真后才应清理全部安全候选。'); + assertEqual( + success.cleanup?.deletedCount, + plan.candidates.length, + 'catalog 和 baseline 验真后才应清理全部安全候选。', + ); const state = readGzipJson(success.statePath); - assertEqual(state.schemaVersion, 2, 'files state 必须迁移为去重后的 v2 契约。'); - assertTrue(!Object.hasOwn(state.latestCatalog, 'files'), 'files state latest ref 不得重复嵌入 files。'); - assertTrue(state.historyCatalogs.every((catalog) => !Object.hasOwn(catalog, 'files')), 'files state history ref 不得重复嵌入 files。'); - assertTrue(!existsSync(success.catalogPath), '已上传并验真的 history catalog 本地 JSON 应被清理。'); - assertTrue(!existsSync(`${success.catalogPath}.gz`), 'history catalog 本地压缩副本也不应保留。'); + assertEqual( + state.schemaVersion, + 2, + 'files state 必须迁移为去重后的 v2 契约。', + ); + assertTrue( + !Object.hasOwn(state.latestCatalog, 'files'), + 'files state latest ref 不得重复嵌入 files。', + ); + assertTrue( + state.historyCatalogs.every((catalog) => !Object.hasOwn(catalog, 'files')), + 'files state history ref 不得重复嵌入 files。', + ); + assertTrue( + !existsSync(success.catalogPath), + '已上传并验真的 history catalog 本地 JSON 应被清理。', + ); + assertTrue( + !existsSync(`${success.catalogPath}.gz`), + 'history catalog 本地压缩副本也不应保留。', + ); const diskResult = JSON.parse(readFileSync(resultFile, 'utf8')); - assertTrue(!Object.hasOwn(diskResult.catalog, 'files'), 'files result 文件不得重复写入完整 files 清单。'); - assertEqual(diskResult.catalog.fileCount, success.fileCount, '紧凑 result 仍应保留文件计数。'); - const compactedLegacyResult = JSON.parse(readFileSync(legacyResultFile, 'utf8')); - assertTrue(!Object.hasOwn(compactedLegacyResult.catalog, 'files'), '旧 result 中重复的 files 清单应在成功运行后压缩。'); - assertEqual(compactedLegacyResult.catalog.symlinkCount, 0, '缺少 symlinks 的旧 result 应按零个符号链接兼容迁移。'); - assertTrue((success.metadataCleanup?.compactedResultCount ?? 0) >= 1, 'metadata 清理应报告已压缩旧 result。'); + assertTrue( + !Object.hasOwn(diskResult.catalog, 'files'), + 'files result 文件不得重复写入完整 files 清单。', + ); + assertEqual( + diskResult.catalog.fileCount, + success.fileCount, + '紧凑 result 仍应保留文件计数。', + ); + const compactedLegacyResult = JSON.parse( + readFileSync(legacyResultFile, 'utf8'), + ); + assertTrue( + !Object.hasOwn(compactedLegacyResult.catalog, 'files'), + '旧 result 中重复的 files 清单应在成功运行后压缩。', + ); + assertEqual( + compactedLegacyResult.catalog.symlinkCount, + 0, + '缺少 symlinks 的旧 result 应按零个符号链接兼容迁移。', + ); + assertTrue( + (success.metadataCleanup?.compactedResultCount ?? 0) >= 1, + 'metadata 清理应报告已压缩旧 result。', + ); const historyCatalogObjectKey = state.historyCatalogs[0].objectKey; harness.objects.delete(historyCatalogObjectKey); let brokenHistoryFailure = null; try { - await runDirectFilesBackup({...common, mode: 'history', uploadManifestFn: harness.uploadManifestFn}); + await runDirectFilesBackup({ + ...common, + mode: 'history', + uploadManifestFn: harness.uploadManifestFn, + }); } catch (error) { brokenHistoryFailure = error; } @@ -544,8 +900,13 @@ async function assertDirectHistoryPublishesCatalogBeforeCleanup() { } async function assertDirectHistoryWithoutCandidatesPublishesLatest() { - const fixture = createHistoryFixture('direct-files-history-empty', {nestedData: false}); - createReplicaHistory(fixture.replicasDir, '1', {snapshots: [10], segments: [0]}); + const fixture = createHistoryFixture('direct-files-history-empty', { + nestedData: false, + }); + createReplicaHistory(fixture.replicasDir, '1', { + snapshots: [10], + segments: [0], + }); const harness = createDirectOssHarness(); const common = { dataDir: fixture.dataDir, @@ -558,13 +919,19 @@ async function assertDirectHistoryWithoutCandidatesPublishesLatest() { uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn, }; - await runDirectFilesBackup({...common, mode: 'full'}); + await runDirectFilesBackup({ ...common, mode: 'full' }); const latestObjectKey = 'database-backups/test-db/latest.json'; harness.objects.delete(latestObjectKey); - const result = await runDirectFilesBackup({...common, mode: 'history'}); + const result = await runDirectFilesBackup({ ...common, mode: 'history' }); assertEqual(result.candidateCount, 0, 'fixture 应没有可归档 history 候选。'); - assertTrue(harness.objects.has(latestObjectKey), 'history 无候选时仍必须从现有 state 发布 latest pointer。'); - assertTrue(harness.verifiedKeys.includes(latestObjectKey), 'history 无候选时 latest pointer 仍必须 HEAD 验真。'); + assertTrue( + harness.objects.has(latestObjectKey), + 'history 无候选时仍必须从现有 state 发布 latest pointer。', + ); + assertTrue( + harness.verifiedKeys.includes(latestObjectKey), + 'history 无候选时 latest pointer 仍必须 HEAD 验真。', + ); } async function assertDirectFilesRestoreDownloadsCatalogAndObjects() { @@ -572,23 +939,39 @@ async function assertDirectFilesRestoreDownloadsCatalogAndObjects() { const dataDir = path.join(root, 'stdb'); const workDir = path.join(root, 'work'); const restoreDir = path.join(root, 'restore'); - mkdirSync(path.join(dataDir, 'empty-directory'), {recursive: true}); - mkdirSync(path.join(dataDir, 'config'), {recursive: true}); - mkdirSync(path.join(dataDir, 'bin', '2.6.0'), {recursive: true}); + mkdirSync(path.join(dataDir, 'empty-directory'), { recursive: true }); + mkdirSync(path.join(dataDir, 'config'), { recursive: true }); + mkdirSync(path.join(dataDir, 'bin', '2.6.0'), { recursive: true }); symlinkSync('2.6.0', path.join(dataDir, 'bin', 'current')); const keyPath = path.join(dataDir, 'config', 'id_ecdsa'); writeFileSync(keyPath, 'private key fixture'); chmodSync(keyPath, 0o640); const harness = createDirectOssHarness(); await runDirectFilesBackup({ - mode: 'full', dataDir, workDir, database: 'test-db', bucket: 'backup-bucket', objectPrefix: 'database-backups', - uploadOptions: {}, uploadFn: harness.uploadFn, uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn, + mode: 'full', + dataDir, + workDir, + database: 'test-db', + bucket: 'backup-bucket', + objectPrefix: 'database-backups', + uploadOptions: {}, + uploadFn: harness.uploadFn, + uploadManifestFn: harness.uploadManifestFn, + verifyFn: harness.verifyFn, }); writeFileSync(keyPath, 'updated private key fixture'); chmodSync(keyPath, 0o640); const latestFull = await runDirectFilesBackup({ - mode: 'full', dataDir, workDir, database: 'test-db', bucket: 'backup-bucket', objectPrefix: 'database-backups', - uploadOptions: {}, uploadFn: harness.uploadFn, uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn, + mode: 'full', + dataDir, + workDir, + database: 'test-db', + bucket: 'backup-bucket', + objectPrefix: 'database-backups', + uploadOptions: {}, + uploadFn: harness.uploadFn, + uploadManifestFn: harness.uploadManifestFn, + verifyFn: harness.verifyFn, }); const restored = await restoreDirectFilesBackup({ statePath: latestFull.statePath, @@ -596,8 +979,9 @@ async function assertDirectFilesRestoreDownloadsCatalogAndObjects() { database: 'test-db', bucket: 'backup-bucket', uploadOptions: {}, - downloadBufferFn: async ({objectKey}) => Buffer.from(harness.objects.get(objectKey)?.body ?? ''), - downloadFileFn: async ({objectKey, destinationPath}) => { + downloadBufferFn: async ({ objectKey }) => + Buffer.from(harness.objects.get(objectKey)?.body ?? ''), + downloadFileFn: async ({ objectKey, destinationPath }) => { const object = harness.objects.get(objectKey); if (!object) { throw new Error(`missing ${objectKey}`); @@ -605,27 +989,56 @@ async function assertDirectFilesRestoreDownloadsCatalogAndObjects() { writeFileSync(destinationPath, object.body); }, }); - assertEqual(restored.downloadedCount, 1, 'files restore 必须从对象存储下载 catalog 中的普通文件。'); - assertEqual(readFileSync(path.join(restoreDir, 'config', 'id_ecdsa'), 'utf8'), 'updated private key fixture', 'files restore 必须按最新 full catalog 的原相对路径恢复内容。'); - assertTrue(existsSync(path.join(restoreDir, 'empty-directory')), 'files restore 必须重建空目录。'); - assertEqual(statSync(path.join(restoreDir, 'config', 'id_ecdsa')).mode & 0o7777, 0o640, 'files restore 必须恢复文件权限。'); - assertTrue(lstatSync(path.join(restoreDir, 'bin', 'current')).isSymbolicLink(), 'files restore 必须重建符号链接。'); - assertEqual(readlinkSync(path.join(restoreDir, 'bin', 'current'), 'utf8'), '2.6.0', 'files restore 必须保留符号链接目标。'); + assertEqual( + restored.downloadedCount, + 1, + 'files restore 必须从对象存储下载 catalog 中的普通文件。', + ); + assertEqual( + readFileSync(path.join(restoreDir, 'config', 'id_ecdsa'), 'utf8'), + 'updated private key fixture', + 'files restore 必须按最新 full catalog 的原相对路径恢复内容。', + ); + assertTrue( + existsSync(path.join(restoreDir, 'empty-directory')), + 'files restore 必须重建空目录。', + ); + assertEqual( + statSync(path.join(restoreDir, 'config', 'id_ecdsa')).mode & 0o7777, + 0o640, + 'files restore 必须恢复文件权限。', + ); + assertTrue( + lstatSync(path.join(restoreDir, 'bin', 'current')).isSymbolicLink(), + 'files restore 必须重建符号链接。', + ); + assertEqual( + readlinkSync(path.join(restoreDir, 'bin', 'current'), 'utf8'), + '2.6.0', + 'files restore 必须保留符号链接目标。', + ); - rmSync(restoreDir, {recursive: true, force: true}); - const legacyRestoreStatePath = path.join(workDir, 'legacy-restore-state.json'); - writeFileSync(legacyRestoreStatePath, `${JSON.stringify({ - ...readGzipJson(latestFull.statePath), - schemaVersion: 1, - })}\n`); + rmSync(restoreDir, { recursive: true, force: true }); + const legacyRestoreStatePath = path.join( + workDir, + 'legacy-restore-state.json', + ); + writeFileSync( + legacyRestoreStatePath, + `${JSON.stringify({ + ...readGzipJson(latestFull.statePath), + schemaVersion: 1, + })}\n`, + ); const legacyRestored = await restoreDirectFilesBackup({ statePath: legacyRestoreStatePath, restoreDir, database: 'test-db', bucket: 'backup-bucket', uploadOptions: {}, - downloadBufferFn: async ({objectKey}) => Buffer.from(harness.objects.get(objectKey)?.body ?? ''), - downloadFileFn: async ({objectKey, destinationPath}) => { + downloadBufferFn: async ({ objectKey }) => + Buffer.from(harness.objects.get(objectKey)?.body ?? ''), + downloadFileFn: async ({ objectKey, destinationPath }) => { const object = harness.objects.get(objectKey); if (!object) { throw new Error(`missing ${objectKey}`); @@ -633,10 +1046,14 @@ async function assertDirectFilesRestoreDownloadsCatalogAndObjects() { writeFileSync(destinationPath, object.body); }, }); - assertEqual(legacyRestored.downloadedCount, 1, 'files restore 必须继续兼容 v1 JSON state。'); + assertEqual( + legacyRestored.downloadedCount, + 1, + 'files restore 必须继续兼容 v1 JSON state。', + ); - rmSync(restoreDir, {recursive: true, force: true}); - const downloadBufferFn = async ({objectKey}) => { + rmSync(restoreDir, { recursive: true, force: true }); + const downloadBufferFn = async ({ objectKey }) => { const object = harness.objects.get(objectKey); if (!object) { throw new Error(`missing ${objectKey}`); @@ -644,7 +1061,7 @@ async function assertDirectFilesRestoreDownloadsCatalogAndObjects() { return Buffer.from(object.body); }; let objectDownloadCount = 0; - const downloadFileFn = async ({objectKey, destinationPath}) => { + const downloadFileFn = async ({ objectKey, destinationPath }) => { objectDownloadCount += 1; const object = harness.objects.get(objectKey); if (!object) { @@ -663,10 +1080,26 @@ async function assertDirectFilesRestoreDownloadsCatalogAndObjects() { downloadFileFn, verifyFn: harness.verifyFn, }); - assertEqual(dryRun.catalogId, latestFull.catalogId, 'OSS-only dry-run 必须选择 latestFullCatalog。'); - assertEqual(dryRun.fileCount, 1, 'OSS-only dry-run 应返回 full catalog 文件数。'); - assertEqual(dryRun.symlinkCount, 1, 'OSS-only dry-run 应返回 full catalog 符号链接数。'); - assertEqual(dryRun.totalSizeBytes, String(Buffer.byteLength('updated private key fixture')), 'OSS-only dry-run 应返回总字节数。'); + assertEqual( + dryRun.catalogId, + latestFull.catalogId, + 'OSS-only dry-run 必须选择 latestFullCatalog。', + ); + assertEqual( + dryRun.fileCount, + 1, + 'OSS-only dry-run 应返回 full catalog 文件数。', + ); + assertEqual( + dryRun.symlinkCount, + 1, + 'OSS-only dry-run 应返回 full catalog 符号链接数。', + ); + assertEqual( + dryRun.totalSizeBytes, + String(Buffer.byteLength('updated private key fixture')), + 'OSS-only dry-run 应返回总字节数。', + ); assertEqual(objectDownloadCount, 0, 'OSS-only dry-run 不得下载数据对象。'); assertTrue(!existsSync(restoreDir), 'OSS-only dry-run 不得创建恢复目录。'); @@ -680,16 +1113,36 @@ async function assertDirectFilesRestoreDownloadsCatalogAndObjects() { downloadFileFn, verifyFn: harness.verifyFn, }); - assertEqual(latestRestored.catalogId, latestFull.catalogId, 'OSS-only restore 必须选择 latestFullCatalog。'); - assertEqual(latestRestored.downloadedCount, 1, 'OSS-only restore 应下载 latest full catalog 的数据对象。'); - assertEqual(readFileSync(path.join(restoreDir, 'config', 'id_ecdsa'), 'utf8'), 'updated private key fixture', 'OSS-only restore 应还原最新 full 内容。'); - assertEqual(readlinkSync(path.join(restoreDir, 'bin', 'current'), 'utf8'), '2.6.0', 'OSS-only restore 应还原符号链接。'); + assertEqual( + latestRestored.catalogId, + latestFull.catalogId, + 'OSS-only restore 必须选择 latestFullCatalog。', + ); + assertEqual( + latestRestored.downloadedCount, + 1, + 'OSS-only restore 应下载 latest full catalog 的数据对象。', + ); + assertEqual( + readFileSync(path.join(restoreDir, 'config', 'id_ecdsa'), 'utf8'), + 'updated private key fixture', + 'OSS-only restore 应还原最新 full 内容。', + ); + assertEqual( + readlinkSync(path.join(restoreDir, 'bin', 'current'), 'utf8'), + '2.6.0', + 'OSS-only restore 应还原符号链接。', + ); } function assertCanonicalQueryAndAuthorizationIncludeMultipartParameters() { - assertEqual(buildCanonicalQuery({uploads: null}), 'uploads', 'InitiateMultipartUpload 必须使用无等号的 uploads 参数。'); assertEqual( - buildCanonicalQuery({uploadId: 'abc+/= xyz', partNumber: 12}), + buildCanonicalQuery({ uploads: null }), + 'uploads', + 'InitiateMultipartUpload 必须使用无等号的 uploads 参数。', + ); + assertEqual( + buildCanonicalQuery({ uploadId: 'abc+/= xyz', partNumber: 12 }), 'partNumber=12&uploadId=abc%2B%2F%3D%20xyz', 'multipart query 必须按 key 排序并使用 RFC3986 编码。', ); @@ -720,9 +1173,13 @@ function assertCanonicalQueryAndAuthorizationIncludeMultipartParameters() { accessKeySecret: 'test-access-secret', headers, date, - queries: {partNumber: 12, uploadId: 'abc+/= xyz'}, + queries: { partNumber: 12, uploadId: 'abc+/= xyz' }, }); - assertNotEqual(withQuery, withoutQuery, 'multipart query 必须参与 V4 Authorization 计算。'); + assertNotEqual( + withQuery, + withoutQuery, + 'multipart query 必须参与 V4 Authorization 计算。', + ); assertEqual( withQuery, 'OSS4-HMAC-SHA256 Credential=test-access-key/20260713/cn-shanghai/oss/aliyun_v4_request,AdditionalHeaders=host,Signature=9323dd3b7272b52f416c4d32115fcc00460eaccdcdaf011575c2502a63a27b1f', @@ -742,12 +1199,41 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() { ]); assertStatus(result, 1, '空间不足时必须失败。'); - assertIncludes(result.stdout, '备份空间预检', '空间不足失败前应打印空间预检。'); - assertIncludes(result.stderr, '剩余空间不足', '空间不足失败应说明剩余空间不足。'); + assertIncludes( + result.stdout, + '备份空间预检', + '空间不足失败前应打印空间预检。', + ); + assertIncludes( + result.stderr, + '剩余空间不足', + '空间不足失败应说明剩余空间不足。', + ); assertFileMissing(fixture.systemctlLog, '空间不足时不能调用 systemctl。'); assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。'); } +function assertStopFailureRetainsRecoveryMarker() { + const fixture = createFixture('stop-failure-marker'); + writeExecutable( + path.join(fixture.binDir, 'systemctl'), + `#!/usr/bin/env bash +printf 'systemctl %s\\n' "$*" >> "${fixture.systemctlLog}" +if [ "$1" = stop ]; then + exit 9 +fi +exit 0 +`, + ); + const result = runBackup(fixture, ['--stop-service', 'spacetimedb.service']); + + assertStatus(result, 1, '停止服务失败时备份必须失败。'); + assertTrue( + existsSync(path.join(fixture.workDir, '.spacetimedb-stopped')), + '停止服务命令失败时必须保留 marker，供 systemd ExecStopPost 兜底恢复。', + ); +} + function assertArchiveFailureStillRestoresDependentServices() { const fixture = createFixture('tar-failure'); const result = runBackup(fixture, [ @@ -764,7 +1250,11 @@ function assertArchiveFailureStillRestoresDependentServices() { ]); assertStatus(result, 1, 'tar 失败时备份脚本必须失败。'); - assertIncludes(result.stderr, 'fake tar failure', 'tar 失败原因应保留在错误输出中。'); + assertIncludes( + result.stderr, + 'fake tar failure', + 'tar 失败原因应保留在错误输出中。', + ); const systemctlLog = readFile(fixture.systemctlLog); const expectedCommands = [ 'systemctl stop spacetimedb.service', @@ -776,6 +1266,10 @@ function assertArchiveFailureStillRestoresDependentServices() { for (const command of expectedCommands) { assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`); } + assertFileMissing( + path.join(fixture.workDir, '.spacetimedb-stopped'), + '正常执行 finally 恢复全部服务后必须清理停库 marker。', + ); } async function assertMultipartUploadRetriesAndVerifiesRemoteLength() { @@ -788,7 +1282,7 @@ async function assertMultipartUploadRetriesAndVerifiesRemoteLength() { Buffer.alloc(17, 'c'), ]); const payloadSha256 = createHash('sha256').update(payload).digest('hex'); - mkdirSync(root, {recursive: true}); + mkdirSync(root, { recursive: true }); writeFileSync(archivePath, payload); const requests = []; @@ -797,30 +1291,50 @@ async function assertMultipartUploadRetriesAndVerifiesRemoteLength() { const uploadId = 'upload+/= id'; const fetchImpl = async (url, options) => { const body = await readRequestBody(options.body); - requests.push({url, method: options.method, headers: options.headers, body}); + requests.push({ + url, + method: options.method, + headers: options.headers, + body, + }); const parsedUrl = new URL(url); if (options.method === 'POST' && parsedUrl.search === '?uploads') { - return new Response(`${uploadId}`, {status: 200}); + return new Response( + `${uploadId}`, + { status: 200 }, + ); } if (options.method === 'PUT') { const partNumber = Number(parsedUrl.searchParams.get('partNumber')); if (partNumber === 1) { firstPartAttempts += 1; if (firstPartAttempts === 1) { - return new Response('ServiceUnavailable', {status: 503}); + return new Response( + 'ServiceUnavailable', + { status: 503 }, + ); } } - return new Response('', {status: 200, headers: {etag: `"etag-${partNumber}"`}}); + return new Response('', { + status: 200, + headers: { etag: `"etag-${partNumber}"` }, + }); } if (options.method === 'POST' && parsedUrl.searchParams.has('uploadId')) { - return new Response('', {status: 200, headers: {etag: '"complete-etag"'}}); + return new Response('', { + status: 200, + headers: { etag: '"complete-etag"' }, + }); } if (options.method === 'HEAD') { - return new Response(null, {status: 200, headers: { - 'content-length': String(payload.length), - 'x-oss-meta-archive-sha256': payloadSha256, - }}); + return new Response(null, { + status: 200, + headers: { + 'content-length': String(payload.length), + 'x-oss-meta-archive-sha256': payloadSha256, + }, + }); } throw new Error(`unexpected request: ${options.method} ${url}`); }; @@ -842,32 +1356,86 @@ async function assertMultipartUploadRetriesAndVerifiesRemoteLength() { randomFn: () => 0, }); - assertEqual(result.uploadMode, 'multipart', '上传结果必须记录 multipart 模式。'); + assertEqual( + result.uploadMode, + 'multipart', + '上传结果必须记录 multipart 模式。', + ); assertEqual(result.partCount, 3, 'multipart 应按配置大小切成三段。'); - assertEqual(result.contentLength, payload.length, '上传结果应保留完整归档长度。'); - assertEqual(result.etag, 'complete-etag', '上传结果应保留 CompleteMultipartUpload ETag。'); + assertEqual( + result.contentLength, + payload.length, + '上传结果应保留完整归档长度。', + ); + assertEqual( + result.etag, + 'complete-etag', + '上传结果应保留 CompleteMultipartUpload ETag。', + ); assertEqual(firstPartAttempts, 2, '503 后应仅重试失败的第一段。'); assertEqual(retryDelays.length, 1, '一次可重试失败应触发一次退避。'); const initiateRequest = requests[0]; - assertTrue(initiateRequest.url.endsWith('?uploads'), 'InitiateMultipartUpload URL 必须使用裸 uploads 参数。'); - assertTrue(!initiateRequest.url.endsWith('?uploads='), 'InitiateMultipartUpload URL 不能把裸参数写成 uploads=。'); - assertEqual(initiateRequest.headers['x-oss-meta-archive-sha256'], payloadSha256, 'multipart 对象必须保存本地归档 SHA-256 元数据。'); - const firstPartRequests = requests.filter(({method, url}) => method === 'PUT' && new URL(url).searchParams.get('partNumber') === '1'); - assertEqual(firstPartRequests.length, 2, '第一段应产生原请求和一次重试。'); - assertBufferEqual(firstPartRequests[0].body, payload.subarray(0, partSizeBytes), '第一段原请求内容必须完整。'); - assertBufferEqual(firstPartRequests[1].body, payload.subarray(0, partSizeBytes), '第一段重试必须重新创建并完整读取 stream。'); assertTrue( - firstPartRequests[0].url.includes('?partNumber=1&uploadId=upload%2B%2F%3D%20id'), + initiateRequest.url.endsWith('?uploads'), + 'InitiateMultipartUpload URL 必须使用裸 uploads 参数。', + ); + assertTrue( + !initiateRequest.url.endsWith('?uploads='), + 'InitiateMultipartUpload URL 不能把裸参数写成 uploads=。', + ); + assertEqual( + initiateRequest.headers['x-oss-meta-archive-sha256'], + payloadSha256, + 'multipart 对象必须保存本地归档 SHA-256 元数据。', + ); + const firstPartRequests = requests.filter( + ({ method, url }) => + method === 'PUT' && new URL(url).searchParams.get('partNumber') === '1', + ); + assertEqual(firstPartRequests.length, 2, '第一段应产生原请求和一次重试。'); + assertBufferEqual( + firstPartRequests[0].body, + payload.subarray(0, partSizeBytes), + '第一段原请求内容必须完整。', + ); + assertBufferEqual( + firstPartRequests[1].body, + payload.subarray(0, partSizeBytes), + '第一段重试必须重新创建并完整读取 stream。', + ); + assertTrue( + firstPartRequests[0].url.includes( + '?partNumber=1&uploadId=upload%2B%2F%3D%20id', + ), 'UploadPart URL 必须使用排序并编码后的 canonical query。', ); - const completeRequest = requests.find(({method, url}) => method === 'POST' && new URL(url).searchParams.has('uploadId')); - assertIncludes(completeRequest?.body.toString('utf8') ?? '', '1"etag-1"', 'Complete XML 应包含第一段 ETag。'); - assertIncludes(completeRequest?.body.toString('utf8') ?? '', '3"etag-3"', 'Complete XML 应包含最后一段 ETag。'); - assertTrue(requests.some(({method}) => method === 'HEAD'), 'Complete 后必须执行签名 HEAD 验证。'); + const completeRequest = requests.find( + ({ method, url }) => + method === 'POST' && new URL(url).searchParams.has('uploadId'), + ); + assertIncludes( + completeRequest?.body.toString('utf8') ?? '', + '1"etag-1"', + 'Complete XML 应包含第一段 ETag。', + ); + assertIncludes( + completeRequest?.body.toString('utf8') ?? '', + '3"etag-3"', + 'Complete XML 应包含最后一段 ETag。', + ); + assertTrue( + requests.some(({ method }) => method === 'HEAD'), + 'Complete 后必须执行签名 HEAD 验证。', + ); for (const request of requests) { - assertTrue(String(request.headers.authorization ?? '').startsWith('OSS4-HMAC-SHA256 '), `${request.method} 请求必须携带 V4 Authorization。`); + assertTrue( + String(request.headers.authorization ?? '').startsWith( + 'OSS4-HMAC-SHA256 ', + ), + `${request.method} 请求必须携带 V4 Authorization。`, + ); } } @@ -877,31 +1445,43 @@ async function assertHeadLengthMismatchAbortsMultipartUpload() { const partSizeBytes = 100 * 1024; const payload = Buffer.alloc(partSizeBytes + 1, 'x'); const payloadSha256 = createHash('sha256').update(payload).digest('hex'); - mkdirSync(root, {recursive: true}); + mkdirSync(root, { recursive: true }); writeFileSync(archivePath, payload); const requests = []; const fetchImpl = async (url, options) => { await readRequestBody(options.body); - requests.push({url, method: options.method}); + requests.push({ url, method: options.method }); const parsedUrl = new URL(url); if (options.method === 'POST' && parsedUrl.search === '?uploads') { - return new Response('mismatch-upload', {status: 200}); + return new Response( + 'mismatch-upload', + { status: 200 }, + ); } if (options.method === 'PUT') { - return new Response('', {status: 200, headers: {etag: `"etag-${parsedUrl.searchParams.get('partNumber')}"`}}); + return new Response('', { + status: 200, + headers: { etag: `"etag-${parsedUrl.searchParams.get('partNumber')}"` }, + }); } if (options.method === 'POST') { - return new Response('', {status: 200, headers: {etag: '"complete-etag"'}}); + return new Response('', { + status: 200, + headers: { etag: '"complete-etag"' }, + }); } if (options.method === 'HEAD') { - return new Response(null, {status: 200, headers: { - 'content-length': String(payload.length - 1), - 'x-oss-meta-archive-sha256': payloadSha256, - }}); + return new Response(null, { + status: 200, + headers: { + 'content-length': String(payload.length - 1), + 'x-oss-meta-archive-sha256': payloadSha256, + }, + }); } if (options.method === 'DELETE') { - return new Response(null, {status: 204}); + return new Response(null, { status: 204 }); } throw new Error(`unexpected request: ${options.method} ${url}`); }; @@ -929,40 +1509,59 @@ async function assertHeadLengthMismatchAbortsMultipartUpload() { } assertTrue(uploadError instanceof Error, 'HEAD 长度不一致时上传必须失败。'); - assertIncludes(uploadError?.message ?? '', 'HEAD 验证长度不一致', 'HEAD 长度不一致错误应保留本地和远端长度。'); - const abortRequest = requests.find(({method}) => method === 'DELETE'); - assertTrue(Boolean(abortRequest), 'HEAD 长度不一致后必须 best-effort AbortMultipartUpload。'); - assertTrue(abortRequest?.url.endsWith('?uploadId=mismatch-upload'), 'AbortMultipartUpload 必须携带同一 uploadId。'); + assertIncludes( + uploadError?.message ?? '', + 'HEAD 验证长度不一致', + 'HEAD 长度不一致错误应保留本地和远端长度。', + ); + const abortRequest = requests.find(({ method }) => method === 'DELETE'); + assertTrue( + Boolean(abortRequest), + 'HEAD 长度不一致后必须 best-effort AbortMultipartUpload。', + ); + assertTrue( + abortRequest?.url.endsWith('?uploadId=mismatch-upload'), + 'AbortMultipartUpload 必须携带同一 uploadId。', + ); } async function assertHeadShaMismatchAbortsMultipartUpload() { const root = path.join(tmpRoot, 'multipart-head-sha-mismatch'); const archivePath = path.join(root, 'backup.tar.gz'); const payload = Buffer.alloc(100 * 1024, 's'); - mkdirSync(root, {recursive: true}); + mkdirSync(root, { recursive: true }); writeFileSync(archivePath, payload); const requests = []; const fetchImpl = async (url, options) => { await readRequestBody(options.body); - requests.push({url, method: options.method}); + requests.push({ url, method: options.method }); const parsedUrl = new URL(url); if (options.method === 'POST' && parsedUrl.search === '?uploads') { - return new Response('sha-mismatch-upload', {status: 200}); + return new Response( + 'sha-mismatch-upload', + { status: 200 }, + ); } if (options.method === 'PUT') { - return new Response('', {status: 200, headers: {etag: '"part-etag"'}}); + return new Response('', { + status: 200, + headers: { etag: '"part-etag"' }, + }); } if (options.method === 'POST') { - return new Response('', {status: 200}); + return new Response('', { status: 200 }); } if (options.method === 'HEAD') { - return new Response(null, {status: 200, headers: { - 'content-length': String(payload.length), - 'x-oss-meta-archive-sha256': '0'.repeat(64), - }}); + return new Response(null, { + status: 200, + headers: { + 'content-length': String(payload.length), + 'x-oss-meta-archive-sha256': '0'.repeat(64), + }, + }); } if (options.method === 'DELETE') { - return new Response(null, {status: 204}); + return new Response(null, { status: 204 }); } throw new Error(`unexpected request: ${options.method} ${url}`); }; @@ -985,9 +1584,16 @@ async function assertHeadShaMismatchAbortsMultipartUpload() { } catch (error) { uploadError = error; } - assertIncludes(uploadError?.message ?? '', 'SHA-256 不一致', 'HEAD SHA-256 不一致时上传必须失败。'); + assertIncludes( + uploadError?.message ?? '', + 'SHA-256 不一致', + 'HEAD SHA-256 不一致时上传必须失败。', + ); assertTrue( - requests.some(({method, url}) => method === 'DELETE' && url.endsWith('?uploadId=sha-mismatch-upload')), + requests.some( + ({ method, url }) => + method === 'DELETE' && url.endsWith('?uploadId=sha-mismatch-upload'), + ), 'HEAD SHA-256 不一致后必须 best-effort AbortMultipartUpload。', ); } @@ -995,9 +1601,14 @@ async function assertHeadShaMismatchAbortsMultipartUpload() { async function assertManifestUploadUsesShaAndHeadVerification() { const root = path.join(tmpRoot, 'manifest-upload'); const manifestPath = path.join(root, 'backup.manifest.json'); - const body = Buffer.from(JSON.stringify({uploadStatus: 'uploaded', catalog: 'x'.repeat(150 * 1024)})); + const body = Buffer.from( + JSON.stringify({ + uploadStatus: 'uploaded', + catalog: 'x'.repeat(150 * 1024), + }), + ); const bodySha256 = createHash('sha256').update(body).digest('hex'); - mkdirSync(root, {recursive: true}); + mkdirSync(root, { recursive: true }); writeFileSync(manifestPath, body); const requests = []; let limitedChunkCount = 0; @@ -1015,37 +1626,68 @@ async function assertManifestUploadUsesShaAndHeadVerification() { randomFn: () => 0, bandwidthLimiter: { wrap(readable) { - return Readable.from((async function* observeLimitedManifest() { - for await (const chunk of readable) { - limitedChunkCount += 1; - limitedBytes += chunk.length; - yield chunk; - } - })(), {objectMode: false}); + return Readable.from( + (async function* observeLimitedManifest() { + for await (const chunk of readable) { + limitedChunkCount += 1; + limitedBytes += chunk.length; + yield chunk; + } + })(), + { objectMode: false }, + ); }, }, fetchImpl: async (url, options) => { const requestBody = await readRequestBody(options.body); - requests.push({url, method: options.method, headers: options.headers, body: requestBody}); + requests.push({ + url, + method: options.method, + headers: options.headers, + body: requestBody, + }); if (options.method === 'PUT') { - return new Response(null, {status: 200}); + return new Response(null, { status: 200 }); } if (options.method === 'HEAD') { - return new Response(null, {status: 200, headers: { - 'x-oss-meta-file-size': String(body.length), - 'x-oss-meta-archive-sha256': bodySha256, - }}); + return new Response(null, { + status: 200, + headers: { + 'x-oss-meta-file-size': String(body.length), + 'x-oss-meta-archive-sha256': bodySha256, + }, + }); } throw new Error(`unexpected request: ${options.method} ${url}`); }, }); - assertEqual(result.archiveSha256, bodySha256, 'manifest 上传结果必须记录本地 SHA-256。'); - assertBufferEqual(requests.find(({method}) => method === 'PUT')?.body, body, 'manifest PUT 必须上传完整 JSON。'); - assertEqual(limitedBytes, body.length, 'manifest 必须完整经过上传带宽限制流。'); - assertTrue(limitedChunkCount > 1, '大型 manifest 必须分块经过限速器，不能整块突发上传。'); - assertTrue(requests.some(({method}) => method === 'HEAD'), 'manifest PUT 后必须执行 HEAD 验真。'); assertEqual( - requests.find(({method}) => method === 'PUT')?.headers['x-oss-meta-file-size'], + result.archiveSha256, + bodySha256, + 'manifest 上传结果必须记录本地 SHA-256。', + ); + assertBufferEqual( + requests.find(({ method }) => method === 'PUT')?.body, + body, + 'manifest PUT 必须上传完整 JSON。', + ); + assertEqual( + limitedBytes, + body.length, + 'manifest 必须完整经过上传带宽限制流。', + ); + assertTrue( + limitedChunkCount > 1, + '大型 manifest 必须分块经过限速器，不能整块突发上传。', + ); + assertTrue( + requests.some(({ method }) => method === 'HEAD'), + 'manifest PUT 后必须执行 HEAD 验真。', + ); + assertEqual( + requests.find(({ method }) => method === 'PUT')?.headers[ + 'x-oss-meta-file-size' + ], String(body.length), 'manifest PUT 必须记录原始字节数，供动态压缩 HEAD 缺少 content-length 时验真。', ); @@ -1054,22 +1696,25 @@ async function assertManifestUploadUsesShaAndHeadVerification() { async function assertMissingPartEtagAbortsMultipartUpload() { const root = path.join(tmpRoot, 'multipart-missing-etag'); const archivePath = path.join(root, 'backup.tar.gz'); - mkdirSync(root, {recursive: true}); + mkdirSync(root, { recursive: true }); writeFileSync(archivePath, Buffer.alloc(100 * 1024, 'e')); const requests = []; const fetchImpl = async (url, options) => { await readRequestBody(options.body); - requests.push({url, method: options.method}); + requests.push({ url, method: options.method }); const parsedUrl = new URL(url); if (options.method === 'POST' && parsedUrl.search === '?uploads') { - return new Response('missing-etag-upload', {status: 200}); + return new Response( + 'missing-etag-upload', + { status: 200 }, + ); } if (options.method === 'PUT') { - return new Response('', {status: 200}); + return new Response('', { status: 200 }); } if (options.method === 'DELETE') { - return new Response(null, {status: 204}); + return new Response(null, { status: 204 }); } throw new Error(`unexpected request: ${options.method} ${url}`); }; @@ -1094,9 +1739,16 @@ async function assertMissingPartEtagAbortsMultipartUpload() { uploadError = error; } - assertIncludes(uploadError?.message ?? '', '响应缺少 ETag', 'UploadPart 缺少 ETag 时必须失败。'); + assertIncludes( + uploadError?.message ?? '', + '响应缺少 ETag', + 'UploadPart 缺少 ETag 时必须失败。', + ); assertTrue( - requests.some(({method, url}) => method === 'DELETE' && url.endsWith('?uploadId=missing-etag-upload')), + requests.some( + ({ method, url }) => + method === 'DELETE' && url.endsWith('?uploadId=missing-etag-upload'), + ), 'UploadPart 缺少 ETag 后必须 AbortMultipartUpload。', ); } @@ -1106,36 +1758,47 @@ async function assertCompleteResponseAmbiguityUsesHeadVerification() { const archivePath = path.join(root, 'backup.tar.gz'); const payload = Buffer.alloc(100 * 1024, 'c'); const payloadSha256 = createHash('sha256').update(payload).digest('hex'); - mkdirSync(root, {recursive: true}); + mkdirSync(root, { recursive: true }); writeFileSync(archivePath, payload); const requests = []; let completeAttempts = 0; const fetchImpl = async (url, options) => { await readRequestBody(options.body); - requests.push({url, method: options.method}); + requests.push({ url, method: options.method }); const parsedUrl = new URL(url); if (options.method === 'POST' && parsedUrl.search === '?uploads') { - return new Response('ambiguous-upload', {status: 200}); + return new Response( + 'ambiguous-upload', + { status: 200 }, + ); } if (options.method === 'PUT') { - return new Response('', {status: 200, headers: {etag: '"part-etag"'}}); + return new Response('', { + status: 200, + headers: { etag: '"part-etag"' }, + }); } if (options.method === 'POST' && parsedUrl.searchParams.has('uploadId')) { completeAttempts += 1; if (completeAttempts === 1) { throw new TypeError('socket closed after remote complete'); } - return new Response('NoSuchUpload', {status: 404}); + return new Response('NoSuchUpload', { + status: 404, + }); } if (options.method === 'HEAD') { - return new Response(null, {status: 200, headers: { - 'content-length': String(payload.length), - 'x-oss-meta-archive-sha256': payloadSha256, - }}); + return new Response(null, { + status: 200, + headers: { + 'content-length': String(payload.length), + 'x-oss-meta-archive-sha256': payloadSha256, + }, + }); } if (options.method === 'DELETE') { - return new Response(null, {status: 204}); + return new Response(null, { status: 204 }); } throw new Error(`unexpected request: ${options.method} ${url}`); }; @@ -1158,13 +1821,23 @@ async function assertCompleteResponseAmbiguityUsesHeadVerification() { }); assertEqual(completeAttempts, 2, 'Complete 网络错误后应按策略重试。'); - assertEqual(result.contentLength, payload.length, 'Complete 结果不确定时应以 HEAD 长度验真收口。'); - assertTrue(requests.some(({method}) => method === 'HEAD'), 'Complete 结果不确定时必须执行 HEAD 验真。'); - assertTrue(!requests.some(({method}) => method === 'DELETE'), 'HEAD 已证实对象完整时不得 Abort 已完成上传。'); + assertEqual( + result.contentLength, + payload.length, + 'Complete 结果不确定时应以 HEAD 长度验真收口。', + ); + assertTrue( + requests.some(({ method }) => method === 'HEAD'), + 'Complete 结果不确定时必须执行 HEAD 验真。', + ); + assertTrue( + !requests.some(({ method }) => method === 'DELETE'), + 'HEAD 已证实对象完整时不得 Abort 已完成上传。', + ); } function assertHistoryDiscoversDevAndProductionLayoutsWithMultipleReplicas() { - const dev = createHistoryFixture('history-dev-layout', {nestedData: false}); + const dev = createHistoryFixture('history-dev-layout', { nestedData: false }); createReplicaHistory(dev.replicasDir, '1', { snapshots: [0, 187, 279], segments: [0, 188, 280], @@ -1173,108 +1846,276 @@ function assertHistoryDiscoversDevAndProductionLayoutsWithMultipleReplicas() { snapshots: [50, 99], segments: [0, 51, 100], }); - const devPlan = discoverHistoryPlan({dataDir: dev.dataDir}); - assertEqual(devPlan.replicas.length, 2, 'history 应逐 replica 计算安全边界。'); - assertEqual(devPlan.candidates.length, 7, '多 replica history 候选数量必须符合 snapshot/segment 边界。'); + const devPlan = discoverHistoryPlan({ dataDir: dev.dataDir }); + assertEqual( + devPlan.replicas.length, + 2, + 'history 应逐 replica 计算安全边界。', + ); + assertEqual( + devPlan.candidates.length, + 7, + '多 replica history 候选数量必须符合 snapshot/segment 边界。', + ); assertTrue( - devPlan.candidates.some(({path}) => path === 'replicas/1/clog/00000000000000000000.stdb.log'), + devPlan.candidates.some( + ({ path }) => path === 'replicas/1/clog/00000000000000000000.stdb.log', + ), 'dev 布局应识别边界 segment 之前的 commitlog。', ); assertTrue( - !devPlan.candidates.some(({path}) => path.includes('00000000000000000188.stdb.log')), + !devPlan.candidates.some(({ path }) => + path.includes('00000000000000000188.stdb.log'), + ), '跨越 latest snapshot 的边界 segment 必须保留。', ); assertTrue( - !devPlan.candidates.some(({path}) => path.includes('00000000000000000279.snapshot_dir')), + !devPlan.candidates.some(({ path }) => + path.includes('00000000000000000279.snapshot_dir'), + ), '每个 replica 的 latest snapshot 必须保留。', ); - const production = createHistoryFixture('history-production-layout', {nestedData: true}); + const production = createHistoryFixture('history-production-layout', { + nestedData: true, + }); createReplicaHistory(production.replicasDir, '7', { snapshots: [10, 20], segments: [0, 11, 21], }); - const productionPlan = discoverHistoryPlan({dataDir: production.dataDir}); - assertEqual(productionPlan.replicasDir, 'data/replicas', 'history 必须兼容 /stdb/data/replicas 布局。'); - assertEqual(productionPlan.candidates.length, 3, 'production 布局应识别一个旧 snapshot 与一对旧 commitlog 文件。'); + const productionPlan = discoverHistoryPlan({ dataDir: production.dataDir }); + assertEqual( + productionPlan.replicasDir, + 'data/replicas', + 'history 必须兼容 /stdb/data/replicas 布局。', + ); + assertEqual( + productionPlan.candidates.length, + 3, + 'production 布局应识别一个旧 snapshot 与一对旧 commitlog 文件。', + ); const importResult = runHistoryDryRun(dev); - assertStatus(importResult, 0, 'history dry-run 应能从已有 uploaded baseline manifest 导入 state。'); - assertTrue(existsSync(dev.statePath), 'history dry-run 应持久化导入后的 baseline state。'); - assertIncludes(importResult.stdout, 'history dry-run', 'history dry-run 应明确说明不会上传或删除。'); + assertStatus( + importResult, + 0, + 'history dry-run 应能从已有 uploaded baseline manifest 导入 state。', + ); + assertTrue( + existsSync(dev.statePath), + 'history dry-run 应持久化导入后的 baseline state。', + ); + assertIncludes( + importResult.stdout, + 'history dry-run', + 'history dry-run 应明确说明不会上传或删除。', + ); for (const candidate of devPlan.candidates) { - assertTrue(existsSync(path.join(dev.dataDir, candidate.path)), `history dry-run 不得删除候选: ${candidate.path}`); + assertTrue( + existsSync(path.join(dev.dataDir, candidate.path)), + `history dry-run 不得删除候选: ${candidate.path}`, + ); } } function assertHistorySkipsReplicaWithoutSnapshotAndRejectsMalformedNames() { - const noSnapshot = createHistoryFixture('history-no-snapshot', {nestedData: false}); - createReplicaHistory(noSnapshot.replicasDir, '1', {snapshots: [], segments: [0]}); - const plan = discoverHistoryPlan({dataDir: noSnapshot.dataDir}); - assertEqual(plan.candidates.length, 0, '没有 snapshot 的 replica 不得产生可删除候选。'); - assertEqual(plan.replicas[0]?.reason, 'no-snapshot', '没有 snapshot 时应记录明确跳过原因。'); + const noSnapshot = createHistoryFixture('history-no-snapshot', { + nestedData: false, + }); + createReplicaHistory(noSnapshot.replicasDir, '1', { + snapshots: [], + segments: [0], + }); + const plan = discoverHistoryPlan({ dataDir: noSnapshot.dataDir }); + assertEqual( + plan.candidates.length, + 0, + '没有 snapshot 的 replica 不得产生可删除候选。', + ); + assertEqual( + plan.replicas[0]?.reason, + 'no-snapshot', + '没有 snapshot 时应记录明确跳过原因。', + ); - const incompleteSnapshot = createHistoryFixture('history-incomplete-snapshot', {nestedData: false}); - const incompleteReplica = createReplicaHistory(incompleteSnapshot.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); - mkdirSync(path.join(incompleteReplica.snapshotsDir, '00000000000000000020.snapshot_dir')); - mkdirSync(path.join(incompleteReplica.snapshotsDir, '00000000000000000030.snapshot_dir')); - writeFileSync(path.join(incompleteReplica.snapshotsDir, '00000000000000000030.snapshot_dir', '00000000000000000030.snapshot_bsatn'), 'locked'); - writeFileSync(path.join(incompleteReplica.snapshotsDir, '00000000000000000030.lock'), `${process.pid}\n`); - const incompletePlan = discoverHistoryPlan({dataDir: incompleteSnapshot.dataDir}); - assertEqual(incompletePlan.replicas[0]?.latestSnapshot, '10', '缺少 snapshot_bsatn 或仍有 lockfile 的目录不得成为 latest snapshot。'); + const incompleteSnapshot = createHistoryFixture( + 'history-incomplete-snapshot', + { nestedData: false }, + ); + const incompleteReplica = createReplicaHistory( + incompleteSnapshot.replicasDir, + '1', + { snapshots: [0, 10], segments: [0, 1, 11] }, + ); + mkdirSync( + path.join( + incompleteReplica.snapshotsDir, + '00000000000000000020.snapshot_dir', + ), + ); + mkdirSync( + path.join( + incompleteReplica.snapshotsDir, + '00000000000000000030.snapshot_dir', + ), + ); + writeFileSync( + path.join( + incompleteReplica.snapshotsDir, + '00000000000000000030.snapshot_dir', + '00000000000000000030.snapshot_bsatn', + ), + 'locked', + ); + writeFileSync( + path.join(incompleteReplica.snapshotsDir, '00000000000000000030.lock'), + `${process.pid}\n`, + ); + const incompletePlan = discoverHistoryPlan({ + dataDir: incompleteSnapshot.dataDir, + }); + assertEqual( + incompletePlan.replicas[0]?.latestSnapshot, + '10', + '缺少 snapshot_bsatn 或仍有 lockfile 的目录不得成为 latest snapshot。', + ); assertTrue( - !incompletePlan.candidates.some(({path}) => path.includes('00000000000000000010.snapshot_dir')), + !incompletePlan.candidates.some(({ path }) => + path.includes('00000000000000000010.snapshot_dir'), + ), '最后一个完整且未锁定的 snapshot 必须保留。', ); - const malformedLog = createHistoryFixture('history-malformed-log', {nestedData: false}); - const malformedLogReplica = createReplicaHistory(malformedLog.replicasDir, '1', {snapshots: [10], segments: [0, 11]}); - writeFileSync(path.join(malformedLogReplica.clogDir, 'broken.stdb.log'), 'broken'); + const malformedLog = createHistoryFixture('history-malformed-log', { + nestedData: false, + }); + const malformedLogReplica = createReplicaHistory( + malformedLog.replicasDir, + '1', + { snapshots: [10], segments: [0, 11] }, + ); + writeFileSync( + path.join(malformedLogReplica.clogDir, 'broken.stdb.log'), + 'broken', + ); assertThrows( - () => discoverHistoryPlan({dataDir: malformedLog.dataDir}), + () => discoverHistoryPlan({ dataDir: malformedLog.dataDir }), 'commitlog 文件名不符合预期', '异常 commitlog 名称必须阻断整个清理计划。', ); } function assertHistoryRequiresBaselineAndProducesDeterministicDeferredBatch() { - const missingBaseline = createHistoryFixture('history-missing-baseline', {nestedData: false}); - createReplicaHistory(missingBaseline.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); - rmSync(missingBaseline.baselineManifestPath, {force: true}); - const missingResult = runHistoryCommand(missingBaseline, ['--dry-run'], {includeBaselineManifest: false}); - assertStatus(missingResult, 1, 'history 没有 baseline state 或 imported manifest 时必须失败。'); - assertIncludes(missingResult.stderr, '缺少已验真 baseline state', 'baseline 门禁失败应给出明确错误。'); + const missingBaseline = createHistoryFixture('history-missing-baseline', { + nestedData: false, + }); + createReplicaHistory(missingBaseline.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); + rmSync(missingBaseline.baselineManifestPath, { force: true }); + const missingResult = runHistoryCommand(missingBaseline, ['--dry-run'], { + includeBaselineManifest: false, + }); + assertStatus( + missingResult, + 1, + 'history 没有 baseline state 或 imported manifest 时必须失败。', + ); + assertIncludes( + missingResult.stderr, + '缺少已验真 baseline state', + 'baseline 门禁失败应给出明确错误。', + ); - const wrongKind = createHistoryFixture('history-wrong-baseline-kind', {nestedData: false}); - createReplicaHistory(wrongKind.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); - const wrongKindManifest = JSON.parse(readFileSync(wrongKind.baselineManifestPath, 'utf8')); + const wrongKind = createHistoryFixture('history-wrong-baseline-kind', { + nestedData: false, + }); + createReplicaHistory(wrongKind.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); + const wrongKindManifest = JSON.parse( + readFileSync(wrongKind.baselineManifestPath, 'utf8'), + ); wrongKindManifest.backupKind = 'spacetimedb-history'; - writeFileSync(wrongKind.baselineManifestPath, `${JSON.stringify(wrongKindManifest)}\n`); + writeFileSync( + wrongKind.baselineManifestPath, + `${JSON.stringify(wrongKindManifest)}\n`, + ); const wrongKindResult = runHistoryDryRun(wrongKind); - assertStatus(wrongKindResult, 1, 'history archive manifest 不得被导入为 full baseline。'); - assertIncludes(wrongKindResult.stderr, 'backupKind 必须是 spacetimedb-data-dir', 'baseline 类型不匹配应失败关闭。'); + assertStatus( + wrongKindResult, + 1, + 'history archive manifest 不得被导入为 full baseline。', + ); + assertIncludes( + wrongKindResult.stderr, + 'backupKind 必须是 spacetimedb-data-dir', + 'baseline 类型不匹配应失败关闭。', + ); - const deterministic = createHistoryFixture('history-deterministic-batch', {nestedData: false}); - createReplicaHistory(deterministic.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const deterministic = createHistoryFixture('history-deterministic-batch', { + nestedData: false, + }); + createReplicaHistory(deterministic.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); importHistoryState(deterministic); const firstResultFile = path.join(deterministic.workDir, 'defer-first.json'); - const secondResultFile = path.join(deterministic.workDir, 'defer-second.json'); - const first = runHistoryCommand(deterministic, ['--defer-upload', '--result-file', firstResultFile]); - const second = runHistoryCommand(deterministic, ['--defer-upload', '--result-file', secondResultFile]); + const secondResultFile = path.join( + deterministic.workDir, + 'defer-second.json', + ); + const first = runHistoryCommand(deterministic, [ + '--defer-upload', + '--result-file', + firstResultFile, + ]); + const second = runHistoryCommand(deterministic, [ + '--defer-upload', + '--result-file', + secondResultFile, + ]); assertStatus(first, 0, '第一次 history defer 应成功生成归档。'); - assertStatus(second, 0, '相同候选重复 history defer 应幂等复用 batch identity。'); + assertStatus( + second, + 0, + '相同候选重复 history defer 应幂等复用 batch identity。', + ); const firstPayload = JSON.parse(readFileSync(firstResultFile, 'utf8')); const secondPayload = JSON.parse(readFileSync(secondResultFile, 'utf8')); - assertEqual(firstPayload.batchId, secondPayload.batchId, '相同 baseline 与候选必须生成确定性 batchId。'); - assertEqual(firstPayload.objectKey, secondPayload.objectKey, '相同 batch 重跑不得制造新的 OSS object key。'); - const archiveListing = spawnSync('tar', ['-tzf', firstPayload.archivePath], {encoding: 'utf8'}); + assertEqual( + firstPayload.batchId, + secondPayload.batchId, + '相同 baseline 与候选必须生成确定性 batchId。', + ); + assertEqual( + firstPayload.objectKey, + secondPayload.objectKey, + '相同 batch 重跑不得制造新的 OSS object key。', + ); + const archiveListing = spawnSync('tar', ['-tzf', firstPayload.archivePath], { + encoding: 'utf8', + }); assertStatus(archiveListing, 0, 'history 归档应可被 tar 正常读取。'); - assertIncludes(archiveListing.stdout, path.basename(firstPayload.manifestPath), 'history 归档内部必须携带安全候选 manifest。'); + assertIncludes( + archiveListing.stdout, + path.basename(firstPayload.manifestPath), + 'history 归档内部必须携带安全候选 manifest。', + ); - const dryRunPending = createHistoryFixture('history-dry-run-pending-cleanup', {nestedData: false}); - createReplicaHistory(dryRunPending.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const dryRunPending = createHistoryFixture( + 'history-dry-run-pending-cleanup', + { nestedData: false }, + ); + createReplicaHistory(dryRunPending.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); const dryRunState = importHistoryState(dryRunPending); - const dryRunPlan = discoverHistoryPlan({dataDir: dryRunPending.dataDir}); + const dryRunPlan = discoverHistoryPlan({ dataDir: dryRunPending.dataDir }); dryRunState.batches.push({ batchId: 'pending-cleanup', objectKey: 'database-backups/test-db/history/pending-cleanup.tar.gz', @@ -1286,57 +2127,117 @@ function assertHistoryRequiresBaselineAndProducesDeterministicDeferredBatch() { }); writeFileSync(dryRunPending.statePath, `${JSON.stringify(dryRunState)}\n`); const pendingDryRunResult = runHistoryDryRun(dryRunPending); - assertStatus(pendingDryRunResult, 0, '存在待清理 uploaded batch 时 history dry-run 仍应只读成功。'); + assertStatus( + pendingDryRunResult, + 0, + '存在待清理 uploaded batch 时 history dry-run 仍应只读成功。', + ); for (const candidate of dryRunPlan.candidates) { - assertTrue(existsSync(path.join(dryRunPending.dataDir, candidate.path)), `history dry-run 不得恢复执行待清理 batch: ${candidate.path}`); + assertTrue( + existsSync(path.join(dryRunPending.dataDir, candidate.path)), + `history dry-run 不得恢复执行待清理 batch: ${candidate.path}`, + ); } } function assertHistoryBackupLockRejectsLiveAndStaleOwners() { - const liveOwner = createHistoryFixture('history-live-lock', {nestedData: false}); - createReplicaHistory(liveOwner.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const liveOwner = createHistoryFixture('history-live-lock', { + nestedData: false, + }); + createReplicaHistory(liveOwner.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); importHistoryState(liveOwner); const liveLockPath = path.join(liveOwner.workDir, 'test-db.backup.lock'); writeFileSync(liveLockPath, `${process.pid}\n`); const liveResult = runHistoryCommand(liveOwner, ['--defer-upload']); - assertStatus(liveResult, 1, '仍存活进程持有 backup lock 时必须拒绝并发备份。'); - assertIncludes(liveResult.stderr, '已有数据库备份进程持有锁', '并发备份失败应报告 lock owner pid。'); + assertStatus( + liveResult, + 1, + '仍存活进程持有 backup lock 时必须拒绝并发备份。', + ); + assertIncludes( + liveResult.stderr, + '已有数据库备份进程持有锁', + '并发备份失败应报告 lock owner pid。', + ); - const staleOwner = createHistoryFixture('history-stale-lock', {nestedData: false}); - createReplicaHistory(staleOwner.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const staleOwner = createHistoryFixture('history-stale-lock', { + nestedData: false, + }); + createReplicaHistory(staleOwner.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); importHistoryState(staleOwner); const staleLockPath = path.join(staleOwner.workDir, 'test-db.backup.lock'); writeFileSync(staleLockPath, '2147483647\n'); const staleResult = runHistoryCommand(staleOwner, ['--defer-upload']); - assertStatus(staleResult, 1, '失效 owner pid 的 backup lock 也必须失败关闭，避免并发抢锁。'); - assertIncludes(staleResult.stderr, '拒绝自动抢锁', '失效 backup lock 应要求人工核对 multipart 与进程。'); - assertTrue(existsSync(staleLockPath), '失效 backup lock 未经人工核对不得自动删除。'); + assertStatus( + staleResult, + 1, + '失效 owner pid 的 backup lock 也必须失败关闭，避免并发抢锁。', + ); + assertIncludes( + staleResult.stderr, + '拒绝自动抢锁', + '失效 backup lock 应要求人工核对 multipart 与进程。', + ); + assertTrue( + existsSync(staleLockPath), + '失效 backup lock 未经人工核对不得自动删除。', + ); } function assertHistoryStatDriftPreventsAnyCleanup() { - const fixture = createHistoryFixture('history-stat-drift', {nestedData: false}); - createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); - const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); - const driftCandidate = plan.candidates.find(({kind}) => kind === 'commitlog'); - const untouchedCandidate = plan.candidates.find(({kind}) => kind === 'snapshot'); - writeFileSync(path.join(fixture.dataDir, driftCandidate.path), 'changed-after-plan'); + const fixture = createHistoryFixture('history-stat-drift', { + nestedData: false, + }); + createReplicaHistory(fixture.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); + const plan = discoverHistoryPlan({ dataDir: fixture.dataDir }); + const driftCandidate = plan.candidates.find( + ({ kind }) => kind === 'commitlog', + ); + const untouchedCandidate = plan.candidates.find( + ({ kind }) => kind === 'snapshot', + ); + writeFileSync( + path.join(fixture.dataDir, driftCandidate.path), + 'changed-after-plan', + ); assertThrows( - () => cleanupHistoryCandidates({dataDir: fixture.dataDir, candidates: plan.candidates}), + () => + cleanupHistoryCandidates({ + dataDir: fixture.dataDir, + candidates: plan.candidates, + }), 'stat 漂移', '任一候选 stat 漂移时必须在删除任何文件前失败。', ); - assertTrue(existsSync(path.join(fixture.dataDir, untouchedCandidate.path)), 'stat 漂移失败时不得删除其他候选。'); + assertTrue( + existsSync(path.join(fixture.dataDir, untouchedCandidate.path)), + 'stat 漂移失败时不得删除其他候选。', + ); } async function assertHistoryUploadFailureDoesNotDeleteSources() { - const fixture = createHistoryFixture('history-upload-failure', {nestedData: false}); - createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const fixture = createHistoryFixture('history-upload-failure', { + nestedData: false, + }); + createReplicaHistory(fixture.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); const state = importHistoryState(fixture); - const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); + const plan = discoverHistoryPlan({ dataDir: fixture.dataDir }); const archivePath = path.join(fixture.workDir, 'history.tar.gz'); const manifestPath = `${archivePath}.manifest.json`; writeFileSync(archivePath, 'history archive'); - const manifest = createHistoryManifest({fixture, state, plan, archivePath}); + const manifest = createHistoryManifest({ fixture, state, plan, archivePath }); writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`); let uploadError = null; @@ -1357,18 +2258,40 @@ async function assertHistoryUploadFailureDoesNotDeleteSources() { } catch (error) { uploadError = error; } - assertIncludes(uploadError?.message ?? '', 'synthetic upload failure', 'history 应保留上传失败原因。'); + assertIncludes( + uploadError?.message ?? '', + 'synthetic upload failure', + 'history 应保留上传失败原因。', + ); for (const candidate of plan.candidates) { - assertTrue(existsSync(path.join(fixture.dataDir, candidate.path)), `上传失败不得删除 history 源文件: ${candidate.path}`); + assertTrue( + existsSync(path.join(fixture.dataDir, candidate.path)), + `上传失败不得删除 history 源文件: ${candidate.path}`, + ); } const stateAfterFailure = JSON.parse(readFileSync(fixture.statePath, 'utf8')); - assertEqual(stateAfterFailure.batches.length, 0, '上传失败不得把 batch 标记为 uploaded。'); + assertEqual( + stateAfterFailure.batches.length, + 0, + '上传失败不得把 batch 标记为 uploaded。', + ); - const manifestFailure = createHistoryFixture('history-manifest-upload-failure', {nestedData: false}); - createReplicaHistory(manifestFailure.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const manifestFailure = createHistoryFixture( + 'history-manifest-upload-failure', + { nestedData: false }, + ); + createReplicaHistory(manifestFailure.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); const manifestFailureState = importHistoryState(manifestFailure); - const manifestFailurePlan = discoverHistoryPlan({dataDir: manifestFailure.dataDir}); - const manifestFailureArchive = path.join(manifestFailure.workDir, 'history.tar.gz'); + const manifestFailurePlan = discoverHistoryPlan({ + dataDir: manifestFailure.dataDir, + }); + const manifestFailureArchive = path.join( + manifestFailure.workDir, + 'history.tar.gz', + ); const manifestFailurePath = `${manifestFailureArchive}.manifest.json`; writeFileSync(manifestFailureArchive, 'history archive'); const manifestFailurePayload = createHistoryManifest({ @@ -1377,7 +2300,10 @@ async function assertHistoryUploadFailureDoesNotDeleteSources() { plan: manifestFailurePlan, archivePath: manifestFailureArchive, }); - writeFileSync(manifestFailurePath, `${JSON.stringify(manifestFailurePayload)}\n`); + writeFileSync( + manifestFailurePath, + `${JSON.stringify(manifestFailurePayload)}\n`, + ); let manifestUploadError = null; try { await uploadHistoryArchiveWithCleanup({ @@ -1400,21 +2326,33 @@ async function assertHistoryUploadFailureDoesNotDeleteSources() { } catch (error) { manifestUploadError = error; } - assertIncludes(manifestUploadError?.message ?? '', 'synthetic manifest upload failure', 'sidecar manifest 上传失败应阻断清理。'); + assertIncludes( + manifestUploadError?.message ?? '', + 'synthetic manifest upload failure', + 'sidecar manifest 上传失败应阻断清理。', + ); for (const candidate of manifestFailurePlan.candidates) { - assertTrue(existsSync(path.join(manifestFailure.dataDir, candidate.path)), `manifest 上传失败不得删除源文件: ${candidate.path}`); + assertTrue( + existsSync(path.join(manifestFailure.dataDir, candidate.path)), + `manifest 上传失败不得删除源文件: ${candidate.path}`, + ); } } async function assertHistorySuccessfulUploadCleansAndIsIdempotent() { - const fixture = createHistoryFixture('history-upload-success', {nestedData: false}); - createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const fixture = createHistoryFixture('history-upload-success', { + nestedData: false, + }); + createReplicaHistory(fixture.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); const state = importHistoryState(fixture); - const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); + const plan = discoverHistoryPlan({ dataDir: fixture.dataDir }); const archivePath = path.join(fixture.workDir, 'history.tar.gz'); const manifestPath = `${archivePath}.manifest.json`; writeFileSync(archivePath, 'history archive'); - const manifest = createHistoryManifest({fixture, state, plan, archivePath}); + const manifest = createHistoryManifest({ fixture, state, plan, archivePath }); writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`); let baselineVerifyCount = 0; const result = await uploadHistoryArchiveWithCleanup({ @@ -1434,7 +2372,7 @@ async function assertHistorySuccessfulUploadCleansAndIsIdempotent() { partSizeBytes: 102400, verifiedAt: '2026-07-16T00:10:00.000Z', }), - manifestUploadFn: async ({objectKey}) => ({ + manifestUploadFn: async ({ objectKey }) => ({ objectKey, contentLength: 512, archiveSha256: 'd'.repeat(64), @@ -1442,30 +2380,54 @@ async function assertHistorySuccessfulUploadCleansAndIsIdempotent() { }), verifyFn: async () => { baselineVerifyCount += 1; - return {verifiedAt: '2026-07-16T00:10:02.000Z'}; + return { verifiedAt: '2026-07-16T00:10:02.000Z' }; }, }); - assertEqual(baselineVerifyCount, 2, 'history 删除源文件前必须重新验真 full baseline 与 sidecar。'); - assertEqual(result.cleanup.deletedCount, plan.candidates.length, '验真上传成功后应删除全部安全候选。'); + assertEqual( + baselineVerifyCount, + 2, + 'history 删除源文件前必须重新验真 full baseline 与 sidecar。', + ); + assertEqual( + result.cleanup.deletedCount, + plan.candidates.length, + '验真上传成功后应删除全部安全候选。', + ); for (const candidate of plan.candidates) { - assertTrue(!existsSync(path.join(fixture.dataDir, candidate.path)), `验真成功后应删除 history 源文件: ${candidate.path}`); + assertTrue( + !existsSync(path.join(fixture.dataDir, candidate.path)), + `验真成功后应删除 history 源文件: ${candidate.path}`, + ); } - const repeatedCleanup = cleanupHistoryCandidates({dataDir: fixture.dataDir, candidates: plan.candidates}); - assertEqual(repeatedCleanup.alreadyMissingCount, plan.candidates.length, '重复清理同一 uploaded batch 应幂等。'); + const repeatedCleanup = cleanupHistoryCandidates({ + dataDir: fixture.dataDir, + candidates: plan.candidates, + }); + assertEqual( + repeatedCleanup.alreadyMissingCount, + plan.candidates.length, + '重复清理同一 uploaded batch 应幂等。', + ); } async function assertHistoryResumeReverifiesArchiveAndManifest() { - const fixture = createHistoryFixture('history-resume-verification', {nestedData: false}); - createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const fixture = createHistoryFixture('history-resume-verification', { + nestedData: false, + }); + createReplicaHistory(fixture.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); const state = importHistoryState(fixture); - const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); + const plan = discoverHistoryPlan({ dataDir: fixture.dataDir }); state.batches.push({ batchId: 'resume-batch', objectKey: 'database-backups/test-db/history/resume.tar.gz', contentLength: 100, archiveSha256: 'e'.repeat(64), verifiedAt: '2026-07-16T00:30:00.000Z', - manifestObjectKey: 'database-backups/test-db/history/resume.tar.gz.manifest.json', + manifestObjectKey: + 'database-backups/test-db/history/resume.tar.gz.manifest.json', manifestContentLength: 200, manifestArchiveSha256: 'f'.repeat(64), manifestVerifiedAt: '2026-07-16T00:30:01.000Z', @@ -1480,21 +2442,36 @@ async function assertHistoryResumeReverifiesArchiveAndManifest() { state, dataDir: fixture.dataDir, verificationOptions: {}, - verifyFn: async ({objectKey}) => { + verifyFn: async ({ objectKey }) => { verifiedKeys.push(objectKey); - return {verifiedAt: '2026-07-16T00:31:00.000Z'}; + return { verifiedAt: '2026-07-16T00:31:00.000Z' }; }, }); - assertEqual(verifiedKeys.length, 2, '续清理前必须重新验真 history archive 与 sidecar manifest。'); + assertEqual( + verifiedKeys.length, + 2, + '续清理前必须重新验真 history archive 与 sidecar manifest。', + ); for (const candidate of plan.candidates) { - assertTrue(!existsSync(path.join(fixture.dataDir, candidate.path)), `续清理验真后应删除候选: ${candidate.path}`); + assertTrue( + !existsSync(path.join(fixture.dataDir, candidate.path)), + `续清理验真后应删除候选: ${candidate.path}`, + ); } - const failure = createHistoryFixture('history-resume-verification-failure', {nestedData: false}); - createReplicaHistory(failure.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const failure = createHistoryFixture('history-resume-verification-failure', { + nestedData: false, + }); + createReplicaHistory(failure.replicasDir, '1', { + snapshots: [0, 10], + segments: [0, 1, 11], + }); const failureState = importHistoryState(failure); - const failurePlan = discoverHistoryPlan({dataDir: failure.dataDir}); - failureState.batches.push({...state.batches[0], candidates: failurePlan.candidates}); + const failurePlan = discoverHistoryPlan({ dataDir: failure.dataDir }); + failureState.batches.push({ + ...state.batches[0], + candidates: failurePlan.candidates, + }); writeFileSync(failure.statePath, `${JSON.stringify(failureState)}\n`); let resumeError = null; try { @@ -1510,87 +2487,140 @@ async function assertHistoryResumeReverifiesArchiveAndManifest() { } catch (error) { resumeError = error; } - assertIncludes(resumeError?.message ?? '', 'synthetic resume HEAD failure', '续清理 OSS 复核失败应保留错误。'); + assertIncludes( + resumeError?.message ?? '', + 'synthetic resume HEAD failure', + '续清理 OSS 复核失败应保留错误。', + ); for (const candidate of failurePlan.candidates) { - assertTrue(existsSync(path.join(failure.dataDir, candidate.path)), `续清理验真失败不得删除候选: ${candidate.path}`); + assertTrue( + existsSync(path.join(failure.dataDir, candidate.path)), + `续清理验真失败不得删除候选: ${candidate.path}`, + ); } } -function createHistoryFixture(name, {nestedData}) { +function createHistoryFixture(name, { nestedData }) { const root = path.join(tmpRoot, name); const dataDir = path.join(root, 'stdb'); - const replicasDir = nestedData ? path.join(dataDir, 'data', 'replicas') : path.join(dataDir, 'replicas'); + const replicasDir = nestedData + ? path.join(dataDir, 'data', 'replicas') + : path.join(dataDir, 'replicas'); const workDir = path.join(root, 'work'); const statePath = path.join(workDir, 'history-state.json'); const baselineManifestPath = path.join(workDir, 'baseline.manifest.json'); - mkdirSync(replicasDir, {recursive: true}); - mkdirSync(workDir, {recursive: true}); - writeFileSync(baselineManifestPath, `${JSON.stringify({ - backupKind: 'spacetimedb-data-dir', - uploadStatus: 'uploaded', - database: 'test-db', + mkdirSync(replicasDir, { recursive: true }); + mkdirSync(workDir, { recursive: true }); + writeFileSync( + baselineManifestPath, + `${JSON.stringify( + { + backupKind: 'spacetimedb-data-dir', + uploadStatus: 'uploaded', + database: 'test-db', + dataDir, + bucket: 'backup-bucket', + objectKey: 'database-backups/test-db/baseline.tar.gz', + manifestObjectKey: + 'database-backups/test-db/baseline.tar.gz.manifest.json', + contentLength: 1234, + archiveSha256: 'a'.repeat(64), + manifestContentLength: 512, + manifestArchiveSha256: '9'.repeat(64), + manifestVerifiedAt: '2026-07-16T00:00:00.500Z', + verifiedAt: '2026-07-16T00:00:00.000Z', + uploadedAt: '2026-07-16T00:00:01.000Z', + }, + null, + 2, + )}\n`, + ); + return { + root, dataDir, - bucket: 'backup-bucket', - objectKey: 'database-backups/test-db/baseline.tar.gz', - manifestObjectKey: 'database-backups/test-db/baseline.tar.gz.manifest.json', - contentLength: 1234, - archiveSha256: 'a'.repeat(64), - manifestContentLength: 512, - manifestArchiveSha256: '9'.repeat(64), - manifestVerifiedAt: '2026-07-16T00:00:00.500Z', - verifiedAt: '2026-07-16T00:00:00.000Z', - uploadedAt: '2026-07-16T00:00:01.000Z', - }, null, 2)}\n`); - return {root, dataDir, replicasDir, workDir, statePath, baselineManifestPath}; + replicasDir, + workDir, + statePath, + baselineManifestPath, + }; } -function createReplicaHistory(replicasDir, replicaId, {snapshots, segments}) { +function createReplicaHistory(replicasDir, replicaId, { snapshots, segments }) { const replicaDir = path.join(replicasDir, replicaId); const snapshotsDir = path.join(replicaDir, 'snapshots'); const clogDir = path.join(replicaDir, 'clog'); - mkdirSync(snapshotsDir, {recursive: true}); - mkdirSync(clogDir, {recursive: true}); + mkdirSync(snapshotsDir, { recursive: true }); + mkdirSync(clogDir, { recursive: true }); for (const transaction of snapshots) { const name = `${String(transaction).padStart(20, '0')}.snapshot_dir`; const snapshotDir = path.join(snapshotsDir, name); - mkdirSync(path.join(snapshotDir, 'objects'), {recursive: true}); - writeFileSync(path.join(snapshotDir, `${String(transaction).padStart(20, '0')}.snapshot_bsatn`), `snapshot-${transaction}`); - writeFileSync(path.join(snapshotDir, 'objects', 'object.bin'), `object-${transaction}`); + mkdirSync(path.join(snapshotDir, 'objects'), { recursive: true }); + writeFileSync( + path.join( + snapshotDir, + `${String(transaction).padStart(20, '0')}.snapshot_bsatn`, + ), + `snapshot-${transaction}`, + ); + writeFileSync( + path.join(snapshotDir, 'objects', 'object.bin'), + `object-${transaction}`, + ); } for (const transaction of segments) { const prefix = String(transaction).padStart(20, '0'); - writeFileSync(path.join(clogDir, `${prefix}.stdb.log`), `log-${transaction}`); - writeFileSync(path.join(clogDir, `${prefix}.stdb.ofs`), `ofs-${transaction}`); + writeFileSync( + path.join(clogDir, `${prefix}.stdb.log`), + `log-${transaction}`, + ); + writeFileSync( + path.join(clogDir, `${prefix}.stdb.ofs`), + `ofs-${transaction}`, + ); } - return {replicaDir, snapshotsDir, clogDir}; + return { replicaDir, snapshotsDir, clogDir }; } function runHistoryDryRun(fixture) { const resultFile = path.join(fixture.workDir, 'dry-run-result.json'); - return runHistoryCommand(fixture, [ - '--result-file', resultFile, - '--dry-run', - ]); + return runHistoryCommand(fixture, ['--result-file', resultFile, '--dry-run']); } -function runHistoryCommand(fixture, extraArgs = [], {includeBaselineManifest = true} = {}) { +function runHistoryCommand( + fixture, + extraArgs = [], + { includeBaselineManifest = true } = {}, +) { const baselineManifestArgs = includeBaselineManifest ? ['--baseline-manifest', fixture.baselineManifestPath] : []; - return spawnSync(process.execPath, [ - BACKUP_SCRIPT, - '--mode', 'history', - '--data-dir', fixture.dataDir, - '--work-dir', fixture.workDir, - '--database', 'test-db', - '--bucket', 'backup-bucket', - '--endpoint', 'oss-cn-shanghai.aliyuncs.com', - '--access-key-id', 'test-access-key', - '--access-key-secret', 'test-access-secret', - '--baseline-state', fixture.statePath, - ...baselineManifestArgs, - ...extraArgs, - ], {encoding: 'utf8'}); + return spawnSync( + process.execPath, + [ + BACKUP_SCRIPT, + '--mode', + 'history', + '--data-dir', + fixture.dataDir, + '--work-dir', + fixture.workDir, + '--database', + 'test-db', + '--bucket', + 'backup-bucket', + '--endpoint', + 'oss-cn-shanghai.aliyuncs.com', + '--access-key-id', + 'test-access-key', + '--access-key-secret', + 'test-access-secret', + '--baseline-state', + fixture.statePath, + ...baselineManifestArgs, + ...extraArgs, + ], + { encoding: 'utf8' }, + ); } function importHistoryState(fixture) { @@ -1599,7 +2629,7 @@ function importHistoryState(fixture) { return JSON.parse(readFileSync(fixture.statePath, 'utf8')); } -function createHistoryManifest({fixture, state, plan, archivePath}) { +function createHistoryManifest({ fixture, state, plan, archivePath }) { return { schemaVersion: 1, backupKind: 'spacetimedb-history', @@ -1640,9 +2670,13 @@ function createFixture(name) { const workDir = path.join(root, 'work'); const systemctlLog = path.join(root, 'systemctl.log'); const tarLog = path.join(root, 'tar.log'); - mkdirSync(binDir, {recursive: true}); - mkdirSync(dataDir, {recursive: true}); - writeFileSync(path.join(dataDir, 'sample.bin'), 'sample backup payload\n', 'utf8'); + mkdirSync(binDir, { recursive: true }); + mkdirSync(dataDir, { recursive: true }); + writeFileSync( + path.join(dataDir, 'sample.bin'), + 'sample backup payload\n', + 'utf8', + ); writeExecutable( path.join(binDir, 'systemctl'), `#!/usr/bin/env bash @@ -1658,7 +2692,7 @@ echo 'fake tar failure' >&2 exit 2 `, ); - return {root, binDir, dataDir, workDir, systemctlLog, tarLog}; + return { root, binDir, dataDir, workDir, systemctlLog, tarLog }; } function runBackup(fixture, extraArgs = []) { @@ -1693,7 +2727,7 @@ function runBackup(fixture, extraArgs = []) { function writeExecutable(filePath, content) { writeFileSync(filePath, content, 'utf8'); - spawnSync('chmod', ['0755', filePath], {encoding: 'utf8'}); + spawnSync('chmod', ['0755', filePath], { encoding: 'utf8' }); } function readFile(filePath) { @@ -1717,7 +2751,9 @@ function assertIncludes(content, expected, reason) { function assertEqual(actual, expected, reason) { if (actual !== expected) { - failures.push(`${reason} 预期: ${String(expected)}，实际: ${String(actual)}`); + failures.push( + `${reason} 预期: ${String(expected)}，实际: ${String(actual)}`, + ); } } @@ -1749,7 +2785,9 @@ function assertThrows(callback, expectedMessage, reason) { function assertBufferEqual(actual, expected, reason) { if (!Buffer.isBuffer(actual) || !actual.equals(expected)) { - failures.push(`${reason} 预期 ${expected.length} bytes，实际 ${actual?.length ?? ''} bytes。`); + failures.push( + `${reason} 预期 ${expected.length} bytes，实际 ${actual?.length ?? ''} bytes。`, + ); } } diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index c9463ace6..bacd4b949 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -821,6 +821,31 @@ const checks = [ reason: '生产冷备份 service 必须用 node -- 分隔脚本参数，避免 Node 22 抢占业务 --env-file。', }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: 'Environment=NODE_OPTIONS=--max-old-space-size=768', + reason: + '备份 Node 进程必须设置独立 heap 上限，避免目录扫描异常拖垮 release 主机。', + }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: + 'Environment=GENARRATIVE_DATABASE_BACKUP_STOP_MARKER=/var/lib/genarrative/database-backups/.spacetimedb-stopped', + reason: + '备份停库 marker 必须固定在受保护的 release work-dir，供 OOM 后 systemd 兜底恢复服务。', + }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: 'MemoryMax=1G', + reason: + '备份 service 必须设置 systemd 内存硬上限，避免异常进程消耗整机内存。', + }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: 'ExecStopPost=/bin/sh -c', + reason: + '备份主进程被 OOM kill 后必须由 systemd 兜底恢复停掉的 SpacetimeDB、API、worker 和 controller。', + }, { file: 'deploy/systemd/genarrative-database-backup.service', excludes: '--storage-format files', @@ -941,10 +966,9 @@ const checks = [ }, { file: 'jenkins/Jenkinsfile.production-server-provision', - excludes: - "params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history'", + includes: 'release 仅允许 archive-full；files-history', reason: - 'release 必须能在显式选择 profile 且 baseline 预检通过后启用 files-history。', + 'release 必须拒绝 files-history，避免逐文件 catalog 扫描再次触发生产内存峰值。', }, { file: 'scripts/database-backup-to-oss.mjs', @@ -954,7 +978,7 @@ const checks = [ { file: 'scripts/database-backup-to-oss.mjs', includes: - 'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter})', + 'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath})', reason: '生产冷备份打包失败时也必须恢复 SpacetimeDB 及依赖服务。', }, { diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index d9a1ff9eb..c64e8657f 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import {spawnSync} from 'node:child_process'; -import {createHash, createHmac} from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { createHash, createHmac } from 'node:crypto'; import { chmodSync, closeSync, @@ -21,20 +21,38 @@ import { symlinkSync, writeFileSync, } from 'node:fs'; -import {basename, dirname, isAbsolute, join, relative, resolve, sep} from 'node:path'; -import {Readable} from 'node:stream'; -import {pipeline} from 'node:stream/promises'; -import {setTimeout as sleep} from 'node:timers/promises'; -import {fileURLToPath} from 'node:url'; -import {gunzipSync, gzipSync} from 'node:zlib'; +import { + basename, + dirname, + isAbsolute, + join, + relative, + resolve, + sep, +} from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; +import { gunzipSync, gzipSync } from 'node:zlib'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const REPO_ROOT = resolve(__dirname, '..'); -const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/data'); -const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups'); +const DEFAULT_LOCAL_DATA_DIR = resolve( + REPO_ROOT, + 'server-rs/.spacetimedb/local/data', +); +const DEFAULT_LOCAL_WORK_DIR = resolve( + REPO_ROOT, + 'server-rs/.data/database-backups', +); const DEFAULT_PRODUCTION_DATA_DIR = '/stdb'; const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups'; +const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join( + DEFAULT_PRODUCTION_WORK_DIR, + '.spacetimedb-stopped', +); const DEFAULT_SPACE_SAFETY_RATIO = 1.1; const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024; const OSS_ALGORITHM = 'OSS4-HMAC-SHA256'; @@ -117,7 +135,7 @@ function loadEnvFile(filePath, target, protectedKeys) { } function loadRepoEnv() { - const env = {...process.env}; + const env = { ...process.env }; const protectedKeys = new Set( Object.entries(process.env) .filter(([, value]) => String(value ?? '').trim()) @@ -289,21 +307,34 @@ function firstNonEmpty(...values) { } function parseDirectFilesConcurrency(rawValue) { - const value = Number(String(rawValue ?? DEFAULT_DIRECT_FILES_CONCURRENCY).trim()); - if (!Number.isSafeInteger(value) || value < 1 || value > MAX_DIRECT_FILES_CONCURRENCY) { - throw new Error(`GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY 必须是 1-${MAX_DIRECT_FILES_CONCURRENCY} 的整数，实际: ${rawValue}`); + const value = Number( + String(rawValue ?? DEFAULT_DIRECT_FILES_CONCURRENCY).trim(), + ); + if ( + !Number.isSafeInteger(value) || + value < 1 || + value > MAX_DIRECT_FILES_CONCURRENCY + ) { + throw new Error( + `GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY 必须是 1-${MAX_DIRECT_FILES_CONCURRENCY} 的整数，实际: ${rawValue}`, + ); } return value; } -export function createUploadBandwidthLimiter(rawValue, {nowFn = Date.now, sleepImpl = sleep} = {}) { +export function createUploadBandwidthLimiter( + rawValue, + { nowFn = Date.now, sleepImpl = sleep } = {}, +) { const normalized = String(rawValue ?? '').trim(); if (!normalized || normalized === '0') { return null; } const maxBytesPerSecond = Number(normalized); if (!Number.isSafeInteger(maxBytesPerSecond) || maxBytesPerSecond < 1024) { - throw new Error(`GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND 必须为空、0 或 >= 1024 的整数，实际: ${rawValue}`); + throw new Error( + `GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND 必须为空、0 或 >= 1024 的整数，实际: ${rawValue}`, + ); } let nextAvailableAtMs = 0; const waitForChunk = async (sizeBytes) => { @@ -319,22 +350,31 @@ export function createUploadBandwidthLimiter(rawValue, {nowFn = Date.now, sleepI return { maxBytesPerSecond, wrap(readable) { - return Readable.from((async function* throttleUpload() { - for await (const chunk of readable) { - await waitForChunk(chunk.length); - yield chunk; - } - })(), {objectMode: false}); + return Readable.from( + (async function* throttleUpload() { + for await (const chunk of readable) { + await waitForChunk(chunk.length); + yield chunk; + } + })(), + { objectMode: false }, + ); }, }; } function createBufferReadStream(buffer, chunkSizeBytes = 64 * 1024) { - return Readable.from((function* readChunks() { - for (let offset = 0; offset < buffer.length; offset += chunkSizeBytes) { - yield buffer.subarray(offset, Math.min(offset + chunkSizeBytes, buffer.length)); - } - })(), {objectMode: false}); + return Readable.from( + (function* readChunks() { + for (let offset = 0; offset < buffer.length; offset += chunkSizeBytes) { + yield buffer.subarray( + offset, + Math.min(offset + chunkSizeBytes, buffer.length), + ); + } + })(), + { objectMode: false }, + ); } function resolvePath(value) { @@ -363,9 +403,12 @@ function timestampForFile(date = new Date()) { return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`; } -function buildBackupNames({database, dataDir, objectPrefix}) { +function buildBackupNames({ database, dataDir, objectPrefix }) { const timestamp = timestampForFile(); - const databasePart = sanitizeObjectPart(database || basename(dataDir), 'spacetimedb'); + const databasePart = sanitizeObjectPart( + database || basename(dataDir), + 'spacetimedb', + ); const fileName = `${databasePart}-${timestamp}.tar.gz`; const prefix = String(objectPrefix || 'database-backups') .trim() @@ -375,24 +418,27 @@ function buildBackupNames({database, dataDir, objectPrefix}) { .map((part) => sanitizeObjectPart(part, 'backup')) .join('/'); const objectKey = [prefix, databasePart, fileName].filter(Boolean).join('/'); - return {fileName, objectKey}; + return { fileName, objectKey }; } function atomicWriteBuffer(filePath, body) { - mkdirSync(dirname(filePath), {recursive: true}); + mkdirSync(dirname(filePath), { recursive: true }); const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`; - writeFileSync(tempPath, body, {mode: 0o600}); + writeFileSync(tempPath, body, { mode: 0o600 }); chmodSync(tempPath, 0o600); renameSync(tempPath, filePath); } function atomicWriteJson(filePath, payload) { - atomicWriteBuffer(filePath, Buffer.from(`${JSON.stringify(payload, null, 2)}\n`, 'utf8')); + atomicWriteBuffer( + filePath, + Buffer.from(`${JSON.stringify(payload, null, 2)}\n`, 'utf8'), + ); } function atomicWriteGzipJson(filePath, payload) { const body = Buffer.from(`${JSON.stringify(payload)}\n`, 'utf8'); - atomicWriteBuffer(filePath, gzipSync(body, {level: 9})); + atomicWriteBuffer(filePath, gzipSync(body, { level: 9 })); } function processIsAlive(pid) { @@ -404,9 +450,12 @@ function processIsAlive(pid) { } } -function acquireBackupLock({workDir, database}) { - mkdirSync(workDir, {recursive: true}); - const lockPath = join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}.backup.lock`); +function acquireBackupLock({ workDir, database }) { + mkdirSync(workDir, { recursive: true }); + const lockPath = join( + workDir, + `${sanitizeObjectPart(database, 'spacetimedb')}.backup.lock`, + ); try { const fd = openSync(lockPath, 'wx', 0o600); writeFileSync(fd, `${process.pid}\n`, 'utf8'); @@ -415,7 +464,7 @@ function acquireBackupLock({workDir, database}) { try { const ownerPid = Number(String(readFileSync(lockPath, 'utf8')).trim()); if (ownerPid === process.pid) { - rmSync(lockPath, {force: true}); + rmSync(lockPath, { force: true }); } } catch { // The lock may already have been removed by the normal exit path. @@ -435,36 +484,53 @@ function acquireBackupLock({workDir, database}) { } } const ownerPid = Number(String(readFileSync(lockPath, 'utf8')).trim()); - if (Number.isSafeInteger(ownerPid) && ownerPid > 0 && processIsAlive(ownerPid)) { + if ( + Number.isSafeInteger(ownerPid) && + ownerPid > 0 && + processIsAlive(ownerPid) + ) { throw new Error(`已有数据库备份进程持有锁: ${lockPath} pid=${ownerPid}`); } - throw new Error(`发现失效数据库备份锁，拒绝自动抢锁；请核对 OSS multipart 与进程后手工删除: ${lockPath} pid=${ownerPid || ''}`); + throw new Error( + `发现失效数据库备份锁，拒绝自动抢锁；请核对 OSS multipart 与进程后手工删除: ${lockPath} pid=${ownerPid || ''}`, + ); } -function historyStatePath({args, env, workDir, database}) { - return resolvePath(firstNonEmpty( - args.baselineState, - env.GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE, - join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-history-state.json`), - )); +function historyStatePath({ args, env, workDir, database }) { + return resolvePath( + firstNonEmpty( + args.baselineState, + env.GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE, + join( + workDir, + `${sanitizeObjectPart(database, 'spacetimedb')}-history-state.json`, + ), + ), + ); } function baselineIdFor(baseline) { - return sha256Hex([ - baseline.bucket, - baseline.objectKey, - baseline.verifiedAt, - baseline.contentLength, - baseline.archiveSha256, - ].join('\0')).slice(0, 24); + return sha256Hex( + [ + baseline.bucket, + baseline.objectKey, + baseline.verifiedAt, + baseline.contentLength, + baseline.archiveSha256, + ].join('\0'), + ).slice(0, 24); } -function normalizeUploadedBaselineManifest(manifest, {database, dataDir}) { +function normalizeUploadedBaselineManifest(manifest, { database, dataDir }) { if (manifest.uploadStatus !== 'uploaded') { - throw new Error(`baseline manifest 必须是 uploaded，实际: ${manifest.uploadStatus ?? ''}`); + throw new Error( + `baseline manifest 必须是 uploaded，实际: ${manifest.uploadStatus ?? ''}`, + ); } if (manifest.backupKind !== 'spacetimedb-data-dir') { - throw new Error(`baseline manifest backupKind 必须是 spacetimedb-data-dir，实际: ${manifest.backupKind ?? ''}`); + throw new Error( + `baseline manifest backupKind 必须是 spacetimedb-data-dir，实际: ${manifest.backupKind ?? ''}`, + ); } const baseline = { backupKind: 'spacetimedb-data-dir', @@ -475,24 +541,28 @@ function normalizeUploadedBaselineManifest(manifest, {database, dataDir}) { verifiedAt: String(manifest.verifiedAt ?? '').trim(), uploadedAt: String(manifest.uploadedAt ?? '').trim(), contentLength: Number(manifest.contentLength), - archiveSha256: String(manifest.archiveSha256 ?? '').trim().toLowerCase(), + archiveSha256: String(manifest.archiveSha256 ?? '') + .trim() + .toLowerCase(), manifestObjectKey: String(manifest.manifestObjectKey ?? '').trim(), manifestContentLength: Number(manifest.manifestContentLength), - manifestArchiveSha256: String(manifest.manifestArchiveSha256 ?? '').trim().toLowerCase(), + manifestArchiveSha256: String(manifest.manifestArchiveSha256 ?? '') + .trim() + .toLowerCase(), manifestVerifiedAt: String(manifest.manifestVerifiedAt ?? '').trim(), }; if ( - !baseline.bucket - || !baseline.objectKey - || !baseline.verifiedAt - || !Number.isSafeInteger(baseline.contentLength) - || baseline.contentLength <= 0 - || !/^[a-f0-9]{64}$/u.test(baseline.archiveSha256) - || !baseline.manifestObjectKey - || !Number.isSafeInteger(baseline.manifestContentLength) - || baseline.manifestContentLength <= 0 - || !/^[a-f0-9]{64}$/u.test(baseline.manifestArchiveSha256) - || !baseline.manifestVerifiedAt + !baseline.bucket || + !baseline.objectKey || + !baseline.verifiedAt || + !Number.isSafeInteger(baseline.contentLength) || + baseline.contentLength <= 0 || + !/^[a-f0-9]{64}$/u.test(baseline.archiveSha256) || + !baseline.manifestObjectKey || + !Number.isSafeInteger(baseline.manifestContentLength) || + baseline.manifestContentLength <= 0 || + !/^[a-f0-9]{64}$/u.test(baseline.manifestArchiveSha256) || + !baseline.manifestVerifiedAt ) { throw new Error('baseline manifest 缺少已验真 OSS 归档或 sidecar 信息。'); } @@ -500,19 +570,23 @@ function normalizeUploadedBaselineManifest(manifest, {database, dataDir}) { return baseline; } -function validateHistoryState(state, {database, dataDir}) { +function validateHistoryState(state, { database, dataDir }) { if (state.schemaVersion !== HISTORY_STATE_SCHEMA_VERSION || !state.baseline) { throw new Error('history state schemaVersion 或 baseline 无效。'); } const baseline = normalizeUploadedBaselineManifest( - {...state.baseline, uploadStatus: 'uploaded'}, - {database, dataDir}, + { ...state.baseline, uploadStatus: 'uploaded' }, + { database, dataDir }, ); if (baseline.database !== database) { - throw new Error(`history state database 不匹配: expected=${database}, actual=${baseline.database}`); + throw new Error( + `history state database 不匹配: expected=${database}, actual=${baseline.database}`, + ); } if (resolvePath(baseline.dataDir) !== resolvePath(dataDir)) { - throw new Error(`history state dataDir 不匹配: expected=${resolvePath(dataDir)}, actual=${resolvePath(baseline.dataDir)}`); + throw new Error( + `history state dataDir 不匹配: expected=${resolvePath(dataDir)}, actual=${resolvePath(baseline.dataDir)}`, + ); } return { ...state, @@ -521,53 +595,82 @@ function validateHistoryState(state, {database, dataDir}) { }; } -function writeBaselineState({statePath, baseline, previousState = null}) { +function writeBaselineState({ statePath, baseline, previousState = null }) { const state = { schemaVersion: HISTORY_STATE_SCHEMA_VERSION, updatedAt: new Date().toISOString(), baseline, - batches: previousState?.baseline?.id === baseline.id && Array.isArray(previousState.batches) - ? previousState.batches - : [], + batches: + previousState?.baseline?.id === baseline.id && + Array.isArray(previousState.batches) + ? previousState.batches + : [], }; atomicWriteJson(statePath, state); return state; } -function loadOrImportHistoryState({args, env, statePath, database, dataDir}) { +function loadOrImportHistoryState({ args, env, statePath, database, dataDir }) { if (existsSync(statePath)) { - return validateHistoryState(readManifest(statePath), {database, dataDir}); + return validateHistoryState(readManifest(statePath), { database, dataDir }); } - const importPath = firstNonEmpty(args.baselineManifest, env.GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST); + const importPath = firstNonEmpty( + args.baselineManifest, + env.GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST, + ); if (!importPath) { - throw new Error(`history 模式缺少已验真 baseline state: ${statePath}；可用 --baseline-manifest 导入已有 uploaded baseline manifest。`); + throw new Error( + `history 模式缺少已验真 baseline state: ${statePath}；可用 --baseline-manifest 导入已有 uploaded baseline manifest。`, + ); } - const baseline = normalizeUploadedBaselineManifest(readManifest(resolvePath(importPath)), {database, dataDir}); - return writeBaselineState({statePath, baseline}); + const baseline = normalizeUploadedBaselineManifest( + readManifest(resolvePath(importPath)), + { database, dataDir }, + ); + return writeBaselineState({ statePath, baseline }); } function assertSafeRelativePath(dataDir, absolutePath) { - const relativePath = relative(resolvePath(dataDir), resolvePath(absolutePath)); - if (!relativePath || relativePath === '..' || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) { + const relativePath = relative( + resolvePath(dataDir), + resolvePath(absolutePath), + ); + if ( + !relativePath || + relativePath === '..' || + relativePath.startsWith(`..${sep}`) || + isAbsolute(relativePath) + ) { throw new Error(`history 候选路径越界或等于数据目录: ${absolutePath}`); } return relativePath.split(sep).join('/'); } function statFingerprint(absolutePath, rootPath = absolutePath) { - const entries = []; + // 候选 snapshot 可能包含数十万条目录项；增量更新摘要，避免把每条 + // fingerprint 字符串同时保存在 entries[] 后再 join，造成一次性内存峰值。 + const fingerprintHash = createHash('sha256'); + let isFirstEntry = true; + let entryCount = 0; let totalSize = 0n; const visit = (currentPath) => { - const stat = lstatSync(currentPath, {bigint: true}); + const stat = lstatSync(currentPath, { bigint: true }); if (stat.isSymbolicLink()) { throw new Error(`history 候选不得包含符号链接: ${currentPath}`); } - const entryPath = currentPath === rootPath ? '.' : relative(rootPath, currentPath).split(sep).join('/'); - const kind = stat.isDirectory() ? 'directory' : stat.isFile() ? 'file' : 'other'; + const entryPath = + currentPath === rootPath + ? '.' + : relative(rootPath, currentPath).split(sep).join('/'); + const kind = stat.isDirectory() + ? 'directory' + : stat.isFile() + ? 'file' + : 'other'; if (kind === 'other') { throw new Error(`history 候选只允许普通文件或目录: ${currentPath}`); } - entries.push([ + const entry = [ entryPath, kind, stat.dev.toString(), @@ -575,7 +678,13 @@ function statFingerprint(absolutePath, rootPath = absolutePath) { stat.mode.toString(), stat.size.toString(), stat.mtimeNs.toString(), - ].join('\0')); + ].join('\0'); + if (!isFirstEntry) { + fingerprintHash.update('\n'); + } + fingerprintHash.update(entry); + isFirstEntry = false; + entryCount += 1; if (stat.isFile()) { totalSize += stat.size; } else { @@ -586,22 +695,34 @@ function statFingerprint(absolutePath, rootPath = absolutePath) { }; visit(rootPath); return { - fingerprint: sha256Hex(entries.join('\n')), + fingerprint: fingerprintHash.digest('hex'), sizeBytes: totalSize.toString(), - entryCount: entries.length, + entryCount, }; } function findReplicasDir(dataDir) { - const candidates = [resolve(dataDir, 'replicas'), resolve(dataDir, 'data', 'replicas')] - .filter((candidate) => existsSync(candidate) && lstatSync(candidate).isDirectory()); + const candidates = [ + resolve(dataDir, 'replicas'), + resolve(dataDir, 'data', 'replicas'), + ].filter( + (candidate) => existsSync(candidate) && lstatSync(candidate).isDirectory(), + ); if (candidates.length !== 1) { - throw new Error(`无法唯一确定 replicas 目录: ${candidates.length === 0 ? '' : candidates.join(', ')}`); + throw new Error( + `无法唯一确定 replicas 目录: ${candidates.length === 0 ? '' : candidates.join(', ')}`, + ); } return candidates[0]; } -function historyCandidate({dataDir, absolutePath, kind, replicaId, transaction}) { +function historyCandidate({ + dataDir, + absolutePath, + kind, + replicaId, + transaction, +}) { const stat = statFingerprint(absolutePath); return { path: assertSafeRelativePath(dataDir, absolutePath), @@ -612,14 +733,16 @@ function historyCandidate({dataDir, absolutePath, kind, replicaId, transaction}) }; } -export function discoverHistoryPlan({dataDir}) { +export function discoverHistoryPlan({ dataDir }) { const resolvedDataDir = resolvePath(dataDir); const replicasDir = findReplicasDir(resolvedDataDir); - const replicaEntries = readdirSync(replicasDir, {withFileTypes: true}); + const replicaEntries = readdirSync(replicasDir, { withFileTypes: true }); const replicas = []; const candidates = []; - for (const replicaEntry of replicaEntries.sort((left, right) => left.name.localeCompare(right.name))) { + for (const replicaEntry of replicaEntries.sort((left, right) => + left.name.localeCompare(right.name), + )) { if (!replicaEntry.isDirectory()) { continue; } @@ -631,93 +754,133 @@ export function discoverHistoryPlan({dataDir}) { const snapshotsDir = join(replicaDir, 'snapshots'); const clogDir = join(replicaDir, 'clog'); if (!existsSync(snapshotsDir) || !lstatSync(snapshotsDir).isDirectory()) { - replicas.push({replicaId, status: 'skipped', reason: 'no-snapshots-directory'}); + replicas.push({ + replicaId, + status: 'skipped', + reason: 'no-snapshots-directory', + }); continue; } - const snapshotEntries = readdirSync(snapshotsDir, {withFileTypes: true}); - const snapshots = snapshotEntries.flatMap((entry) => { - const match = /^(\d{20})\.snapshot_dir$/u.exec(entry.name); - if (!match) { - return []; - } - const transaction = BigInt(match[1]); - if (transaction > 0xffff_ffff_ffff_ffffn) { - throw new Error(`snapshot transaction 超出 u64: ${entry.name}`); - } - if (!entry.isDirectory()) { - throw new Error(`snapshot 候选必须是目录: ${join(snapshotsDir, entry.name)}`); - } - const snapshotDir = join(snapshotsDir, entry.name); - const lockPath = join(snapshotsDir, `${match[1]}.lock`); - const snapshotFile = join(snapshotDir, `${match[1]}.snapshot_bsatn`); - if (existsSync(lockPath) || !existsSync(snapshotFile) || !lstatSync(snapshotFile).isFile()) { - return []; - } - return [{name: entry.name, transaction}]; - }).sort((left, right) => left.transaction < right.transaction ? -1 : left.transaction > right.transaction ? 1 : 0); + const snapshotEntries = readdirSync(snapshotsDir, { withFileTypes: true }); + const snapshots = snapshotEntries + .flatMap((entry) => { + const match = /^(\d{20})\.snapshot_dir$/u.exec(entry.name); + if (!match) { + return []; + } + const transaction = BigInt(match[1]); + if (transaction > 0xffff_ffff_ffff_ffffn) { + throw new Error(`snapshot transaction 超出 u64: ${entry.name}`); + } + if (!entry.isDirectory()) { + throw new Error( + `snapshot 候选必须是目录: ${join(snapshotsDir, entry.name)}`, + ); + } + const snapshotDir = join(snapshotsDir, entry.name); + const lockPath = join(snapshotsDir, `${match[1]}.lock`); + const snapshotFile = join(snapshotDir, `${match[1]}.snapshot_bsatn`); + if ( + existsSync(lockPath) || + !existsSync(snapshotFile) || + !lstatSync(snapshotFile).isFile() + ) { + return []; + } + return [{ name: entry.name, transaction }]; + }) + .sort((left, right) => + left.transaction < right.transaction + ? -1 + : left.transaction > right.transaction + ? 1 + : 0, + ); if (snapshots.length === 0) { - replicas.push({replicaId, status: 'skipped', reason: 'no-snapshot'}); + replicas.push({ replicaId, status: 'skipped', reason: 'no-snapshot' }); continue; } if (!existsSync(clogDir) || !lstatSync(clogDir).isDirectory()) { throw new Error(`replica ${replicaId} 缺少 clog 目录。`); } const segmentFiles = new Map(); - for (const entry of readdirSync(clogDir, {withFileTypes: true})) { + for (const entry of readdirSync(clogDir, { withFileTypes: true })) { const match = /^(\d{20})\.stdb\.(log|ofs)$/u.exec(entry.name); if (!match) { throw new Error(`commitlog 文件名不符合预期: ${entry.name}`); } if (!entry.isFile()) { - throw new Error(`commitlog 候选必须是普通文件: ${join(clogDir, entry.name)}`); + throw new Error( + `commitlog 候选必须是普通文件: ${join(clogDir, entry.name)}`, + ); } const transaction = BigInt(match[1]); if (transaction > 0xffff_ffff_ffff_ffffn) { throw new Error(`commitlog transaction 超出 u64: ${entry.name}`); } const key = transaction.toString(); - const group = segmentFiles.get(key) ?? {transaction}; + const group = segmentFiles.get(key) ?? { transaction }; group[match[2]] = entry.name; segmentFiles.set(key, group); } for (const group of segmentFiles.values()) { if (group.ofs && !group.log) { - throw new Error(`commitlog offset 缺少对应 log: replica=${replicaId}, transaction=${group.transaction}`); + throw new Error( + `commitlog offset 缺少对应 log: replica=${replicaId}, transaction=${group.transaction}`, + ); } } const segments = [...segmentFiles.values()] .filter((group) => group.log) - .sort((left, right) => left.transaction < right.transaction ? -1 : left.transaction > right.transaction ? 1 : 0); + .sort((left, right) => + left.transaction < right.transaction + ? -1 + : left.transaction > right.transaction + ? 1 + : 0, + ); const latestSnapshot = snapshots.at(-1).transaction; - const boundarySegment = segments.filter((segment) => segment.transaction <= latestSnapshot).at(-1); + const boundarySegment = segments + .filter((segment) => segment.transaction <= latestSnapshot) + .at(-1); if (!boundarySegment) { - throw new Error(`replica ${replicaId} 无法找到覆盖 latest snapshot ${latestSnapshot} 的 commitlog 边界。`); + throw new Error( + `replica ${replicaId} 无法找到覆盖 latest snapshot ${latestSnapshot} 的 commitlog 边界。`, + ); } for (const snapshot of snapshots.slice(0, -1)) { - candidates.push(historyCandidate({ - dataDir: resolvedDataDir, - absolutePath: join(snapshotsDir, snapshot.name), - kind: 'snapshot', - replicaId, - transaction: snapshot.transaction, - })); - } - for (const segment of segments.filter((item) => item.transaction < boundarySegment.transaction)) { - candidates.push(historyCandidate({ - dataDir: resolvedDataDir, - absolutePath: join(clogDir, segment.log), - kind: 'commitlog', - replicaId, - transaction: segment.transaction, - })); - if (segment.ofs) { - candidates.push(historyCandidate({ + candidates.push( + historyCandidate({ dataDir: resolvedDataDir, - absolutePath: join(clogDir, segment.ofs), - kind: 'commitlog-offset', + absolutePath: join(snapshotsDir, snapshot.name), + kind: 'snapshot', + replicaId, + transaction: snapshot.transaction, + }), + ); + } + for (const segment of segments.filter( + (item) => item.transaction < boundarySegment.transaction, + )) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.log), + kind: 'commitlog', replicaId, transaction: segment.transaction, - })); + }), + ); + if (segment.ofs) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.ofs), + kind: 'commitlog-offset', + replicaId, + transaction: segment.transaction, + }), + ); } } replicas.push({ @@ -733,7 +896,9 @@ export function discoverHistoryPlan({dataDir}) { replicasDir: assertSafeRelativePath(resolvedDataDir, replicasDir), replicas, candidates, - totalSizeBytes: candidates.reduce((sum, item) => sum + BigInt(item.sizeBytes), 0n).toString(), + totalSizeBytes: candidates + .reduce((sum, item) => sum + BigInt(item.sizeBytes), 0n) + .toString(), }; } @@ -762,7 +927,9 @@ function parseByteSize(rawValue, label) { } const match = /^(\d+)(?:\s*([KMGTPE]?)(?:I?B?)?)?$/iu.exec(value); if (!match) { - throw new Error(`${label} 必须是字节数或 K/M/G/T/P/E 后缀大小，实际: ${rawValue}`); + throw new Error( + `${label} 必须是字节数或 K/M/G/T/P/E 后缀大小，实际: ${rawValue}`, + ); } const [, amountText, unitText = ''] = match; const multipliers = { @@ -779,11 +946,11 @@ function parseByteSize(rawValue, label) { function formatBytes(bytes) { const value = BigInt(bytes); - const gib = Number(value) / (1024 ** 3); + const gib = Number(value) / 1024 ** 3; if (gib >= 1) { return `${gib.toFixed(1)}GiB`; } - const mib = Number(value) / (1024 ** 2); + const mib = Number(value) / 1024 ** 2; if (mib >= 1) { return `${mib.toFixed(1)}MiB`; } @@ -792,7 +959,9 @@ function formatBytes(bytes) { function getDirectorySizeBytes(dataDir) { const result = runCommand('du', ['-sk', dataDir]); - const [sizeKbText] = String(result.stdout ?? '').trim().split(/\s+/u); + const [sizeKbText] = String(result.stdout ?? '') + .trim() + .split(/\s+/u); if (!sizeKbText || !/^\d+$/u.test(sizeKbText)) { throw new Error(`无法解析数据目录大小: ${result.stdout}`); } @@ -800,7 +969,7 @@ function getDirectorySizeBytes(dataDir) { } function getAvailableBytes(fileSystemPath) { - const stat = statfsSync(fileSystemPath, {bigint: true}); + const stat = statfsSync(fileSystemPath, { bigint: true }); return stat.bavail * stat.bsize; } @@ -811,35 +980,51 @@ function parseSafetyRatio(rawValue) { } const ratio = Number(value); if (!Number.isFinite(ratio) || ratio < 1) { - throw new Error(`GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO 必须是 >= 1 的数字，实际: ${rawValue}`); + throw new Error( + `GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO 必须是 >= 1 的数字，实际: ${rawValue}`, + ); } return ratio; } -function calculateRequiredFreeBytes({dataSizeBytes, args, env}) { +function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) { const explicitMinFreeBytes = parseByteSize( - firstNonEmpty(args.minFreeBytes, env.GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES), + firstNonEmpty( + args.minFreeBytes, + env.GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES, + ), 'GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES', ); if (explicitMinFreeBytes !== null) { return explicitMinFreeBytes; } - const ratio = parseSafetyRatio(env.GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO); + const ratio = parseSafetyRatio( + env.GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO, + ); const ratioBasisPoints = BigInt(Math.ceil(ratio * 10000)); const ratioRequirement = (dataSizeBytes * ratioBasisPoints + 9999n) / 10000n; const extraFreeBytes = parseByteSize( - firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_EXTRA_FREE_BYTES, String(DEFAULT_EXTRA_FREE_BYTES)), + firstNonEmpty( + env.GENARRATIVE_DATABASE_BACKUP_EXTRA_FREE_BYTES, + String(DEFAULT_EXTRA_FREE_BYTES), + ), 'GENARRATIVE_DATABASE_BACKUP_EXTRA_FREE_BYTES', ); const extraRequirement = dataSizeBytes + extraFreeBytes; - return ratioRequirement > extraRequirement ? ratioRequirement : extraRequirement; + return ratioRequirement > extraRequirement + ? ratioRequirement + : extraRequirement; } -function assertSufficientWorkDirSpace({dataDir, workDir, args, env}) { - mkdirSync(workDir, {recursive: true}); +function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) { + mkdirSync(workDir, { recursive: true }); const dataSizeBytes = getDirectorySizeBytes(dataDir); const availableBytes = getAvailableBytes(workDir); - const requiredFreeBytes = calculateRequiredFreeBytes({dataSizeBytes, args, env}); + const requiredFreeBytes = calculateRequiredFreeBytes({ + dataSizeBytes, + args, + env, + }); console.log( `[database-backup] 备份空间预检: data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, ); @@ -857,19 +1042,30 @@ function assertSufficientWorkDirSpace({dataDir, workDir, args, env}) { } } -function assertSufficientHistoryWorkDirSpace({historySizeBytes, workDir, args, env}) { - mkdirSync(workDir, {recursive: true}); +function assertSufficientHistoryWorkDirSpace({ + historySizeBytes, + workDir, + args, + env, +}) { + mkdirSync(workDir, { recursive: true }); const availableBytes = getAvailableBytes(workDir); - const requiredFreeBytes = calculateRequiredFreeBytes({dataSizeBytes: BigInt(historySizeBytes), args, env}); + const requiredFreeBytes = calculateRequiredFreeBytes({ + dataSizeBytes: BigInt(historySizeBytes), + args, + env, + }); console.log( `[database-backup] history 空间预检: candidates=${formatBytes(historySizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, ); if (availableBytes < requiredFreeBytes) { - throw new Error(`history 工作目录剩余空间不足: available=${formatBytes(availableBytes)}；required=${formatBytes(requiredFreeBytes)}`); + throw new Error( + `history 工作目录剩余空间不足: available=${formatBytes(availableBytes)}；required=${formatBytes(requiredFreeBytes)}`, + ); } } -function collectRestartServicesAfterBackup({args, env}) { +function collectRestartServicesAfterBackup({ args, env }) { const serviceNames = [ ...String(env.GENARRATIVE_DATABASE_BACKUP_RESTART_SERVICE_AFTER ?? '') .split(',') @@ -880,12 +1076,40 @@ function collectRestartServicesAfterBackup({args, env}) { return [...new Set(serviceNames.filter(Boolean))]; } -function stopServiceIfNeeded(serviceName) { +function databaseBackupStopMarkerPath(workDir) { + return resolvePath( + firstNonEmpty( + process.env.GENARRATIVE_DATABASE_BACKUP_STOP_MARKER, + workDir === DEFAULT_PRODUCTION_WORK_DIR + ? DEFAULT_DATABASE_BACKUP_STOP_MARKER + : join(workDir, '.spacetimedb-stopped'), + ), + ); +} + +function writeDatabaseBackupStopMarker(markerPath, serviceName) { + atomicWriteJson(markerPath, { + serviceName, + pid: process.pid, + stoppedAt: new Date().toISOString(), + }); +} + +function clearDatabaseBackupStopMarker(markerPath) { + if (markerPath) { + rmSync(markerPath, { force: true }); + } +} + +function stopServiceIfNeeded(serviceName, stopMarkerPath) { if (!serviceName) { return false; } console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`); - runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'}); + writeDatabaseBackupStopMarker(stopMarkerPath, serviceName); + // stop 命令失败时仍保留 marker：systemd 的 ExecStopPost 需要它判断是否要 + // 兜底恢复，不能因为当前进程还能捕获异常就抹掉上一次停库证据。 + runCommand('systemctl', ['stop', serviceName], { stdio: 'inherit' }); return true; } @@ -894,7 +1118,7 @@ function startServiceIfNeeded(serviceName, wasStopped) { return; } console.log(`[database-backup] 恢复服务: ${serviceName}`); - runCommand('systemctl', ['start', serviceName], {stdio: 'inherit'}); + runCommand('systemctl', ['start', serviceName], { stdio: 'inherit' }); } function restartServicesAfterBackup(serviceNames) { @@ -905,17 +1129,25 @@ function restartServicesAfterBackup(serviceNames) { } console.log(`[database-backup] 冷备份后重启依赖服务: ${serviceName}`); try { - runCommand('systemctl', ['restart', serviceName], {stdio: 'inherit'}); + runCommand('systemctl', ['restart', serviceName], { stdio: 'inherit' }); } catch (error) { errors.push(error); } } if (errors.length > 0) { - throw new AggregateError(errors, `冷备份后重启依赖服务失败: ${errors.map((error) => error.message).join('; ')}`); + throw new AggregateError( + errors, + `冷备份后重启依赖服务失败: ${errors.map((error) => error.message).join('; ')}`, + ); } } -function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}) { +function restoreServicesAfterBackup({ + stopService, + serviceStopped, + restartServicesAfter, + stopMarkerPath, +}) { const errors = []; try { startServiceIfNeeded(stopService, serviceStopped); @@ -928,11 +1160,15 @@ function restoreServicesAfterBackup({stopService, serviceStopped, restartService errors.push(error); } if (errors.length > 0) { - throw new AggregateError(errors, `恢复冷备份相关服务失败: ${errors.map((error) => error.message).join('; ')}`); + throw new AggregateError( + errors, + `恢复冷备份相关服务失败: ${errors.map((error) => error.message).join('; ')}`, + ); } + clearDatabaseBackupStopMarker(stopMarkerPath); } -function createArchive({dataDir, workDir, fileName}) { +function createArchive({ dataDir, workDir, fileName }) { if (!existsSync(dataDir)) { throw new Error(`数据库数据目录不存在: ${dataDir}`); } @@ -940,32 +1176,38 @@ function createArchive({dataDir, workDir, fileName}) { if (!stat.isDirectory()) { throw new Error(`数据库数据路径不是目录: ${dataDir}`); } - mkdirSync(workDir, {recursive: true}); + mkdirSync(workDir, { recursive: true }); const archivePath = resolve(workDir, fileName); const parentDir = dirname(dataDir); const entryName = basename(dataDir); console.log(`[database-backup] 打包: ${dataDir} -> ${archivePath}`); - runCommand('tar', ['-czf', archivePath, '-C', parentDir, entryName], {stdio: 'inherit'}); + runCommand('tar', ['-czf', archivePath, '-C', parentDir, entryName], { + stdio: 'inherit', + }); verifyArchive(archivePath); return archivePath; } function verifyArchive(archivePath) { console.log(`[database-backup] 校验归档: ${archivePath}`); - runCommand('tar', ['-tzf', archivePath], {stdio: 'ignore'}); + runCommand('tar', ['-tzf', archivePath], { stdio: 'ignore' }); } -function historyBatchId({baselineId, plan}) { - const identity = plan.candidates.map((candidate) => [ - candidate.path, - candidate.kind, - candidate.transaction, - candidate.fingerprint, - ].join('\0')).join('\n'); +function historyBatchId({ baselineId, plan }) { + const identity = plan.candidates + .map((candidate) => + [ + candidate.path, + candidate.kind, + candidate.transaction, + candidate.fingerprint, + ].join('\0'), + ) + .join('\n'); return sha256Hex(`${baselineId}\0${identity}`).slice(0, 32); } -function buildHistoryNames({database, objectPrefix, baselineId, batchId}) { +function buildHistoryNames({ database, objectPrefix, baselineId, batchId }) { const databasePart = sanitizeObjectPart(database, 'spacetimedb'); const prefix = String(objectPrefix || 'database-backups') .trim() @@ -977,30 +1219,52 @@ function buildHistoryNames({database, objectPrefix, baselineId, batchId}) { const fileName = `${databasePart}-history-${batchId}.tar.gz`; return { fileName, - objectKey: [prefix, databasePart, 'history', baselineId, fileName].filter(Boolean).join('/'), + objectKey: [prefix, databasePart, 'history', baselineId, fileName] + .filter(Boolean) + .join('/'), }; } -function createHistoryArchive({dataDir, workDir, fileName, manifestPath, candidates}) { - mkdirSync(workDir, {recursive: true}); +function createHistoryArchive({ + dataDir, + workDir, + fileName, + manifestPath, + candidates, +}) { + mkdirSync(workDir, { recursive: true }); const archivePath = resolve(workDir, fileName); const candidatePaths = candidates.map((candidate) => candidate.path); - console.log(`[database-backup] 打包 history: ${candidatePaths.length} 个候选 -> ${archivePath}`); - runCommand('tar', [ - '-czf', - archivePath, - '-C', - dataDir, - ...candidatePaths, - '-C', - dirname(manifestPath), - basename(manifestPath), - ], {stdio: 'inherit'}); + console.log( + `[database-backup] 打包 history: ${candidatePaths.length} 个候选 -> ${archivePath}`, + ); + runCommand( + 'tar', + [ + '-czf', + archivePath, + '-C', + dataDir, + ...candidatePaths, + '-C', + dirname(manifestPath), + basename(manifestPath), + ], + { stdio: 'inherit' }, + ); verifyArchive(archivePath); return archivePath; } -function recordHistoryBatch({statePath, state, manifest, uploadResult, manifestUpload, status, cleanedAt = ''}) { +function recordHistoryBatch({ + statePath, + state, + manifest, + uploadResult, + manifestUpload, + status, + cleanedAt = '', +}) { const batch = { batchId: manifest.batchId, objectKey: uploadResult.objectKey, @@ -1016,9 +1280,11 @@ function recordHistoryBatch({statePath, state, manifest, uploadResult, manifestU cleanedAt, candidates: manifest.candidates, }; - const batches = state.batches.filter((item) => item.batchId !== batch.batchId); + const batches = state.batches.filter( + (item) => item.batchId !== batch.batchId, + ); batches.push(batch); - const nextState = {...state, updatedAt: new Date().toISOString(), batches}; + const nextState = { ...state, updatedAt: new Date().toISOString(), batches }; atomicWriteJson(statePath, nextState); return nextState; } @@ -1027,9 +1293,14 @@ function candidateKey(candidate) { return `${candidate.kind}\0${candidate.path}`; } -export function cleanupHistoryCandidates({dataDir, candidates}) { - const currentPlan = discoverHistoryPlan({dataDir}); - const eligible = new Map(currentPlan.candidates.map((candidate) => [candidateKey(candidate), candidate])); +export function cleanupHistoryCandidates({ dataDir, candidates }) { + const currentPlan = discoverHistoryPlan({ dataDir }); + const eligible = new Map( + currentPlan.candidates.map((candidate) => [ + candidateKey(candidate), + candidate, + ]), + ); const existing = []; for (const candidate of candidates) { const absolutePath = resolve(dataDir, candidate.path); @@ -1039,27 +1310,41 @@ export function cleanupHistoryCandidates({dataDir, candidates}) { } const current = eligible.get(candidateKey(candidate)); if (!current) { - throw new Error(`history 候选已不在当前安全边界内，拒绝删除: ${candidate.path}`); + throw new Error( + `history 候选已不在当前安全边界内，拒绝删除: ${candidate.path}`, + ); } const currentStat = statFingerprint(absolutePath); - if (currentStat.fingerprint !== candidate.fingerprint || currentStat.sizeBytes !== candidate.sizeBytes) { + if ( + currentStat.fingerprint !== candidate.fingerprint || + currentStat.sizeBytes !== candidate.sizeBytes + ) { throw new Error(`history 候选 stat 漂移，拒绝删除: ${candidate.path}`); } - existing.push({candidate, absolutePath}); + existing.push({ candidate, absolutePath }); } existing.sort((left, right) => { - const priority = {'commitlog-offset': 0, commitlog: 1, snapshot: 2}; - return (priority[left.candidate.kind] ?? 3) - (priority[right.candidate.kind] ?? 3) - || left.candidate.path.localeCompare(right.candidate.path); + const priority = { 'commitlog-offset': 0, commitlog: 1, snapshot: 2 }; + return ( + (priority[left.candidate.kind] ?? 3) - + (priority[right.candidate.kind] ?? 3) || + left.candidate.path.localeCompare(right.candidate.path) + ); }); - for (const {candidate, absolutePath} of existing) { - rmSync(absolutePath, {recursive: candidate.kind === 'snapshot', force: false}); + for (const { candidate, absolutePath } of existing) { + rmSync(absolutePath, { + recursive: candidate.kind === 'snapshot', + force: false, + }); console.log(`[database-backup] 已清理 history 源文件: ${candidate.path}`); } - return {deletedCount: existing.length, alreadyMissingCount: candidates.length - existing.length}; + return { + deletedCount: existing.length, + alreadyMissingCount: candidates.length - existing.length, + }; } -function writeManifest({manifestPath, payload}) { +function writeManifest({ manifestPath, payload }) { writeFileSync(manifestPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); } @@ -1086,15 +1371,21 @@ async function sha256FileHex(filePath) { return hash.digest('hex'); } -function directFilesStatePath({workDir, database}) { - return join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-files-state.json.gz`); +function directFilesStatePath({ workDir, database }) { + return join( + workDir, + `${sanitizeObjectPart(database, 'spacetimedb')}-files-state.json.gz`, + ); } -function legacyDirectFilesStatePath({workDir, database}) { - return join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-files-state.json`); +function legacyDirectFilesStatePath({ workDir, database }) { + return join( + workDir, + `${sanitizeObjectPart(database, 'spacetimedb')}-files-state.json`, + ); } -function directCatalogLocalPaths({workDir, database, catalog}) { +function directCatalogLocalPaths({ workDir, database, catalog }) { const baseName = `${sanitizeObjectPart(database, 'spacetimedb')}-${catalog.mode}-${catalog.catalogId}.catalog.json`; return { jsonPath: join(workDir, baseName), @@ -1102,8 +1393,12 @@ function directCatalogLocalPaths({workDir, database, catalog}) { }; } -function readLocalDirectCatalog({workDir, database, catalog}) { - const {jsonPath, gzipPath} = directCatalogLocalPaths({workDir, database, catalog}); +function readLocalDirectCatalog({ workDir, database, catalog }) { + const { jsonPath, gzipPath } = directCatalogLocalPaths({ + workDir, + database, + catalog, + }); let body = null; if (existsSync(jsonPath)) { body = readFileSync(jsonPath); @@ -1113,16 +1408,21 @@ function readLocalDirectCatalog({workDir, database, catalog}) { if (!body) { return null; } - if (body.length !== catalog.contentLength || sha256Hex(body) !== catalog.sha256) { - throw new Error(`本地 files catalog 长度或 SHA 与 state 引用不匹配: ${jsonPath}`); + if ( + body.length !== catalog.contentLength || + sha256Hex(body) !== catalog.sha256 + ) { + throw new Error( + `本地 files catalog 长度或 SHA 与 state 引用不匹配: ${jsonPath}`, + ); } const payload = JSON.parse(body.toString('utf8')); if ( - payload.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION - || payload.database !== database - || payload.mode !== catalog.mode - || payload.catalogId !== catalog.catalogId - || !Array.isArray(payload.files) + payload.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION || + payload.database !== database || + payload.mode !== catalog.mode || + payload.catalogId !== catalog.catalogId || + !Array.isArray(payload.files) ) { throw new Error(`本地 files catalog 与 state 引用不匹配: ${jsonPath}`); } @@ -1131,32 +1431,45 @@ function readLocalDirectCatalog({workDir, database, catalog}) { function readJsonOrGzip(filePath) { const body = readFileSync(filePath); - const decoded = filePath.endsWith('.gz') || (body[0] === 0x1f && body[1] === 0x8b) - ? gunzipSync(body) - : body; + const decoded = + filePath.endsWith('.gz') || (body[0] === 0x1f && body[1] === 0x8b) + ? gunzipSync(body) + : body; return JSON.parse(decoded.toString('utf8')); } -function compactDirectCatalogFile({workDir, database, catalog}) { - const {jsonPath, gzipPath} = directCatalogLocalPaths({workDir, database, catalog}); +function compactDirectCatalogFile({ workDir, database, catalog }) { + const { jsonPath, gzipPath } = directCatalogLocalPaths({ + workDir, + database, + catalog, + }); if (!existsSync(jsonPath)) { - return existsSync(gzipPath) ? {compressed: false, gzipPath} : null; + return existsSync(gzipPath) ? { compressed: false, gzipPath } : null; } const body = readFileSync(jsonPath); - atomicWriteBuffer(gzipPath, gzipSync(body, {level: 9})); - rmSync(jsonPath, {force: false}); - return {compressed: true, gzipPath}; + atomicWriteBuffer(gzipPath, gzipSync(body, { level: 9 })); + rmSync(jsonPath, { force: false }); + return { compressed: true, gzipPath }; } -function compactDirectFilesLocalMetadata({workDir, database, nextState, transientCatalogPaths = []}) { +function compactDirectFilesLocalMetadata({ + workDir, + database, + nextState, + transientCatalogPaths = [], +}) { const keepCatalog = nextState.latestCatalog; const keepCatalogIds = new Set([keepCatalog?.catalogId].filter(Boolean)); const databasePart = sanitizeObjectPart(database, 'spacetimedb'); - const catalogPattern = new RegExp(`^${databasePart.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&')}-(full|history)-([a-f0-9]{64})\\.catalog\\.json(?:\\.gz)?$`, 'u'); + const catalogPattern = new RegExp( + `^${databasePart.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&')}-(full|history)-([a-f0-9]{64})\\.catalog\\.json(?:\\.gz)?$`, + 'u', + ); let compressedCatalogCount = 0; let deletedCatalogCount = 0; let compactedResultCount = 0; - for (const entry of readdirSync(workDir, {withFileTypes: true})) { + for (const entry of readdirSync(workDir, { withFileTypes: true })) { if (!entry.isFile()) { continue; } @@ -1164,25 +1477,32 @@ function compactDirectFilesLocalMetadata({workDir, database, nextState, transien if (!match) { continue; } - const catalog = {mode: match[1], catalogId: match[2]}; + const catalog = { mode: match[1], catalogId: match[2] }; if (keepCatalogIds.has(catalog.catalogId) && catalog.mode === 'full') { - readLocalDirectCatalog({workDir, database, catalog: keepCatalog}); - if (!entry.name.endsWith('.gz') && compactDirectCatalogFile({workDir, database, catalog})?.compressed) { + readLocalDirectCatalog({ workDir, database, catalog: keepCatalog }); + if ( + !entry.name.endsWith('.gz') && + compactDirectCatalogFile({ workDir, database, catalog })?.compressed + ) { compressedCatalogCount += 1; } continue; } - rmSync(join(workDir, entry.name), {force: false}); + rmSync(join(workDir, entry.name), { force: false }); deletedCatalogCount += 1; } for (const filePath of transientCatalogPaths) { if (existsSync(filePath)) { - rmSync(filePath, {force: false}); + rmSync(filePath, { force: false }); deletedCatalogCount += 1; } } - for (const entry of readdirSync(workDir, {withFileTypes: true})) { - if (!entry.isFile() || !entry.name.endsWith('.json') || entry.name.endsWith('.catalog.json')) { + for (const entry of readdirSync(workDir, { withFileTypes: true })) { + if ( + !entry.isFile() || + !entry.name.endsWith('.json') || + entry.name.endsWith('.catalog.json') + ) { continue; } const filePath = join(workDir, entry.name); @@ -1193,18 +1513,24 @@ function compactDirectFilesLocalMetadata({workDir, database, nextState, transien continue; } if ( - payload?.catalog?.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION - || payload.catalog.database !== database - || payload.catalog.bucket !== nextState.bucket - || !Array.isArray(payload.catalog.files) + payload?.catalog?.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION || + payload.catalog.database !== database || + payload.catalog.bucket !== nextState.bucket || + !Array.isArray(payload.catalog.files) ) { continue; } atomicWriteJson(filePath, compactDirectFilesResult(payload)); compactedResultCount += 1; } - const result = {compressedCatalogCount, deletedCatalogCount, compactedResultCount}; - console.log(`[database-backup] files 本地元数据清理: ${JSON.stringify(result)}`); + const result = { + compressedCatalogCount, + deletedCatalogCount, + compactedResultCount, + }; + console.log( + `[database-backup] files 本地元数据清理: ${JSON.stringify(result)}`, + ); return result; } @@ -1216,11 +1542,13 @@ function normalizeObjectPrefix(objectPrefix, database) { .filter(Boolean) .map((part) => sanitizeObjectPart(part, 'backup')) .join('/'); - return [prefix, sanitizeObjectPart(database, 'spacetimedb')].filter(Boolean).join('/'); + return [prefix, sanitizeObjectPart(database, 'spacetimedb')] + .filter(Boolean) + .join('/'); } function directFileIdentity(filePath) { - const stat = lstatSync(filePath, {bigint: true}); + const stat = lstatSync(filePath, { bigint: true }); if (!stat.isFile() || stat.isSymbolicLink()) { throw new Error(`files 模式只允许普通文件: ${filePath}`); } @@ -1234,43 +1562,63 @@ function directFileIdentity(filePath) { } function sameDirectFileIdentity(left, right) { - return left.dev === right.dev - && left.ino === right.ino - && left.size === right.size - && left.mtimeNs === right.mtimeNs - && left.mode === right.mode; + return ( + left.dev === right.dev && + left.ino === right.ino && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.mode === right.mode + ); } -export async function collectDirectFileEntries({dataDir, candidates = null, objectPrefix, database}) { +export async function collectDirectFileEntries({ + dataDir, + candidates = null, + objectPrefix, + database, +}) { const resolvedDataDir = resolvePath(dataDir); - if (!existsSync(resolvedDataDir) || !lstatSync(resolvedDataDir).isDirectory()) { + if ( + !existsSync(resolvedDataDir) || + !lstatSync(resolvedDataDir).isDirectory() + ) { throw new Error(`files 数据目录不存在或不是目录: ${resolvedDataDir}`); } const files = new Map(); const symlinks = new Map(); const directories = new Set(['.']); - const roots = candidates === null - ? [{absolutePath: resolvedDataDir, relativePath: '.'}] - : candidates.map((candidate) => ({ - absolutePath: resolve(resolvedDataDir, candidate.path), - relativePath: assertSafeRelativePath(resolvedDataDir, resolve(resolvedDataDir, candidate.path)), - })); + const roots = + candidates === null + ? [{ absolutePath: resolvedDataDir, relativePath: '.' }] + : candidates.map((candidate) => ({ + absolutePath: resolve(resolvedDataDir, candidate.path), + relativePath: assertSafeRelativePath( + resolvedDataDir, + resolve(resolvedDataDir, candidate.path), + ), + })); const visit = async (absolutePath, relativePath) => { const stat = lstatSync(absolutePath); if (stat.isSymbolicLink()) { const target = readlinkSync(absolutePath, 'utf8'); if (!target || isAbsolute(target)) { - throw new Error(`files 模式只允许 data-dir 内部的相对符号链接: ${absolutePath} -> ${target}`); + throw new Error( + `files 模式只允许 data-dir 内部的相对符号链接: ${absolutePath} -> ${target}`, + ); } - assertSafeRelativePath(resolvedDataDir, resolve(dirname(absolutePath), target)); - symlinks.set(relativePath, {path: relativePath, target}); + assertSafeRelativePath( + resolvedDataDir, + resolve(dirname(absolutePath), target), + ); + symlinks.set(relativePath, { path: relativePath, target }); return; } if (stat.isDirectory()) { directories.add(relativePath); for (const name of readdirSync(absolutePath).sort()) { - const childRelative = relativePath === '.' ? name : `${relativePath}/${name}`; + const childRelative = + relativePath === '.' ? name : `${relativePath}/${name}`; await visit(join(absolutePath, name), childRelative); } return; @@ -1285,17 +1633,25 @@ export async function collectDirectFileEntries({dataDir, candidates = null, obje throw new Error(`files 扫描期间源文件发生变化: ${relativePath}`); } const basePrefix = normalizeObjectPrefix(objectPrefix, database); - files.set(relativePath, { + const file = { path: relativePath, sizeBytes: Number(after.size), sha256, mode: after.mode, objectKey: `${basePrefix}/files/sha256/${sha256.slice(0, 2)}/${sha256}`, - sourceStat: after, + }; + // 上传前后的 inode/stat 仍用于防止在线扫描漂移，但设为不可枚举，避免 + // 把仅供本地校验的副本再次写入 catalog 或 result JSON。 + Object.defineProperty(file, 'sourceStat', { + value: after, + enumerable: false, }); + files.set(relativePath, file); }; - for (const root of roots.sort((left, right) => left.relativePath.localeCompare(right.relativePath))) { + for (const root of roots.sort((left, right) => + left.relativePath.localeCompare(right.relativePath), + )) { if (!existsSync(root.absolutePath)) { throw new Error(`files 候选在扫描前消失: ${root.relativePath}`); } @@ -1303,23 +1659,64 @@ export async function collectDirectFileEntries({dataDir, candidates = null, obje } return { directories: [...directories].sort(), - files: [...files.values()].sort((left, right) => left.path.localeCompare(right.path)), - symlinks: [...symlinks.values()].sort((left, right) => left.path.localeCompare(right.path)), + files: [...files.values()].sort((left, right) => + left.path.localeCompare(right.path), + ), + symlinks: [...symlinks.values()].sort((left, right) => + left.path.localeCompare(right.path), + ), }; } -function directCatalogIdentity({mode, baselineCatalogId, rootName, directories, files, symlinks}) { - return sha256Hex(JSON.stringify({ - mode, - baselineCatalogId: baselineCatalogId || '', - rootName, - directories, - files: files.map(({path, sizeBytes, sha256, mode, objectKey}) => ({path, sizeBytes, sha256, mode, objectKey})), - symlinks, - })); +function directCatalogIdentity({ + mode, + baselineCatalogId, + rootName, + directories, + files, + symlinks, +}) { + // 不把数十万条文件元数据先拼成一个巨型 JSON 字符串；分段写入 hash + // 保持与 JSON.stringify 同样的字段顺序和转义结果，同时把峰值降到单条记录。 + const hash = createHash('sha256'); + hash.update('{"mode":'); + hash.update(JSON.stringify(mode)); + hash.update(',"baselineCatalogId":'); + hash.update(JSON.stringify(baselineCatalogId || '')); + hash.update(',"rootName":'); + hash.update(JSON.stringify(rootName)); + hash.update(',"directories":'); + updateJsonArrayHash(hash, directories, (directory) => + JSON.stringify(directory), + ); + hash.update(',"files":'); + updateJsonArrayHash(hash, files, (file) => + JSON.stringify({ + path: file.path, + sizeBytes: file.sizeBytes, + sha256: file.sha256, + mode: file.mode, + objectKey: file.objectKey, + }), + ); + hash.update(',"symlinks":'); + updateJsonArrayHash(hash, symlinks, (symlink) => JSON.stringify(symlink)); + hash.update('}'); + return hash.digest('hex'); } -function readDirectFilesState(statePath, {database, bucket}) { +function updateJsonArrayHash(hash, values, serialize) { + hash.update('['); + values.forEach((value, index) => { + if (index > 0) { + hash.update(','); + } + hash.update(serialize(value)); + }); + hash.update(']'); +} + +function readDirectFilesState(statePath, { database, bucket }) { const candidates = statePath.endsWith('.gz') ? [statePath, statePath.slice(0, -3)] : [statePath, `${statePath}.gz`]; @@ -1329,24 +1726,30 @@ function readDirectFilesState(statePath, {database, bucket}) { } const state = readJsonOrGzip(existingPath); if ( - ![LEGACY_DIRECT_FILES_STATE_SCHEMA_VERSION, DIRECT_FILES_STATE_SCHEMA_VERSION].includes(state.schemaVersion) - || state.backupKind !== 'spacetimedb-direct-files-state' - || state.database !== database - || state.bucket !== bucket + ![ + LEGACY_DIRECT_FILES_STATE_SCHEMA_VERSION, + DIRECT_FILES_STATE_SCHEMA_VERSION, + ].includes(state.schemaVersion) || + state.backupKind !== 'spacetimedb-direct-files-state' || + state.database !== database || + state.bucket !== bucket ) { throw new Error(`files state 与本次数据源或 bucket 不匹配: ${statePath}`); } return state; } -function directPreviousFiles({state, workDir, database}) { +function directPreviousFiles({ state, workDir, database }) { if (!state?.latestCatalog) { return []; } if (Array.isArray(state?.latestCatalog?.files)) { return state.latestCatalog.files; } - return readLocalDirectCatalog({workDir, database, catalog: state?.latestCatalog})?.files ?? []; + return ( + readLocalDirectCatalog({ workDir, database, catalog: state?.latestCatalog }) + ?.files ?? [] + ); } async function ensureDirectObject({ @@ -1360,9 +1763,11 @@ async function ensureDirectObject({ }) { const absolutePath = resolve(dataDir, file.path); assertSafeRelativePath(dataDir, absolutePath); - if (previousFile?.sha256 === file.sha256 - && previousFile?.sizeBytes === file.sizeBytes - && previousFile?.objectKey === file.objectKey) { + if ( + previousFile?.sha256 === file.sha256 && + previousFile?.sizeBytes === file.sizeBytes && + previousFile?.objectKey === file.objectKey + ) { if (verifyCatalogReuse) { await verifyFn({ ...uploadOptions, @@ -1383,7 +1788,7 @@ async function ensureDirectObject({ contentLength: file.sizeBytes, archiveSha256: file.sha256, }); - return {status: 'oss-reused', objectKey: file.objectKey}; + return { status: 'oss-reused', objectKey: file.objectKey }; } catch (error) { if (error?.status !== 404) { throw error; @@ -1406,10 +1811,16 @@ async function ensureDirectObject({ if (!sameDirectFileIdentity(afterUpload, file.sourceStat)) { throw new Error(`files 上传期间源文件 stat 漂移: ${file.path}`); } - return {status: 'uploaded', objectKey: file.objectKey}; + return { status: 'uploaded', objectKey: file.objectKey }; } -async function ensureDirectManifest({manifestPath, objectKey, uploadOptions, uploadManifestFn, verifyFn}) { +async function ensureDirectManifest({ + manifestPath, + objectKey, + uploadOptions, + uploadManifestFn, + verifyFn, +}) { const body = readFileSync(manifestPath); const archiveSha256 = sha256Hex(body); try { @@ -1419,13 +1830,19 @@ async function ensureDirectManifest({manifestPath, objectKey, uploadOptions, upl contentLength: body.length, archiveSha256, }); - return {objectKey, contentLength: body.length, archiveSha256, verifiedAt: verification.verifiedAt, reused: true}; + return { + objectKey, + contentLength: body.length, + archiveSha256, + verifiedAt: verification.verifiedAt, + reused: true, + }; } catch (error) { if (error?.status !== 404) { throw error; } } - return uploadManifestFn({manifestPath, ...uploadOptions, objectKey}); + return uploadManifestFn({ manifestPath, ...uploadOptions, objectKey }); } function directCatalogRef(catalog) { @@ -1439,7 +1856,7 @@ function directCatalogRef(catalog) { }; } -function normalizeDirectFilesState({state, dataDir, database, bucket}) { +function normalizeDirectFilesState({ state, dataDir, database, bucket }) { return { schemaVersion: DIRECT_FILES_STATE_SCHEMA_VERSION, backupKind: 'spacetimedb-direct-files-state', @@ -1447,18 +1864,26 @@ function normalizeDirectFilesState({state, dataDir, database, bucket}) { dataDir, bucket, updatedAt: new Date().toISOString(), - baselineCatalog: assertDirectCatalogRef(state?.baselineCatalog, 'full', 'baseline full'), - latestCatalog: assertDirectCatalogRef(state?.latestCatalog, 'full', 'latest full'), - historyCatalogs: (state?.historyCatalogs ?? []).map((catalog) => ( - assertDirectCatalogRef(catalog, 'history', 'history') - )), + baselineCatalog: assertDirectCatalogRef( + state?.baselineCatalog, + 'full', + 'baseline full', + ), + latestCatalog: assertDirectCatalogRef( + state?.latestCatalog, + 'full', + 'latest full', + ), + historyCatalogs: (state?.historyCatalogs ?? []).map((catalog) => + assertDirectCatalogRef(catalog, 'history', 'history'), + ), }; } -function persistDirectFilesState({statePath, legacyStatePath, state}) { +function persistDirectFilesState({ statePath, legacyStatePath, state }) { atomicWriteGzipJson(statePath, state); if (legacyStatePath !== statePath && existsSync(legacyStatePath)) { - rmSync(legacyStatePath, {force: false}); + rmSync(legacyStatePath, { force: false }); } } @@ -1466,7 +1891,7 @@ function compactDirectFilesResult(result) { if (!result.catalog) { return result; } - const {catalog, ...rest} = result; + const { catalog, ...rest } = result; return { ...rest, catalog: { @@ -1480,34 +1905,40 @@ function compactDirectFilesResult(result) { baselineCatalogId: catalog.baselineCatalogId, rootName: catalog.rootName, fileCount: Array.isArray(catalog.files) ? catalog.files.length : 0, - symlinkCount: Array.isArray(catalog.symlinks) ? catalog.symlinks.length : 0, + symlinkCount: Array.isArray(catalog.symlinks) + ? catalog.symlinks.length + : 0, }, }; } function assertDirectCatalogRef(catalog, expectedMode, label) { if ( - !catalog - || catalog.mode !== expectedMode - || !/^[a-f0-9]{64}$/u.test(catalog.catalogId) - || typeof catalog.objectKey !== 'string' - || !catalog.objectKey - || !Number.isSafeInteger(catalog.contentLength) - || catalog.contentLength <= 0 - || !/^[a-f0-9]{64}$/u.test(catalog.sha256) - || typeof catalog.verifiedAt !== 'string' - || !catalog.verifiedAt + !catalog || + catalog.mode !== expectedMode || + !/^[a-f0-9]{64}$/u.test(catalog.catalogId) || + typeof catalog.objectKey !== 'string' || + !catalog.objectKey || + !Number.isSafeInteger(catalog.contentLength) || + catalog.contentLength <= 0 || + !/^[a-f0-9]{64}$/u.test(catalog.sha256) || + typeof catalog.verifiedAt !== 'string' || + !catalog.verifiedAt ) { throw new Error(`files ${label} catalog ref 无效。`); } return directCatalogRef(catalog); } -function buildDirectFilesLatest({database, bucket, state}) { - const latestFullCatalog = assertDirectCatalogRef(state?.latestCatalog, 'full', 'latest full'); - const historyCatalogs = (state?.historyCatalogs ?? []).map((catalog) => ( - assertDirectCatalogRef(catalog, 'history', 'history') - )); +function buildDirectFilesLatest({ database, bucket, state }) { + const latestFullCatalog = assertDirectCatalogRef( + state?.latestCatalog, + 'full', + 'latest full', + ); + const historyCatalogs = (state?.historyCatalogs ?? []).map((catalog) => + assertDirectCatalogRef(catalog, 'history', 'history'), + ); return { schemaVersion: DIRECT_FILES_LATEST_SCHEMA_VERSION, backupKind: 'spacetimedb-direct-files-latest', @@ -1519,20 +1950,26 @@ function buildDirectFilesLatest({database, bucket, state}) { }; } -function validateDirectFilesLatest(latest, {database, bucket}) { +function validateDirectFilesLatest(latest, { database, bucket }) { if ( - latest?.schemaVersion !== DIRECT_FILES_LATEST_SCHEMA_VERSION - || latest.backupKind !== 'spacetimedb-direct-files-latest' - || latest.database !== database - || latest.bucket !== bucket - || !Array.isArray(latest.historyCatalogs) + latest?.schemaVersion !== DIRECT_FILES_LATEST_SCHEMA_VERSION || + latest.backupKind !== 'spacetimedb-direct-files-latest' || + latest.database !== database || + latest.bucket !== bucket || + !Array.isArray(latest.historyCatalogs) ) { throw new Error('files latest pointer 契约无效。'); } return { ...latest, - latestFullCatalog: assertDirectCatalogRef(latest.latestFullCatalog, 'full', 'latest full'), - historyCatalogs: latest.historyCatalogs.map((catalog) => assertDirectCatalogRef(catalog, 'history', 'history')), + latestFullCatalog: assertDirectCatalogRef( + latest.latestFullCatalog, + 'full', + 'latest full', + ), + historyCatalogs: latest.historyCatalogs.map((catalog) => + assertDirectCatalogRef(catalog, 'history', 'history'), + ), }; } @@ -1546,8 +1983,11 @@ async function publishDirectFilesLatest({ uploadManifestFn, verifyFn, }) { - const latest = buildDirectFilesLatest({database, bucket, state}); - for (const catalogRef of [latest.latestFullCatalog, ...latest.historyCatalogs]) { + const latest = buildDirectFilesLatest({ database, bucket, state }); + for (const catalogRef of [ + latest.latestFullCatalog, + ...latest.historyCatalogs, + ]) { await verifyFn({ ...uploadOptions, objectKey: catalogRef.objectKey, @@ -1555,10 +1995,17 @@ async function publishDirectFilesLatest({ archiveSha256: catalogRef.sha256, }); } - const latestPath = join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-latest.json`); + const latestPath = join( + workDir, + `${sanitizeObjectPart(database, 'spacetimedb')}-latest.json`, + ); const latestObjectKey = `${normalizeObjectPrefix(objectPrefix, database)}/latest.json`; - writeManifest({manifestPath: latestPath, payload: latest}); - const uploaded = await uploadManifestFn({manifestPath: latestPath, ...uploadOptions, objectKey: latestObjectKey}); + writeManifest({ manifestPath: latestPath, payload: latest }); + const uploaded = await uploadManifestFn({ + manifestPath: latestPath, + ...uploadOptions, + objectKey: latestObjectKey, + }); const verification = await verifyFn({ ...uploadOptions, objectKey: latestObjectKey, @@ -1590,29 +2037,46 @@ export async function runDirectFilesBackup({ verifyFn = verifyOssObject, concurrency = 1, }) { - mkdirSync(workDir, {recursive: true}); - const statePath = directFilesStatePath({workDir, database}); - const legacyStatePath = legacyDirectFilesStatePath({workDir, database}); - const state = readDirectFilesState(statePath, {database, bucket}); - if (mode === 'history' && (!state?.baselineCatalog || state?.latestCatalog?.mode !== 'full')) { - throw new Error(`files history 模式缺少已发布 full baseline catalog: ${statePath}`); + mkdirSync(workDir, { recursive: true }); + const statePath = directFilesStatePath({ workDir, database }); + const legacyStatePath = legacyDirectFilesStatePath({ workDir, database }); + const state = readDirectFilesState(statePath, { database, bucket }); + if ( + mode === 'history' && + (!state?.baselineCatalog || state?.latestCatalog?.mode !== 'full') + ) { + throw new Error( + `files history 模式缺少已发布 full baseline catalog: ${statePath}`, + ); } - const plan = mode === 'history' ? discoverHistoryPlan({dataDir}) : null; + const plan = mode === 'history' ? discoverHistoryPlan({ dataDir }) : null; const collected = await collectDirectFileEntries({ dataDir, candidates: plan?.candidates ?? null, objectPrefix, database, }); - const baselineCatalogId = mode === 'history' ? (state?.baselineCatalog?.catalogId ?? '') : ''; + const baselineCatalogId = + mode === 'history' ? (state?.baselineCatalog?.catalogId ?? '') : ''; const rootName = basename(dataDir); - const catalogId = directCatalogIdentity({mode, baselineCatalogId, rootName, ...collected}); + const catalogId = directCatalogIdentity({ + mode, + baselineCatalogId, + rootName, + ...collected, + }); const basePrefix = normalizeObjectPrefix(objectPrefix, database); const catalogObjectKey = `${basePrefix}/catalogs/${mode}/${catalogId}.json`; - const catalogPath = join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-${mode}-${catalogId}.catalog.json`); + const catalogPath = join( + workDir, + `${sanitizeObjectPart(database, 'spacetimedb')}-${mode}-${catalogId}.catalog.json`, + ); const catalog = { schemaVersion: DIRECT_FILES_CATALOG_SCHEMA_VERSION, - backupKind: mode === 'full' ? 'spacetimedb-data-dir-files' : 'spacetimedb-history-files', + backupKind: + mode === 'full' + ? 'spacetimedb-data-dir-files' + : 'spacetimedb-history-files', database, bucket, mode, @@ -1621,10 +2085,10 @@ export async function runDirectFilesBackup({ baselineCatalogId, rootName, directories: collected.directories, - files: collected.files.map(({sourceStat: _sourceStat, ...file}) => file), + files: collected.files, symlinks: collected.symlinks, }; - writeManifest({manifestPath: catalogPath, payload: catalog}); + writeManifest({ manifestPath: catalogPath, payload: catalog }); const summary = { statePath, catalogPath, @@ -1632,16 +2096,22 @@ export async function runDirectFilesBackup({ catalogId, fileCount: collected.files.length, symlinkCount: collected.symlinks.length, - totalSizeBytes: collected.files.reduce((sum, file) => sum + BigInt(file.sizeBytes), 0n).toString(), + totalSizeBytes: collected.files + .reduce((sum, file) => sum + BigInt(file.sizeBytes), 0n) + .toString(), candidateCount: plan?.candidates.length ?? 0, }; if (resultFile) { - atomicWriteJson(resolvePath(resultFile), {...summary, dryRun}); + atomicWriteJson(resolvePath(resultFile), { ...summary, dryRun }); } - console.log(`[database-backup] files ${mode}: files=${summary.fileCount}, symlinks=${summary.symlinkCount}, size=${formatBytes(summary.totalSizeBytes)}, catalog=${catalogId}`); + console.log( + `[database-backup] files ${mode}: files=${summary.fileCount}, symlinks=${summary.symlinkCount}, size=${formatBytes(summary.totalSizeBytes)}, catalog=${catalogId}`, + ); if (dryRun) { - console.log('[database-backup] files dry-run，仅扫描并生成本地 catalog，不上传或删除。'); - return {...summary, catalog, uploadedCount: 0, reusedCount: 0}; + console.log( + '[database-backup] files dry-run，仅扫描并生成本地 catalog，不上传或删除。', + ); + return { ...summary, catalog, uploadedCount: 0, reusedCount: 0 }; } if (mode === 'history' && plan.candidates.length === 0) { await verifyFn({ @@ -1660,8 +2130,17 @@ export async function runDirectFilesBackup({ uploadManifestFn, verifyFn, }); - const compactedState = normalizeDirectFilesState({state, dataDir, database, bucket}); - persistDirectFilesState({statePath, legacyStatePath, state: compactedState}); + const compactedState = normalizeDirectFilesState({ + state, + dataDir, + database, + bucket, + }); + persistDirectFilesState({ + statePath, + legacyStatePath, + state: compactedState, + }); const metadataCleanup = compactDirectFilesLocalMetadata({ workDir, database, @@ -1679,13 +2158,24 @@ export async function runDirectFilesBackup({ metadataCleanup, }; if (resultFile) { - atomicWriteJson(resolvePath(resultFile), compactDirectFilesResult(emptyResult)); + atomicWriteJson( + resolvePath(resultFile), + compactDirectFilesResult(emptyResult), + ); } return emptyResult; } - if (state?.latestCatalog?.catalogId === catalogId && state.latestCatalog.mode === mode) { - await verifyFn({...uploadOptions, objectKey: state.latestCatalog.objectKey, contentLength: state.latestCatalog.contentLength, archiveSha256: state.latestCatalog.sha256}); + if ( + state?.latestCatalog?.catalogId === catalogId && + state.latestCatalog.mode === mode + ) { + await verifyFn({ + ...uploadOptions, + objectKey: state.latestCatalog.objectKey, + contentLength: state.latestCatalog.contentLength, + archiveSha256: state.latestCatalog.sha256, + }); if (mode === 'full') { const latestPointer = await publishDirectFilesLatest({ workDir, @@ -1697,8 +2187,17 @@ export async function runDirectFilesBackup({ uploadManifestFn, verifyFn, }); - const compactedState = normalizeDirectFilesState({state, dataDir, database, bucket}); - persistDirectFilesState({statePath, legacyStatePath, state: compactedState}); + const compactedState = normalizeDirectFilesState({ + state, + dataDir, + database, + bucket, + }); + persistDirectFilesState({ + statePath, + legacyStatePath, + state: compactedState, + }); const metadataCleanup = compactDirectFilesLocalMetadata({ workDir, database, @@ -1726,13 +2225,18 @@ export async function runDirectFilesBackup({ }); } - const previousFiles = new Map(directPreviousFiles({state, workDir, database}).map((file) => [file.path, file])); + const previousFiles = new Map( + directPreviousFiles({ state, workDir, database }).map((file) => [ + file.path, + file, + ]), + ); let uploadedCount = 0; let reusedCount = 0; let nextIndex = 0; let completedCount = 0; const workerCount = Math.min(concurrency, collected.files.length); - const workers = Array.from({length: workerCount}, async () => { + const workers = Array.from({ length: workerCount }, async () => { while (nextIndex < collected.files.length) { const index = nextIndex; nextIndex += 1; @@ -1752,8 +2256,14 @@ export async function runDirectFilesBackup({ reusedCount += 1; } completedCount += 1; - if (collected.files.length <= 100 || completedCount % 1000 === 0 || completedCount === collected.files.length) { - console.log(`[database-backup] files 进度: ${completedCount}/${collected.files.length} (${result.status}) ${file.path}`); + if ( + collected.files.length <= 100 || + completedCount % 1000 === 0 || + completedCount === collected.files.length + ) { + console.log( + `[database-backup] files 进度: ${completedCount}/${collected.files.length} (${result.status}) ${file.path}`, + ); } } }); @@ -1765,7 +2275,12 @@ export async function runDirectFilesBackup({ uploadManifestFn, verifyFn, }); - await verifyFn({...uploadOptions, objectKey: catalogObjectKey, contentLength: catalogUpload.contentLength, archiveSha256: catalogUpload.archiveSha256}); + await verifyFn({ + ...uploadOptions, + objectKey: catalogObjectKey, + contentLength: catalogUpload.contentLength, + archiveSha256: catalogUpload.archiveSha256, + }); if (mode === 'history') { await verifyFn({ @@ -1791,15 +2306,18 @@ export async function runDirectFilesBackup({ bucket, updatedAt: new Date().toISOString(), baselineCatalog: directCatalogRef(state?.baselineCatalog ?? catalogRef), - latestCatalog: directCatalogRef(mode === 'full' ? catalogRef : state.latestCatalog), - historyCatalogs: mode === 'history' - ? [ - ...(state.historyCatalogs ?? []) - .filter((item) => item.catalogId !== catalogId) - .map((item) => directCatalogRef(item)), - directCatalogRef(catalogRef), - ] - : (state?.historyCatalogs ?? []).map((item) => directCatalogRef(item)), + latestCatalog: directCatalogRef( + mode === 'full' ? catalogRef : state.latestCatalog, + ), + historyCatalogs: + mode === 'history' + ? [ + ...(state.historyCatalogs ?? []) + .filter((item) => item.catalogId !== catalogId) + .map((item) => directCatalogRef(item)), + directCatalogRef(catalogRef), + ] + : (state?.historyCatalogs ?? []).map((item) => directCatalogRef(item)), }; const latestPointer = await publishDirectFilesLatest({ workDir, @@ -1811,7 +2329,7 @@ export async function runDirectFilesBackup({ uploadManifestFn, verifyFn, }); - persistDirectFilesState({statePath, legacyStatePath, state: nextState}); + persistDirectFilesState({ statePath, legacyStatePath, state: nextState }); const metadataCleanup = compactDirectFilesLocalMetadata({ workDir, database, @@ -1819,7 +2337,10 @@ export async function runDirectFilesBackup({ }); let cleanup = null; if (mode === 'history') { - cleanup = cleanupHistoryCandidates({dataDir, candidates: plan.candidates}); + cleanup = cleanupHistoryCandidates({ + dataDir, + candidates: plan.candidates, + }); } const finalResult = { ...summary, @@ -1831,12 +2352,15 @@ export async function runDirectFilesBackup({ metadataCleanup, }; if (resultFile) { - atomicWriteJson(resolvePath(resultFile), compactDirectFilesResult(finalResult)); + atomicWriteJson( + resolvePath(resultFile), + compactDirectFilesResult(finalResult), + ); } return finalResult; } -async function downloadOssBuffer({objectKey, uploadOptions}) { +async function downloadOssBuffer({ objectKey, uploadOptions }) { const response = await signedOssRequest({ ...ossRequestDefaults(uploadOptions), method: 'GET', @@ -1846,7 +2370,7 @@ async function downloadOssBuffer({objectKey, uploadOptions}) { return Buffer.from(await response.arrayBuffer()); } -async function downloadOssFile({objectKey, destinationPath, uploadOptions}) { +async function downloadOssFile({ objectKey, destinationPath, uploadOptions }) { const response = await signedOssRequest({ ...ossRequestDefaults(uploadOptions), method: 'GET', @@ -1854,50 +2378,67 @@ async function downloadOssFile({objectKey, destinationPath, uploadOptions}) { operation: '下载对象', }); const tempPath = `${destinationPath}.partial-${process.pid}`; - rmSync(tempPath, {force: true}); + rmSync(tempPath, { force: true }); try { if (response.body) { - await pipeline(Readable.fromWeb(response.body), createWriteStream(tempPath, {mode: 0o600})); + await pipeline( + Readable.fromWeb(response.body), + createWriteStream(tempPath, { mode: 0o600 }), + ); } else { - writeFileSync(tempPath, Buffer.alloc(0), {mode: 0o600}); + writeFileSync(tempPath, Buffer.alloc(0), { mode: 0o600 }); } renameSync(tempPath, destinationPath); } catch (error) { - rmSync(tempPath, {force: true}); + rmSync(tempPath, { force: true }); throw error; } } -async function loadDirectFilesCatalog({catalogRef, database, bucket, uploadOptions, downloadBufferFn}) { - const catalogBody = await downloadBufferFn({objectKey: catalogRef.objectKey, uploadOptions}); - if (catalogBody.length !== catalogRef.contentLength || sha256Hex(catalogBody) !== catalogRef.sha256) { - throw new Error(`files restore catalog 长度或 SHA-256 不一致: ${catalogRef.objectKey}`); +async function loadDirectFilesCatalog({ + catalogRef, + database, + bucket, + uploadOptions, + downloadBufferFn, +}) { + const catalogBody = await downloadBufferFn({ + objectKey: catalogRef.objectKey, + uploadOptions, + }); + if ( + catalogBody.length !== catalogRef.contentLength || + sha256Hex(catalogBody) !== catalogRef.sha256 + ) { + throw new Error( + `files restore catalog 长度或 SHA-256 不一致: ${catalogRef.objectKey}`, + ); } const catalog = JSON.parse(catalogBody.toString('utf8')); if ( - catalog.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION - || catalog.backupKind !== 'spacetimedb-data-dir-files' - || catalog.database !== database - || catalog.bucket !== bucket - || catalog.catalogId !== catalogRef.catalogId - || !Array.isArray(catalog.directories) - || !Array.isArray(catalog.files) + catalog.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION || + catalog.backupKind !== 'spacetimedb-data-dir-files' || + catalog.database !== database || + catalog.bucket !== bucket || + catalog.catalogId !== catalogRef.catalogId || + !Array.isArray(catalog.directories) || + !Array.isArray(catalog.files) ) { throw new Error(`files restore catalog 契约无效: ${catalogRef.objectKey}`); } - return {...catalog, symlinks: catalog.symlinks ?? []}; + return { ...catalog, symlinks: catalog.symlinks ?? [] }; } function assertDirectCatalogFile(file, index) { if ( - !file - || typeof file.path !== 'string' - || !Number.isSafeInteger(file.sizeBytes) - || file.sizeBytes < 0 - || !/^[a-f0-9]{64}$/u.test(file.sha256) - || typeof file.objectKey !== 'string' - || !file.objectKey - || !Number.isSafeInteger(file.mode) + !file || + typeof file.path !== 'string' || + !Number.isSafeInteger(file.sizeBytes) || + file.sizeBytes < 0 || + !/^[a-f0-9]{64}$/u.test(file.sha256) || + typeof file.objectKey !== 'string' || + !file.objectKey || + !Number.isSafeInteger(file.mode) ) { throw new Error(`files restore catalog 文件项无效: index=${index}`); } @@ -1905,18 +2446,21 @@ function assertDirectCatalogFile(file, index) { function assertDirectCatalogSymlink(symlink, index, restoreDir) { if ( - !symlink - || typeof symlink.path !== 'string' - || !symlink.path - || typeof symlink.target !== 'string' - || !symlink.target - || isAbsolute(symlink.target) + !symlink || + typeof symlink.path !== 'string' || + !symlink.path || + typeof symlink.target !== 'string' || + !symlink.target || + isAbsolute(symlink.target) ) { throw new Error(`files restore catalog 符号链接项无效: index=${index}`); } const destinationPath = resolve(restoreDir, symlink.path); assertSafeRelativePath(restoreDir, destinationPath); - assertSafeRelativePath(restoreDir, resolve(dirname(destinationPath), symlink.target)); + assertSafeRelativePath( + restoreDir, + resolve(dirname(destinationPath), symlink.target), + ); } async function restoreDirectFilesCatalog({ @@ -1929,8 +2473,12 @@ async function restoreDirectFilesCatalog({ }) { const resolvedRestoreDir = resolvePath(restoreDir); catalog.files.forEach(assertDirectCatalogFile); - catalog.symlinks.forEach((symlink, index) => assertDirectCatalogSymlink(symlink, index, resolvedRestoreDir)); - const totalSizeBytes = catalog.files.reduce((sum, file) => sum + BigInt(file.sizeBytes), 0n).toString(); + catalog.symlinks.forEach((symlink, index) => + assertDirectCatalogSymlink(symlink, index, resolvedRestoreDir), + ); + const totalSizeBytes = catalog.files + .reduce((sum, file) => sum + BigInt(file.sizeBytes), 0n) + .toString(); if (dryRun) { const result = { restoreDir: resolvedRestoreDir, @@ -1947,14 +2495,14 @@ async function restoreDirectFilesCatalog({ } return result; } - mkdirSync(resolvedRestoreDir, {recursive: true, mode: 0o700}); + mkdirSync(resolvedRestoreDir, { recursive: true, mode: 0o700 }); for (const directoryPath of catalog.directories) { if (directoryPath === '.') { continue; } const absolutePath = resolve(resolvedRestoreDir, directoryPath); assertSafeRelativePath(resolvedRestoreDir, absolutePath); - mkdirSync(absolutePath, {recursive: true}); + mkdirSync(absolutePath, { recursive: true }); } let downloadedCount = 0; @@ -1962,35 +2510,47 @@ async function restoreDirectFilesCatalog({ for (const [index, file] of catalog.files.entries()) { const destinationPath = resolve(resolvedRestoreDir, file.path); assertSafeRelativePath(resolvedRestoreDir, destinationPath); - mkdirSync(dirname(destinationPath), {recursive: true}); + mkdirSync(dirname(destinationPath), { recursive: true }); let reusable = false; if (existsSync(destinationPath) && lstatSync(destinationPath).isFile()) { const stat = statSync(destinationPath); - reusable = stat.size === file.sizeBytes && await sha256FileHex(destinationPath) === file.sha256; + reusable = + stat.size === file.sizeBytes && + (await sha256FileHex(destinationPath)) === file.sha256; } if (reusable) { reusedCount += 1; } else { - rmSync(destinationPath, {force: true}); - await downloadFileFn({objectKey: file.objectKey, destinationPath, uploadOptions}); + rmSync(destinationPath, { force: true }); + await downloadFileFn({ + objectKey: file.objectKey, + destinationPath, + uploadOptions, + }); const stat = statSync(destinationPath); const sha256 = await sha256FileHex(destinationPath); if (stat.size !== file.sizeBytes || sha256 !== file.sha256) { - rmSync(destinationPath, {force: true}); - throw new Error(`files restore 对象长度或 SHA-256 不一致: ${file.path}`); + rmSync(destinationPath, { force: true }); + throw new Error( + `files restore 对象长度或 SHA-256 不一致: ${file.path}`, + ); } downloadedCount += 1; } chmodSync(destinationPath, file.mode & 0o7777); - console.log(`[database-backup] files restore: ${index + 1}/${catalog.files.length} (${reusable ? 'reused' : 'downloaded'}) ${file.path}`); + console.log( + `[database-backup] files restore: ${index + 1}/${catalog.files.length} (${reusable ? 'reused' : 'downloaded'}) ${file.path}`, + ); } for (const symlink of catalog.symlinks) { const destinationPath = resolve(resolvedRestoreDir, symlink.path); assertSafeRelativePath(resolvedRestoreDir, destinationPath); - mkdirSync(dirname(destinationPath), {recursive: true}); - rmSync(destinationPath, {recursive: true, force: true}); + mkdirSync(dirname(destinationPath), { recursive: true }); + rmSync(destinationPath, { recursive: true, force: true }); symlinkSync(symlink.target, destinationPath); - console.log(`[database-backup] files restore: symlink ${symlink.path} -> ${symlink.target}`); + console.log( + `[database-backup] files restore: symlink ${symlink.path} -> ${symlink.target}`, + ); } const result = { restoreDir: resolvedRestoreDir, @@ -2018,12 +2578,25 @@ export async function restoreDirectFilesBackup({ downloadBufferFn = downloadOssBuffer, downloadFileFn = downloadOssFile, }) { - const state = readDirectFilesState(resolvePath(statePath), {database, bucket}); + const state = readDirectFilesState(resolvePath(statePath), { + database, + bucket, + }); if (!state?.latestCatalog || state.latestCatalog.mode !== 'full') { throw new Error(`files restore 缺少 full baseline catalog: ${statePath}`); } - const catalogRef = assertDirectCatalogRef(state.latestCatalog, 'full', 'latest full'); - const catalog = await loadDirectFilesCatalog({catalogRef, database, bucket, uploadOptions, downloadBufferFn}); + const catalogRef = assertDirectCatalogRef( + state.latestCatalog, + 'full', + 'latest full', + ); + const catalog = await loadDirectFilesCatalog({ + catalogRef, + database, + bucket, + uploadOptions, + downloadBufferFn, + }); return restoreDirectFilesCatalog({ catalog, restoreDir, @@ -2047,7 +2620,10 @@ export async function restoreDirectFilesLatest({ verifyFn = verifyOssObject, }) { const latestObjectKey = `${normalizeObjectPrefix(objectPrefix, database)}/latest.json`; - const latestBody = await downloadBufferFn({objectKey: latestObjectKey, uploadOptions}); + const latestBody = await downloadBufferFn({ + objectKey: latestObjectKey, + uploadOptions, + }); const latestSha256 = sha256Hex(latestBody); await verifyFn({ ...uploadOptions, @@ -2055,7 +2631,10 @@ export async function restoreDirectFilesLatest({ contentLength: latestBody.length, archiveSha256: latestSha256, }); - const latest = validateDirectFilesLatest(JSON.parse(latestBody.toString('utf8')), {database, bucket}); + const latest = validateDirectFilesLatest( + JSON.parse(latestBody.toString('utf8')), + { database, bucket }, + ); const catalogRef = latest.latestFullCatalog; await verifyFn({ ...uploadOptions, @@ -2063,7 +2642,13 @@ export async function restoreDirectFilesLatest({ contentLength: catalogRef.contentLength, archiveSha256: catalogRef.sha256, }); - const catalog = await loadDirectFilesCatalog({catalogRef, database, bucket, uploadOptions, downloadBufferFn}); + const catalog = await loadDirectFilesCatalog({ + catalogRef, + database, + bucket, + uploadOptions, + downloadBufferFn, + }); return restoreDirectFilesCatalog({ catalog, restoreDir, @@ -2093,7 +2678,12 @@ function formatOssDate(date) { function encodePath(path) { return path .split('/') - .map((segment) => encodeURIComponent(segment).replace(/[!'()*]/gu, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`)) + .map((segment) => + encodeURIComponent(segment).replace( + /[!'()*]/gu, + (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`, + ), + ) .join('/'); } @@ -2106,7 +2696,10 @@ function encodeQueryComponent(value) { export function buildCanonicalQuery(queries = {}) { return Object.entries(queries) - .map(([key, value]) => [encodeQueryComponent(key), value === null ? null : encodeQueryComponent(value)]) + .map(([key, value]) => [ + encodeQueryComponent(key), + value === null ? null : encodeQueryComponent(value), + ]) .sort(([leftKey, leftValue], [rightKey, rightValue]) => { if (leftKey !== rightKey) { return leftKey < rightKey ? -1 : 1; @@ -2115,7 +2708,7 @@ export function buildCanonicalQuery(queries = {}) { const right = rightValue ?? ''; return left === right ? 0 : left < right ? -1 : 1; }) - .map(([key, value]) => value === null ? key : `${key}=${value}`) + .map(([key, value]) => (value === null ? key : `${key}=${value}`)) .join('&'); } @@ -2123,13 +2716,26 @@ function canonicalHeaderValue(value) { return String(value).trim().replace(/\s+/gu, ' '); } -export function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, accessKeySecret, headers, date, queries = {}}) { +export function buildAuthorization({ + method, + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + headers, + date, + queries = {}, +}) { const region = regionFromEndpoint(endpoint); const scopeDate = formatScopeDate(date); const scope = `${scopeDate}/${region}/${OSS_SERVICE}/${OSS_REQUEST}`; const canonicalUri = `/${encodeURIComponent(bucket)}/${encodePath(objectKey)}`; const signedHeaders = Object.fromEntries( - Object.entries(headers).map(([key, value]) => [key.toLowerCase(), canonicalHeaderValue(value)]), + Object.entries(headers).map(([key, value]) => [ + key.toLowerCase(), + canonicalHeaderValue(value), + ]), ); const canonicalHeaders = Object.entries(signedHeaders) .sort(([left], [right]) => left.localeCompare(right)) @@ -2144,8 +2750,16 @@ export function buildAuthorization({method, bucket, endpoint, objectKey, accessK additionalHeaders, UNSIGNED_PAYLOAD, ].join('\n'); - const stringToSign = [OSS_ALGORITHM, headers['x-oss-date'], scope, sha256Hex(canonicalRequest)].join('\n'); - const signature = hmac(Buffer.from(`aliyun_v4${accessKeySecret}`, 'utf8'), scopeDate); + const stringToSign = [ + OSS_ALGORITHM, + headers['x-oss-date'], + scope, + sha256Hex(canonicalRequest), + ].join('\n'); + const signature = hmac( + Buffer.from(`aliyun_v4${accessKeySecret}`, 'utf8'), + scopeDate, + ); const regionKey = hmac(signature, region); const serviceKey = hmac(regionKey, OSS_SERVICE); const signingKey = hmac(serviceKey, OSS_REQUEST); @@ -2153,7 +2767,7 @@ export function buildAuthorization({method, bucket, endpoint, objectKey, accessK return `${OSS_ALGORITHM} Credential=${accessKeyId}/${scope},AdditionalHeaders=${additionalHeaders},Signature=${finalSignature}`; } -function buildOssUrl({bucket, endpoint, objectKey, queries = {}}) { +function buildOssUrl({ bucket, endpoint, objectKey, queries = {} }) { const canonicalQuery = buildCanonicalQuery(queries); return `https://${bucket}.${endpoint}/${encodePath(objectKey)}${canonicalQuery ? `?${canonicalQuery}` : ''}`; } @@ -2162,8 +2776,11 @@ function isRetryableOssStatus(status) { return RETRYABLE_OSS_HTTP_STATUSES.has(status); } -function retryDelayMs({attempt, baseDelayMs, maxDelayMs, randomFn}) { - const ceiling = Math.min(maxDelayMs, baseDelayMs * (2 ** Math.max(0, attempt - 1))); +function retryDelayMs({ attempt, baseDelayMs, maxDelayMs, randomFn }) { + const ceiling = Math.min( + maxDelayMs, + baseDelayMs * 2 ** Math.max(0, attempt - 1), + ); return Math.floor(randomFn() * ceiling); } @@ -2215,7 +2832,7 @@ async function signedOssRequest({ retryBaseDelayMs, retryMaxDelayMs, }) { - const targetUrl = buildOssUrl({bucket, endpoint, objectKey, queries}); + const targetUrl = buildOssUrl({ bucket, endpoint, objectKey, queries }); let lastError = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { @@ -2237,12 +2854,12 @@ async function signedOssRequest({ date: now, queries, }); - const requestHeaders = {...signedHeaders, authorization}; + const requestHeaders = { ...signedHeaders, authorization }; if (contentLength !== undefined) { requestHeaders['content-length'] = String(contentLength); } const body = bodyFactory ? bodyFactory() : undefined; - const requestOptions = {method, headers: requestHeaders}; + const requestOptions = { method, headers: requestHeaders }; if (body !== undefined) { requestOptions.body = body; requestOptions.duplex = 'half'; @@ -2252,7 +2869,10 @@ async function signedOssRequest({ try { response = await fetchImpl(targetUrl, requestOptions); } catch (error) { - lastError = new Error(`OSS ${operation}请求失败: oss://${bucket}/${objectKey}`, {cause: error}); + lastError = new Error( + `OSS ${operation}请求失败: oss://${bucket}/${objectKey}`, + { cause: error }, + ); } if (response?.ok) { @@ -2271,16 +2891,27 @@ async function signedOssRequest({ if (!retryable || attempt >= maxAttempts) { throw lastError; } - const delayMs = retryDelayMs({attempt, baseDelayMs: retryBaseDelayMs, maxDelayMs: retryMaxDelayMs, randomFn}); - console.warn(`[database-backup] OSS ${operation}失败，${delayMs}ms 后重试 (${attempt}/${maxAttempts})`); + const delayMs = retryDelayMs({ + attempt, + baseDelayMs: retryBaseDelayMs, + maxDelayMs: retryMaxDelayMs, + randomFn, + }); + console.warn( + `[database-backup] OSS ${operation}失败，${delayMs}ms 后重试 (${attempt}/${maxAttempts})`, + ); await sleepImpl(delayMs); } - throw lastError ?? new Error(`OSS ${operation}失败: oss://${bucket}/${objectKey}`); + throw ( + lastError ?? new Error(`OSS ${operation}失败: oss://${bucket}/${objectKey}`) + ); } function readXmlTag(xml, tagName) { - const match = new RegExp(`<${tagName}>([\\s\\S]*?)`, 'u').exec(xml); + const match = new RegExp(`<${tagName}>([\\s\\S]*?)`, 'u').exec( + xml, + ); if (!match) { return ''; } @@ -2304,19 +2935,26 @@ function escapeXml(value) { function buildCompleteMultipartBody(parts) { const partXml = parts - .map(({partNumber, etag}) => [ - '', - `${partNumber}`, - `${escapeXml(etag)}`, - '', - ].join('')) + .map(({ partNumber, etag }) => + [ + '', + `${partNumber}`, + `${escapeXml(etag)}`, + '', + ].join(''), + ) .join(''); return `${partXml}`; } function resolveMultipartPartSize(fileSize, configuredPartSize) { - if (!Number.isSafeInteger(configuredPartSize) || configuredPartSize < OSS_MIN_MULTIPART_PART_SIZE_BYTES) { - throw new Error(`OSS multipart part size 必须是 >= ${OSS_MIN_MULTIPART_PART_SIZE_BYTES} 的安全整数`); + if ( + !Number.isSafeInteger(configuredPartSize) || + configuredPartSize < OSS_MIN_MULTIPART_PART_SIZE_BYTES + ) { + throw new Error( + `OSS multipart part size 必须是 >= ${OSS_MIN_MULTIPART_PART_SIZE_BYTES} 的安全整数`, + ); } const minimumForPartLimit = Math.ceil(fileSize / OSS_MAX_MULTIPART_PARTS); const partSize = Math.max(configuredPartSize, minimumForPartLimit); @@ -2326,7 +2964,11 @@ function resolveMultipartPartSize(fileSize, configuredPartSize) { return partSize; } -async function verifyUploadedObject({requestOptions, expectedContentLength, expectedArchiveSha256}) { +async function verifyUploadedObject({ + requestOptions, + expectedContentLength, + expectedArchiveSha256, +}) { const response = await signedOssRequest({ ...requestOptions, method: 'HEAD', @@ -2344,17 +2986,27 @@ async function verifyUploadedObject({requestOptions, expectedContentLength, expe } const remoteContentLength = Number(effectiveLengthHeader); if (remoteContentLength !== expectedContentLength) { - throw new Error(`OSS HEAD 验证长度不一致: local=${expectedContentLength}, remote=${remoteContentLength}`); + throw new Error( + `OSS HEAD 验证长度不一致: local=${expectedContentLength}, remote=${remoteContentLength}`, + ); } const remoteArchiveSha256 = String( - response.headers.get('x-oss-meta-file-sha256') - ?? response.headers.get('x-oss-meta-archive-sha256') - ?? '', - ).trim().toLowerCase(); + response.headers.get('x-oss-meta-file-sha256') ?? + response.headers.get('x-oss-meta-archive-sha256') ?? + '', + ) + .trim() + .toLowerCase(); if (remoteArchiveSha256 !== expectedArchiveSha256) { - throw new Error(`OSS HEAD 验证 SHA-256 不一致: local=${expectedArchiveSha256}, remote=${remoteArchiveSha256 || ''}`); + throw new Error( + `OSS HEAD 验证 SHA-256 不一致: local=${expectedArchiveSha256}, remote=${remoteArchiveSha256 || ''}`, + ); } - return {verifiedAt: new Date().toISOString(), remoteContentLength, remoteArchiveSha256}; + return { + verifiedAt: new Date().toISOString(), + remoteContentLength, + remoteArchiveSha256, + }; } export async function verifyOssObject({ @@ -2388,22 +3040,28 @@ export async function verifyOssObject({ return verifyUploadedObject({ requestOptions, expectedContentLength: Number(contentLength), - expectedArchiveSha256: String(archiveSha256 ?? '').trim().toLowerCase(), + expectedArchiveSha256: String(archiveSha256 ?? '') + .trim() + .toLowerCase(), }); } -async function abortMultipartUpload({requestOptions, uploadId}) { +async function abortMultipartUpload({ requestOptions, uploadId }) { try { await signedOssRequest({ ...requestOptions, method: 'DELETE', - queries: {uploadId}, + queries: { uploadId }, operation: 'AbortMultipartUpload', maxAttempts: Math.min(2, requestOptions.maxAttempts), }); - console.warn(`[database-backup] 已清理失败的 multipart upload: ${uploadId}`); + console.warn( + `[database-backup] 已清理失败的 multipart upload: ${uploadId}`, + ); } catch (error) { - console.warn(`[database-backup] 清理 multipart upload 失败: ${error.message}`); + console.warn( + `[database-backup] 清理 multipart upload 失败: ${error.message}`, + ); } } @@ -2430,9 +3088,12 @@ export async function uploadArchive({ }) { const fileStat = statSync(archivePath); if (!fileStat.isFile() || (!allowEmpty && fileStat.size <= 0)) { - throw new Error(`待上传备份必须是${allowEmpty ? '' : '非空'}普通文件: ${archivePath}`); + throw new Error( + `待上传备份必须是${allowEmpty ? '' : '非空'}普通文件: ${archivePath}`, + ); } - const verifiedArchiveSha256 = archiveSha256 || await sha256FileHex(archivePath); + const verifiedArchiveSha256 = + archiveSha256 || (await sha256FileHex(archivePath)); if (!/^[a-f0-9]{64}$/u.test(verifiedArchiveSha256)) { throw new Error(`归档 SHA-256 无效: ${verifiedArchiveSha256}`); } @@ -2465,20 +3126,36 @@ export async function uploadArchive({ bodyFactory: () => Buffer.alloc(0), operation: '上传空文件', }); - const verification = await verifyUploadedObject({requestOptions, expectedContentLength: 0, expectedArchiveSha256: verifiedArchiveSha256}); - return {bucket, objectKey, contentLength: 0, archiveSha256: verifiedArchiveSha256, etag: '', uploadMode: 'single', partCount: 1, partSizeBytes: 0, verifiedAt: verification.verifiedAt}; + const verification = await verifyUploadedObject({ + requestOptions, + expectedContentLength: 0, + expectedArchiveSha256: verifiedArchiveSha256, + }); + return { + bucket, + objectKey, + contentLength: 0, + archiveSha256: verifiedArchiveSha256, + etag: '', + uploadMode: 'single', + partCount: 1, + partSizeBytes: 0, + verifiedAt: verification.verifiedAt, + }; } const partSize = resolveMultipartPartSize(fileStat.size, partSizeBytes); const partCount = Math.ceil(fileStat.size / partSize); let uploadId = ''; let uploadCompleted = false; - console.log(`[database-backup] multipart 上传 OSS: oss://${bucket}/${objectKey} (${partCount} parts)`); + console.log( + `[database-backup] multipart 上传 OSS: oss://${bucket}/${objectKey} (${partCount} parts)`, + ); try { const initiateResponse = await signedOssRequest({ ...requestOptions, method: 'POST', - queries: {uploads: null}, + queries: { uploads: null }, headers: { 'content-type': contentType, 'x-oss-meta-archive-sha256': verifiedArchiveSha256, @@ -2501,21 +3178,25 @@ export async function uploadArchive({ const response = await signedOssRequest({ ...requestOptions, method: 'PUT', - queries: {partNumber, uploadId}, - headers: {'content-type': 'application/octet-stream'}, + queries: { partNumber, uploadId }, + headers: { 'content-type': 'application/octet-stream' }, contentLength, bodyFactory: () => { - const stream = createReadStream(archivePath, {start, end}); + const stream = createReadStream(archivePath, { start, end }); return bandwidthLimiter ? bandwidthLimiter.wrap(stream) : stream; }, operation: `UploadPart ${partNumber}/${partCount}`, }); const etag = response.headers.get('etag'); if (!etag) { - throw new Error(`OSS UploadPart ${partNumber}/${partCount} 响应缺少 ETag`); + throw new Error( + `OSS UploadPart ${partNumber}/${partCount} 响应缺少 ETag`, + ); } - parts.push({partNumber, etag}); - console.log(`[database-backup] multipart 进度: ${partNumber}/${partCount}`); + parts.push({ partNumber, etag }); + console.log( + `[database-backup] multipart 进度: ${partNumber}/${partCount}`, + ); } const completeBody = buildCompleteMultipartBody(parts); @@ -2524,19 +3205,25 @@ export async function uploadArchive({ completeResponse = await signedOssRequest({ ...requestOptions, method: 'POST', - queries: {uploadId}, - headers: {'content-type': 'application/xml'}, + queries: { uploadId }, + headers: { 'content-type': 'application/xml' }, contentLength: Buffer.byteLength(completeBody), bodyFactory: () => completeBody, operation: 'CompleteMultipartUpload', }); const completeResponseText = await completeResponse.text(); if (/)/u.test(completeResponseText)) { - throw new Error(`OSS CompleteMultipartUpload 返回错误: ${completeResponseText.slice(0, 500)}`); + throw new Error( + `OSS CompleteMultipartUpload 返回错误: ${completeResponseText.slice(0, 500)}`, + ); } } catch (completeError) { try { - await verifyUploadedObject({requestOptions, expectedContentLength: fileStat.size, expectedArchiveSha256: verifiedArchiveSha256}); + await verifyUploadedObject({ + requestOptions, + expectedContentLength: fileStat.size, + expectedArchiveSha256: verifiedArchiveSha256, + }); completeResponse = null; } catch { throw completeError; @@ -2562,7 +3249,7 @@ export async function uploadArchive({ }; } catch (error) { if (uploadId && !uploadCompleted) { - await abortMultipartUpload({requestOptions, uploadId}); + await abortMultipartUpload({ requestOptions, uploadId }); } throw error; } @@ -2590,7 +3277,9 @@ export async function uploadDirectFile({ }) { const fileStat = statSync(archivePath); if (!fileStat.isFile() || (!allowEmpty && fileStat.size <= 0)) { - throw new Error(`待上传备份必须是${allowEmpty ? '' : '非空'}普通文件: ${archivePath}`); + throw new Error( + `待上传备份必须是${allowEmpty ? '' : '非空'}普通文件: ${archivePath}`, + ); } if (fileStat.size > DIRECT_FILES_SINGLE_PUT_MAX_BYTES) { return uploadArchive({ @@ -2614,7 +3303,8 @@ export async function uploadDirectFile({ bandwidthLimiter, }); } - const verifiedArchiveSha256 = archiveSha256 || await sha256FileHex(archivePath); + const verifiedArchiveSha256 = + archiveSha256 || (await sha256FileHex(archivePath)); if (!/^[a-f0-9]{64}$/u.test(verifiedArchiveSha256)) { throw new Error(`归档 SHA-256 无效: ${verifiedArchiveSha256}`); } @@ -2724,10 +3414,15 @@ export async function uploadManifestFile({ expectedContentLength: body.length, expectedArchiveSha256: archiveSha256, }); - return {objectKey, contentLength: body.length, archiveSha256, verifiedAt: verification.verifiedAt}; + return { + objectKey, + contentLength: body.length, + archiveSha256, + verifiedAt: verification.verifiedAt, + }; } -function uploadedManifestPayload({manifest, database, result}) { +function uploadedManifestPayload({ manifest, database, result }) { return { ...manifest, database, @@ -2761,8 +3456,12 @@ export async function uploadHistoryArchiveWithCleanup({ ...uploadOptions, backupKind: 'spacetimedb-history', }); - const uploadedManifest = uploadedManifestPayload({manifest, database: manifest.database, result}); - writeManifest({manifestPath, payload: uploadedManifest}); + const uploadedManifest = uploadedManifestPayload({ + manifest, + database: manifest.database, + result, + }); + writeManifest({ manifestPath, payload: uploadedManifest }); const manifestUpload = await manifestUploadFn({ manifestPath, ...uploadOptions, @@ -2771,13 +3470,15 @@ export async function uploadHistoryArchiveWithCleanup({ uploadedManifest.manifestVerifiedAt = manifestUpload.verifiedAt; uploadedManifest.manifestContentLength = manifestUpload.contentLength; uploadedManifest.manifestArchiveSha256 = manifestUpload.archiveSha256; - writeManifest({manifestPath, payload: uploadedManifest}); + writeManifest({ manifestPath, payload: uploadedManifest }); let state = validateHistoryState(readManifest(statePath), { database: uploadedManifest.database, dataDir: uploadedManifest.dataDir, }); if (state.baseline.id !== uploadedManifest.baselineId) { - throw new Error(`history manifest baselineId 与 state 不匹配: manifest=${uploadedManifest.baselineId}, state=${state.baseline.id}`); + throw new Error( + `history manifest baselineId 与 state 不匹配: manifest=${uploadedManifest.baselineId}, state=${state.baseline.id}`, + ); } await verifyFn({ ...uploadOptions, @@ -2814,69 +3515,111 @@ export async function uploadHistoryArchiveWithCleanup({ status: 'cleaned', cleanedAt: new Date().toISOString(), }); - return {result, uploadedManifest, cleanup, state}; + return { result, uploadedManifest, cleanup, state }; } -export function discoverDeferredArchiveUploads({workDir, database, includeUploaded = false}) { +export function discoverDeferredArchiveUploads({ + workDir, + database, + includeUploaded = false, +}) { const resolvedWorkDir = resolvePath(workDir); if (!existsSync(resolvedWorkDir)) { - return {archives: [], missingArchives: []}; + return { archives: [], missingArchives: [] }; } const archives = []; const missingArchives = []; const manifestSuffix = '.tar.gz.manifest.json'; const expectedDatabase = String(database || '').trim(); - const entries = readdirSync(resolvedWorkDir, {withFileTypes: true}) - .filter((candidate) => candidate.isFile() && candidate.name.endsWith(manifestSuffix)) + const entries = readdirSync(resolvedWorkDir, { withFileTypes: true }) + .filter( + (candidate) => + candidate.isFile() && candidate.name.endsWith(manifestSuffix), + ) .sort((left, right) => left.name.localeCompare(right.name, 'en')); for (const entry of entries) { const manifestPath = join(resolvedWorkDir, entry.name); const manifest = readManifest(manifestPath); const uploadStatus = String(manifest.uploadStatus || '').trim(); - if (!['deferred', 'pending'].includes(uploadStatus) && !(includeUploaded && uploadStatus === 'uploaded')) { + if ( + !['deferred', 'pending'].includes(uploadStatus) && + !(includeUploaded && uploadStatus === 'uploaded') + ) { continue; } - if (expectedDatabase && String(manifest.database || '').trim() !== expectedDatabase) { + if ( + expectedDatabase && + String(manifest.database || '').trim() !== expectedDatabase + ) { continue; } if (!manifest.archivePath) { throw new Error(`deferred 备份清单缺少 archivePath: ${manifestPath}`); } const archivePath = resolvePath(manifest.archivePath); - if (dirname(archivePath) !== resolvedWorkDir || manifestPath !== `${archivePath}.manifest.json`) { + if ( + dirname(archivePath) !== resolvedWorkDir || + manifestPath !== `${archivePath}.manifest.json` + ) { throw new Error(`deferred 备份路径与清单不匹配: ${manifestPath}`); } - const candidate = {archivePath, manifestPath, manifest}; + const candidate = { archivePath, manifestPath, manifest }; if (existsSync(archivePath)) { const archiveStat = lstatSync(archivePath); if (!archiveStat.isFile() || archiveStat.isSymbolicLink()) { - throw new Error(`deferred 备份归档必须是非符号链接的普通文件: ${archivePath}`); + throw new Error( + `deferred 备份归档必须是非符号链接的普通文件: ${archivePath}`, + ); } archives.push(candidate); } else { missingArchives.push(candidate); } } - return {archives, missingArchives}; + return { archives, missingArchives }; } -async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix, bandwidthLimiter}) { +async function uploadExistingArchive({ + args, + env, + bucket, + endpoint, + accessKeyId, + accessKeySecret, + objectPrefix, + bandwidthLimiter, +}) { const archivePath = resolvePath(args.uploadArchive); if (!existsSync(archivePath)) { throw new Error(`待上传备份文件不存在: ${archivePath}`); } - const manifestPath = resolvePath(args.manifestFile || `${archivePath}.manifest.json`); + const manifestPath = resolvePath( + args.manifestFile || `${archivePath}.manifest.json`, + ); const manifest = existsSync(manifestPath) ? readManifest(manifestPath) : {}; - const dataDir = firstNonEmpty(manifest.dataDir, env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, DEFAULT_PRODUCTION_DATA_DIR); - const database = firstNonEmpty(args.database, manifest.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir)); - const objectKey = firstNonEmpty(args.objectKey, manifest.objectKey, buildBackupNames({database, dataDir, objectPrefix}).objectKey); + const dataDir = firstNonEmpty( + manifest.dataDir, + env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, + DEFAULT_PRODUCTION_DATA_DIR, + ); + const database = firstNonEmpty( + args.database, + manifest.database, + env.GENARRATIVE_SPACETIME_DATABASE, + basename(dataDir), + ); + const objectKey = firstNonEmpty( + args.objectKey, + manifest.objectKey, + buildBackupNames({ database, dataDir, objectPrefix }).objectKey, + ); if (manifest.backupKind !== 'spacetimedb-history') { manifest.backupKind = 'spacetimedb-data-dir'; manifest.baselineStatePath = firstNonEmpty( manifest.baselineStatePath, - historyStatePath({args, env, workDir: dirname(archivePath), database}), + historyStatePath({ args, env, workDir: dirname(archivePath), database }), ); } @@ -2888,10 +3631,12 @@ async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, return; } - const statePath = resolvePath(firstNonEmpty( - manifest.baselineStatePath, - historyStatePath({args, env, workDir: dirname(archivePath), database}), - )); + const statePath = resolvePath( + firstNonEmpty( + manifest.baselineStatePath, + historyStatePath({ args, env, workDir: dirname(archivePath), database }), + ), + ); let result; let uploadedAt; if (manifest.backupKind === 'spacetimedb-history') { @@ -2900,16 +3645,37 @@ async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, manifestPath, manifest, statePath, - uploadOptions: {bucket, endpoint, objectKey, accessKeyId, accessKeySecret, bandwidthLimiter}, + uploadOptions: { + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + bandwidthLimiter, + }, }); result = historyResult.result; uploadedAt = historyResult.uploadedManifest.uploadedAt; - console.log(`[database-backup] history 上传并清理完成: ${JSON.stringify(historyResult.cleanup)}`); + console.log( + `[database-backup] history 上传并清理完成: ${JSON.stringify(historyResult.cleanup)}`, + ); } else { - result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret, bandwidthLimiter}); - const uploadedManifest = uploadedManifestPayload({manifest, database, result}); + result = await uploadArchive({ + archivePath, + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + bandwidthLimiter, + }); + const uploadedManifest = uploadedManifestPayload({ + manifest, + database, + result, + }); uploadedAt = uploadedManifest.uploadedAt; - writeManifest({manifestPath, payload: uploadedManifest}); + writeManifest({ manifestPath, payload: uploadedManifest }); const manifestUpload = await uploadManifestFile({ manifestPath, bucket, @@ -2922,50 +3688,81 @@ async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, uploadedManifest.manifestVerifiedAt = manifestUpload.verifiedAt; uploadedManifest.manifestContentLength = manifestUpload.contentLength; uploadedManifest.manifestArchiveSha256 = manifestUpload.archiveSha256; - writeManifest({manifestPath, payload: uploadedManifest}); + writeManifest({ manifestPath, payload: uploadedManifest }); const previousState = existsSync(statePath) - ? validateHistoryState(readManifest(statePath), {database, dataDir}) + ? validateHistoryState(readManifest(statePath), { database, dataDir }) : null; - const baseline = normalizeUploadedBaselineManifest(uploadedManifest, {database, dataDir}); - writeBaselineState({statePath, baseline, previousState}); + const baseline = normalizeUploadedBaselineManifest(uploadedManifest, { + database, + dataDir, + }); + writeBaselineState({ statePath, baseline, previousState }); console.log(`[database-backup] 已写入 baseline state: ${statePath}`); } console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`); if (args.resultFile) { - writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, statePath, ...result, uploadedAt}, null, 2)}\n`, 'utf8'); + writeFileSync( + resolvePath(args.resultFile), + `${JSON.stringify({ archivePath, manifestPath, statePath, ...result, uploadedAt }, null, 2)}\n`, + 'utf8', + ); } - const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true'; + const keepLocal = + args.keepLocal || + String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '') + .trim() + .toLowerCase() === 'true'; if (!keepLocal) { - rmSync(archivePath, {force: true}); - rmSync(manifestPath, {force: true}); - console.log('[database-backup] 已删除本地临时备份文件；如需保留请设置 --keep-local。'); + rmSync(archivePath, { force: true }); + rmSync(manifestPath, { force: true }); + console.log( + '[database-backup] 已删除本地临时备份文件；如需保留请设置 --keep-local。', + ); } else { console.log(`[database-backup] 已保留本地备份: ${archivePath}`); console.log(`[database-backup] 已保留备份清单: ${manifestPath}`); } } -async function uploadDeferredArchives({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix, database, bandwidthLimiter}) { +async function uploadDeferredArchives({ + args, + env, + bucket, + endpoint, + accessKeyId, + accessKeySecret, + objectPrefix, + database, + bandwidthLimiter, +}) { const workDir = resolvePath(args.uploadDeferredDir); - const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true'; - const {archives, missingArchives} = discoverDeferredArchiveUploads({ + const keepLocal = + args.keepLocal || + String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '') + .trim() + .toLowerCase() === 'true'; + const { archives, missingArchives } = discoverDeferredArchiveUploads({ workDir, database, includeUploaded: !keepLocal, }); - for (const {manifestPath} of missingArchives) { - console.warn(`[database-backup] deferred 清单对应的本地归档不存在，跳过: ${manifestPath}`); + for (const { manifestPath } of missingArchives) { + console.warn( + `[database-backup] deferred 清单对应的本地归档不存在，跳过: ${manifestPath}`, + ); } if (archives.length === 0) { console.log(`[database-backup] 没有可补偿的本地归档: ${workDir}`); return; } - console.log(`[database-backup] 开始串行上传待补偿本地归档: count=${archives.length}`); - for (const {archivePath, manifestPath} of archives) { + console.log( + `[database-backup] 开始串行上传待补偿本地归档: count=${archives.length}`, + ); + for (const { archivePath, manifestPath } of archives) { await uploadExistingArchive({ - args: {...args, uploadArchive: archivePath, manifestFile: manifestPath}, + args: { ...args, uploadArchive: archivePath, manifestFile: manifestPath }, env, bucket, endpoint, @@ -2975,17 +3772,29 @@ async function uploadDeferredArchives({args, env, bucket, endpoint, accessKeyId, bandwidthLimiter, }); } - console.log(`[database-backup] 待补偿本地归档上传完成: count=${archives.length}`); + console.log( + `[database-backup] 待补偿本地归档上传完成: count=${archives.length}`, + ); } -async function publishExistingManifest({args, bucket, endpoint, accessKeyId, accessKeySecret, bandwidthLimiter}) { +async function publishExistingManifest({ + args, + bucket, + endpoint, + accessKeyId, + accessKeySecret, + bandwidthLimiter, +}) { const manifestPath = resolvePath(args.publishManifest); const manifest = readManifest(manifestPath); if (manifest.uploadStatus !== 'uploaded' || !manifest.objectKey) { - throw new Error('只允许发布 uploadStatus=uploaded 且包含 objectKey 的备份 manifest。'); + throw new Error( + '只允许发布 uploadStatus=uploaded 且包含 objectKey 的备份 manifest。', + ); } - manifest.manifestObjectKey = manifest.manifestObjectKey || `${manifest.objectKey}.manifest.json`; - writeManifest({manifestPath, payload: manifest}); + manifest.manifestObjectKey = + manifest.manifestObjectKey || `${manifest.objectKey}.manifest.json`; + writeManifest({ manifestPath, payload: manifest }); const result = await uploadManifestFile({ manifestPath, bucket, @@ -2998,16 +3807,28 @@ async function publishExistingManifest({args, bucket, endpoint, accessKeyId, acc manifest.manifestVerifiedAt = result.verifiedAt; manifest.manifestContentLength = result.contentLength; manifest.manifestArchiveSha256 = result.archiveSha256; - writeManifest({manifestPath, payload: manifest}); - console.log(`[database-backup] manifest 上传并验真完成: ${JSON.stringify(result)}`); + writeManifest({ manifestPath, payload: manifest }); + console.log( + `[database-backup] manifest 上传并验真完成: ${JSON.stringify(result)}`, + ); } -export async function resumeUploadedHistoryBatch({statePath, state, dataDir, verificationOptions, verifyFn = verifyOssObject}) { - const pendingBatch = state.batches.find((batch) => batch.status === 'uploaded'); +export async function resumeUploadedHistoryBatch({ + statePath, + state, + dataDir, + verificationOptions, + verifyFn = verifyOssObject, +}) { + const pendingBatch = state.batches.find( + (batch) => batch.status === 'uploaded', + ); if (!pendingBatch) { return state; } - console.log(`[database-backup] 重试已上传 history 批次的本地清理: ${pendingBatch.batchId}`); + console.log( + `[database-backup] 重试已上传 history 批次的本地清理: ${pendingBatch.batchId}`, + ); await verifyFn({ ...verificationOptions, objectKey: pendingBatch.objectKey, @@ -3020,7 +3841,10 @@ export async function resumeUploadedHistoryBatch({statePath, state, dataDir, ver contentLength: pendingBatch.manifestContentLength, archiveSha256: pendingBatch.manifestArchiveSha256, }); - const cleanup = cleanupHistoryCandidates({dataDir, candidates: pendingBatch.candidates}); + const cleanup = cleanupHistoryCandidates({ + dataDir, + candidates: pendingBatch.candidates, + }); const manifest = { batchId: pendingBatch.batchId, uploadedAt: pendingBatch.uploadedAt, @@ -3047,7 +3871,9 @@ export async function resumeUploadedHistoryBatch({statePath, state, dataDir, ver status: 'cleaned', cleanedAt: new Date().toISOString(), }); - console.log(`[database-backup] 已完成 history 清理重试: ${JSON.stringify(cleanup)}`); + console.log( + `[database-backup] 已完成 history 清理重试: ${JSON.stringify(cleanup)}`, + ); return nextState; } @@ -3065,10 +3891,18 @@ async function runHistoryBackup({ keepLocal, bandwidthLimiter, }) { - const statePath = historyStatePath({args, env, workDir, database}); - let state = loadOrImportHistoryState({args, env, statePath, database, dataDir}); + const statePath = historyStatePath({ args, env, workDir, database }); + let state = loadOrImportHistoryState({ + args, + env, + statePath, + database, + dataDir, + }); if (!args.dryRun && !args.deferUpload) { - console.log(`[database-backup] 重新验真 full baseline: oss://${state.baseline.bucket}/${state.baseline.objectKey}`); + console.log( + `[database-backup] 重新验真 full baseline: oss://${state.baseline.bucket}/${state.baseline.objectKey}`, + ); await verifyOssObject({ bucket: state.baseline.bucket, endpoint, @@ -3091,17 +3925,27 @@ async function runHistoryBackup({ statePath, state, dataDir, - verificationOptions: {bucket, endpoint, accessKeyId, accessKeySecret}, + verificationOptions: { bucket, endpoint, accessKeyId, accessKeySecret }, }); } - const plan = discoverHistoryPlan({dataDir}); - console.log(`[database-backup] history replicas: ${JSON.stringify(plan.replicas)}`); - console.log(`[database-backup] history 候选: count=${plan.candidates.length}, size=${formatBytes(plan.totalSizeBytes)}`); + const plan = discoverHistoryPlan({ dataDir }); + console.log( + `[database-backup] history replicas: ${JSON.stringify(plan.replicas)}`, + ); + console.log( + `[database-backup] history 候选: count=${plan.candidates.length}, size=${formatBytes(plan.totalSizeBytes)}`, + ); if (args.resultFile) { - writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({statePath, baseline: state.baseline, ...plan}, null, 2)}\n`, 'utf8'); + writeFileSync( + resolvePath(args.resultFile), + `${JSON.stringify({ statePath, baseline: state.baseline, ...plan }, null, 2)}\n`, + 'utf8', + ); } if (args.dryRun) { - console.log('[database-backup] history dry-run，仅输出安全候选，不打包、上传或删除。'); + console.log( + '[database-backup] history dry-run，仅输出安全候选，不打包、上传或删除。', + ); return; } if (plan.candidates.length === 0) { @@ -3109,9 +3953,14 @@ async function runHistoryBackup({ return; } - assertSufficientHistoryWorkDirSpace({historySizeBytes: plan.totalSizeBytes, workDir, args, env}); - const batchId = historyBatchId({baselineId: state.baseline.id, plan}); - const {fileName, objectKey} = buildHistoryNames({ + assertSufficientHistoryWorkDirSpace({ + historySizeBytes: plan.totalSizeBytes, + workDir, + args, + env, + }); + const batchId = historyBatchId({ baselineId: state.baseline.id, plan }); + const { fileName, objectKey } = buildHistoryNames({ database, objectPrefix, baselineId: state.baseline.id, @@ -3136,7 +3985,7 @@ async function runHistoryBackup({ totalSizeBytes: plan.totalSizeBytes, uploadStatus: args.deferUpload ? 'deferred' : 'pending', }; - writeManifest({manifestPath, payload: manifest}); + writeManifest({ manifestPath, payload: manifest }); createHistoryArchive({ dataDir, workDir, @@ -3146,9 +3995,15 @@ async function runHistoryBackup({ }); if (args.deferUpload) { - console.log(`[database-backup] 已生成 history 归档，延后上传且未清理源文件: ${archivePath}`); + console.log( + `[database-backup] 已生成 history 归档，延后上传且未清理源文件: ${archivePath}`, + ); if (args.resultFile) { - writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, statePath, bucket, objectKey, batchId}, null, 2)}\n`, 'utf8'); + writeFileSync( + resolvePath(args.resultFile), + `${JSON.stringify({ archivePath, manifestPath, statePath, bucket, objectKey, batchId }, null, 2)}\n`, + 'utf8', + ); } return; } @@ -3158,22 +4013,39 @@ async function runHistoryBackup({ manifestPath, manifest, statePath, - uploadOptions: {bucket, endpoint, objectKey, accessKeyId, accessKeySecret, bandwidthLimiter}, + uploadOptions: { + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + bandwidthLimiter, + }, }); - console.log(`[database-backup] history 上传并清理完成: ${JSON.stringify(historyResult.cleanup)}`); + console.log( + `[database-backup] history 上传并清理完成: ${JSON.stringify(historyResult.cleanup)}`, + ); if (args.resultFile) { - writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({ - archivePath, - manifestPath, - statePath, - batchId, - ...historyResult.result, - uploadedAt: historyResult.uploadedManifest.uploadedAt, - }, null, 2)}\n`, 'utf8'); + writeFileSync( + resolvePath(args.resultFile), + `${JSON.stringify( + { + archivePath, + manifestPath, + statePath, + batchId, + ...historyResult.result, + uploadedAt: historyResult.uploadedManifest.uploadedAt, + }, + null, + 2, + )}\n`, + 'utf8', + ); } if (!keepLocal) { - rmSync(archivePath, {force: true}); - rmSync(manifestPath, {force: true}); + rmSync(archivePath, { force: true }); + rmSync(manifestPath, { force: true }); console.log('[database-backup] 已删除本地 history 临时归档和清单。'); } } @@ -3181,44 +4053,96 @@ async function runHistoryBackup({ async function main() { const args = parseArgs(process.argv.slice(2)); const env = loadEffectiveEnv(args.envFiles); - const isProductionLike = existsSync(DEFAULT_PRODUCTION_DATA_DIR) && process.platform !== 'win32'; - const dataDir = resolvePath(firstNonEmpty( - args.dataDir, - env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, - isProductionLike ? DEFAULT_PRODUCTION_DATA_DIR : DEFAULT_LOCAL_DATA_DIR, - )); - const workDir = resolvePath(firstNonEmpty( - args.workDir, - args.uploadDeferredDir, - env.GENARRATIVE_DATABASE_BACKUP_WORK_DIR, - isProductionLike ? DEFAULT_PRODUCTION_WORK_DIR : DEFAULT_LOCAL_WORK_DIR, - )); - const bucket = firstNonEmpty(args.bucket, env.GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET, env.ALIYUN_OSS_BUCKET); - const endpoint = normalizeEndpoint(firstNonEmpty(args.endpoint, env.GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT, env.ALIYUN_OSS_ENDPOINT)); - const accessKeyId = firstNonEmpty(args.accessKeyId, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID, env.ALIYUN_OSS_ACCESS_KEY_ID); - const accessKeySecret = firstNonEmpty(args.accessKeySecret, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET, env.ALIYUN_OSS_ACCESS_KEY_SECRET); - const objectPrefix = firstNonEmpty(args.objectPrefix, env.GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX, 'database-backups'); - const database = firstNonEmpty(args.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir)); - const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true'; - const storageFormat = firstNonEmpty(args.storageFormat, env.GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT, 'archive'); - const directFilesConcurrency = parseDirectFilesConcurrency(env.GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY); - const uploadBandwidthLimiter = createUploadBandwidthLimiter(env.GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND); + const isProductionLike = + existsSync(DEFAULT_PRODUCTION_DATA_DIR) && process.platform !== 'win32'; + const dataDir = resolvePath( + firstNonEmpty( + args.dataDir, + env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, + isProductionLike ? DEFAULT_PRODUCTION_DATA_DIR : DEFAULT_LOCAL_DATA_DIR, + ), + ); + const workDir = resolvePath( + firstNonEmpty( + args.workDir, + args.uploadDeferredDir, + env.GENARRATIVE_DATABASE_BACKUP_WORK_DIR, + isProductionLike ? DEFAULT_PRODUCTION_WORK_DIR : DEFAULT_LOCAL_WORK_DIR, + ), + ); + const bucket = firstNonEmpty( + args.bucket, + env.GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET, + env.ALIYUN_OSS_BUCKET, + ); + const endpoint = normalizeEndpoint( + firstNonEmpty( + args.endpoint, + env.GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT, + env.ALIYUN_OSS_ENDPOINT, + ), + ); + const accessKeyId = firstNonEmpty( + args.accessKeyId, + env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID, + env.ALIYUN_OSS_ACCESS_KEY_ID, + ); + const accessKeySecret = firstNonEmpty( + args.accessKeySecret, + env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET, + env.ALIYUN_OSS_ACCESS_KEY_SECRET, + ); + const objectPrefix = firstNonEmpty( + args.objectPrefix, + env.GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX, + 'database-backups', + ); + const database = firstNonEmpty( + args.database, + env.GENARRATIVE_SPACETIME_DATABASE, + basename(dataDir), + ); + const keepLocal = + args.keepLocal || + String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '') + .trim() + .toLowerCase() === 'true'; + const storageFormat = firstNonEmpty( + args.storageFormat, + env.GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT, + 'archive', + ); + const directFilesConcurrency = parseDirectFilesConcurrency( + env.GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY, + ); + const uploadBandwidthLimiter = createUploadBandwidthLimiter( + env.GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND, + ); if (!['full', 'history'].includes(args.mode)) { throw new Error(`--mode 只能是 full 或 history，实际: ${args.mode}`); } if (!['archive', 'files'].includes(storageFormat)) { - throw new Error(`--storage-format 只能是 archive 或 files，实际: ${storageFormat}`); + throw new Error( + `--storage-format 只能是 archive 或 files，实际: ${storageFormat}`, + ); } - for (const [label, value] of Object.entries({bucket, endpoint, accessKeyId, accessKeySecret})) { + for (const [label, value] of Object.entries({ + bucket, + endpoint, + accessKeyId, + accessKeySecret, + })) { if (!value) { throw new Error(`缺少 ${label} 配置`); } } if (args.restoreFilesState && args.restoreFilesLatest) { - throw new Error('--restore-files-state 与 --restore-files-latest 不能同时使用。'); + throw new Error( + '--restore-files-state 与 --restore-files-latest 不能同时使用。', + ); } if (args.restoreFilesState) { if (!args.restoreDir) { @@ -3229,7 +4153,7 @@ async function main() { restoreDir: args.restoreDir, database, bucket, - uploadOptions: {bucket, endpoint, accessKeyId, accessKeySecret}, + uploadOptions: { bucket, endpoint, accessKeyId, accessKeySecret }, resultFile: args.resultFile, dryRun: args.dryRun, }); @@ -3244,26 +4168,35 @@ async function main() { database, bucket, objectPrefix, - uploadOptions: {bucket, endpoint, accessKeyId, accessKeySecret}, + uploadOptions: { bucket, endpoint, accessKeyId, accessKeySecret }, resultFile: args.resultFile, dryRun: args.dryRun, }); return; } if (args.restoreDir) { - throw new Error('--restore-dir 只能与 --restore-files-state 或 --restore-files-latest 一起使用。'); + throw new Error( + '--restore-dir 只能与 --restore-files-state 或 --restore-files-latest 一起使用。', + ); } if (args.uploadArchive && args.uploadDeferredDir) { throw new Error('--upload-archive 与 --upload-deferred-dir 不能同时使用。'); } if (!args.dryRun) { - const lockPath = acquireBackupLock({workDir, database}); + const lockPath = acquireBackupLock({ workDir, database }); console.log(`[database-backup] 已获取进程锁: ${lockPath}`); } if (args.publishManifest) { - await publishExistingManifest({args, bucket, endpoint, accessKeyId, accessKeySecret, bandwidthLimiter: uploadBandwidthLimiter}); + await publishExistingManifest({ + args, + bucket, + endpoint, + accessKeyId, + accessKeySecret, + bandwidthLimiter: uploadBandwidthLimiter, + }); return; } @@ -3298,16 +4231,24 @@ async function main() { if (storageFormat === 'files') { if (args.deferUpload) { - throw new Error('files 模式无需本地归档且不支持 --defer-upload；失败后使用同一 work-dir 重跑即可续传。'); + throw new Error( + 'files 模式无需本地归档且不支持 --defer-upload；失败后使用同一 work-dir 重跑即可续传。', + ); } - const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE); - const restartServicesAfter = collectRestartServicesAfterBackup({args, env}); + const stopService = + args.stopService || + firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE); + const restartServicesAfter = collectRestartServicesAfterBackup({ + args, + env, + }); + const stopMarkerPath = databaseBackupStopMarkerPath(workDir); let serviceStopped = false; let backupError = null; let restoreError = null; try { if (args.mode === 'full' && !args.dryRun) { - serviceStopped = stopServiceIfNeeded(stopService); + serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); } await runDirectFilesBackup({ mode: args.mode, @@ -3318,7 +4259,13 @@ async function main() { objectPrefix, dryRun: args.dryRun, resultFile: args.resultFile, - uploadOptions: {bucket, endpoint, accessKeyId, accessKeySecret, bandwidthLimiter: uploadBandwidthLimiter}, + uploadOptions: { + bucket, + endpoint, + accessKeyId, + accessKeySecret, + bandwidthLimiter: uploadBandwidthLimiter, + }, concurrency: directFilesConcurrency, }); } catch (error) { @@ -3326,7 +4273,12 @@ async function main() { } finally { try { if (serviceStopped) { - restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}); + restoreServicesAfterBackup({ + stopService, + serviceStopped, + restartServicesAfter, + stopMarkerPath, + }); } else if (!backupError && args.mode === 'full' && !args.dryRun) { restartServicesAfterBackup(restartServicesAfter); } @@ -3335,7 +4287,10 @@ async function main() { } } if (backupError && restoreError) { - throw new AggregateError([backupError, restoreError], `files 备份失败，且恢复依赖服务时也失败: ${backupError.message}; ${restoreError.message}`); + throw new AggregateError( + [backupError, restoreError], + `files 备份失败，且恢复依赖服务时也失败: ${backupError.message}; ${restoreError.message}`, + ); } if (backupError) { throw backupError; @@ -3364,7 +4319,11 @@ async function main() { return; } - const {fileName, objectKey} = buildBackupNames({database, dataDir, objectPrefix}); + const { fileName, objectKey } = buildBackupNames({ + database, + dataDir, + objectPrefix, + }); console.log(`[database-backup] 数据目录: ${dataDir}`); console.log(`[database-backup] 本地临时目录: ${workDir}`); console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`); @@ -3378,18 +4337,26 @@ async function main() { let serviceStopped = false; let backupError = null; let restoreError = null; - const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE); - const restartServicesAfter = collectRestartServicesAfterBackup({args, env}); + const stopService = + args.stopService || + firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE); + const restartServicesAfter = collectRestartServicesAfterBackup({ args, env }); + const stopMarkerPath = databaseBackupStopMarkerPath(workDir); try { - assertSufficientWorkDirSpace({dataDir, workDir, args, env}); - serviceStopped = stopServiceIfNeeded(stopService); - archivePath = createArchive({dataDir, workDir, fileName}); + assertSufficientWorkDirSpace({ dataDir, workDir, args, env }); + serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); + archivePath = createArchive({ dataDir, workDir, fileName }); } catch (error) { backupError = error; } finally { try { if (serviceStopped) { - restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}); + restoreServicesAfterBackup({ + stopService, + serviceStopped, + restartServicesAfter, + stopMarkerPath, + }); } else if (!backupError) { restartServicesAfterBackup(restartServicesAfter); } @@ -3399,7 +4366,10 @@ async function main() { } if (backupError) { if (restoreError) { - throw new AggregateError([backupError, restoreError], `数据库备份失败，且恢复依赖服务时也失败: ${backupError.message}; ${restoreError.message}`); + throw new AggregateError( + [backupError, restoreError], + `数据库备份失败，且恢复依赖服务时也失败: ${backupError.message}; ${restoreError.message}`, + ); } throw backupError; } @@ -3408,7 +4378,7 @@ async function main() { } const manifestPath = `${archivePath}.manifest.json`; - const baselineStatePath = historyStatePath({args, env, workDir, database}); + const baselineStatePath = historyStatePath({ args, env, workDir, database }); const fullManifest = { backupKind: 'spacetimedb-data-dir', createdAt: new Date().toISOString(), @@ -3429,7 +4399,11 @@ async function main() { console.log(`[database-backup] 已生成本地冷备份，延后上传: ${archivePath}`); console.log(`[database-backup] 已写入备份清单: ${manifestPath}`); if (args.resultFile) { - writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, baselineStatePath, bucket, objectKey}, null, 2)}\n`, 'utf8'); + writeFileSync( + resolvePath(args.resultFile), + `${JSON.stringify({ archivePath, manifestPath, baselineStatePath, bucket, objectKey }, null, 2)}\n`, + 'utf8', + ); } return; } @@ -3444,8 +4418,12 @@ async function main() { bandwidthLimiter: uploadBandwidthLimiter, }); console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`); - const uploadedManifest = uploadedManifestPayload({manifest: fullManifest, database, result}); - writeManifest({manifestPath, payload: uploadedManifest}); + const uploadedManifest = uploadedManifestPayload({ + manifest: fullManifest, + database, + result, + }); + writeManifest({ manifestPath, payload: uploadedManifest }); const manifestUpload = await uploadManifestFile({ manifestPath, bucket, @@ -3458,18 +4436,26 @@ async function main() { uploadedManifest.manifestVerifiedAt = manifestUpload.verifiedAt; uploadedManifest.manifestContentLength = manifestUpload.contentLength; uploadedManifest.manifestArchiveSha256 = manifestUpload.archiveSha256; - writeManifest({manifestPath, payload: uploadedManifest}); + writeManifest({ manifestPath, payload: uploadedManifest }); const previousState = existsSync(baselineStatePath) - ? validateHistoryState(readManifest(baselineStatePath), {database, dataDir}) + ? validateHistoryState(readManifest(baselineStatePath), { + database, + dataDir, + }) : null; - const baseline = normalizeUploadedBaselineManifest(uploadedManifest, {database, dataDir}); - writeBaselineState({statePath: baselineStatePath, baseline, previousState}); + const baseline = normalizeUploadedBaselineManifest(uploadedManifest, { + database, + dataDir, + }); + writeBaselineState({ statePath: baselineStatePath, baseline, previousState }); console.log(`[database-backup] 已写入 baseline state: ${baselineStatePath}`); if (!keepLocal) { - rmSync(archivePath, {force: true}); - rmSync(manifestPath, {force: true}); - console.log('[database-backup] 已删除本地临时备份文件；如需保留请设置 --keep-local。'); + rmSync(archivePath, { force: true }); + rmSync(manifestPath, { force: true }); + console.log( + '[database-backup] 已删除本地临时备份文件；如需保留请设置 --keep-local。', + ); } else { console.log(`[database-backup] 已保留本地备份: ${archivePath}`); console.log(`[database-backup] 已保留备份清单: ${manifestPath}`); @@ -3483,7 +4469,9 @@ function formatErrorDetails(error) { return ['code', 'errno', 'syscall', 'hostname', 'host', 'port', 'address'] .map((field) => { const value = error[field]; - return value === undefined || value === null || value === '' ? '' : `${field}=${String(value)}`; + return value === undefined || value === null || value === '' + ? '' + : `${field}=${String(value)}`; }) .filter(Boolean) .join(' '); @@ -3506,9 +4494,14 @@ function describeError(error) { } if (current instanceof AggregateError) { current.errors.slice(0, 3).forEach((item, index) => { - const itemText = item instanceof Error ? `${item.name}: ${item.message}` : String(item); + const itemText = + item instanceof Error + ? `${item.name}: ${item.message}` + : String(item); const itemDetails = formatErrorDetails(item); - lines.push(`${label}.errors[${index}]: ${itemText}${itemDetails ? ` (${itemDetails})` : ''}`); + lines.push( + `${label}.errors[${index}]: ${itemText}${itemDetails ? ` (${itemDetails})` : ''}`, + ); }); } current = current.cause; @@ -3516,7 +4509,10 @@ function describeError(error) { return lines; } -if (process.argv[1] && realpathSync(resolve(process.argv[1])) === realpathSync(__filename)) { +if ( + process.argv[1] && + realpathSync(resolve(process.argv[1])) === realpathSync(__filename) +) { main().catch((error) => { for (const line of describeError(error)) { console.error(`[database-backup] ${line}`); diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 894458fd2..259b6f6c8 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -78,6 +78,10 @@ validate_database_backup_profile() { exit 1 ;; esac + if [[ "${DEPLOY_TARGET}" == "release" && "${DATABASE_BACKUP_PROFILE}" == "files-history" ]]; then + echo "[server-provision] release 仅允许 archive-full；files-history 会把整棵历史目录加载到 Node 内存，需先完成流式 catalog 改造后才能重新启用。" >&2 + exit 1 + fi if [[ ! "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" =~ ^/var/lib/genarrative/database-backups/[A-Za-z0-9._/-]+$ || "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" == *..* ]]; then echo "[server-provision] DATABASE_BACKUP_FILES_HISTORY_WORK_DIR 必须是 /var/lib/genarrative/database-backups/ 下不含连续点号的绝对路径，当前值: ${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" >&2 exit 1 diff --git a/scripts/spacetime-maintain-external-generation-jobs.mjs b/scripts/spacetime-maintain-external-generation-jobs.mjs index 69ead2f3c..49a6c8625 100644 --- a/scripts/spacetime-maintain-external-generation-jobs.mjs +++ b/scripts/spacetime-maintain-external-generation-jobs.mjs @@ -8,23 +8,29 @@ import { } from './spacetime-migration-common.mjs'; const MAX_BATCH_SIZE = 25; +const DEFAULT_RETENTION_DAYS = 30; +const MICROS_PER_DAY = 86_400_000_000; function usage() { return `用法： node scripts/spacetime-maintain-external-generation-jobs.mjs --database [选项] 默认只 dry-run 一批历史终态任务 payload 压缩，不修改数据库。 +使用 --prune-history 时改为清理已确认通知且超过保留期的历史任务、摘要与事件。 公共选项： --database 目标数据库（必填，也可用 GENARRATIVE_SPACETIME_DATABASE） --server spacetime CLI server 名或 URL --server-url 显式 server URL - --limit <1-${MAX_BATCH_SIZE}> 单批任务数，默认 10 + --limit <1-${MAX_BATCH_SIZE}> 单批任务数，默认 10 --cursor-job-id 从上一批 next_cursor_job_id 继续 --apply 执行写入；省略时始终 dry-run --backfill-summaries 改为回填轻量摘要投影 + --prune-history 改为清理已确认通知的终态历史 --owner-user-id 仅摘要回填可选，限定 owner - --completed-before-micros 仅 payload 压缩可选，限定终态完成时间 + --source-module 仅历史清理可选，默认 editor-canvas + --retention-days 仅历史清理可选，默认 ${DEFAULT_RETENTION_DAYS} 天 + --completed-before-micros 限定终态完成时间；历史清理默认按 retention-days 计算 --help 显示帮助 必须使用已授权 migration operator 的 spacetime CLI 登录态。脚本每次只处理一批； @@ -40,6 +46,9 @@ function parseOptions(argv) { database: process.env.GENARRATIVE_SPACETIME_DATABASE || '', limit: 10, ownerUserId: '', + pruneHistory: false, + retentionDays: DEFAULT_RETENTION_DAYS, + sourceModule: 'editor-canvas', passthrough: [], server: process.env.GENARRATIVE_SPACETIME_SERVER || '', serverUrl: process.env.GENARRATIVE_SPACETIME_SERVER_URL || '', @@ -82,6 +91,15 @@ function parseOptions(argv) { options.apply = true; } else if (arg === '--backfill-summaries') { options.backfillSummaries = true; + } else if (arg === '--prune-history') { + options.pruneHistory = true; + } else if (arg === '--source-module') { + options.sourceModule = readValue(arg).trim(); + if (!options.sourceModule) { + throw new Error('--source-module 不能为空。'); + } + } else if (arg === '--retention-days') { + options.retentionDays = parsePositiveInteger(readValue(arg), arg); } else if (arg === '--help' || arg === '-h') { options.help = true; } else { @@ -95,12 +113,49 @@ function parseOptions(argv) { if (options.ownerUserId && !options.backfillSummaries) { throw new Error('--owner-user-id 只能与 --backfill-summaries 一起使用。'); } + if (options.backfillSummaries && options.pruneHistory) { + throw new Error('--backfill-summaries 与 --prune-history 不能同时使用。'); + } + if (options.sourceModule !== 'editor-canvas' && !options.pruneHistory) { + throw new Error('--source-module 只能与 --prune-history 一起使用。'); + } + if ( + options.retentionDays !== DEFAULT_RETENTION_DAYS && + !options.pruneHistory + ) { + throw new Error('--retention-days 只能与 --prune-history 一起使用。'); + } if (options.completedBeforeMicros !== null && options.backfillSummaries) { throw new Error('--completed-before-micros 不能用于摘要回填。'); } + if ( + options.completedBeforeMicros !== null && + options.pruneHistory && + options.retentionDays !== DEFAULT_RETENTION_DAYS + ) { + throw new Error( + '--completed-before-micros 与 --retention-days 不能同时使用。', + ); + } return options; } +function resolveRetentionCutoffMicros(options) { + if (!options.pruneHistory) { + return options.completedBeforeMicros; + } + if (options.completedBeforeMicros !== null) { + return options.completedBeforeMicros; + } + const cutoff = Date.now() * 1000 - options.retentionDays * MICROS_PER_DAY; + if (!Number.isSafeInteger(cutoff)) { + throw new Error( + '--retention-days 计算出的 completed_before_micros 超出安全整数范围。', + ); + } + return cutoff; +} + try { const options = parseOptions(process.argv.slice(2)); if (options.help) { @@ -113,24 +168,36 @@ try { ); } - const procedureName = options.backfillSummaries - ? 'backfill_external_generation_job_summaries_and_return' - : 'compact_external_generation_job_payloads_and_return'; - const input = options.backfillSummaries + const completedBeforeMicros = resolveRetentionCutoffMicros(options); + + const procedureName = options.pruneHistory + ? 'prune_external_generation_job_history_and_return' + : options.backfillSummaries + ? 'backfill_external_generation_job_summaries_and_return' + : 'compact_external_generation_job_payloads_and_return'; + const input = options.pruneHistory ? { - owner_user_id: encodeSpacetimeCliOption(options.ownerUserId || null), + source_module: options.sourceModule, limit: options.limit, cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), + completed_before_micros: completedBeforeMicros, dry_run: !options.apply, } - : { - dry_run: !options.apply, - limit: options.limit, - cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), - completed_before_micros: encodeSpacetimeCliOption( - options.completedBeforeMicros, - ), - }; + : options.backfillSummaries + ? { + owner_user_id: encodeSpacetimeCliOption(options.ownerUserId || null), + limit: options.limit, + cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), + dry_run: !options.apply, + } + : { + dry_run: !options.apply, + limit: options.limit, + cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null), + completed_before_micros: encodeSpacetimeCliOption( + completedBeforeMicros, + ), + }; const result = await callSpacetimeProcedureViaCli( options, procedureName, @@ -138,10 +205,29 @@ try { ); ensureProcedureOk(result); - console.log(JSON.stringify({ procedure: procedureName, ...result }, null, 2)); - const pendingApplyCount = options.backfillSummaries - ? Number(result.selected_count ?? 0) - : Number(result.matched_count ?? 0); + console.log( + JSON.stringify( + { + procedure: procedureName, + ...(options.pruneHistory + ? { + source_module: options.sourceModule, + completed_before_micros: completedBeforeMicros, + ...(options.completedBeforeMicros === null + ? { retention_days: options.retentionDays } + : {}), + } + : {}), + ...result, + }, + null, + 2, + ), + ); + const pendingApplyCount = + options.pruneHistory || options.backfillSummaries + ? Number(result.selected_count ?? 0) + : Number(result.matched_count ?? 0); if (result.has_more && options.apply) { console.log( `仍有后续批次；下一次追加 --cursor-job-id ${result.next_cursor_job_id ?? ''}。`, @@ -150,8 +236,11 @@ try { const currentCursor = options.cursorJobId ? `保留 --cursor-job-id ${options.cursorJobId}` : '仍从首批开始'; + const cutoffHint = options.pruneHistory + ? `并固定 --completed-before-micros ${completedBeforeMicros}` + : ''; console.log( - `当前仅 dry-run；请${currentCursor}并追加 --apply 重跑同一批。apply 成功后再使用其 next_cursor_job_id 进入下一批。`, + `当前仅 dry-run；请${currentCursor}${cutoffHint}并追加 --apply 重跑同一批。apply 成功后再使用其 next_cursor_job_id 进入下一批。`, ); } } catch (error) { diff --git a/scripts/spacetime-migration-common.mjs b/scripts/spacetime-migration-common.mjs index 6cc7cbbe4..1f2341ad4 100644 --- a/scripts/spacetime-migration-common.mjs +++ b/scripts/spacetime-migration-common.mjs @@ -9,11 +9,13 @@ export function parseArgs(argv) { 'GENARRATIVE_SPACETIME_MIGRATION_CHUNK_SIZE', ), database: process.env.GENARRATIVE_SPACETIME_DATABASE || '', - bootstrapSecret: process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET || '', + bootstrapSecret: + process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET || '', bootstrapSecretFile: process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET_FILE || '', includeTables: [], - operatorIdentity: process.env.GENARRATIVE_SPACETIME_MIGRATION_OPERATOR_IDENTITY || '', + operatorIdentity: + process.env.GENARRATIVE_SPACETIME_MIGRATION_OPERATOR_IDENTITY || '', passthrough: [], note: '', server: process.env.GENARRATIVE_SPACETIME_SERVER || '', @@ -142,7 +144,9 @@ export function buildSpacetimeCallArgs(options, procedureName, input) { export async function callSpacetimeProcedure(options, procedureName, input) { if (!options.database) { - throw new Error('必须传入 --database，或设置 GENARRATIVE_SPACETIME_DATABASE。'); + throw new Error( + '必须传入 --database，或设置 GENARRATIVE_SPACETIME_DATABASE。', + ); } validateSpacetimeDatabaseName(options.database); @@ -196,7 +200,9 @@ export async function createSpacetimeWebIdentity(options) { const text = await response.text(); if (!response.ok) { - throw new Error(`SpacetimeDB identity HTTP ${response.status}: ${trimPreview(text)}`); + throw new Error( + `SpacetimeDB identity HTTP ${response.status}: ${trimPreview(text)}`, + ); } let payload; @@ -209,16 +215,25 @@ export async function createSpacetimeWebIdentity(options) { } const identity = - payload.identity ?? payload.Identity ?? payload.identity_hex ?? payload.identityHex; + payload.identity ?? + payload.Identity ?? + payload.identity_hex ?? + payload.identityHex; const token = payload.token ?? payload.Token; if (typeof identity !== 'string' || typeof token !== 'string') { - throw new Error(`SpacetimeDB identity 响应缺少 identity/token: ${trimPreview(text)}`); + throw new Error( + `SpacetimeDB identity 响应缺少 identity/token: ${trimPreview(text)}`, + ); } return { identity, token }; } -export async function callSpacetimeProcedureAuto(options, procedureName, input) { +export async function callSpacetimeProcedureAuto( + options, + procedureName, + input, +) { if (options.useHttp) { return callSpacetimeProcedure(options, procedureName, input); } @@ -226,7 +241,11 @@ export async function callSpacetimeProcedureAuto(options, procedureName, input) return callSpacetimeProcedureViaCli(options, procedureName, input); } -export async function callSpacetimeProcedureViaCli(options, procedureName, input) { +export async function callSpacetimeProcedureViaCli( + options, + procedureName, + input, +) { const args = buildSpacetimeCallArgs(options, procedureName, input); const output = await runSpacetimeCli(args); return parseProcedureResult(output, procedureName); @@ -335,7 +354,8 @@ function normalizeSatsProduct(value, procedureName) { } if ( - procedureName === 'normalize_editor_character_animation_metadata_and_return' && + procedureName === + 'normalize_editor_character_animation_metadata_and_return' && value.length === 19 ) { return { @@ -427,6 +447,24 @@ function normalizeSatsProduct(value, procedureName) { }; } + if ( + procedureName === 'prune_external_generation_job_history_and_return' && + value.length === 10 + ) { + return { + ok: normalizeSatsValue(value[0]), + dry_run: normalizeSatsValue(value[1]), + scanned_count: normalizeSatsValue(value[2]), + selected_count: normalizeSatsValue(value[3]), + deleted_job_count: normalizeSatsValue(value[4]), + deleted_summary_count: normalizeSatsValue(value[5]), + deleted_event_count: normalizeSatsValue(value[6]), + next_cursor_job_id: normalizeSatsOption(value[7]), + has_more: normalizeSatsValue(value[8]), + error_message: normalizeSatsOption(value[9]), + }; + } + if (value.length === 3) { return { ok: normalizeSatsValue(value[0]), @@ -497,7 +535,10 @@ function normalizeSatsValue(value) { if (value && typeof value === 'object') { return Object.fromEntries( - Object.entries(value).map(([key, entry]) => [key, normalizeSatsValue(entry)]), + Object.entries(value).map(([key, entry]) => [ + key, + normalizeSatsValue(entry), + ]), ); } @@ -581,7 +622,9 @@ export function resolveServerUrl(options) { return 'http://127.0.0.1:3101'; } - throw new Error(`未知 SpacetimeDB server: ${server}。请改用 --server-url 显式传入地址。`); + throw new Error( + `未知 SpacetimeDB server: ${server}。请改用 --server-url 显式传入地址。`, + ); } function resolveCliServer(options) { @@ -635,7 +678,11 @@ function runSpacetimeCli(args) { return; } if (code !== 0) { - reject(new Error(`spacetime call 失败，退出码 ${code}: ${trimPreview(output)}`)); + reject( + new Error( + `spacetime call 失败，退出码 ${code}: ${trimPreview(output)}`, + ), + ); return; } diff --git a/scripts/spacetime-migration-common.test.ts b/scripts/spacetime-migration-common.test.ts index 6317bd5d9..689abe321 100644 --- a/scripts/spacetime-migration-common.test.ts +++ b/scripts/spacetime-migration-common.test.ts @@ -223,4 +223,35 @@ describe('SpacetimeDB CLI SATS option encoding', () => { expect(objectResult.batch_sha256).toBe('d'.repeat(64)); expect(objectResult).not.toHaveProperty('batch_sha_256'); }); + + it('normalizes external generation history prune tuple results', () => { + const result = parseProcedureResult( + JSON.stringify([ + true, + false, + 25, + 2, + 2, + 2, + 10, + [0, 'job-25'], + true, + [0, '清理失败'], + ]), + 'prune_external_generation_job_history_and_return', + ); + + expect(result).toEqual({ + ok: true, + dry_run: false, + scanned_count: 25, + selected_count: 2, + deleted_job_count: 2, + deleted_summary_count: 2, + deleted_event_count: 10, + next_cursor_job_id: 'job-25', + has_more: true, + error_message: '清理失败', + }); + }); }); diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index 2d2f169b1..7d79543c0 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -5435,6 +5435,7 @@ dependencies = [ "shared-contracts", "spacetimedb", "spacetimedb-lib", + "time", ] [[package]] diff --git a/server-rs/crates/api-server/src/admin.rs b/server-rs/crates/api-server/src/admin.rs index 49751fd92..5afb9ccf2 100644 --- a/server-rs/crates/api-server/src/admin.rs +++ b/server-rs/crates/api-server/src/admin.rs @@ -657,6 +657,10 @@ pub async fn admin_list_editor_assets( Extension(_admin): Extension, Query(query): Query, ) -> Result, AppError> { + state + .refresh_auth_store_from_spacetime() + .await + .map_err(map_admin_spacetime_error)?; let page_size = query .limit .unwrap_or(ADMIN_EDITOR_ASSET_DEFAULT_LIMIT) diff --git a/server-rs/crates/api-server/src/admin_recharge.rs b/server-rs/crates/api-server/src/admin_recharge.rs index 28ad682ec..28492509e 100644 --- a/server-rs/crates/api-server/src/admin_recharge.rs +++ b/server-rs/crates/api-server/src/admin_recharge.rs @@ -64,6 +64,7 @@ pub async fn admin_list_recharge_orders( Extension(_admin): Extension, Query(query): Query, ) -> Result, Response> { + refresh_auth_projection(&state, &request_context).await?; let user_id = resolve_optional_user_id(&state, query.user_id, query.public_user_code) .map_err(|error| error_response(&request_context, error))?; let input = build_runtime_profile_recharge_order_admin_list_input( @@ -112,6 +113,7 @@ pub async fn admin_get_user_detail( Extension(admin): Extension, Query(query): Query, ) -> Result, Response> { + refresh_auth_projection(&state, &request_context).await?; let user = resolve_user(&state, query.user_id, query.public_user_code) .map_err(|error| error_response(&request_context, error))?; let wallet_detail = state @@ -181,6 +183,7 @@ pub async fn admin_reconcile_user_consumption( Extension(admin): Extension, Json(payload): Json, ) -> Result, Response> { + refresh_auth_projection(&state, &request_context).await?; let user = resolve_user(&state, Some(payload.user_id), None) .map_err(|error| error_response(&request_context, error))?; let input = build_runtime_profile_wallet_consumption_reconcile_input( @@ -589,6 +592,7 @@ pub async fn admin_update_wallet_restriction( Extension(admin): Extension, Json(payload): Json, ) -> Result, Response> { + refresh_auth_projection(&state, &request_context).await?; let user = resolve_user(&state, Some(payload.user_id), None) .map_err(|error| error_response(&request_context, error))?; let input = build_runtime_profile_wallet_manual_restriction_upsert_input( @@ -1040,6 +1044,22 @@ fn map_hold(hold: RuntimeProfileRechargeRefundHoldSnapshot) -> AdminRechargeRefu } } +async fn refresh_auth_projection( + state: &AppState, + request_context: &RequestContext, +) -> Result<(), Response> { + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + error_response( + request_context, + AppError::from_status(StatusCode::BAD_GATEWAY) + .with_message(format!("刷新用户认证信息失败：{error}")), + ) + }) +} + fn resolve_optional_user_id( state: &AppState, user_id: Option, diff --git a/server-rs/crates/api-server/src/asset_billing.rs b/server-rs/crates/api-server/src/asset_billing.rs index 11e7369eb..e0ce18d6d 100644 --- a/server-rs/crates/api-server/src/asset_billing.rs +++ b/server-rs/crates/api-server/src/asset_billing.rs @@ -499,87 +499,127 @@ async fn refund_asset_operation_points_with_job_id( external_generation_claim_attempt: Option, ) -> Result<(), AppError> { let created_at_micros = current_utc_micros(); - let metadata_json = wallet_metadata_json( - external_generation_job_id.as_deref(), + let current_attempt_is_owned_by_failure_transaction = + external_generation_job_id.as_deref().is_some_and(|job_id| { + current_external_generation_billing_context().is_some_and(|context| { + context.job_id == job_id + && Some(context.claim_attempt) == external_generation_claim_attempt + }) + }); + if current_attempt_is_owned_by_failure_transaction { + // 队列当前 attempt 的 refund 由 fail_external_generation_job transaction 原子写入 + // SpacetimeDB outbox；这里不能先写另一笔独立退款，避免任务成功写回后被误退。 + return Ok(()); + } + let settlement_reason = if external_generation_job_id.is_some() { + "stale_attempt_recovery" + } else { + "asset_operation_failed" + }; + let enqueue_input = module_runtime::build_runtime_profile_wallet_refund_outbox_enqueue_input( + owner_user_id.clone(), + points_cost, + ledger_id.clone(), + created_at_micros, + asset_kind.clone(), + asset_id.clone(), + settlement_reason.to_string(), + external_generation_job_id.clone(), external_generation_claim_attempt, - ); - let result = state + ) + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "provider": "profile-wallet-refund-outbox", + "message": error.to_string(), + })) + })?; + let enqueue_result = state .spacetime_client() - .refund_profile_wallet_points_with_metadata( - owner_user_id.clone(), - points_cost, - ledger_id.clone(), - created_at_micros, - metadata_json, - ) + .enqueue_profile_wallet_refund_outbox(enqueue_input) .await; - if let Err(error) = result { - let refund_error = error.to_string(); - let app_error = map_asset_operation_wallet_error(error); - if let Some(outbox) = state.wallet_refund_outbox() { - match outbox - .enqueue(WalletRefundOutboxRecord { - owner_user_id: owner_user_id.clone(), - amount: points_cost, - ledger_id: ledger_id.clone(), - created_at_micros, - asset_kind: asset_kind.clone(), - asset_id: asset_id.clone(), - external_generation_job_id: external_generation_job_id.clone(), - }) - .await - { - Ok(WalletRefundOutboxEnqueueOutcome::Enqueued) => { - tracing::warn!( - owner_user_id, - asset_kind, - asset_id, - external_generation_job_id, - ledger_id, - error = %refund_error, - "资产操作失败后的泥点退款立即执行失败，已写入 wallet refund outbox" - ); - } - Ok(WalletRefundOutboxEnqueueOutcome::Dropped { reason }) => { - tracing::error!( - owner_user_id, - asset_kind, - asset_id, - external_generation_job_id, - ledger_id, - reason, - error = %refund_error, - "资产操作失败后的泥点退款立即执行失败，且 wallet refund outbox 因容量限制丢弃" - ); - } - Err(outbox_error) => { - tracing::error!( - owner_user_id, - asset_kind, - asset_id, - external_generation_job_id, - ledger_id, - refund_error = %refund_error, - outbox_error = %outbox_error, - "资产操作失败后的泥点退款立即执行失败，且写入 wallet refund outbox 失败" - ); - } - } - } else { - tracing::error!( + match enqueue_result { + Ok(_) => { + tracing::info!( owner_user_id, asset_kind, asset_id, external_generation_job_id, external_generation_claim_attempt, ledger_id, - error = %refund_error, - "资产操作失败后的泥点退款失败，且 wallet refund outbox 未启用" + "资产操作失败后的泥点退款已写入 SpacetimeDB refund outbox" ); + Ok(()) } - return Err(app_error); + Err(error) if should_use_wallet_refund_emergency_spool(&error) => { + let refund_error = error.to_string(); + let app_error = map_asset_operation_wallet_error(error); + if let Some(outbox) = state.wallet_refund_outbox() { + match outbox + .enqueue(WalletRefundOutboxRecord { + owner_user_id: owner_user_id.clone(), + amount: points_cost, + ledger_id: ledger_id.clone(), + created_at_micros, + asset_kind: asset_kind.clone(), + asset_id: asset_id.clone(), + settlement_reason: settlement_reason.to_string(), + external_generation_job_id: external_generation_job_id.clone(), + external_generation_claim_attempt, + }) + .await + { + Ok(WalletRefundOutboxEnqueueOutcome::Enqueued) => { + tracing::warn!( + owner_user_id, + asset_kind, + asset_id, + external_generation_job_id, + ledger_id, + error = %refund_error, + "SpacetimeDB refund outbox 不可达，已写入本机 emergency spool" + ); + } + Ok(WalletRefundOutboxEnqueueOutcome::OverflowEnqueued { reason }) => { + tracing::error!( + owner_user_id, + asset_kind, + asset_id, + external_generation_job_id, + ledger_id, + reason, + error = %refund_error, + "SpacetimeDB refund outbox 不可达，退款已写入本机 emergency spool overflow 文件；需监控并尽快恢复库内队列" + ); + } + Err(outbox_error) => { + tracing::error!( + owner_user_id, + asset_kind, + asset_id, + external_generation_job_id, + ledger_id, + refund_error = %refund_error, + outbox_error = %outbox_error, + "SpacetimeDB refund outbox 不可达，且写入本机 emergency spool 失败" + ); + } + } + } else { + tracing::error!( + owner_user_id, + asset_kind, + asset_id, + external_generation_job_id, + external_generation_claim_attempt, + ledger_id, + error = %refund_error, + "SpacetimeDB refund outbox 不可达，且本机 emergency spool 未启用" + ); + } + Err(app_error) + } + Err(error) => Err(map_asset_operation_wallet_error(error)), } - Ok(()) } fn current_external_generation_billing_context() -> Option { @@ -683,6 +723,22 @@ pub(crate) fn should_skip_asset_operation_billing_for_connectivity( } } +fn should_use_wallet_refund_emergency_spool(error: &SpacetimeClientError) -> bool { + match error { + SpacetimeClientError::ConnectDropped | SpacetimeClientError::Timeout(_) => true, + SpacetimeClientError::Build(message) + | SpacetimeClientError::Procedure(message) + | SpacetimeClientError::Runtime(message) => { + message.contains("503") + || message.contains("Service Unavailable") + || message.contains("Failed to connect") + || message.contains("WebSocket") + || message.contains("连接已断开") + || message.contains("连接在返回结果前已断开") + } + } +} + fn current_utc_micros() -> i64 { time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64 / 1_000 } @@ -838,6 +894,24 @@ mod tests { )); } + #[test] + fn wallet_refund_emergency_spool_requires_database_unavailability() { + assert!(should_use_wallet_refund_emergency_spool( + &SpacetimeClientError::ConnectDropped + )); + assert!(should_use_wallet_refund_emergency_spool( + &SpacetimeClientError::Runtime("503 Service Unavailable".to_string()) + )); + assert!(!should_use_wallet_refund_emergency_spool( + &SpacetimeClientError::Procedure( + "No such procedure: enqueue_profile_wallet_refund_outbox_and_return".to_string(), + ) + )); + assert!(!should_use_wallet_refund_emergency_spool( + &SpacetimeClientError::Procedure("泥点余额不足".to_string()) + )); + } + #[test] fn asset_operation_wallet_insufficient_balance_is_public_message() { for domain_message in [ diff --git a/server-rs/crates/api-server/src/auth.rs b/server-rs/crates/api-server/src/auth.rs index cac3b201d..7ccaf0e5f 100644 --- a/server-rs/crates/api-server/src/auth.rs +++ b/server-rs/crates/api-server/src/auth.rs @@ -19,6 +19,7 @@ use serde_json::{Value, json}; use shared_contracts::auth::RuntimeGuestTokenResponse; #[cfg(any())] use shared_kernel::{format_rfc3339, new_uuid_simple_string}; +#[cfg(test)] use time::OffsetDateTime; use tracing::warn; @@ -145,6 +146,16 @@ pub async fn require_bearer_auth( let Some(authenticated) = authenticate_request(&state, headers, request_id).await? else { return Err(AppError::from_status(StatusCode::UNAUTHORIZED)); }; + // JWT 会话校验走 SpacetimeDB；随后刷新用户/身份投影，保证所有受保护路由在 + // 读取进程内工作集时都不会依赖粘性会话命中创建或更新它的 API 节点。 + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + warn!(error = %error, "受保护请求刷新认证投影失败"); + AppError::from_status(StatusCode::SERVICE_UNAVAILABLE) + .with_message("认证状态服务暂不可用") + })?; request.extensions_mut().insert(authenticated.clone()); let mut response = next.run(request).await; @@ -236,54 +247,77 @@ async fn authenticate_request( ); AppError::from_status(StatusCode::UNAUTHORIZED) })?; - let current_user = state - .auth_user_service() - .get_user_by_id(claims.user_id()) - .map_err(|error| { - warn!( - %request_id, - error = %error, - "Bearer JWT 用户快照读取失败" - ); - AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) - })?; - let Some(current_user) = current_user else { - warn!( - %request_id, - user_id = %claims.user_id(), - "Bearer JWT 对应用户不存在" - ); - return Err(AppError::from_status(StatusCode::UNAUTHORIZED)); - }; - if current_user.token_version != claims.token_version() { - warn!( - %request_id, - user_id = %claims.user_id(), - token_version = claims.token_version(), - current_token_version = current_user.token_version, - "Bearer JWT 版本已失效" - ); - return Err(AppError::from_status(StatusCode::UNAUTHORIZED) - .with_message("当前登录态已失效，请重新登录")); - } - + #[cfg(not(test))] let session_is_active = state - .refresh_session_service() - .is_session_active_for_user( - claims.user_id(), - claims.session_id(), - OffsetDateTime::now_utc(), - ) + .spacetime_client() + .validate_auth_session(spacetime_client::AuthSessionValidationRecordInput { + user_id: claims.user_id().to_string(), + session_id: claims.session_id().to_string(), + token_version: claims.token_version(), + }) + .await .map_err(|error| { warn!( %request_id, user_id = %claims.user_id(), session_id = %claims.session_id(), error = %error, - "Bearer JWT refresh session 状态读取失败" + "Bearer JWT SpacetimeDB 会话状态读取失败" ); AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) })?; + + #[cfg(test)] + let session_is_active = { + let current_user = state + .auth_user_service() + .get_user_by_id(claims.user_id()) + .map_err(|error| { + warn!( + %request_id, + error = %error, + "Bearer JWT 用户快照读取失败" + ); + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + })?; + let Some(current_user) = current_user else { + warn!( + %request_id, + user_id = %claims.user_id(), + "Bearer JWT 对应用户不存在" + ); + return Err(AppError::from_status(StatusCode::UNAUTHORIZED)); + }; + if current_user.token_version != claims.token_version() { + warn!( + %request_id, + user_id = %claims.user_id(), + token_version = claims.token_version(), + current_token_version = current_user.token_version, + "Bearer JWT 版本已失效" + ); + return Err(AppError::from_status(StatusCode::UNAUTHORIZED) + .with_message("当前登录态已失效，请重新登录")); + } + + state + .refresh_session_service() + .is_session_active_for_user( + claims.user_id(), + claims.session_id(), + OffsetDateTime::now_utc(), + ) + .map_err(|error| { + warn!( + %request_id, + user_id = %claims.user_id(), + session_id = %claims.session_id(), + error = %error, + "Bearer JWT refresh session 状态读取失败" + ); + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + })? + }; if !session_is_active { warn!( %request_id, diff --git a/server-rs/crates/api-server/src/auth_public_user.rs b/server-rs/crates/api-server/src/auth_public_user.rs index e07b4f6ab..d7f676dfb 100644 --- a/server-rs/crates/api-server/src/auth_public_user.rs +++ b/server-rs/crates/api-server/src/auth_public_user.rs @@ -15,6 +15,13 @@ pub async fn get_public_user_by_code( Extension(request_context): Extension, Path(code): Path, ) -> Result, AppError> { + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新认证状态失败：{error}")) + })?; let user = state .password_entry_service() .get_user_by_public_user_code(&code) @@ -41,6 +48,14 @@ pub async fn get_public_user_by_id( return Err(AppError::from_status(StatusCode::BAD_REQUEST).with_message("用户 ID 不能为空")); } + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新认证状态失败：{error}")) + })?; + let user = state .auth_user_service() .get_user_by_id(user_id) diff --git a/server-rs/crates/api-server/src/external_generation_worker.rs b/server-rs/crates/api-server/src/external_generation_worker.rs index b27804a9d..fa80d9b32 100644 --- a/server-rs/crates/api-server/src/external_generation_worker.rs +++ b/server-rs/crates/api-server/src/external_generation_worker.rs @@ -14,6 +14,7 @@ use spacetime_client::{ ExternalGenerationJobRenewLeaseRecordInput, ExternalGenerationQueueWakeSubscription, }; use tokio::{ + sync::{OwnedSemaphorePermit, Semaphore}, task::{JoinHandle, JoinSet}, time::sleep, }; @@ -92,6 +93,10 @@ pub(crate) async fn run_external_generation_worker(state: AppState) -> Result<() let concurrency = state.config.external_generation_worker_concurrency.max(1); let poll_interval = state.config.external_generation_worker_poll_interval; let lease = state.config.external_generation_worker_lease; + // 超时任务不能立即取消（在途 procedure 仍可能写回），因此执行容量必须同时 + // 约束 active 与 detached work；否则每次超时都会释放 tasks 槽位，实际内存占用 + // 会超过配置并发。 + let work_slots = std::sync::Arc::new(Semaphore::new(concurrency)); let mut tasks = JoinSet::new(); let mut shutdown = external_generation_worker_shutdown_signal(); let mut queue_wake = None; @@ -113,16 +118,29 @@ pub(crate) async fn run_external_generation_worker(state: AppState) -> Result<() ); loop { + // 持续有队列任务时不会进入等待分支，因此必须在每轮主动回收已完成的 + // JoinHandle；否则 permit 虽已归还，JoinSet 仍会保留每个历史任务的句柄。 + reap_finished_external_generation_worker_tasks(&mut tasks); ensure_external_generation_queue_wake_subscription(&state, &mut queue_wake).await; - while tasks.len() >= concurrency { - if await_worker_task_or_shutdown(&mut tasks, &mut shutdown).await { - drain_external_generation_worker_tasks(&mut tasks).await; - return Ok(()); + while work_slots.available_permits() == 0 { + tokio::select! { + _ = shutdown.as_mut() => { + drain_external_generation_worker_tasks(&mut tasks).await; + return Ok(()); + } + permit = work_slots.clone().acquire_owned() => { + if permit.is_err() { + drain_external_generation_worker_tasks(&mut tasks).await; + return Ok(()); + } + // 只用 acquire 作为容量变化唤醒信号，许可立即归还；真正领取任务 + // 时在下方按返回的 job 数量逐个 try_acquire。 + } } } - let available = concurrency.saturating_sub(tasks.len()).max(1); + let available = work_slots.available_permits().max(1); let now_micros = current_utc_micros(); let lease_expires_at_micros = now_micros.saturating_add(duration_micros_i64(lease)); @@ -178,9 +196,13 @@ pub(crate) async fn run_external_generation_worker(state: AppState) -> Result<() for job in jobs { let state = state.clone(); let worker_id = worker_id.clone(); + let permit = work_slots + .clone() + .try_acquire_owned() + .expect("claimed job must have an execution capacity permit"); tasks.spawn(async move { if let Err(error) = - process_external_generation_job(state, worker_id, lease, job).await + process_external_generation_job(state, worker_id, lease, job, permit).await { error!(error = %error, "external generation worker 执行任务失败"); } @@ -255,13 +277,11 @@ async fn await_worker_task(tasks: &mut JoinSet<()>) { } } -async fn await_worker_task_or_shutdown( - tasks: &mut JoinSet<()>, - shutdown: &mut ExternalGenerationShutdownSignal, -) -> bool { - tokio::select! { - _ = shutdown.as_mut() => true, - _ = await_worker_task(tasks) => false, +fn reap_finished_external_generation_worker_tasks(tasks: &mut JoinSet<()>) { + while let Some(result) = tasks.try_join_next() { + if let Err(error) = result { + error!(error = %error, "external generation worker 子任务 panic"); + } } } @@ -326,6 +346,7 @@ async fn process_external_generation_job( worker_id: String, lease: Duration, job: ExternalGenerationJobRecord, + permit: OwnedSemaphorePermit, ) -> Result<(), String> { let heartbeat_interval = external_generation_worker_heartbeat_interval(lease); let job_timeout = external_generation_worker_job_timeout(&state.config, job.job_kind.as_str()); @@ -377,13 +398,14 @@ async fn process_external_generation_job( job_id = %job.job_id, job_kind = %job.job_kind, timeout_seconds = job_timeout.as_secs(), - "external generation worker 任务超过执行预算，停止续租并释放 worker 槽位，在途执行交由租约仲裁" + "external generation worker 任务超过执行预算，停止续租并保留 worker 槽位，在途执行交由租约仲裁" ); detach_external_generation_work_until_lease_expiry( work_handle, &job, lease, "任务超过执行预算", + Some(permit), ); Err(message) } @@ -393,6 +415,7 @@ async fn process_external_generation_job( &job, lease, "任务租约续期失败", + Some(permit), ); Err(error) } @@ -417,6 +440,7 @@ fn detach_external_generation_work_until_lease_expiry( job: &ExternalGenerationJobRecord, lease: Duration, reason: &'static str, + permit: Option, ) { let job_id = job.job_id.clone(); let job_kind = job.job_kind.clone(); @@ -445,6 +469,9 @@ fn detach_external_generation_work_until_lease_expiry( ), Err(_) => { work_handle.abort(); + // 仅调用 abort 不会从 JoinHandle/JoinSet 中消费完成结果；等待被取消 + // 的 handle，确保 permit 与任务句柄在同一生命周期内一起释放。 + let _ = work_handle.await; warn!( job_id = %job_id, job_kind = %job_kind, @@ -454,6 +481,9 @@ fn detach_external_generation_work_until_lease_expiry( ); } } + // 保持执行许可直到 work 真正结束或被取消，避免超时任务脱管后继续 + // 累积图片/音频响应占用。 + drop(permit); }); } @@ -1972,6 +2002,7 @@ mod tests { &job, Duration::from_millis(200), "任务超过执行预算", + None, ); tokio::time::sleep(Duration::from_millis(100)).await; @@ -1981,6 +2012,61 @@ mod tests { ); } + #[tokio::test] + async fn worker_detached_work_keeps_execution_slot_until_finished() { + let slots = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let permit = slots + .clone() + .acquire_owned() + .await + .expect("the only execution slot should be available"); + let work_handle = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(20)).await; + Ok(()) + }); + let job = external_generation_job_record_fixture(Some("lease-1")); + + detach_external_generation_work_until_lease_expiry( + work_handle, + &job, + Duration::from_millis(200), + "任务超过执行预算", + Some(permit), + ); + + assert!( + slots.try_acquire().is_err(), + "脱管 work 完成前不得重新领取执行容量" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + slots.try_acquire().is_ok(), + "脱管 work 完成后应归还执行容量" + ); + } + + #[tokio::test] + async fn worker_reaps_completed_tasks_while_queue_remains_busy() { + let mut tasks = JoinSet::new(); + let completed = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + const TASK_COUNT: usize = 128; + for _ in 0..TASK_COUNT { + let completed = completed.clone(); + tasks.spawn(async move { + completed.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }); + } + + while completed.load(std::sync::atomic::Ordering::SeqCst) < TASK_COUNT { + tokio::task::yield_now().await; + } + assert_eq!(tasks.len(), TASK_COUNT); + + reap_finished_external_generation_worker_tasks(&mut tasks); + + assert!(tasks.is_empty(), "已完成任务的 JoinHandle 应在每轮被回收"); + } + #[tokio::test] async fn worker_detached_work_is_aborted_after_lease_arbitration_window() { let connection = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); @@ -1999,6 +2085,7 @@ mod tests { &job, Duration::from_millis(10), "任务超过执行预算", + None, ); let reacquired = diff --git a/server-rs/crates/api-server/src/main.rs b/server-rs/crates/api-server/src/main.rs index 1d73da766..441936e1e 100644 --- a/server-rs/crates/api-server/src/main.rs +++ b/server-rs/crates/api-server/src/main.rs @@ -529,21 +529,24 @@ async fn finalize_shutdown(context: ShutdownContext) { } if let Some(outbox) = context.wallet_refund_outbox { - info!(timeout_ms, "api-server 退出前 flush wallet refund outbox"); + info!( + timeout_ms, + "api-server 退出前 flush wallet refund emergency spool" + ); match timeout(context.outbox_flush_timeout, outbox.flush_for_shutdown()).await { Ok(Ok(())) => { - info!("api-server 退出前 wallet refund outbox flush 完成"); + info!("api-server 退出前 wallet refund emergency spool flush 完成"); } Ok(Err(error)) => { warn!( error = %error, - "api-server 退出前 wallet refund outbox flush 未完成，已保留本地文件等待下次启动重试" + "api-server 退出前 wallet refund emergency spool flush 未完成，已保留本地文件等待下次启动重试" ); } Err(_) => { warn!( timeout_ms, - "api-server 退出前 wallet refund outbox flush 超时，已保留本地文件等待下次启动重试" + "api-server 退出前 wallet refund emergency spool flush 超时，已保留本地文件等待下次启动重试" ); } } @@ -557,6 +560,7 @@ fn spawn_common_app_state_background_workers(state: &AppState) { if let Some(outbox) = state.wallet_refund_outbox() { outbox.spawn_worker(); } + state.profile_wallet_refund_outbox_worker().spawn_worker(); } fn spawn_http_app_state_background_workers(state: &AppState, process_role: ProcessRole) { diff --git a/server-rs/crates/api-server/src/password_entry.rs b/server-rs/crates/api-server/src/password_entry.rs index f2e82660e..d1acc699c 100644 --- a/server-rs/crates/api-server/src/password_entry.rs +++ b/server-rs/crates/api-server/src/password_entry.rs @@ -27,6 +27,13 @@ pub async fn password_entry( headers: HeaderMap, Json(payload): Json, ) -> Result { + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新认证状态失败：{error}")) + })?; let input = PasswordEntryInput { country_code: payload.country_code, pure_phone_number: payload.pure_phone_number, diff --git a/server-rs/crates/api-server/src/password_management.rs b/server-rs/crates/api-server/src/password_management.rs index 70f620791..068e65c7b 100644 --- a/server-rs/crates/api-server/src/password_management.rs +++ b/server-rs/crates/api-server/src/password_management.rs @@ -9,6 +9,7 @@ use shared_contracts::auth::{ PasswordChangeRequest, PasswordChangeResponse, PasswordResetRequest, PasswordResetResponse, }; use time::OffsetDateTime; +use tracing::warn; use crate::{ api_response::json_success_body, @@ -81,7 +82,17 @@ pub async fn reset_password( ); } - let result = state + // reset_password 消费的是跨节点共享的短期验证码；先恢复正式投影， + // 避免发码节点与消费节点的本机工作集不一致。 + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新短信验证码状态失败：{error}")) + })?; + + let result = match state .phone_auth_service() .reset_password( ResetPasswordInput { @@ -93,7 +104,17 @@ pub async fn reset_password( OffsetDateTime::now_utc(), ) .await - .map_err(map_phone_auth_error)?; + { + Ok(result) => result, + Err(error) => { + if let Err(sync_error) = state.sync_auth_store_tables_to_spacetime().await { + warn!(error = %sync_error, "重置密码失败后的短信验证码状态同步失败"); + return Err(AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message("同步短信验证码状态失败")); + } + return Err(map_phone_auth_error(error)); + } + }; let session_client = resolve_session_client_context(&headers); let signed_session = create_auth_session( &state, diff --git a/server-rs/crates/api-server/src/phone_auth.rs b/server-rs/crates/api-server/src/phone_auth.rs index a0e9cd753..37e77acec 100644 --- a/server-rs/crates/api-server/src/phone_auth.rs +++ b/server-rs/crates/api-server/src/phone_auth.rs @@ -50,16 +50,33 @@ pub async fn send_phone_code( phone_input_masked = phone_input_masked.as_str(), "收到手机号验证码发送请求" ); + let send_input = SendPhoneCodeInput { + country_code: payload.country_code, + pure_phone_number: payload.pure_phone_number, + scene: scene.clone(), + }; + let send_now = OffsetDateTime::now_utc(); + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新短信验证码状态失败：{error}")) + })?; + state + .phone_auth_service() + .reserve_code_send(&send_input, send_now) + .map_err(map_phone_auth_error)?; + state + .sync_auth_store_tables_to_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("占用短信验证码发送窗口失败：{error}")) + })?; let result = match state .phone_auth_service() - .send_code( - SendPhoneCodeInput { - country_code: payload.country_code, - pure_phone_number: payload.pure_phone_number, - scene: scene.clone(), - }, - OffsetDateTime::now_utc(), - ) + .send_code_after_authoritative_reservation(send_input, send_now) .await { Ok(result) => { @@ -91,6 +108,14 @@ pub async fn send_phone_code( } }; + state + .sync_auth_store_tables_to_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("同步短信验证码状态失败：{error}")) + })?; + Ok(json_success_body( Some(&request_context), PhoneSendCodeResponse { @@ -114,19 +139,44 @@ pub async fn phone_login( AppError::from_status(StatusCode::BAD_REQUEST).with_message("手机号登录暂未启用") ); } - let invite_code = payload.invite_code.clone(); - let result = match state - .phone_auth_service() - .login( - PhoneLoginInput { - country_code: payload.country_code, - pure_phone_number: payload.pure_phone_number, - verify_code: payload.code, - }, - OffsetDateTime::now_utc(), - ) + state + .refresh_auth_store_from_spacetime() .await - { + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新短信验证码状态失败：{error}")) + })?; + let invite_code = payload.invite_code.clone(); + let login_input = PhoneLoginInput { + country_code: payload.country_code, + pure_phone_number: payload.pure_phone_number, + verify_code: payload.code, + }; + let login_now = OffsetDateTime::now_utc(); + let login_result = state + .phone_auth_service() + .login(login_input.clone(), login_now) + .await; + let login_result = match login_result { + Err(PhoneAuthError::VerifyCodeNotFound) => { + if let Err(sync_error) = state.refresh_auth_store_from_spacetime().await { + warn!( + request_id = request_context.request_id(), + operation = request_context.operation(), + error = %sync_error, + "手机号验证码未命中后的认证投影刷新失败" + ); + return Err(AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message("刷新短信验证码状态失败")); + } + state + .phone_auth_service() + .login(login_input, login_now) + .await + } + result => result, + }; + let result = match login_result { Ok(result) => { info!( request_id = request_context.request_id(), @@ -142,6 +192,16 @@ pub async fn phone_login( result } Err(error) => { + if let Err(sync_error) = state.sync_auth_store_tables_to_spacetime().await { + warn!( + request_id = request_context.request_id(), + operation = request_context.operation(), + error = %sync_error, + "手机号验证码登录失败后的状态同步失败" + ); + return Err(AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message("同步短信验证码状态失败")); + } warn!( request_id = request_context.request_id(), operation = request_context.operation(), diff --git a/server-rs/crates/api-server/src/profile_recharge_expiration_listener.rs b/server-rs/crates/api-server/src/profile_recharge_expiration_listener.rs index 5804203c3..070ac5752 100644 --- a/server-rs/crates/api-server/src/profile_recharge_expiration_listener.rs +++ b/server-rs/crates/api-server/src/profile_recharge_expiration_listener.rs @@ -265,6 +265,14 @@ async fn process_expired_virtual_payment_order( state: &AppState, order: &RuntimeProfileRechargeOrderRecord, ) -> Result<(), ExpirationCompensationError> { + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + ExpirationCompensationError::Runtime(format!( + "failed to refresh auth projection for virtual payment query: {error}" + )) + })?; let identity = state .wechat_auth_service() .get_identity_by_user_id(&order.user_id) diff --git a/server-rs/crates/api-server/src/refresh_session.rs b/server-rs/crates/api-server/src/refresh_session.rs index 4cdd87bb8..c6c783f67 100644 --- a/server-rs/crates/api-server/src/refresh_session.rs +++ b/server-rs/crates/api-server/src/refresh_session.rs @@ -39,6 +39,16 @@ pub async fn refresh_session( let next_refresh_token = platform_auth::create_refresh_session_token(); let next_refresh_token_hash = hash_refresh_session_token(&next_refresh_token); + // refresh_session 是跨节点的正式认证入口；先加载最新投影，再在本机工作集执行领域轮换， + // 避免请求落到旧节点时把合法 refresh cookie 误判为不存在。 + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新认证状态失败：{error}")) + })?; + let rotated = match state.refresh_session_service().rotate_session( RotateRefreshSessionInput { refresh_token_hash: refresh_token_hash.clone(), diff --git a/server-rs/crates/api-server/src/runtime_profile.rs b/server-rs/crates/api-server/src/runtime_profile.rs index 290e5b12e..67625908d 100644 --- a/server-rs/crates/api-server/src/runtime_profile.rs +++ b/server-rs/crates/api-server/src/runtime_profile.rs @@ -1650,6 +1650,13 @@ async fn resolve_wechat_identity_for_payment( state: &AppState, user_id: &str, ) -> Result { + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新微信认证状态失败：{error}")) + })?; if let Some(identity) = state .wechat_auth_service() .get_identity_by_user_id(user_id) diff --git a/server-rs/crates/api-server/src/state.rs b/server-rs/crates/api-server/src/state.rs index 501182c91..284b437cb 100644 --- a/server-rs/crates/api-server/src/state.rs +++ b/server-rs/crates/api-server/src/state.rs @@ -8,7 +8,7 @@ use std::{ fmt, sync::{ Arc, - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}, }, }; @@ -35,7 +35,7 @@ use spacetime_client::{ SpacetimeClient, SpacetimeClientConfig, SpacetimeClientError, SpacetimeClientHealthSnapshot, }; use time::OffsetDateTime; -use tokio::sync::{Semaphore, broadcast}; +use tokio::sync::{Mutex as AsyncMutex, Semaphore, broadcast}; use tracing::{info, warn}; use crate::config::AppConfig; @@ -45,7 +45,7 @@ use crate::editor_generation_config::{ EditorGenerationPricingUnit, }; use crate::tracking_outbox::TrackingOutbox; -use crate::wallet_refund_outbox::WalletRefundOutbox; +use crate::wallet_refund_outbox::{ProfileWalletRefundOutboxWorker, WalletRefundOutbox}; use crate::wechat::pay::{build_wechat_pay_config, map_wechat_pay_init_error}; use crate::wechat::provider::build_wechat_provider; use crate::work_author::{ @@ -273,6 +273,14 @@ pub struct AppStateInner { oss_client: Option, #[cfg_attr(test, allow(dead_code))] auth_store: InMemoryAuthStore, + /// 当前进程工作集所基于的正式认证投影版本；跨节点写入使用它做 CAS。 + #[cfg_attr(test, allow(dead_code))] + auth_projection_version: AtomicI64, + /// 最近一次确认写入正式投影时对应的工作集 revision；不一致表示有待重试的本地变更。 + #[cfg_attr(test, allow(dead_code))] + auth_projection_synced_revision: AtomicU64, + #[cfg_attr(test, allow(dead_code))] + auth_projection_sync_lock: AsyncMutex<()>, password_entry_service: PasswordEntryService, refresh_session_service: RefreshSessionService, auth_user_service: AuthUserService, @@ -289,6 +297,7 @@ pub struct AppStateInner { puzzle_gallery_cache: PuzzleGalleryCache, tracking_outbox: Option>, wallet_refund_outbox: Option>, + profile_wallet_refund_outbox_worker: Arc, editor_generation_pricing_store: EditorGenerationPricingStore, llm_client: Option, vector_engine_llm_client: Option, @@ -505,12 +514,13 @@ impl AppState { pub fn new_with_empty_auth_store(config: AppConfig) -> Result { // 中文注释：api-server 不再把本地 auth-store.json 当作用户认证真相源，启动恢复只允许来自 SpacetimeDB。 - Self::new_with_auth_store(config, InMemoryAuthStore::default()) + Self::new_with_auth_store(config, InMemoryAuthStore::default(), 0) } fn new_with_auth_store( config: AppConfig, auth_store: InMemoryAuthStore, + auth_projection_version: i64, ) -> Result { let auth_jwt_config = JwtConfig::new( config.jwt_issuer.clone(), @@ -561,7 +571,11 @@ impl AppState { ORPHAN_WORK_AUTHOR_PUBLIC_USER_CODE, ) .map_err(|error| AppStateInitError::AuthStore(error.to_string()))?; - let phone_auth_service = PhoneAuthService::new(auth_store.clone(), sms_provider); + let phone_auth_service = PhoneAuthService::new_with_verify_code_salt( + auth_store.clone(), + sms_provider, + config.jwt_secret.clone(), + ); let wechat_auth_state_service = WechatAuthStateService::new(auth_store.clone(), config.wechat_state_ttl_minutes); let wechat_auth_service = WechatAuthService::new(auth_store.clone()); @@ -577,6 +591,8 @@ impl AppState { let tracking_outbox = TrackingOutbox::from_config(&config, spacetime_client.clone()); let wallet_refund_outbox = WalletRefundOutbox::from_config(&config, spacetime_client.clone()); + let profile_wallet_refund_outbox_worker = + ProfileWalletRefundOutboxWorker::from_config(&config, spacetime_client.clone()); let editor_generation_pricing_store = EditorGenerationPricingStore::load( config.editor_generation_pricing_override_path.clone(), ) @@ -602,6 +618,10 @@ impl AppState { let editor_oss_http_client = build_editor_oss_http_client()?; let http_request_permit_pools = HttpRequestPermitPools::from_config(&config); let (profile_recharge_order_updates, _) = broadcast::channel(128); + // `ensure_orphan_work_owner_user` 只为公开作品作者回退提供进程内占位账号， + // 不属于正式认证投影；将当前工作集 revision 作为已同步起点，首次认证请求会 + // 先按正式投影刷新并自然丢弃该占位账号，避免把它误当成待提交认证变更。 + let initial_auth_store_revision = auth_store.revision(); Ok(Self(Arc::new(AppStateInner { config, @@ -629,6 +649,9 @@ impl AppState { test_external_background_removal_enqueue: Arc::new(Mutex::new(None)), oss_client, auth_store, + auth_projection_version: AtomicI64::new(auth_projection_version), + auth_projection_synced_revision: AtomicU64::new(initial_auth_store_revision), + auth_projection_sync_lock: AsyncMutex::new(()), password_entry_service, refresh_session_service, auth_user_service, @@ -644,6 +667,7 @@ impl AppState { puzzle_gallery_cache: PuzzleGalleryCache::new(), tracking_outbox, wallet_refund_outbox, + profile_wallet_refund_outbox_worker, editor_generation_pricing_store, llm_client, vector_engine_llm_client, @@ -1288,30 +1312,138 @@ impl AppState { return Ok(()); #[cfg(not(test))] - let updated_at_micros = i64::try_from( - OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000, - ) - .map_err(|_| SpacetimeClientError::Runtime("认证状态更新时间超出 i64 范围".to_string()))?; + let _sync_guard = self.auth_projection_sync_lock.lock().await; #[cfg(not(test))] - let projection = self - .auth_store - .export_projection_view(updated_at_micros) - .map_err(SpacetimeClientError::Runtime)?; - // 当前仍由 module-auth 的进程内工作集执行业务规则；这里只用 typed projection 同步正式认证表。 - #[cfg(not(test))] - if let Err(error) = self - .spacetime_client - .sync_auth_store_projection(projection) - .await - { - warn!( - error = %error, - "认证投影同步 SpacetimeDB 正式表失败，当前认证流程中止" - ); - return Err(error); + for attempt in 0..3 { + let base_updated_at_micros = self.auth_projection_version.load(Ordering::Acquire); + let now_updated_at_micros = + i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000).map_err( + |_| SpacetimeClientError::Runtime("认证状态更新时间超出 i64 范围".to_string()), + )?; + let updated_at_micros = if now_updated_at_micros > base_updated_at_micros { + now_updated_at_micros + } else { + base_updated_at_micros.checked_add(1).ok_or_else(|| { + SpacetimeClientError::Runtime("认证状态版本超出 i64 范围".to_string()) + })? + }; + let (mut projection, attempted_revision) = self + .auth_store + .export_projection_view_with_revision(updated_at_micros) + .map_err(SpacetimeClientError::Runtime)?; + projection.base_updated_at_micros = base_updated_at_micros; + + // 当前仍由 module-auth 的进程内工作集执行业务规则；这里只用 typed projection 同步正式认证表。 + match self + .spacetime_client + .sync_auth_store_projection(projection) + .await + { + Ok(_) => { + self.auth_projection_version + .store(updated_at_micros, Ordering::Release); + if self.auth_store.revision() == attempted_revision { + self.auth_projection_synced_revision + .store(attempted_revision, Ordering::Release); + return Ok(()); + } + warn!( + attempt, + "认证投影同步期间工作集发生变化，将继续同步最新工作集" + ); + continue; + } + Err(error) => { + warn!( + error = %error, + "认证投影同步 SpacetimeDB 正式表失败，当前认证流程中止" + ); + // 当前请求已经失败；只要同步尝试期间没有新的本地变更，恢复为 + // 数据库快照，避免一次 CAS 冲突把本节点永久留在“待同步”状态。 + if self.auth_store.revision() != attempted_revision { + warn!( + "认证投影同步失败期间工作集发生并发变化，跳过自动恢复以避免覆盖未提交变更" + ); + } else if let Ok(current_projection) = self + .spacetime_client + .export_auth_store_projection_from_tables() + .await + { + match self.auth_store.refresh_from_projection_view_if_revision( + current_projection.clone(), + attempted_revision, + ) { + Ok(true) => { + self.auth_projection_version + .store(current_projection.updated_at_micros, Ordering::Release); + self.auth_projection_synced_revision + .store(self.auth_store.revision(), Ordering::Release); + } + Ok(false) => { + warn!( + "认证投影同步冲突期间工作集发生并发变化，跳过自动恢复以避免覆盖未提交变更" + ); + } + Err(refresh_error) => { + warn!( + error = %refresh_error, + "认证投影同步冲突后恢复进程内工作集失败" + ); + } + } + } + return Err(error); + } + } } #[cfg(not(test))] - Ok(()) + Err(SpacetimeClientError::Runtime( + "认证工作集在同步期间持续发生变化，未能完成投影同步".to_string(), + )) + } + + /// 在认证主链路执行前，从正式投影刷新一次本地工作集，避免请求落到另一节点后 + /// 因本机工作集滞后而必须依赖粘性会话才能成功。 + pub async fn refresh_auth_store_from_spacetime(&self) -> Result<(), SpacetimeClientError> { + #[cfg(test)] + return Ok(()); + + #[cfg(not(test))] + { + // 上一次业务操作可能已经改了工作集，但在返回响应前遇到数据库暂时不可用。 + // 先重试提交这份待同步变更，避免只读请求把节点永久卡在 pending 状态。 + if self.auth_projection_synced_revision.load(Ordering::Acquire) + != self.auth_store.revision() + { + self.sync_auth_store_tables_to_spacetime().await?; + } + let _sync_guard = self.auth_projection_sync_lock.lock().await; + let expected_revision = self.auth_store.revision(); + if self.auth_projection_synced_revision.load(Ordering::Acquire) != expected_revision { + return Err(SpacetimeClientError::Runtime( + "认证工作集存在待同步变更，跳过只读刷新".to_string(), + )); + } + let projection = self + .spacetime_client + .export_auth_store_projection_from_tables() + .await?; + let updated_at_micros = projection.updated_at_micros; + let refreshed = self + .auth_store + .refresh_from_projection_view_if_revision(projection, expected_revision) + .map_err(SpacetimeClientError::Runtime)?; + if !refreshed { + return Err(SpacetimeClientError::Runtime( + "认证工作集刷新期间发生并发变更".to_string(), + )); + } + self.auth_projection_version + .store(updated_at_micros, Ordering::Release); + self.auth_projection_synced_revision + .store(self.auth_store.revision(), Ordering::Release); + Ok(()) + } } pub async fn try_restore_auth_store_from_spacetime( @@ -1319,6 +1451,11 @@ impl AppState { ) -> Result { let spacetime_client = SpacetimeClient::new(spacetime_client_config_for_startup_restore(&config)); + initialize_editor_generation_runtime_service_identity_for_startup( + &config, + &spacetime_client, + ) + .await?; let mut spacetime_restore_available = false; let mut restore_errors = Vec::new(); @@ -1332,7 +1469,11 @@ impl AppState { projection, AuthStoreRestoreSource::SpacetimeTables, )? { - let state = Self::new_with_auth_store(config, candidate.auth_store)?; + let state = Self::new_with_auth_store( + config, + candidate.auth_store, + candidate.updated_at_micros.unwrap_or_default(), + )?; info!( source = candidate.source.as_str(), updated_at_micros = candidate.updated_at_micros, @@ -1415,6 +1556,10 @@ impl AppState { self.wallet_refund_outbox.clone() } + pub fn profile_wallet_refund_outbox_worker(&self) -> Arc { + self.profile_wallet_refund_outbox_worker.clone() + } + pub fn llm_client(&self) -> Option<&LlmClient> { self.llm_client.as_ref() } @@ -1845,6 +1990,9 @@ fn auth_store_candidate_from_projection_view( if projection.users.is_empty() && projection.identities.is_empty() && projection.refresh_sessions.is_empty() + && projection.phone_codes.is_empty() + && projection.wechat_states.is_empty() + && projection.updated_at_micros == 0 { return Ok(None); } @@ -1886,6 +2034,34 @@ fn spacetime_client_config_for_startup_restore(config: &AppConfig) -> SpacetimeC } } +async fn initialize_editor_generation_runtime_service_identity_for_startup( + config: &AppConfig, + spacetime_client: &SpacetimeClient, +) -> Result<(), AppStateInitError> { + let pricing_store = + EditorGenerationPricingStore::load(config.editor_generation_pricing_override_path.clone()) + .map_err(|error| AppStateInitError::DependencyUnavailable(error.to_string()))?; + let fallback = pricing_store + .snapshot() + .map_err(|error| AppStateInitError::DependencyUnavailable(error.to_string()))?; + let models = editor_generation_pricing_to_records(&fallback) + .map_err(|error| AppStateInitError::DependencyUnavailable(error.to_string()))?; + spacetime_client + .initialize_editor_generation_pricing_config_if_missing( + editor_generation_pricing_upsert_input( + config, + "system:editor-generation-pricing".to_string(), + models, + crate::editor_project::current_utc_micros(), + ), + ) + .await + .map_err(|error| { + AppStateInitError::DependencyUnavailable(format!("初始化模型定价服务身份失败：{error}")) + })?; + Ok(()) +} + impl fmt::Display for AppStateInitError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { diff --git a/server-rs/crates/api-server/src/tracking_outbox.rs b/server-rs/crates/api-server/src/tracking_outbox.rs index 0f85a68c9..272f08df4 100644 --- a/server-rs/crates/api-server/src/tracking_outbox.rs +++ b/server-rs/crates/api-server/src/tracking_outbox.rs @@ -140,6 +140,10 @@ impl TrackingOutbox { pub fn spawn_worker(self: Arc) { tokio::spawn(async move { + if let Err(error) = self.flush_sealed_files_once().await { + warn!(error = %error, "tracking outbox 启动恢复写入 SpacetimeDB 失败，将保留文件等待重试"); + } + loop { tokio::select! { _ = sleep(self.flush_interval) => { @@ -657,6 +661,45 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[tokio::test] + async fn worker_flushes_existing_active_file_immediately_on_startup() { + let dir = test_dir("worker-startup"); + std::fs::create_dir_all(&dir).unwrap(); + let active_path = dir.join(ACTIVE_FILE_NAME); + let record = TrackingOutboxRecord { + event: sample_event("startup-event"), + }; + std::fs::write(&active_path, serde_json::to_vec(&record).unwrap()).unwrap(); + + let outbox = test_outbox(dir.clone(), 500, 1024 * 1024); + outbox.spawn_worker(); + + for _ in 0..100 { + if !active_path.exists() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + + assert!( + !active_path.exists(), + "worker should recover active file without waiting for interval" + ); + let sealed_count = std::fs::read_dir(&dir) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(SEALED_FILE_PREFIX)) + }) + .count(); + assert_eq!(sealed_count, 1); + + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn directory_size_excludes_quarantined_corrupt_files() { let dir = test_dir("directory-size"); diff --git a/server-rs/crates/api-server/src/wallet_refund_outbox.rs b/server-rs/crates/api-server/src/wallet_refund_outbox.rs index 7c27580f3..6d79051ff 100644 --- a/server-rs/crates/api-server/src/wallet_refund_outbox.rs +++ b/server-rs/crates/api-server/src/wallet_refund_outbox.rs @@ -19,6 +19,7 @@ use tracing::{debug, warn}; use crate::config::AppConfig; const PENDING_FILE_PREFIX: &str = "refund-"; +const OVERFLOW_FILE_PREFIX: &str = "refund-overflow-"; const CORRUPT_FILE_PREFIX: &str = "corrupt-"; const TEMP_FILE_PREFIX: &str = "tmp-"; const OUTBOX_FILE_EXTENSION: &str = ".json"; @@ -34,7 +35,72 @@ pub struct WalletRefundOutbox { flush_notify: Arc, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone)] +pub struct ProfileWalletRefundOutboxWorker { + batch_size: u32, + flush_interval: Duration, + spacetime_client: SpacetimeClient, + worker_id: String, +} + +impl ProfileWalletRefundOutboxWorker { + pub fn from_config(config: &AppConfig, spacetime_client: SpacetimeClient) -> Arc { + Arc::new(Self { + batch_size: config + .wallet_refund_outbox_batch_size + .max(1) + .min(u32::MAX as usize) as u32, + flush_interval: config.wallet_refund_outbox_flush_interval, + spacetime_client, + worker_id: format!("api-server-refund-outbox-{}", std::process::id()), + }) + } + + pub fn spawn_worker(self: Arc) { + tokio::spawn(async move { + self.process_once().await; + loop { + sleep(self.flush_interval).await; + self.process_once().await; + } + }); + } + + async fn process_once(&self) { + match self + .spacetime_client + .process_profile_wallet_refund_outbox(self.worker_id.clone(), self.batch_size) + .await + { + Ok(result) if result.failed_count > 0 => { + warn!( + worker_id = %self.worker_id, + processed_count = result.processed_count, + retry_count = result.retry_count, + failed_count = result.failed_count, + "profile wallet refund outbox 处理部分失败，将按库内 available_at 重试" + ); + } + Ok(result) if result.processed_count > 0 => { + debug!( + worker_id = %self.worker_id, + processed_count = result.processed_count, + "profile wallet refund outbox 已完成库内退款" + ); + } + Ok(_) => {} + Err(error) => { + warn!( + worker_id = %self.worker_id, + error = %error, + "profile wallet refund outbox worker 暂时无法连接 SpacetimeDB" + ); + } + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] pub(crate) struct WalletRefundOutboxRecord { pub owner_user_id: String, pub amount: u64, @@ -42,14 +108,18 @@ pub(crate) struct WalletRefundOutboxRecord { pub created_at_micros: i64, pub asset_kind: String, pub asset_id: String, + #[serde(default = "default_settlement_reason")] + pub settlement_reason: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub external_generation_job_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub external_generation_claim_attempt: Option, } #[derive(Debug)] pub enum WalletRefundOutboxEnqueueOutcome { Enqueued, - Dropped { reason: &'static str }, + OverflowEnqueued { reason: &'static str }, } #[derive(Debug)] @@ -84,7 +154,14 @@ impl WalletRefundOutbox { fs::create_dir_all(&self.dir).await?; let pending_path = self.pending_path_for_ledger(&record.ledger_id); - if fs::metadata(&pending_path).await.is_ok() { + let overflow_path = self.overflow_path_for_ledger(&record.ledger_id); + if self + .reuse_existing_pending_file(&pending_path, &record) + .await? + || self + .reuse_existing_pending_file(&overflow_path, &record) + .await? + { self.flush_notify.notify_one(); return Ok(WalletRefundOutboxEnqueueOutcome::Enqueued); } @@ -92,11 +169,12 @@ impl WalletRefundOutbox { let bytes = serde_json::to_vec(&record)?; let line_bytes = bytes.len().min(u64::MAX as usize) as u64; let current_bytes = directory_size_if_exists(&self.dir).unwrap_or(0); - if current_bytes.saturating_add(line_bytes) > self.max_bytes { - return Ok(WalletRefundOutboxEnqueueOutcome::Dropped { - reason: "max_bytes", - }); - } + let overflow = current_bytes.saturating_add(line_bytes) > self.max_bytes; + let target_path = if overflow { + &overflow_path + } else { + &pending_path + }; let temp_path = self.temp_path(); let mut file = OpenOptions::new() @@ -108,19 +186,111 @@ impl WalletRefundOutbox { file.flush().await?; file.sync_data().await?; drop(file); - if fs::metadata(&pending_path).await.is_ok() { + if self + .reuse_existing_pending_file(&pending_path, &record) + .await? + || self + .reuse_existing_pending_file(&overflow_path, &record) + .await? + { let _ = fs::remove_file(&temp_path).await; self.flush_notify.notify_one(); return Ok(WalletRefundOutboxEnqueueOutcome::Enqueued); } - fs::rename(&temp_path, &pending_path).await?; - sync_directory_metadata(&self.dir).await?; - self.flush_notify.notify_one(); - Ok(WalletRefundOutboxEnqueueOutcome::Enqueued) + for _ in 0..2 { + match fs::hard_link(&temp_path, target_path).await { + Ok(()) => { + sync_directory_metadata(&self.dir).await?; + remove_file_and_sync(&temp_path, &self.dir).await?; + self.flush_notify.notify_one(); + return Ok(if overflow { + WalletRefundOutboxEnqueueOutcome::OverflowEnqueued { + reason: "max_bytes", + } + } else { + WalletRefundOutboxEnqueueOutcome::Enqueued + }); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + if self + .reuse_existing_pending_file(target_path, &record) + .await? + { + remove_file_and_sync(&temp_path, &self.dir).await?; + self.flush_notify.notify_one(); + return Ok(WalletRefundOutboxEnqueueOutcome::Enqueued); + } + } + Err(error) => return Err(error.into()), + } + } + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "refund pending path could not be installed: {}", + target_path.display() + ), + ) + .into()) + } + + async fn reuse_existing_pending_file( + &self, + pending_path: &Path, + expected: &WalletRefundOutboxRecord, + ) -> Result { + let metadata = match fs::metadata(pending_path).await { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + }; + if !metadata.is_file() { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "refund pending path is not a regular file: {}", + pending_path.display() + ), + ) + .into()); + } + + match read_refund_record(pending_path).await { + Ok(existing) if existing == *expected => Ok(true), + Ok(existing) if existing.ledger_id == expected.ledger_id => Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "refund ledger {} 已存在但退款事实不一致: {}", + expected.ledger_id, + pending_path.display() + ), + ) + .into()), + Ok(_) => Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!("refund ledger hash collision at {}", pending_path.display()), + ) + .into()), + Err(error) if error.is_data_corruption() => { + // Preserve the malformed durable file for inspection, then allow this + // enqueue to install a valid file for the same ledger id. + self.quarantine_file(pending_path).await?; + warn!( + source = %pending_path.display(), + "wallet refund outbox 已隔离损坏 pending 文件，继续写入新的幂等退款记录" + ); + Ok(false) + } + Err(error) => Err(error), + } } pub fn spawn_worker(self: Arc) { tokio::spawn(async move { + if let Err(error) = self.flush_pending_files_once().await { + warn!(error = %error, "wallet refund outbox 启动恢复退款失败，将保留文件等待重试"); + } + loop { tokio::select! { _ = sleep(self.flush_interval) => { @@ -144,6 +314,7 @@ impl WalletRefundOutbox { async fn flush_pending_files_once(&self) -> Result<(), WalletRefundOutboxError> { fs::create_dir_all(&self.dir).await?; + self.recover_temporary_files().await?; let pending_files = self.list_pending_files().await?; for path in pending_files.into_iter().take(self.batch_size) { let record = match read_refund_record(&path).await { @@ -163,15 +334,28 @@ impl WalletRefundOutbox { Err(error) => return Err(error), }; - match self - .spacetime_client - .refund_profile_wallet_points_with_metadata( + let enqueue_input = + module_runtime::build_runtime_profile_wallet_refund_outbox_enqueue_input( record.owner_user_id.clone(), record.amount, record.ledger_id.clone(), record.created_at_micros, - refund_metadata_json(record.external_generation_job_id.as_deref()), + record.asset_kind.clone(), + record.asset_id.clone(), + record.settlement_reason.clone(), + record.external_generation_job_id.clone(), + record + .external_generation_claim_attempt + .or_else(|| infer_external_generation_claim_attempt(&record)), ) + .map_err(|error| { + WalletRefundOutboxError::Spacetime(SpacetimeClientError::Runtime( + error.to_string(), + )) + })?; + match self + .spacetime_client + .enqueue_profile_wallet_refund_outbox(enqueue_input) .await { Ok(_) => { @@ -188,7 +372,7 @@ impl WalletRefundOutbox { asset_id = %record.asset_id, external_generation_job_id = ?record.external_generation_job_id, path = %path.display(), - "wallet refund outbox 退款已重放并删除文件" + "wallet refund emergency spool 已恢复到 SpacetimeDB outbox 并删除文件" ); } Err(error) => return Err(WalletRefundOutboxError::Spacetime(error)), @@ -197,6 +381,159 @@ impl WalletRefundOutbox { Ok(()) } + async fn recover_temporary_files(&self) -> Result<(), WalletRefundOutboxError> { + let _guard = self.enqueue_lock.lock().await; + let temporary_files = self.list_temporary_files().await?; + 'temporary_files: for path in temporary_files { + let record = match read_refund_record(&path).await { + Ok(record) => record, + Err(error) if error.is_data_corruption() => { + self.quarantine_file(&path).await?; + warn!( + error = %error, + source = %path.display(), + "wallet refund outbox 崩溃遗留临时文件无法解析，已隔离" + ); + continue; + } + Err(error) => return Err(error), + }; + + let pending_path = self.pending_path_for_ledger(&record.ledger_id); + let overflow_path = self.overflow_path_for_ledger(&record.ledger_id); + for existing_path in [&pending_path, &overflow_path] { + match self + .reuse_existing_pending_file(existing_path, &record) + .await + { + Ok(true) => { + remove_file_and_sync(&path, &self.dir).await?; + debug!( + ledger_id = %record.ledger_id, + source = %path.display(), + target = %existing_path.display(), + "wallet refund outbox 临时文件与已有幂等文件重复，已删除临时副本" + ); + continue 'temporary_files; + } + Ok(false) => {} + Err(WalletRefundOutboxError::Io(error)) + if error.kind() == std::io::ErrorKind::AlreadyExists => + { + self.quarantine_file(&path).await?; + warn!( + ledger_id = %record.ledger_id, + source = %path.display(), + target = %existing_path.display(), + error = %error, + "wallet refund outbox 临时文件与现有幂等文件事实冲突，已隔离临时文件" + ); + continue 'temporary_files; + } + Err(error) => return Err(error), + } + } + + let temp_bytes = fs::metadata(&path).await?.len(); + let record_bytes = serde_json::to_vec(&record)?; + let current_bytes = directory_size_if_exists(&self.dir) + .unwrap_or(0) + .saturating_sub(temp_bytes); + let target_path = + if current_bytes.saturating_add(record_bytes.len() as u64) > self.max_bytes { + &overflow_path + } else { + &pending_path + }; + + match fs::hard_link(&path, target_path).await { + Ok(()) => { + sync_directory_metadata(&self.dir).await?; + remove_file_and_sync(&path, &self.dir).await?; + debug!( + ledger_id = %record.ledger_id, + source = %path.display(), + target = %target_path.display(), + "wallet refund outbox 崩溃遗留临时文件已恢复为幂等退款文件" + ); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + match self.reuse_existing_pending_file(target_path, &record).await { + Ok(true) => { + // Another writer won the same ledger id with identical facts. Keep + // the first durable file and remove only this duplicate temporary + // link. + remove_file_and_sync(&path, &self.dir).await?; + } + Ok(false) => match fs::hard_link(&path, target_path).await { + Ok(()) => { + sync_directory_metadata(&self.dir).await?; + remove_file_and_sync(&path, &self.dir).await?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + // The file may have been completed by another process after the + // scan. + continue; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + self.quarantine_file(&path).await?; + warn!( + ledger_id = %record.ledger_id, + source = %path.display(), + target = %target_path.display(), + "wallet refund outbox 临时文件无法与现有幂等文件合并，已隔离临时文件" + ); + } + Err(error) => return Err(error.into()), + }, + Err(WalletRefundOutboxError::Io(error)) + if error.kind() == std::io::ErrorKind::AlreadyExists => + { + self.quarantine_file(&path).await?; + warn!( + ledger_id = %record.ledger_id, + source = %path.display(), + target = %target_path.display(), + error = %error, + "wallet refund outbox 临时文件与现有幂等文件事实冲突，已隔离临时文件" + ); + } + Err(error) => return Err(error), + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + // The file may have been completed by another process after the scan. + continue; + } + Err(error) => return Err(error.into()), + } + } + Ok(()) + } + + async fn list_temporary_files(&self) -> Result, WalletRefundOutboxError> { + let mut entries = fs::read_dir(&self.dir).await?; + let mut files = Vec::new(); + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let Some(name) = path.file_name().and_then(|value| value.to_str()) else { + continue; + }; + if name.starts_with(TEMP_FILE_PREFIX) && name.ends_with(OUTBOX_FILE_EXTENSION) { + files.push(path); + } + } + files.sort(); + Ok(files) + } + + async fn quarantine_file(&self, path: &Path) -> Result<(), WalletRefundOutboxError> { + let corrupt_path = self.corrupt_path_for(path); + fs::rename(path, &corrupt_path).await?; + sync_directory_metadata(&self.dir).await?; + Ok(()) + } + async fn list_pending_files(&self) -> Result, WalletRefundOutboxError> { let mut entries = fs::read_dir(&self.dir).await?; let mut files = Vec::new(); @@ -205,7 +542,9 @@ impl WalletRefundOutbox { let Some(name) = path.file_name().and_then(|value| value.to_str()) else { continue; }; - if name.starts_with(PENDING_FILE_PREFIX) && name.ends_with(OUTBOX_FILE_EXTENSION) { + if (name.starts_with(PENDING_FILE_PREFIX) || name.starts_with(OVERFLOW_FILE_PREFIX)) + && name.ends_with(OUTBOX_FILE_EXTENSION) + { files.push(path); } } @@ -220,6 +559,13 @@ impl WalletRefundOutbox { )) } + fn overflow_path_for_ledger(&self, ledger_id: &str) -> PathBuf { + self.dir.join(format!( + "{OVERFLOW_FILE_PREFIX}{}{OUTBOX_FILE_EXTENSION}", + ledger_id_hash(ledger_id) + )) + } + fn temp_path(&self) -> PathBuf { self.dir.join(format!( "{TEMP_FILE_PREFIX}{}-{uuid}{OUTBOX_FILE_EXTENSION}", @@ -241,18 +587,17 @@ impl WalletRefundOutbox { } } -fn refund_metadata_json(external_generation_job_id: Option<&str>) -> String { - let Some(external_generation_job_id) = external_generation_job_id - .map(str::trim) - .filter(|value| !value.is_empty()) - else { - return module_runtime::PROFILE_INVITE_CODE_METADATA_DEFAULT_JSON.to_string(); - }; +fn default_settlement_reason() -> String { + "emergency_spool_replay".to_string() +} - serde_json::json!({ - "externalGenerationJobId": external_generation_job_id, - }) - .to_string() +fn infer_external_generation_claim_attempt(record: &WalletRefundOutboxRecord) -> Option { + let job_id = record.external_generation_job_id.as_deref()?.trim(); + let prefix = format!("asset_operation_refund:external_generation_job:{job_id}:attempt:"); + record + .ledger_id + .strip_prefix(&prefix) + .and_then(|value| value.parse::().ok()) } impl fmt::Debug for WalletRefundOutbox { @@ -311,7 +656,7 @@ fn directory_size_if_exists(path: &Path) -> Result { let mut total = 0u64; for entry in std::fs::read_dir(path)? { let entry = entry?; - if !is_pending_outbox_file_name(&entry.file_name()) { + if !is_capped_outbox_file_name(&entry.file_name()) { continue; } let metadata = entry.metadata()?; @@ -335,10 +680,29 @@ fn ledger_id_hash(ledger_id: &str) -> String { fn is_pending_outbox_file_name(name: &std::ffi::OsStr) -> bool { name.to_str().is_some_and(|value| { - value.starts_with(PENDING_FILE_PREFIX) && value.ends_with(OUTBOX_FILE_EXTENSION) + (value.starts_with(PENDING_FILE_PREFIX) + || value.starts_with(OVERFLOW_FILE_PREFIX) + || value.starts_with(TEMP_FILE_PREFIX)) + && value.ends_with(OUTBOX_FILE_EXTENSION) }) } +fn is_capped_outbox_file_name(name: &std::ffi::OsStr) -> bool { + name.to_str().is_some_and(|value| { + ((value.starts_with(PENDING_FILE_PREFIX) && !value.starts_with(OVERFLOW_FILE_PREFIX)) + || value.starts_with(TEMP_FILE_PREFIX)) + && value.ends_with(OUTBOX_FILE_EXTENSION) + }) +} + +async fn remove_file_and_sync(path: &Path, dir: &Path) -> Result<(), WalletRefundOutboxError> { + match fs::remove_file(path).await { + Ok(()) => sync_directory_metadata(dir).await, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + async fn sync_directory_metadata(path: &Path) -> Result<(), WalletRefundOutboxError> { let path = path.to_path_buf(); tokio::task::spawn_blocking(move || { @@ -362,7 +726,9 @@ mod tests { created_at_micros: 1_713_680_000_000_000, asset_kind: "puzzle_initial_image".to_string(), asset_id: "asset-1".to_string(), + settlement_reason: "worker_attempt_failed".to_string(), external_generation_job_id: Some("extgen-test".to_string()), + external_generation_claim_attempt: Some(1), } } @@ -416,7 +782,35 @@ mod tests { } #[tokio::test] - async fn enqueue_drops_when_outbox_exceeds_max_bytes() { + async fn enqueue_rejects_conflicting_existing_ledger_file() { + let dir = test_dir("conflicting-ledger"); + let outbox = test_outbox(dir.clone(), 1024 * 1024); + outbox.enqueue(sample_record("ledger-1")).await.unwrap(); + + let mut conflicting = sample_record("ledger-1"); + conflicting.amount += 1; + let error = outbox + .enqueue(conflicting) + .await + .expect_err("conflicting refund must fail"); + assert!( + matches!(error, WalletRefundOutboxError::Io(error) if error.kind() == std::io::ErrorKind::AlreadyExists) + ); + + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn legacy_external_refund_record_infers_missing_claim_attempt() { + let mut record = + sample_record("asset_operation_refund:external_generation_job:extgen-test:attempt:7"); + record.external_generation_claim_attempt = None; + + assert_eq!(infer_external_generation_claim_attempt(&record), Some(7)); + } + + #[tokio::test] + async fn enqueue_uses_durable_overflow_file_when_outbox_exceeds_max_bytes() { let dir = test_dir("max-bytes"); let outbox = test_outbox(dir.clone(), 1); @@ -424,11 +818,18 @@ mod tests { assert!(matches!( outcome, - WalletRefundOutboxEnqueueOutcome::Dropped { + WalletRefundOutboxEnqueueOutcome::OverflowEnqueued { reason: "max_bytes" } )); - assert!(!dir.exists() || std::fs::read_dir(&dir).unwrap().next().is_none()); + assert!(outbox.overflow_path_for_ledger("ledger-1").is_file()); + assert_eq!(directory_size_if_exists(&dir).unwrap(), 0); + assert_eq!( + read_refund_record(&outbox.overflow_path_for_ledger("ledger-1")) + .await + .unwrap(), + sample_record("ledger-1") + ); let _ = std::fs::remove_dir_all(dir); } @@ -459,6 +860,36 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[tokio::test] + async fn enqueue_does_not_silently_accept_corrupt_pending_file() { + let dir = test_dir("corrupt-pending-enqueue"); + std::fs::create_dir_all(&dir).unwrap(); + let outbox = test_outbox(dir.clone(), 1024 * 1024); + let record = sample_record("ledger-corrupt-pending"); + let pending_path = outbox.pending_path_for_ledger(&record.ledger_id); + std::fs::write(&pending_path, b"{not-json}").unwrap(); + + outbox.enqueue(record.clone()).await.unwrap(); + + assert_eq!( + read_refund_record(&pending_path).await.unwrap().ledger_id, + record.ledger_id + ); + let corrupt_count = std::fs::read_dir(&dir) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(CORRUPT_FILE_PREFIX)) + }) + .count(); + assert_eq!(corrupt_count, 1); + + let _ = std::fs::remove_dir_all(dir); + } + #[tokio::test] async fn shutdown_flush_keeps_file_when_spacetime_is_unavailable() { let dir = test_dir("shutdown"); @@ -480,4 +911,132 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + + #[tokio::test] + async fn flush_recovers_valid_crash_left_temp_file() { + let dir = test_dir("recover-temp"); + std::fs::create_dir_all(&dir).unwrap(); + let outbox = test_outbox(dir.clone(), 1024 * 1024); + let record = sample_record("ledger-temp"); + let temp_path = outbox.temp_path(); + std::fs::write(&temp_path, serde_json::to_vec(&record).unwrap()).unwrap(); + + let result = outbox.flush_pending_files_once().await; + + assert!(matches!(result, Err(WalletRefundOutboxError::Spacetime(_)))); + assert!(!temp_path.exists()); + assert!(outbox.pending_path_for_ledger(&record.ledger_id).exists()); + + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn flush_recovers_crash_left_temp_file_into_overflow_when_capped() { + let dir = test_dir("recover-temp-overflow"); + std::fs::create_dir_all(&dir).unwrap(); + let outbox = test_outbox(dir.clone(), 1); + let existing = sample_record("ledger-existing"); + std::fs::write( + outbox.pending_path_for_ledger(&existing.ledger_id), + serde_json::to_vec(&existing).unwrap(), + ) + .unwrap(); + let record = sample_record("ledger-temp-overflow"); + let temp_path = outbox.temp_path(); + std::fs::write(&temp_path, serde_json::to_vec(&record).unwrap()).unwrap(); + + let result = outbox.flush_pending_files_once().await; + + assert!(matches!(result, Err(WalletRefundOutboxError::Spacetime(_)))); + assert!(!temp_path.exists()); + assert!(outbox.overflow_path_for_ledger(&record.ledger_id).exists()); + + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn flush_quarantines_conflicting_crash_left_temp_file() { + let dir = test_dir("recover-conflicting-temp"); + std::fs::create_dir_all(&dir).unwrap(); + let outbox = test_outbox(dir.clone(), 1024 * 1024); + let record = sample_record("ledger-temp-conflict"); + let mut conflicting = record.clone(); + conflicting.amount += 1; + let pending_path = outbox.pending_path_for_ledger(&record.ledger_id); + let temp_path = outbox.temp_path(); + std::fs::write(&pending_path, serde_json::to_vec(&conflicting).unwrap()).unwrap(); + std::fs::write(&temp_path, serde_json::to_vec(&record).unwrap()).unwrap(); + + let result = outbox.flush_pending_files_once().await; + + assert!(matches!(result, Err(WalletRefundOutboxError::Spacetime(_)))); + assert!(!temp_path.exists()); + assert_eq!( + read_refund_record(&pending_path).await.unwrap(), + conflicting + ); + let corrupt_count = std::fs::read_dir(&dir) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(CORRUPT_FILE_PREFIX)) + }) + .count(); + assert_eq!(corrupt_count, 1); + + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn flush_quarantines_corrupt_crash_left_temp_file() { + let dir = test_dir("recover-corrupt-temp"); + std::fs::create_dir_all(&dir).unwrap(); + let outbox = test_outbox(dir.clone(), 1024 * 1024); + let temp_path = outbox.temp_path(); + std::fs::write(&temp_path, b"{not-json}").unwrap(); + + outbox.flush_pending_files_once().await.unwrap(); + + assert!(!temp_path.exists()); + let corrupt_count = std::fs::read_dir(&dir) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(CORRUPT_FILE_PREFIX)) + }) + .count(); + assert_eq!(corrupt_count, 1); + + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn worker_recovers_temp_file_immediately_on_startup() { + let dir = test_dir("worker-startup"); + std::fs::create_dir_all(&dir).unwrap(); + let outbox = test_outbox(dir.clone(), 1024 * 1024); + let record = sample_record("ledger-worker-startup"); + let temp_path = outbox.temp_path(); + std::fs::write(&temp_path, serde_json::to_vec(&record).unwrap()).unwrap(); + + outbox.clone().spawn_worker(); + + for _ in 0..100 { + if !temp_path.exists() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + + assert!(!temp_path.exists()); + assert!(outbox.pending_path_for_ledger(&record.ledger_id).exists()); + + let _ = std::fs::remove_dir_all(dir); + } } diff --git a/server-rs/crates/api-server/src/wechat/auth.rs b/server-rs/crates/api-server/src/wechat/auth.rs index 420438146..5f553f51f 100644 --- a/server-rs/crates/api-server/src/wechat/auth.rs +++ b/server-rs/crates/api-server/src/wechat/auth.rs @@ -16,6 +16,7 @@ use shared_contracts::auth::{ }; use shared_kernel::normalize_optional_string; use time::OffsetDateTime; +use tracing::warn; use url::Url; use crate::{ @@ -42,6 +43,13 @@ pub async fn start_wechat_login( if !state.config.wechat_auth_enabled { return Err(AppError::from_status(StatusCode::BAD_REQUEST).with_message("微信登录暂未启用")); } + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新微信认证状态失败：{error}")) + })?; let user_agent = headers .get("user-agent") .and_then(|value| value.to_str().ok()) @@ -62,6 +70,13 @@ pub async fn start_wechat_login( OffsetDateTime::now_utc(), ) .map_err(map_wechat_auth_error)?; + state + .sync_auth_store_tables_to_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("同步微信登录状态失败：{error}")) + })?; let authorization_url = state .wechat_provider() .build_authorization_url( @@ -107,6 +122,13 @@ pub async fn start_wechat_bind( OffsetDateTime::now_utc(), ) .map_err(map_wechat_auth_error)?; + state + .sync_auth_store_tables_to_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("同步微信绑定状态失败：{error}")) + })?; let authorization_url = state .wechat_provider() .build_authorization_url( @@ -149,11 +171,59 @@ pub async fn handle_wechat_callback( .into_response()); } - let consumed = match state + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + warn!( + request_id = request_context.request_id(), + operation = request_context.operation(), + error = %error, + "微信回调前刷新认证投影失败" + ); + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message("刷新微信认证状态失败") + })?; + + let consume_result = state .wechat_auth_state_service() - .consume_state(&state_token, OffsetDateTime::now_utc()) - { + .consume_state(&state_token, OffsetDateTime::now_utc()); + let consumed = match consume_result { Ok(value) => value, + Err(WechatAuthError::StateNotFound) => { + if let Err(error) = state.refresh_auth_store_from_spacetime().await { + warn!( + request_id = request_context.request_id(), + operation = request_context.operation(), + error = %error, + "微信 state 未命中后的认证投影刷新失败" + ); + return Ok(Redirect::to(&build_auth_result_redirect_url( + &fallback_redirect, + &[ + ("auth_provider", "wechat"), + ("auth_error", "微信登录状态已失效，请重新发起登录。"), + ], + )) + .into_response()); + } + match state + .wechat_auth_state_service() + .consume_state(&state_token, OffsetDateTime::now_utc()) + { + Ok(value) => value, + Err(_) => { + return Ok(Redirect::to(&build_auth_result_redirect_url( + &fallback_redirect, + &[ + ("auth_provider", "wechat"), + ("auth_error", "微信登录状态已失效，请重新发起登录。"), + ], + )) + .into_response()); + } + } + } Err(_) => { return Ok(Redirect::to(&build_auth_result_redirect_url( &fallback_redirect, @@ -165,6 +235,22 @@ pub async fn handle_wechat_callback( .into_response()); } }; + if let Err(error) = state.sync_auth_store_tables_to_spacetime().await { + warn!( + request_id = request_context.request_id(), + operation = request_context.operation(), + error = %error, + "微信回调消费 state 后同步失败" + ); + return Ok(Redirect::to(&build_auth_result_redirect_url( + &fallback_redirect, + &[ + ("auth_provider", "wechat"), + ("auth_error", "微信登录服务暂时不可用，请稍后重试。"), + ], + )) + .into_response()); + } let redirect_path = consumed.state.redirect_path.clone(); let session_client = resolve_session_client_context(&headers); @@ -293,7 +379,7 @@ pub async fn bind_wechat_phone( .ok_or_else(|| { AppError::from_status(StatusCode::BAD_REQUEST).with_message("缺少短信验证码") })?; - state + match state .phone_auth_service() .bind_wechat_phone( BindWechatPhoneInput { @@ -306,7 +392,17 @@ pub async fn bind_wechat_phone( OffsetDateTime::now_utc(), ) .await - .map_err(map_wechat_bind_phone_error)? + { + Ok(result) => result, + Err(error) => { + if let Err(sync_error) = state.sync_auth_store_tables_to_spacetime().await { + warn!(error = %sync_error, "微信绑定手机号失败后的短信验证码状态同步失败"); + return Err(AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message("同步短信验证码状态失败")); + } + return Err(map_wechat_bind_phone_error(error)); + } + } }; let session_client = resolve_session_client_context(&headers); let signed_session = create_auth_session( @@ -365,6 +461,13 @@ pub async fn login_wechat_mini_program( if !state.config.wechat_auth_enabled { return Err(AppError::from_status(StatusCode::BAD_REQUEST).with_message("微信登录暂未启用")); } + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新微信认证状态失败：{error}")) + })?; let code = payload.code.trim(); if code.is_empty() { return Err( diff --git a/server-rs/crates/api-server/src/wechat/pay.rs b/server-rs/crates/api-server/src/wechat/pay.rs index 556c8f00e..3cc839884 100644 --- a/server-rs/crates/api-server/src/wechat/pay.rs +++ b/server-rs/crates/api-server/src/wechat/pay.rs @@ -239,6 +239,10 @@ async fn confirm_virtual_payment_recharge_order( )); } + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| WechatPayError::Upstream(format!("刷新认证状态失败：{error}")))?; let identity = state .wechat_auth_service() .get_identity_by_user_id(&order.user_id) diff --git a/server-rs/crates/api-server/src/wechat/subscribe_message.rs b/server-rs/crates/api-server/src/wechat/subscribe_message.rs index 8946afaf1..863c0185b 100644 --- a/server-rs/crates/api-server/src/wechat/subscribe_message.rs +++ b/server-rs/crates/api-server/src/wechat/subscribe_message.rs @@ -56,6 +56,13 @@ async fn send_generation_result_subscribe_message( AppError::from_status(StatusCode::SERVICE_UNAVAILABLE) .with_message("微信订阅消息模板 ID 未配置") })?; + state + .refresh_auth_store_from_spacetime() + .await + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_message(format!("刷新微信认证状态失败：{error}")) + })?; let user = state .auth_user_service() .get_user_by_id(&message.owner_user_id) diff --git a/server-rs/crates/module-ai/src/application/service.rs b/server-rs/crates/module-ai/src/application/service.rs index 713f3cd33..6f42f1d69 100644 --- a/server-rs/crates/module-ai/src/application/service.rs +++ b/server-rs/crates/module-ai/src/application/service.rs @@ -28,7 +28,7 @@ impl AiTaskService { validate_task_create_input(&input).map_err(AiTaskServiceError::Field)?; let snapshot = AiTaskSnapshot { - task_id: input.task_id.clone(), + task_id: normalize_required_string(input.task_id).unwrap_or_default(), task_kind: input.task_kind, owner_user_id: normalize_required_string(input.owner_user_id).unwrap_or_default(), request_label: normalize_required_string(input.request_label).unwrap_or_default(), diff --git a/server-rs/crates/module-ai/src/application/store.rs b/server-rs/crates/module-ai/src/application/store.rs index f2d0c1175..35a9725a9 100644 --- a/server-rs/crates/module-ai/src/application/store.rs +++ b/server-rs/crates/module-ai/src/application/store.rs @@ -1,10 +1,12 @@ use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, sync::{Arc, Mutex}, }; use crate::{ AiTaskServiceError, AiTaskSnapshot, AiTaskStageStatus, AiTaskStatus, AiTextChunkSnapshot, + MAX_AI_TASK_RETAINED_OUTPUT_BYTES, MAX_AI_TASK_RETAINED_TASKS, MAX_AI_TASK_TEXT_OUTPUT_BYTES, + validate_ai_task_snapshot_memory_limits, }; use super::ensure_task_is_not_terminal; @@ -17,7 +19,9 @@ pub struct InMemoryAiTaskStore { #[derive(Debug, Default)] struct InMemoryAiTaskStoreState { tasks: HashMap, - text_chunks: HashMap>, + // Keep only the ordered deltas needed to handle an out-of-order chunk. + // Completed tasks drop this map immediately; it is not a second durable log. + text_chunks: HashMap>>, } impl InMemoryAiTaskStore { @@ -34,7 +38,48 @@ impl InMemoryAiTaskStore { return Err(AiTaskServiceError::TaskAlreadyExists); } - state.text_chunks.insert(task.task_id.clone(), Vec::new()); + validate_task_memory_limits(&task)?; + + let oldest_terminal = if state.tasks.len() >= MAX_AI_TASK_RETAINED_TASKS { + let oldest_terminal = state + .tasks + .values() + .filter(|value| value.status.is_terminal()) + .min_by_key(|value| value.completed_at_micros.or(Some(value.updated_at_micros))) + .map(|value| value.task_id.clone()); + if oldest_terminal.is_none() { + return Err(AiTaskServiceError::Store( + "AI 任务仓储已达到内存容量上限".to_string(), + )); + } + oldest_terminal + } else { + None + }; + + let retained_output_bytes = retained_output_bytes(&state) + .saturating_sub( + oldest_terminal + .as_deref() + .and_then(|task_id| state.tasks.get(task_id)) + .map(task_output_bytes) + .unwrap_or_default(), + ) + .saturating_add(task_output_bytes(&task)); + if retained_output_bytes > MAX_AI_TASK_RETAINED_OUTPUT_BYTES { + return Err(AiTaskServiceError::Store( + "AI 任务仓储输出工作集超过内存上限".to_string(), + )); + } + + if let Some(task_id) = oldest_terminal { + state.tasks.remove(&task_id); + state.text_chunks.remove(&task_id); + } + + state + .text_chunks + .insert(task.task_id.clone(), HashMap::new()); state.tasks.insert(task.task_id.clone(), task.clone()); Ok(task) } @@ -51,12 +96,44 @@ impl InMemoryAiTaskStore { .inner .lock() .map_err(|_| AiTaskServiceError::Store("AI 任务仓储锁已中毒".to_string()))?; - let task = state - .tasks - .get_mut(task_id.trim()) - .ok_or(AiTaskServiceError::TaskNotFound)?; - apply(task)?; - Ok(task.clone()) + let (previous_task, snapshot) = { + let task = state + .tasks + .get_mut(task_id.trim()) + .ok_or(AiTaskServiceError::TaskNotFound)?; + let previous_task = task.clone(); + if let Err(error) = apply(task) { + *task = previous_task; + return Err(error); + } + (previous_task, task.clone()) + }; + if let Err(error) = validate_task_memory_limits(&snapshot) { + state + .tasks + .insert(task_id.trim().to_string(), previous_task); + return Err(error); + } + let released_text_chunks = if snapshot.status.is_terminal() { + state.text_chunks.remove(task_id.trim()) + } else { + None + }; + let retained_output_bytes = retained_output_bytes(&state); + if retained_output_bytes > MAX_AI_TASK_RETAINED_OUTPUT_BYTES { + state + .tasks + .insert(task_id.trim().to_string(), previous_task); + if let Some(text_chunks) = released_text_chunks { + state + .text_chunks + .insert(task_id.trim().to_string(), text_chunks); + } + return Err(AiTaskServiceError::Store( + "AI 任务仓储输出工作集超过内存上限".to_string(), + )); + } + Ok(snapshot) } pub(super) fn append_text_chunk( @@ -67,13 +144,82 @@ impl InMemoryAiTaskStore { .inner .lock() .map_err(|_| AiTaskServiceError::Store("AI 任务仓储锁已中毒".to_string()))?; - { + if chunk.delta_text.len() > MAX_AI_TASK_TEXT_OUTPUT_BYTES { + return Err(AiTaskServiceError::Store( + "AI 任务文本输出超过内存上限".to_string(), + )); + } + let (previous_stage_output_bytes, previous_latest_output_bytes, previous_task) = { + let task = state + .tasks + .get(&chunk.task_id) + .ok_or(AiTaskServiceError::TaskNotFound)?; + ensure_task_is_not_terminal(task.status)?; + let stage = task + .stages + .iter() + .find(|stage| stage.stage_kind == chunk.stage_kind) + .ok_or(AiTaskServiceError::StageNotFound)?; + ( + stage.text_output.as_ref().map_or(0, String::len), + task.latest_text_output.as_ref().map_or(0, String::len), + task.clone(), + ) + }; + + let (previous_chunk, aggregated_bytes, aggregated_text) = { + let chunks = state + .text_chunks + .get_mut(&chunk.task_id) + .ok_or(AiTaskServiceError::TaskNotFound)?; + let stage_chunks = chunks.entry(chunk.stage_kind).or_default(); + if !stage_chunks.contains_key(&chunk.sequence) + && stage_chunks.len() >= crate::MAX_AI_TASK_TEXT_CHUNKS_PER_STAGE + { + return Err(AiTaskServiceError::Store( + "AI 任务文本 chunk 数量超过内存上限".to_string(), + )); + } + let previous_chunk = stage_chunks.insert(chunk.sequence, chunk.delta_text.clone()); + let aggregated_bytes = stage_chunks + .values() + .fold(0_usize, |total, delta| total.saturating_add(delta.len())); + let mut aggregated_text = String::with_capacity(aggregated_bytes); + for delta in stage_chunks.values() { + aggregated_text.push_str(delta); + } + (previous_chunk, aggregated_bytes, aggregated_text) + }; + if aggregated_bytes > MAX_AI_TASK_TEXT_OUTPUT_BYTES { + rollback_text_chunk(&mut state, &chunk, previous_chunk); + return Err(AiTaskServiceError::Store( + "AI 任务文本输出超过内存上限".to_string(), + )); + } + + let projected_retained_output_bytes = retained_output_bytes(&state) + .saturating_sub(previous_stage_output_bytes) + .saturating_sub(previous_latest_output_bytes) + .saturating_add(aggregated_bytes.saturating_mul(2)); + if projected_retained_output_bytes > MAX_AI_TASK_RETAINED_OUTPUT_BYTES { + rollback_text_chunk(&mut state, &chunk, previous_chunk); + return Err(AiTaskServiceError::Store( + "AI 任务仓储输出工作集超过内存上限".to_string(), + )); + } + + let normalized_output = if aggregated_text.trim().is_empty() { + None + } else { + Some(aggregated_text) + }; + + let snapshot = { let task = state .tasks .get_mut(&chunk.task_id) .ok_or(AiTaskServiceError::TaskNotFound)?; ensure_task_is_not_terminal(task.status)?; - let stage = task .stages .iter_mut() @@ -83,45 +229,23 @@ impl InMemoryAiTaskStore { stage.status = AiTaskStageStatus::Running; stage.started_at_micros = Some(chunk.created_at_micros); } - task.status = AiTaskStatus::Running; task.started_at_micros .get_or_insert(chunk.created_at_micros); - } - - let chunks = state - .text_chunks - .get_mut(&chunk.task_id) - .ok_or(AiTaskServiceError::TaskNotFound)?; - chunks.push(chunk.clone()); - chunks.sort_by_key(|value| value.sequence); - - let aggregated_text = chunks - .iter() - .filter(|value| value.stage_kind == chunk.stage_kind) - .map(|value| value.delta_text.as_str()) - .collect::>() - .join(""); - let normalized_output = if aggregated_text.trim().is_empty() { - None - } else { - Some(aggregated_text) + stage.text_output = normalized_output.clone(); + task.latest_text_output = normalized_output; + task.updated_at_micros = chunk.created_at_micros; + task.version += 1; + task.clone() }; - - let task = state - .tasks - .get_mut(&chunk.task_id) - .ok_or(AiTaskServiceError::TaskNotFound)?; - let stage = task - .stages - .iter_mut() - .find(|stage| stage.stage_kind == chunk.stage_kind) - .ok_or(AiTaskServiceError::StageNotFound)?; - stage.text_output = normalized_output.clone(); - task.latest_text_output = normalized_output; - task.updated_at_micros = chunk.created_at_micros; - task.version += 1; - Ok(task.clone()) + if let Err(error) = validate_task_memory_limits(&snapshot) + .and_then(|_| validate_retained_output_bytes(&state)) + { + state.tasks.insert(chunk.task_id.clone(), previous_task); + rollback_text_chunk(&mut state, &chunk, previous_chunk); + return Err(error); + } + Ok(snapshot) } pub(super) fn get_task(&self, task_id: &str) -> Result { @@ -136,3 +260,246 @@ impl InMemoryAiTaskStore { .ok_or(AiTaskServiceError::TaskNotFound) } } + +fn rollback_text_chunk( + state: &mut InMemoryAiTaskStoreState, + chunk: &AiTextChunkSnapshot, + previous_chunk: Option, +) { + if let Some(stage_chunks) = state + .text_chunks + .get_mut(&chunk.task_id) + .and_then(|chunks| chunks.get_mut(&chunk.stage_kind)) + { + if let Some(previous_chunk) = previous_chunk { + stage_chunks.insert(chunk.sequence, previous_chunk); + } else { + stage_chunks.remove(&chunk.sequence); + } + } +} + +fn retained_output_bytes(state: &InMemoryAiTaskStoreState) -> usize { + let snapshot_bytes = state.tasks.values().fold(0_usize, |total, task| { + total.saturating_add(task_output_bytes(task)) + }); + state + .text_chunks + .values() + .fold(snapshot_bytes, |total, stages| { + stages.values().fold(total, |stage_total, chunks| { + chunks.values().fold(stage_total, |chunk_total, delta| { + chunk_total.saturating_add(delta.len()) + }) + }) + }) +} + +fn validate_retained_output_bytes( + state: &InMemoryAiTaskStoreState, +) -> Result<(), AiTaskServiceError> { + if retained_output_bytes(state) > MAX_AI_TASK_RETAINED_OUTPUT_BYTES { + return Err(AiTaskServiceError::Store( + "AI 任务仓储输出工作集超过内存上限".to_string(), + )); + } + Ok(()) +} + +fn task_output_bytes(task: &AiTaskSnapshot) -> usize { + let task_metadata = task + .task_id + .len() + .saturating_add(task.owner_user_id.len()) + .saturating_add(task.request_label.len()) + .saturating_add(task.source_module.len()) + .saturating_add(task.source_entity_id.as_ref().map_or(0, String::len)) + .saturating_add(task.stages.iter().fold(0_usize, |total, stage| { + total + .saturating_add(stage.label.len()) + .saturating_add(stage.detail.len()) + })); + let request_payload = task.request_payload_json.as_ref().map_or(0, String::len); + let failure_message = task.failure_message.as_ref().map_or(0, String::len); + let result_references = task + .result_references + .iter() + .fold(0_usize, |total, reference| { + total + .saturating_add(reference.result_ref_id.len()) + .saturating_add(reference.task_id.len()) + .saturating_add(reference.reference_id.len()) + .saturating_add(reference.label.as_ref().map_or(0, String::len)) + }); + let latest_text = task.latest_text_output.as_ref().map_or(0, String::len); + let latest_structured = task + .latest_structured_payload_json + .as_ref() + .map_or(0, String::len); + let stage_bytes = task.stages.iter().fold(0_usize, |total, stage| { + let text = stage.text_output.as_ref().map_or(0, String::len); + let structured = stage + .structured_payload_json + .as_ref() + .map_or(0, String::len); + let warnings = stage + .warning_messages + .iter() + .fold(0_usize, |warning_total, warning| { + warning_total.saturating_add(warning.len()) + }); + total + .saturating_add(text) + .saturating_add(structured) + .saturating_add(warnings) + }); + task_metadata + .saturating_add(request_payload) + .saturating_add(failure_message) + .saturating_add(result_references) + .saturating_add(latest_text) + .saturating_add(latest_structured) + .saturating_add(stage_bytes) +} + +fn validate_task_memory_limits(task: &AiTaskSnapshot) -> Result<(), AiTaskServiceError> { + validate_ai_task_snapshot_memory_limits(task) + .map_err(|message| AiTaskServiceError::Store(message.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build_running_task(task_id: &str) -> AiTaskSnapshot { + AiTaskSnapshot { + task_id: task_id.to_string(), + task_kind: crate::AiTaskKind::CharacterChat, + owner_user_id: "user-1".to_string(), + request_label: "测试任务".to_string(), + source_module: "test".to_string(), + source_entity_id: None, + request_payload_json: None, + status: AiTaskStatus::Running, + failure_message: None, + stages: vec![crate::AiTaskStageSnapshot { + stage_kind: crate::AiTaskStageKind::RequestModel, + label: "请求模型".to_string(), + detail: "测试".to_string(), + order: 0, + status: AiTaskStageStatus::Running, + text_output: None, + structured_payload_json: None, + warning_messages: Vec::new(), + started_at_micros: Some(1), + completed_at_micros: None, + }], + result_references: Vec::new(), + latest_text_output: None, + latest_structured_payload_json: None, + version: 1, + created_at_micros: 1, + started_at_micros: Some(1), + completed_at_micros: None, + updated_at_micros: 1, + } + } + + #[test] + fn append_text_chunk_rejects_excessive_chunk_count() { + let store = InMemoryAiTaskStore::default(); + let task_id = "task-chunk-limit"; + let task = build_running_task(task_id); + let mut state = store.inner.lock().expect("store lock should be available"); + state.tasks.insert(task_id.to_string(), task); + state.text_chunks.insert( + task_id.to_string(), + HashMap::from([( + crate::AiTaskStageKind::RequestModel, + (1..=crate::MAX_AI_TASK_TEXT_CHUNKS_PER_STAGE as u32) + .map(|sequence| (sequence, "a".to_string())) + .collect(), + )]), + ); + drop(state); + + let error = store + .append_text_chunk(AiTextChunkSnapshot { + chunk_id: "chunk-overflow".to_string(), + task_id: task_id.to_string(), + stage_kind: crate::AiTaskStageKind::RequestModel, + sequence: crate::MAX_AI_TASK_TEXT_CHUNKS_PER_STAGE as u32 + 1, + delta_text: "b".to_string(), + created_at_micros: 2, + }) + .expect_err("a new chunk beyond the count cap should fail"); + assert!(matches!( + error, + AiTaskServiceError::Store(message) if message.contains("chunk 数量") + )); + } + + #[test] + fn terminal_failure_releases_chunks_before_global_cap_check() { + let store = InMemoryAiTaskStore::default(); + let target_id = "task-terminal-release"; + let target = build_running_task(target_id); + let mut state = store.inner.lock().expect("store lock should be available"); + state.tasks.insert(target_id.to_string(), target); + state.text_chunks.insert( + target_id.to_string(), + HashMap::from([( + crate::AiTaskStageKind::RequestModel, + BTreeMap::from([(1, "t".repeat(crate::MAX_AI_TASK_TEXT_OUTPUT_BYTES))]), + )]), + ); + + let desired_retained = crate::MAX_AI_TASK_RETAINED_OUTPUT_BYTES + .saturating_sub(crate::MAX_AI_TASK_FAILURE_MESSAGE_BYTES) + .saturating_add(crate::MAX_AI_TASK_FAILURE_MESSAGE_BYTES / 2) + .saturating_sub(4 * 1024); + let mut filler_index = 0_u32; + while retained_output_bytes(&state) < desired_retained { + let remaining = desired_retained.saturating_sub(retained_output_bytes(&state)); + let bytes = remaining.min(crate::MAX_AI_TASK_TEXT_OUTPUT_BYTES); + if bytes == 0 { + break; + } + let filler_id = format!("task-filler-{filler_index}"); + filler_index += 1; + state + .tasks + .insert(filler_id.clone(), build_running_task(&filler_id)); + state.text_chunks.insert( + filler_id, + HashMap::from([( + crate::AiTaskStageKind::RequestModel, + BTreeMap::from([(1, "f".repeat(bytes))]), + )]), + ); + } + let retained_before_failure = retained_output_bytes(&state); + assert!(retained_before_failure <= crate::MAX_AI_TASK_RETAINED_OUTPUT_BYTES); + assert!( + retained_before_failure + crate::MAX_AI_TASK_FAILURE_MESSAGE_BYTES + > crate::MAX_AI_TASK_RETAINED_OUTPUT_BYTES + ); + drop(state); + + let failed = store + .update_task(target_id, |task| { + task.status = AiTaskStatus::Failed; + task.failure_message = Some("f".repeat(crate::MAX_AI_TASK_FAILURE_MESSAGE_BYTES)); + task.completed_at_micros = Some(2); + task.updated_at_micros = 2; + task.version += 1; + Ok(()) + }) + .expect("terminal transition should release chunks before checking the cap"); + assert_eq!(failed.status, AiTaskStatus::Failed); + assert_eq!( + failed.failure_message.as_deref().map(str::len), + Some(crate::MAX_AI_TASK_FAILURE_MESSAGE_BYTES) + ); + } +} diff --git a/server-rs/crates/module-ai/src/domain.rs b/server-rs/crates/module-ai/src/domain.rs index a9931048f..abafd7616 100644 --- a/server-rs/crates/module-ai/src/domain.rs +++ b/server-rs/crates/module-ai/src/domain.rs @@ -1,4 +1,5 @@ mod ids; +mod limits; mod stages; mod types; @@ -8,6 +9,17 @@ pub use ids::{ generate_ai_task_stage_id, generate_ai_text_chunk_id, normalize_optional_text, normalize_string_list, }; +pub use limits::{ + MAX_AI_TASK_FAILURE_MESSAGE_BYTES, MAX_AI_TASK_ID_BYTES, MAX_AI_TASK_OWNER_USER_ID_BYTES, + MAX_AI_TASK_REFERENCE_ID_BYTES, MAX_AI_TASK_REFERENCE_LABEL_BYTES, + MAX_AI_TASK_REQUEST_LABEL_BYTES, MAX_AI_TASK_REQUEST_PAYLOAD_BYTES, + MAX_AI_TASK_RESULT_REFERENCES, MAX_AI_TASK_RETAINED_OUTPUT_BYTES, MAX_AI_TASK_RETAINED_TASKS, + MAX_AI_TASK_SOURCE_ENTITY_ID_BYTES, MAX_AI_TASK_SOURCE_MODULE_BYTES, + MAX_AI_TASK_STAGE_DETAIL_BYTES, MAX_AI_TASK_STAGE_LABEL_BYTES, + MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES, MAX_AI_TASK_TEXT_CHUNKS_PER_STAGE, + MAX_AI_TASK_TEXT_OUTPUT_BYTES, MAX_AI_TASK_WARNING_BYTES, + validate_ai_task_snapshot_memory_limits, +}; pub use types::{ AiResultReferenceKind, AiResultReferenceSnapshot, AiTaskKind, AiTaskSnapshot, AiTaskStageBlueprint, AiTaskStageKind, AiTaskStageSnapshot, AiTaskStageStatus, AiTaskStatus, diff --git a/server-rs/crates/module-ai/src/domain/limits.rs b/server-rs/crates/module-ai/src/domain/limits.rs new file mode 100644 index 000000000..16d3a46c2 --- /dev/null +++ b/server-rs/crates/module-ai/src/domain/limits.rs @@ -0,0 +1,117 @@ +use super::types::AiTaskSnapshot; + +pub const MAX_AI_TASK_RETAINED_TASKS: usize = 1024; +pub const MAX_AI_TASK_ID_BYTES: usize = 256; +pub const MAX_AI_TASK_OWNER_USER_ID_BYTES: usize = 256; +pub const MAX_AI_TASK_REQUEST_LABEL_BYTES: usize = 4 * 1024; +pub const MAX_AI_TASK_SOURCE_MODULE_BYTES: usize = 256; +pub const MAX_AI_TASK_SOURCE_ENTITY_ID_BYTES: usize = 512; +pub const MAX_AI_TASK_STAGE_LABEL_BYTES: usize = 4 * 1024; +pub const MAX_AI_TASK_STAGE_DETAIL_BYTES: usize = 8 * 1024; +pub const MAX_AI_TASK_TEXT_OUTPUT_BYTES: usize = 512 * 1024; +// provider 产生大量细小流式增量时，限制行和索引开销。 +pub const MAX_AI_TASK_TEXT_CHUNKS_PER_STAGE: usize = 8 * 1024; +pub const MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES: usize = 512 * 1024; +pub const MAX_AI_TASK_WARNING_BYTES: usize = 64 * 1024; +pub const MAX_AI_TASK_REQUEST_PAYLOAD_BYTES: usize = 512 * 1024; +pub const MAX_AI_TASK_FAILURE_MESSAGE_BYTES: usize = 64 * 1024; +pub const MAX_AI_TASK_RESULT_REFERENCES: usize = 64; +pub const MAX_AI_TASK_REFERENCE_ID_BYTES: usize = 512; +pub const MAX_AI_TASK_REFERENCE_LABEL_BYTES: usize = 2 * 1024; +pub const MAX_AI_TASK_RETAINED_OUTPUT_BYTES: usize = 64 * 1024 * 1024; + +pub fn validate_ai_task_snapshot_memory_limits(task: &AiTaskSnapshot) -> Result<(), &'static str> { + if task.task_id.len() > MAX_AI_TASK_ID_BYTES { + return Err("AI 任务 ID 超过内存上限"); + } + if task.owner_user_id.len() > MAX_AI_TASK_OWNER_USER_ID_BYTES { + return Err("AI 任务用户 ID 超过内存上限"); + } + if task.request_label.len() > MAX_AI_TASK_REQUEST_LABEL_BYTES { + return Err("AI 任务请求标签超过内存上限"); + } + if task.source_module.len() > MAX_AI_TASK_SOURCE_MODULE_BYTES { + return Err("AI 任务来源模块超过内存上限"); + } + if task + .source_entity_id + .as_ref() + .is_some_and(|entity_id| entity_id.len() > MAX_AI_TASK_SOURCE_ENTITY_ID_BYTES) + { + return Err("AI 任务来源实体 ID 超过内存上限"); + } + if task.stages.iter().any(|stage| { + stage.label.len() > MAX_AI_TASK_STAGE_LABEL_BYTES + || stage.detail.len() > MAX_AI_TASK_STAGE_DETAIL_BYTES + }) { + return Err("AI 任务阶段元数据超过内存上限"); + } + if task + .request_payload_json + .as_ref() + .is_some_and(|payload| payload.len() > MAX_AI_TASK_REQUEST_PAYLOAD_BYTES) + { + return Err("AI 任务请求 payload 超过内存上限"); + } + if task + .failure_message + .as_ref() + .is_some_and(|message| message.len() > MAX_AI_TASK_FAILURE_MESSAGE_BYTES) + { + return Err("AI 任务失败消息超过内存上限"); + } + if task.stages.iter().any(|stage| { + stage + .text_output + .as_ref() + .is_some_and(|text| text.len() > MAX_AI_TASK_TEXT_OUTPUT_BYTES) + }) || task + .latest_text_output + .as_ref() + .is_some_and(|text| text.len() > MAX_AI_TASK_TEXT_OUTPUT_BYTES) + { + return Err("AI 任务文本输出超过内存上限"); + } + if task.stages.iter().any(|stage| { + stage + .structured_payload_json + .as_ref() + .is_some_and(|payload| payload.len() > MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES) + }) || task + .latest_structured_payload_json + .as_ref() + .is_some_and(|payload| payload.len() > MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES) + { + return Err("AI 任务结构化输出超过内存上限"); + } + if task.stages.iter().any(|stage| { + stage + .warning_messages + .iter() + .fold(0_usize, |total, warning| { + total.saturating_add(warning.len()) + }) + > MAX_AI_TASK_WARNING_BYTES + }) { + return Err("AI 任务 warning 输出超过内存上限"); + } + if task.result_references.len() > MAX_AI_TASK_RESULT_REFERENCES { + return Err("AI 任务结果引用数量超过内存上限"); + } + if task + .result_references + .iter() + .any(|reference| reference.reference_id.len() > MAX_AI_TASK_REFERENCE_ID_BYTES) + { + return Err("AI 任务结果引用 ID 超过内存上限"); + } + if task.result_references.iter().any(|reference| { + reference + .label + .as_ref() + .is_some_and(|label| label.len() > MAX_AI_TASK_REFERENCE_LABEL_BYTES) + }) { + return Err("AI 任务结果引用标签超过内存上限"); + } + Ok(()) +} diff --git a/server-rs/crates/module-ai/src/lib.rs b/server-rs/crates/module-ai/src/lib.rs index e69b5609f..a3311e4d9 100644 --- a/server-rs/crates/module-ai/src/lib.rs +++ b/server-rs/crates/module-ai/src/lib.rs @@ -14,9 +14,17 @@ pub use domain::{ AI_RESULT_REF_ID_PREFIX, AI_TASK_ID_PREFIX, AI_TASK_STAGE_ID_PREFIX, AI_TEXT_CHUNK_ID_PREFIX, AiResultReferenceKind, AiResultReferenceSnapshot, AiTaskKind, AiTaskSnapshot, AiTaskStageBlueprint, AiTaskStageKind, AiTaskStageSnapshot, AiTaskStageStatus, AiTaskStatus, - AiTextChunkSnapshot, INITIAL_AI_TASK_VERSION, generate_ai_result_ref_id, generate_ai_task_id, - generate_ai_task_stage_id, generate_ai_text_chunk_id, normalize_optional_text, - normalize_string_list, + AiTextChunkSnapshot, INITIAL_AI_TASK_VERSION, MAX_AI_TASK_FAILURE_MESSAGE_BYTES, + MAX_AI_TASK_ID_BYTES, MAX_AI_TASK_OWNER_USER_ID_BYTES, MAX_AI_TASK_REFERENCE_ID_BYTES, + MAX_AI_TASK_REFERENCE_LABEL_BYTES, MAX_AI_TASK_REQUEST_LABEL_BYTES, + MAX_AI_TASK_REQUEST_PAYLOAD_BYTES, MAX_AI_TASK_RESULT_REFERENCES, + MAX_AI_TASK_RETAINED_OUTPUT_BYTES, MAX_AI_TASK_RETAINED_TASKS, + MAX_AI_TASK_SOURCE_ENTITY_ID_BYTES, MAX_AI_TASK_SOURCE_MODULE_BYTES, + MAX_AI_TASK_STAGE_DETAIL_BYTES, MAX_AI_TASK_STAGE_LABEL_BYTES, + MAX_AI_TASK_STRUCTURED_OUTPUT_BYTES, MAX_AI_TASK_TEXT_CHUNKS_PER_STAGE, + MAX_AI_TASK_TEXT_OUTPUT_BYTES, MAX_AI_TASK_WARNING_BYTES, generate_ai_result_ref_id, + generate_ai_task_id, generate_ai_task_stage_id, generate_ai_text_chunk_id, + normalize_optional_text, normalize_string_list, validate_ai_task_snapshot_memory_limits, }; pub use errors::{AiTaskFieldError, AiTaskServiceError}; pub use events::AiTaskDomainEvent; diff --git a/server-rs/crates/module-ai/src/tests.rs b/server-rs/crates/module-ai/src/tests.rs index 766320035..a1ba8f6cb 100644 --- a/server-rs/crates/module-ai/src/tests.rs +++ b/server-rs/crates/module-ai/src/tests.rs @@ -43,6 +43,32 @@ fn create_task_rejects_duplicate_stage_blueprints() { assert_eq!(error, AiTaskFieldError::DuplicateStageBlueprint); } +#[test] +fn create_task_rejects_oversized_request_payload() { + let service = build_service(); + let mut input = build_create_input(AiTaskKind::StoryGeneration); + input.request_payload_json = Some("x".repeat(MAX_AI_TASK_REQUEST_PAYLOAD_BYTES + 1)); + + let error = service + .create_task(input) + .expect_err("request payload over the memory cap should fail"); + assert!( + matches!(error, AiTaskServiceError::Store(message) if message.contains("请求 payload")) + ); +} + +#[test] +fn create_task_rejects_oversized_request_metadata() { + let service = build_service(); + let mut input = build_create_input(AiTaskKind::StoryGeneration); + input.request_label = "x".repeat(MAX_AI_TASK_REQUEST_LABEL_BYTES + 1); + + let error = service + .create_task(input) + .expect_err("request metadata over the memory cap should fail"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("请求标签"))); +} + #[test] fn generate_ai_task_stage_id_contains_task_and_stage_slug() { let stage_id = generate_ai_task_stage_id("aitask_demo", AiTaskStageKind::NormalizeResult); @@ -112,6 +138,47 @@ fn append_text_chunk_aggregates_stream_output_by_stage() { assert_eq!(second_chunk.sequence, 2); } +#[test] +fn append_text_chunk_rejects_output_over_stage_memory_limit_without_mutating_task() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::CharacterChat)) + .expect("task should create"); + let max_output = "a".repeat(512 * 1024); + + let (updated, _) = service + .append_text_chunk( + &task.task_id, + AiTaskStageKind::RequestModel, + 1, + max_output.clone(), + task.created_at_micros + 1, + ) + .expect("the stage limit itself should be accepted"); + assert_eq!( + updated.latest_text_output.as_deref().map(str::len), + Some(max_output.len()) + ); + + let error = service + .append_text_chunk( + &task.task_id, + AiTaskStageKind::RequestModel, + 2, + "b".to_string(), + task.created_at_micros + 2, + ) + .expect_err("output beyond the stage limit should fail"); + assert!(matches!(error, AiTaskServiceError::Store(_))); + let after_rejection = service + .get_task(&task.task_id) + .expect("task should remain readable"); + assert_eq!( + after_rejection.latest_text_output.as_deref().map(str::len), + Some(max_output.len()) + ); +} + #[test] fn complete_stage_updates_latest_outputs() { let service = build_service(); @@ -147,6 +214,150 @@ fn complete_stage_updates_latest_outputs() { assert_eq!(stage.warning_messages, vec!["使用了 fallback 选项池"]); } +#[test] +fn complete_stage_rejects_oversized_text_output_without_mutating_task() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::StoryGeneration)) + .expect("task should create"); + let oversized = "x".repeat(512 * 1024 + 1); + + let error = service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id.clone(), + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: Some(oversized), + structured_payload_json: None, + warning_messages: Vec::new(), + completed_at_micros: task.created_at_micros + 1, + }) + .expect_err("text output over the per-stage cap should fail"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("文本输出"))); + + let unchanged = service + .get_task(&task.task_id) + .expect("task should remain readable"); + let stage = unchanged + .stages + .iter() + .find(|stage| stage.stage_kind == AiTaskStageKind::NormalizeResult) + .expect("normalize stage should exist"); + assert_eq!(stage.status, AiTaskStageStatus::Pending); + assert!(stage.text_output.is_none()); + assert!(unchanged.latest_text_output.is_none()); +} + +#[test] +fn complete_stage_rejects_oversized_structured_output_without_mutating_task() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::StoryGeneration)) + .expect("task should create"); + let oversized = "x".repeat(512 * 1024 + 1); + + let error = service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id.clone(), + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: None, + structured_payload_json: Some(oversized), + warning_messages: Vec::new(), + completed_at_micros: task.created_at_micros + 1, + }) + .expect_err("structured output over the per-stage cap should fail"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("结构化输出"))); + + let unchanged = service + .get_task(&task.task_id) + .expect("task should remain readable"); + let stage = unchanged + .stages + .iter() + .find(|stage| stage.stage_kind == AiTaskStageKind::NormalizeResult) + .expect("normalize stage should exist"); + assert_eq!(stage.status, AiTaskStageStatus::Pending); + assert!(stage.structured_payload_json.is_none()); + assert!(unchanged.latest_structured_payload_json.is_none()); +} + +#[test] +fn complete_stage_rejects_oversized_warning_output_without_mutating_task() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::StoryGeneration)) + .expect("task should create"); + let oversized_warning = "w".repeat(64 * 1024 + 1); + + let error = service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id.clone(), + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: None, + structured_payload_json: None, + warning_messages: vec![oversized_warning], + completed_at_micros: task.created_at_micros + 1, + }) + .expect_err("warning output over the per-stage cap should fail"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("warning"))); + + let unchanged = service + .get_task(&task.task_id) + .expect("task should remain readable"); + let stage = unchanged + .stages + .iter() + .find(|stage| stage.stage_kind == AiTaskStageKind::NormalizeResult) + .expect("normalize stage should exist"); + assert_eq!(stage.status, AiTaskStageStatus::Pending); + assert!(stage.warning_messages.is_empty()); +} + +#[test] +fn complete_stage_enforces_global_retained_output_cap() { + let service = build_service(); + let structured_payload = "x".repeat(512 * 1024); + for index in 0..63 { + let task = service + .create_task(AiTaskCreateInput { + task_id: format!("task-structured-cap-{index}"), + ..build_create_input(AiTaskKind::StoryGeneration) + }) + .expect("task should create"); + service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id, + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: None, + structured_payload_json: Some(structured_payload.clone()), + warning_messages: Vec::new(), + completed_at_micros: task.created_at_micros + 1, + }) + .expect("63 MiB retained output should remain within the global cap"); + } + + let task = service + .create_task(AiTaskCreateInput { + task_id: "task-structured-cap-overflow".to_string(), + ..build_create_input(AiTaskKind::StoryGeneration) + }) + .expect("the overflow candidate task itself should create"); + let error = service + .complete_stage(AiStageCompletionInput { + task_id: task.task_id.clone(), + stage_kind: AiTaskStageKind::NormalizeResult, + text_output: None, + structured_payload_json: Some(structured_payload), + warning_messages: Vec::new(), + completed_at_micros: task.created_at_micros + 1, + }) + .expect_err("global retained output cap should reject the overflow"); + assert!(matches!(error, AiTaskServiceError::Store(message) if message.contains("工作集"))); + let unchanged = service + .get_task(&task.task_id) + .expect("overflow task should remain readable"); + assert!(unchanged.latest_structured_payload_json.is_none()); +} + #[test] fn attach_result_reference_appends_binding() { let service = build_service(); @@ -172,6 +383,47 @@ fn attach_result_reference_appends_binding() { assert_eq!(updated.result_references[0].reference_id, "profile_001"); } +#[test] +fn attach_result_reference_rejects_unbounded_reference_growth() { + let service = build_service(); + let task = service + .create_task(build_create_input(AiTaskKind::CustomWorldGeneration)) + .expect("task should create"); + + for index in 0..MAX_AI_TASK_RESULT_REFERENCES { + service + .attach_result_reference( + &task.task_id, + AiResultReferenceKind::CustomWorldProfile, + format!("profile_{index}"), + None, + task.created_at_micros + index as i64 + 1, + ) + .expect("references within the cap should attach"); + } + + let error = service + .attach_result_reference( + &task.task_id, + AiResultReferenceKind::CustomWorldProfile, + "profile_overflow".to_string(), + None, + task.created_at_micros + MAX_AI_TASK_RESULT_REFERENCES as i64 + 1, + ) + .expect_err("references over the cap should fail"); + assert!( + matches!(error, AiTaskServiceError::Store(message) if message.contains("结果引用数量")) + ); + + let unchanged = service + .get_task(&task.task_id) + .expect("task should remain readable"); + assert_eq!( + unchanged.result_references.len(), + MAX_AI_TASK_RESULT_REFERENCES + ); +} + #[test] fn fail_and_cancel_task_move_into_terminal_states() { let service = build_service(); diff --git a/server-rs/crates/module-auth/src/domain.rs b/server-rs/crates/module-auth/src/domain.rs index e602d5dd5..4a29af646 100644 --- a/server-rs/crates/module-auth/src/domain.rs +++ b/server-rs/crates/module-auth/src/domain.rs @@ -81,7 +81,7 @@ pub struct PhoneNumberSnapshot { } /// 手机验证码使用场景。 -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum PhoneAuthScene { Login, BindPhone, @@ -101,7 +101,7 @@ impl PhoneAuthScene { } /// 微信授权入口场景。 -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum WechatAuthScene { Desktop, WechatInApp, @@ -187,6 +187,13 @@ pub struct AuthStoreProjectionView { pub users: Vec, pub identities: Vec, pub refresh_sessions: Vec, + #[serde(default)] + pub phone_codes: Vec, + #[serde(default)] + pub wechat_states: Vec, + /// 当前进程工作集所基于的正式投影版本，用于事务内 CAS。 + #[serde(default)] + pub base_updated_at_micros: i64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -230,6 +237,31 @@ pub struct AuthStoreProjectionRefreshSession { pub last_seen_at: String, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuthStoreProjectionPhoneCode { + pub phone_number: String, + pub scene: String, + pub verify_code_hash: String, + pub expires_at: String, + pub last_sent_at: String, + pub failed_attempts: u32, + pub provider_out_id: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuthStoreProjectionWechatState { + pub wechat_state_id: String, + pub state_token: String, + pub redirect_path: String, + pub scene: String, + pub request_user_agent: Option, + pub bind_user_id: Option, + pub expires_at: String, + pub consumed_at: Option, + pub created_at: String, + pub updated_at: String, +} + pub fn validate_password(password: &str) -> Result<(), PasswordEntryError> { let length = password.chars().count(); if !(PASSWORD_MIN_LENGTH..=PASSWORD_MAX_LENGTH).contains(&length) { diff --git a/server-rs/crates/module-auth/src/lib.rs b/server-rs/crates/module-auth/src/lib.rs index 6336495d0..c69c9cc7d 100644 --- a/server-rs/crates/module-auth/src/lib.rs +++ b/server-rs/crates/module-auth/src/lib.rs @@ -12,7 +12,10 @@ pub use events::*; use std::{ collections::{HashMap, HashSet}, - sync::{Arc, Mutex}, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + }, }; use platform_auth::{ @@ -28,9 +31,17 @@ use shared_kernel::{ use time::{Duration, OffsetDateTime}; use tracing::{info, warn}; +const DEFAULT_PHONE_VERIFY_CODE_SALT: &str = "genarrative-phone-verify-code-v1"; +const PHONE_CODE_RESERVATION_MARKER: &str = "__genarrative_phone_code_reservation__"; +const MAX_ACTIVE_WECHAT_AUTH_STATES: usize = 1024; +const REFRESH_SESSION_STALE_RETENTION: Duration = Duration::days(1); +const MAX_REFRESH_SESSIONS: usize = 8_192; +const MAX_PHONE_CODES: usize = 4_096; + #[derive(Clone, Debug)] pub struct InMemoryAuthStore { inner: Arc>, + revision: Arc, } #[derive(Debug)] @@ -131,6 +142,24 @@ fn parse_auth_binding_status(value: &str) -> AuthBindingStatus { } } +fn parse_phone_auth_scene(value: &str) -> Option { + match value.trim() { + "login" => Some(PhoneAuthScene::Login), + "bind_phone" => Some(PhoneAuthScene::BindPhone), + "change_phone" => Some(PhoneAuthScene::ChangePhone), + "reset_password" => Some(PhoneAuthScene::ResetPassword), + _ => None, + } +} + +fn parse_wechat_auth_scene(value: &str) -> Option { + match value.trim() { + "desktop" => Some(WechatAuthScene::Desktop), + "wechat_in_app" => Some(WechatAuthScene::WechatInApp), + _ => None, + } +} + fn next_sequence_from_public_user_code(public_user_code: &str) -> u64 { public_user_code .trim() @@ -364,6 +393,7 @@ impl RefreshSessionService { input: CreateRefreshSessionInput, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; self.store .find_by_user_id(&input.user_id) .map_err(map_password_store_error)? @@ -400,6 +430,7 @@ impl RefreshSessionService { input: RotateRefreshSessionInput, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; let Some(refresh_token_hash) = normalize_required_string(&input.refresh_token_hash) else { return Err(RefreshSessionError::MissingToken); }; @@ -454,6 +485,7 @@ impl RefreshSessionService { user_id: &str, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; self.store .find_by_user_id(user_id) .map_err(map_password_store_error)? @@ -468,6 +500,7 @@ impl RefreshSessionService { input: RevokeRefreshSessionByUserInput, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; self.store .find_by_user_id(&input.user_id) .map_err(map_password_store_error)? @@ -492,6 +525,7 @@ impl RefreshSessionService { session_id: &str, now: OffsetDateTime, ) -> Result { + self.store.prune_stale_sessions(now)?; self.store .is_session_active_for_user(user_id, session_id.trim(), now) } @@ -499,10 +533,19 @@ impl RefreshSessionService { impl PhoneAuthService { pub fn new(store: InMemoryAuthStore, sms_provider: SmsAuthProvider) -> Self { + Self::new_with_verify_code_salt(store, sms_provider, DEFAULT_PHONE_VERIFY_CODE_SALT) + } + + /// 使用部署级稳定盐值构造服务，确保验证码投影恢复到另一节点后仍可校验。 + pub fn new_with_verify_code_salt( + store: InMemoryAuthStore, + sms_provider: SmsAuthProvider, + verify_code_salt: impl Into, + ) -> Self { Self { store, sms_provider, - verify_code_salt: new_uuid_simple_string(), + verify_code_salt: verify_code_salt.into(), } } @@ -511,6 +554,67 @@ impl PhoneAuthService { input: SendPhoneCodeInput, now: OffsetDateTime, ) -> Result { + self.send_code_inner(input, now, true).await + } + + /// 在 provider 调用前由 api-server 先同步该占位记录，以便 SpacetimeDB 的 + /// projection CAS 原子占用跨节点冷却窗口。 + pub fn reserve_code_send( + &self, + input: &SendPhoneCodeInput, + now: OffsetDateTime, + ) -> Result<(), PhoneAuthError> { + self.store.prune_expired_phone_codes(now)?; + let scene = input.scene.clone(); + validate_mainland_china_country_code(input.country_code.as_deref())?; + let normalized_phone = normalize_mainland_china_phone_number(&input.pure_phone_number)?; + self.store + .ensure_phone_code_not_cooling_down(&normalized_phone.e164, &scene, now)?; + let expires_at = now + .checked_add(Duration::minutes(SMS_CODE_TTL_MINUTES)) + .ok_or_else(|| PhoneAuthError::Store("短信验证码过期时间计算溢出".to_string()))?; + let expires_at = format_rfc3339(expires_at).map_err(|message| { + PhoneAuthError::Store(format!("短信验证码过期时间格式化失败：{message}")) + })?; + let last_sent_at = format_rfc3339(now).map_err(|message| { + PhoneAuthError::Store(format!("短信验证码发送时间格式化失败：{message}")) + })?; + let verify_code_hash = hash_phone_verify_code( + &self.verify_code_salt, + &normalized_phone.e164, + &scene, + PHONE_CODE_RESERVATION_MARKER, + ); + self.store.upsert_phone_code( + StoredPhoneCode { + phone_number: normalized_phone.e164, + scene, + verify_code_hash, + expires_at, + last_sent_at, + failed_attempts: 0, + provider_out_id: None, + }, + now, + ) + } + + /// 仅供完成权威占用后的 provider 调用使用；占用已由 projection CAS 校验。 + pub async fn send_code_after_authoritative_reservation( + &self, + input: SendPhoneCodeInput, + now: OffsetDateTime, + ) -> Result { + self.send_code_inner(input, now, false).await + } + + async fn send_code_inner( + &self, + input: SendPhoneCodeInput, + now: OffsetDateTime, + check_local_cooldown: bool, + ) -> Result { + self.store.prune_expired_phone_codes(now)?; let scene = input.scene.clone(); validate_mainland_china_country_code(input.country_code.as_deref())?; let normalized_phone = normalize_mainland_china_phone_number(&input.pure_phone_number)?; @@ -523,8 +627,12 @@ impl PhoneAuthService { phone_national_masked = normalized_phone.masked_national_number.as_str(), "手机号验证码发送准备调用 provider" ); + if check_local_cooldown { + self.store + .ensure_phone_code_not_cooling_down(&normalized_phone.e164, &scene, now)?; + } self.store - .ensure_phone_code_not_cooling_down(&normalized_phone.e164, &scene, now)?; + .ensure_phone_code_capacity(&normalized_phone.e164, &scene)?; let expires_at = now .checked_add(Duration::minutes(SMS_CODE_TTL_MINUTES)) .ok_or_else(|| PhoneAuthError::Store("短信验证码过期时间计算溢出".to_string()))?; @@ -787,6 +895,7 @@ impl WechatAuthStateService { input: CreateWechatAuthStateInput, now: OffsetDateTime, ) -> Result { + self.store.prune_wechat_states(now)?; let created_at = format_rfc3339(now).map_err(|message| { WechatAuthError::Store(format!("微信 state 时间格式化失败：{message}")) })?; @@ -808,7 +917,7 @@ impl WechatAuthStateService { created_at: created_at.clone(), updated_at: created_at, }; - self.store.insert_wechat_state(state.clone())?; + self.store.insert_wechat_state(state.clone(), now)?; Ok(CreateWechatAuthStateResult { state }) } @@ -987,6 +1096,7 @@ impl Default for InMemoryAuthStore { fn default() -> Self { Self { inner: Arc::new(Mutex::new(InMemoryAuthStoreState::default())), + revision: Arc::new(AtomicU64::new(0)), } } } @@ -1021,6 +1131,8 @@ impl InMemoryAuthStoreState { let mut wechat_identity_by_provider_uid = HashMap::new(); let mut user_id_by_provider_union_id = HashMap::new(); let mut phone_number_by_user_id = HashMap::new(); + let mut phone_codes_by_key = HashMap::new(); + let mut wechat_states_by_token = HashMap::new(); for user in &view.users { if let Some(phone_number) = normalize_optional_string(user.phone_number_e164.clone()) { @@ -1063,10 +1175,25 @@ impl InMemoryAuthStoreState { } } + let now = OffsetDateTime::now_utc(); + let mut retained_refresh_session_count = 0_usize; for session in view.refresh_sessions { if !existing_user_ids.contains(&session.user_id) { continue; } + if should_prune_refresh_session_fields( + &session.expires_at, + session.revoked_at.as_deref(), + now, + ) { + continue; + } + retained_refresh_session_count += 1; + if retained_refresh_session_count > MAX_REFRESH_SESSIONS { + return Err(format!( + "认证投影中的 refresh session 数量超过内存上限（最多 {MAX_REFRESH_SESSIONS} 条）" + )); + } let client_info = serde_json::from_str::(&session.client_info_json) .map_err(|error| format!("解析 refresh session 客户端信息失败：{error}"))?; @@ -1093,6 +1220,46 @@ impl InMemoryAuthStoreState { ); } + for phone_code in view.phone_codes { + let scene = parse_phone_auth_scene(&phone_code.scene) + .ok_or_else(|| format!("未知短信验证码场景：{}", phone_code.scene))?; + let key = build_phone_code_key(&phone_code.phone_number, &scene); + phone_codes_by_key.insert( + key, + StoredPhoneCode { + phone_number: phone_code.phone_number, + scene, + verify_code_hash: phone_code.verify_code_hash, + expires_at: phone_code.expires_at, + last_sent_at: phone_code.last_sent_at, + failed_attempts: phone_code.failed_attempts, + provider_out_id: phone_code.provider_out_id, + }, + ); + } + + for wechat_state in view.wechat_states { + let scene = parse_wechat_auth_scene(&wechat_state.scene) + .ok_or_else(|| format!("未知微信授权 state 场景：{}", wechat_state.scene))?; + wechat_states_by_token.insert( + wechat_state.state_token.clone(), + StoredWechatAuthState { + state: WechatAuthStateRecord { + wechat_state_id: wechat_state.wechat_state_id, + state_token: wechat_state.state_token, + redirect_path: wechat_state.redirect_path, + scene, + request_user_agent: wechat_state.request_user_agent, + bind_user_id: wechat_state.bind_user_id, + expires_at: wechat_state.expires_at, + consumed_at: wechat_state.consumed_at, + created_at: wechat_state.created_at, + updated_at: wechat_state.updated_at, + }, + }, + ); + } + for user in view.users { let wechat_identity = wechat_identity_by_provider_uid .values() @@ -1140,8 +1307,8 @@ impl InMemoryAuthStoreState { phone_to_user_id, sessions_by_id, session_id_by_refresh_token_hash, - phone_codes_by_key: HashMap::new(), - wechat_states_by_token: HashMap::new(), + phone_codes_by_key, + wechat_states_by_token, wechat_identity_by_provider_uid, user_id_by_provider_union_id, }) @@ -1153,20 +1320,47 @@ impl InMemoryAuthStoreState { self.phone_to_user_id = next_state.phone_to_user_id; self.sessions_by_id = next_state.sessions_by_id; self.session_id_by_refresh_token_hash = next_state.session_id_by_refresh_token_hash; + self.phone_codes_by_key = next_state.phone_codes_by_key; + self.wechat_states_by_token = next_state.wechat_states_by_token; self.wechat_identity_by_provider_uid = next_state.wechat_identity_by_provider_uid; self.user_id_by_provider_union_id = next_state.user_id_by_provider_union_id; } } +fn prune_expired_short_lived_state( + state: &mut InMemoryAuthStoreState, + now: OffsetDateTime, +) -> bool { + let phone_code_count = state.phone_codes_by_key.len(); + state.phone_codes_by_key.retain(|_, code| { + parse_rfc3339(&code.expires_at) + .map(|expires_at| expires_at > now) + .unwrap_or(true) + }); + let wechat_state_count = state.wechat_states_by_token.len(); + state.wechat_states_by_token.retain(|_, stored| { + parse_rfc3339(&stored.state.expires_at) + .map(|expires_at| expires_at > now) + .unwrap_or(true) + }); + phone_code_count != state.phone_codes_by_key.len() + || wechat_state_count != state.wechat_states_by_token.len() +} + impl InMemoryAuthStore { pub fn from_projection_view(view: AuthStoreProjectionView) -> Result { Ok(Self { inner: Arc::new(Mutex::new(InMemoryAuthStoreState::from_projection_view( view, )?)), + revision: Arc::new(AtomicU64::new(0)), }) } + pub fn revision(&self) -> u64 { + self.revision.load(Ordering::Acquire) + } + pub fn refresh_from_projection_view( &self, view: AuthStoreProjectionView, @@ -1177,18 +1371,44 @@ impl InMemoryAuthStore { .lock() .map_err(|_| "认证仓储锁已中毒".to_string())?; state.apply_persistent_state(next_state); + self.revision.fetch_add(1, Ordering::Release); Ok(()) } + pub fn refresh_from_projection_view_if_revision( + &self, + view: AuthStoreProjectionView, + expected_revision: u64, + ) -> Result { + let next_state = InMemoryAuthStoreState::from_projection_view(view)?; + let mut state = self + .inner + .lock() + .map_err(|_| "认证仓储锁已中毒".to_string())?; + if self.revision.load(Ordering::Acquire) != expected_revision { + return Ok(false); + } + state.apply_persistent_state(next_state); + self.revision.fetch_add(1, Ordering::Release); + + Ok(true) + } + pub fn export_projection_view( &self, updated_at_micros: i64, ) -> Result { - let state = self + self.prune_stale_sessions(OffsetDateTime::now_utc()) + .map_err(|error| error.to_string())?; + let mut state = self .inner .lock() .map_err(|_| "认证仓储锁已中毒".to_string())?; + let pruned = prune_expired_short_lived_state(&mut state, OffsetDateTime::now_utc()); + if pruned { + self.revision.fetch_add(1, Ordering::Release); + } let users = state .users_by_username .values() @@ -1253,17 +1473,97 @@ impl InMemoryAuthStore { }) }) .collect::, String>>()?; + let phone_codes = state + .phone_codes_by_key + .values() + .map(|stored| AuthStoreProjectionPhoneCode { + phone_number: stored.phone_number.clone(), + scene: stored.scene.as_str().to_string(), + verify_code_hash: stored.verify_code_hash.clone(), + expires_at: stored.expires_at.clone(), + last_sent_at: stored.last_sent_at.clone(), + failed_attempts: stored.failed_attempts, + provider_out_id: stored.provider_out_id.clone(), + }) + .collect(); + let wechat_states = state + .wechat_states_by_token + .values() + .map(|stored| AuthStoreProjectionWechatState { + wechat_state_id: stored.state.wechat_state_id.clone(), + state_token: stored.state.state_token.clone(), + redirect_path: stored.state.redirect_path.clone(), + scene: stored.state.scene.as_str().to_string(), + request_user_agent: stored.state.request_user_agent.clone(), + bind_user_id: stored.state.bind_user_id.clone(), + expires_at: stored.state.expires_at.clone(), + consumed_at: stored.state.consumed_at.clone(), + created_at: stored.state.created_at.clone(), + updated_at: stored.state.updated_at.clone(), + }) + .collect(); Ok(AuthStoreProjectionView { + base_updated_at_micros: 0, updated_at_micros, users, identities, refresh_sessions, + phone_codes, + wechat_states, }) } + pub fn export_projection_view_with_revision( + &self, + updated_at_micros: i64, + ) -> Result<(AuthStoreProjectionView, u64), String> { + for _ in 0..3 { + let before = self.revision.load(Ordering::Acquire); + let view = self.export_projection_view(updated_at_micros)?; + let after = self.revision.load(Ordering::Acquire); + if before == after { + return Ok((view, after)); + } + } + Err("认证工作集在导出期间持续发生变化".to_string()) + } + + fn prune_stale_sessions(&self, now: OffsetDateTime) -> Result<(), RefreshSessionError> { + let mut state = self + .inner + .lock() + .map_err(|_| RefreshSessionError::Store("会话仓储锁已中毒".to_string()))?; + let stale_session_ids = state + .sessions_by_id + .iter() + .filter(|(_, stored)| should_prune_refresh_session(&stored.session, now)) + .map(|(session_id, _)| session_id.clone()) + .collect::>(); + if stale_session_ids.is_empty() { + return Ok(()); + } + + for session_id in stale_session_ids { + let Some(stored) = state.sessions_by_id.remove(&session_id) else { + continue; + }; + if state + .session_id_by_refresh_token_hash + .get(&stored.session.refresh_token_hash) + .is_some_and(|mapped_id| mapped_id == &session_id) + { + state + .session_id_by_refresh_token_hash + .remove(&stored.session.refresh_token_hash); + } + } + self.persist_refresh_state(&state) + } + fn persist_state(&self, state: &InMemoryAuthStoreState) -> Result<(), String> { let _ = state; + self.revision.fetch_add(1, Ordering::Release); Ok(()) } @@ -1894,6 +2194,11 @@ impl InMemoryAuthStore { "refresh token hash 已存在，无法重复创建会话".to_string(), )); } + if state.sessions_by_id.len() >= MAX_REFRESH_SESSIONS { + return Err(RefreshSessionError::Store( + "refresh session 内存容量已达到上限".to_string(), + )); + } state.session_id_by_refresh_token_hash.insert( session.refresh_token_hash.clone(), @@ -1918,7 +2223,39 @@ impl InMemoryAuthStore { .map_err(|_| PhoneAuthError::Store("短信验证码仓储锁已中毒".to_string()))?; // 手机号和业务场景共同决定同一份验证码快照，重复发送时直接覆盖旧值。 let key = build_phone_code_key(&code.phone_number, &code.scene); + if !state.phone_codes_by_key.contains_key(&key) + && state.phone_codes_by_key.len() >= MAX_PHONE_CODES + { + return Err(PhoneAuthError::Store( + "短信验证码内存容量已达到上限，请稍后重试".to_string(), + )); + } state.phone_codes_by_key.insert(key, code); + self.persist_phone_state(&state)?; + Ok(()) + } + + fn prune_expired_phone_codes(&self, now: OffsetDateTime) -> Result<(), PhoneAuthError> { + let mut state = self + .inner + .lock() + .map_err(|_| PhoneAuthError::Store("短信验证码仓储锁已中毒".to_string()))?; + let expired_keys = state + .phone_codes_by_key + .iter() + .filter_map(|(key, stored)| { + OffsetDateTime::parse( + &stored.expires_at, + &time::format_description::well_known::Rfc3339, + ) + .ok() + .filter(|expires_at| *expires_at <= now) + .map(|_| key.clone()) + }) + .collect::>(); + for key in expired_keys { + state.phone_codes_by_key.remove(&key); + } Ok(()) } @@ -1961,6 +2298,26 @@ impl InMemoryAuthStore { }) } + fn ensure_phone_code_capacity( + &self, + phone_number: &str, + scene: &PhoneAuthScene, + ) -> Result<(), PhoneAuthError> { + let state = self + .inner + .lock() + .map_err(|_| PhoneAuthError::Store("短信验证码仓储锁已中毒".to_string()))?; + let key = build_phone_code_key(phone_number, scene); + if state.phone_codes_by_key.contains_key(&key) + || state.phone_codes_by_key.len() < MAX_PHONE_CODES + { + return Ok(()); + } + Err(PhoneAuthError::Store( + "短信验证码内存容量已达到上限，请稍后重试".to_string(), + )) + } + fn get_active_phone_code( &self, phone_number: &str, @@ -2000,6 +2357,7 @@ impl InMemoryAuthStore { .map_err(|_| PhoneAuthError::Store("短信验证码仓储锁已中毒".to_string()))?; let key = build_phone_code_key(phone_number, scene); state.phone_codes_by_key.remove(&key); + self.persist_phone_state(&state)?; Ok(()) } @@ -2019,34 +2377,44 @@ impl InMemoryAuthStore { let next_failed_attempts = stored.failed_attempts.saturating_add(1); if next_failed_attempts >= SMS_CODE_MAX_FAILED_ATTEMPTS { state.phone_codes_by_key.remove(&key); + self.persist_phone_state(&state)?; return Err(PhoneAuthError::VerifyAttemptsExceeded); } if let Some(current) = state.phone_codes_by_key.get_mut(&key) { current.failed_attempts = next_failed_attempts; } + self.persist_phone_state(&state)?; Err(PhoneAuthError::InvalidVerifyCode) } fn insert_wechat_state( &self, state_record: WechatAuthStateRecord, + now: OffsetDateTime, ) -> Result<(), WechatAuthError> { let mut state = self .inner .lock() .map_err(|_| WechatAuthError::Store("微信 state 仓储锁已中毒".to_string()))?; + prune_expired_short_lived_state(&mut state, now); if state .wechat_states_by_token .contains_key(&state_record.state_token) { return Err(WechatAuthError::Store("微信 state 已存在".to_string())); } + if state.wechat_states_by_token.len() >= MAX_ACTIVE_WECHAT_AUTH_STATES { + return Err(WechatAuthError::Store( + "微信登录请求过多，请稍后重试".to_string(), + )); + } state.wechat_states_by_token.insert( state_record.state_token.clone(), StoredWechatAuthState { state: state_record, }, ); + self.persist_wechat_state(&state)?; Ok(()) } @@ -2084,7 +2452,10 @@ impl InMemoryAuthStore { .ok_or(WechatAuthError::StateNotFound)?; current.state.consumed_at = Some(now_iso.clone()); current.state.updated_at = now_iso; - Ok(current.clone()) + let consumed = current.clone(); + state.wechat_states_by_token.remove(state_token.trim()); + self.persist_wechat_state(&state)?; + Ok(consumed) } fn bind_wechat_phone_to_user( @@ -2405,6 +2776,33 @@ impl InMemoryAuthStore { Ok(()) } + fn prune_wechat_states(&self, now: OffsetDateTime) -> Result<(), WechatAuthError> { + let mut state = self + .inner + .lock() + .map_err(|_| WechatAuthError::Store("微信 state 仓储锁已中毒".to_string()))?; + let stale_tokens = state + .wechat_states_by_token + .iter() + .filter_map(|(token, stored)| { + if stored.state.consumed_at.is_some() { + return Some(token.clone()); + } + OffsetDateTime::parse( + &stored.state.expires_at, + &time::format_description::well_known::Rfc3339, + ) + .ok() + .filter(|expires_at| *expires_at <= now) + .map(|_| token.clone()) + }) + .collect::>(); + for token in stale_tokens { + state.wechat_states_by_token.remove(&token); + } + Ok(()) + } + fn revoke_session_by_user_and_session_id( &self, user_id: &str, @@ -2568,6 +2966,25 @@ impl InMemoryAuthStore { } } +fn should_prune_refresh_session(session: &RefreshSessionRecord, now: OffsetDateTime) -> bool { + should_prune_refresh_session_fields(&session.expires_at, session.revoked_at.as_deref(), now) +} + +fn should_prune_refresh_session_fields( + expires_at: &str, + revoked_at: Option<&str>, + now: OffsetDateTime, +) -> bool { + let stale_before = now.saturating_sub(REFRESH_SESSION_STALE_RETENTION); + if let Some(revoked_at) = revoked_at { + return OffsetDateTime::parse(revoked_at, &time::format_description::well_known::Rfc3339) + .is_ok_and(|timestamp| timestamp <= stale_before); + } + + OffsetDateTime::parse(expires_at, &time::format_description::well_known::Rfc3339) + .is_ok_and(|timestamp| timestamp <= stale_before) +} + fn map_sms_provider_error_to_phone_error(error: SmsProviderError) -> PhoneAuthError { match error { SmsProviderError::InvalidVerifyCode => PhoneAuthError::InvalidVerifyCode, @@ -2783,10 +3200,13 @@ mod tests { fn empty_projection_store() -> InMemoryAuthStore { InMemoryAuthStore::from_projection_view(AuthStoreProjectionView { + base_updated_at_micros: 0, updated_at_micros: 0, users: vec![], identities: vec![], refresh_sessions: vec![], + phone_codes: vec![], + wechat_states: vec![], }) .expect("projection should restore") } @@ -3208,6 +3628,7 @@ mod tests { async fn phone_login_reuses_user_restored_from_projection() { let phone_service = build_phone_service( InMemoryAuthStore::from_projection_view(AuthStoreProjectionView { + base_updated_at_micros: 0, updated_at_micros: 1, users: vec![projection_user( "user_existing_phone", @@ -3216,6 +3637,8 @@ mod tests { )], identities: vec![], refresh_sessions: vec![], + phone_codes: vec![], + wechat_states: vec![], }) .expect("projection should restore"), ); @@ -3298,6 +3721,98 @@ mod tests { assert_eq!(rotated.user.id, user.id); } + #[tokio::test] + async fn projection_roundtrip_preserves_phone_code_and_wechat_state() { + let store = InMemoryAuthStore::default(); + let phone_service = build_phone_service(store.clone()); + let wechat_state_service = WechatAuthStateService::new(store.clone(), 5); + let now = OffsetDateTime::now_utc(); + + phone_service + .send_code( + SendPhoneCodeInput { + country_code: None, + pure_phone_number: "13800138040".to_string(), + scene: PhoneAuthScene::Login, + }, + now, + ) + .await + .expect("phone code should send before projection export"); + let created_state = wechat_state_service + .create_state( + CreateWechatAuthStateInput { + redirect_path: "/studio".to_string(), + scene: WechatAuthScene::Desktop, + request_user_agent: Some("test-agent".to_string()), + bind_user_id: None, + }, + now, + ) + .expect("wechat state should be created before projection export"); + + let projection = store + .export_projection_view(1) + .expect("projection export should include short-lived auth state"); + assert_eq!(projection.phone_codes.len(), 1); + assert_eq!(projection.wechat_states.len(), 1); + + let restored_store = InMemoryAuthStore::from_projection_view(projection) + .expect("projection should restore short-lived auth state"); + let restored_phone_service = build_phone_service(restored_store.clone()); + let login = restored_phone_service + .login( + PhoneLoginInput { + country_code: None, + pure_phone_number: "13800138040".to_string(), + verify_code: DEFAULT_SMS_MOCK_VERIFY_CODE.to_string(), + }, + now + Duration::seconds(1), + ) + .await + .expect("restored phone code should verify"); + assert!(login.created); + + let consumed_state = WechatAuthStateService::new(restored_store, 5) + .consume_state(&created_state.state.state_token, now + Duration::seconds(1)) + .expect("restored wechat state should be consumable"); + assert_eq!(consumed_state.state.redirect_path, "/studio"); + } + + #[test] + fn wechat_state_creation_is_bounded_before_projection_sync() { + let store = InMemoryAuthStore::default(); + let service = WechatAuthStateService::new(store, 5); + let now = OffsetDateTime::now_utc(); + + for index in 0..MAX_ACTIVE_WECHAT_AUTH_STATES { + service + .create_state( + CreateWechatAuthStateInput { + redirect_path: format!("/studio?attempt={index}"), + scene: WechatAuthScene::Desktop, + request_user_agent: None, + bind_user_id: None, + }, + now, + ) + .expect("active wechat state should fit within the projection budget"); + } + + let error = service + .create_state( + CreateWechatAuthStateInput { + redirect_path: "/studio".to_string(), + scene: WechatAuthScene::Desktop, + request_user_agent: None, + bind_user_id: None, + }, + now, + ) + .expect_err("wechat state creation must reject an unbounded projection"); + assert!(matches!(error, WechatAuthError::Store(message) if message.contains("请求过多"))); + } + #[tokio::test] async fn refresh_from_projection_view_merges_session_created_by_another_process() { let source_store = InMemoryAuthStore::default(); @@ -3344,19 +3859,45 @@ mod tests { ) .expect("refreshed session active check should succeed") ); - assert!(matches!( - local_phone_service - .send_code( - SendPhoneCodeInput { - country_code: None, - pure_phone_number: "13800138034".to_string(), - scene: PhoneAuthScene::Login, - }, - local_now + Duration::seconds(5), - ) - .await, - Err(PhoneAuthError::SendCoolingDown { .. }) - )); + // 刷新到数据库正式投影后，短期认证状态也以数据库快照为准；本地未同步的验证码 + // 不得继续留在工作集里，避免消费已被其他节点清理的验证码。 + local_phone_service + .send_code( + SendPhoneCodeInput { + country_code: None, + pure_phone_number: "13800138034".to_string(), + scene: PhoneAuthScene::Login, + }, + local_now + Duration::seconds(5), + ) + .await + .expect("phone code should be resendable after authoritative refresh"); + } + + #[test] + fn conditional_projection_refresh_rejects_stale_revision() { + let store = InMemoryAuthStore::default(); + let projection = AuthStoreProjectionView { + base_updated_at_micros: 0, + updated_at_micros: 1, + users: vec![], + identities: vec![], + refresh_sessions: vec![], + phone_codes: vec![], + wechat_states: vec![], + }; + + assert_eq!(store.revision(), 0); + store + .refresh_from_projection_view(projection.clone()) + .expect("initial projection refresh should succeed"); + assert_eq!(store.revision(), 1); + assert!( + !store + .refresh_from_projection_view_if_revision(projection, 0) + .expect("stale projection refresh should be checked without error") + ); + assert_eq!(store.revision(), 1); } #[tokio::test] @@ -3463,6 +4004,30 @@ mod tests { } } + #[tokio::test] + async fn authoritative_phone_code_reservation_blocks_duplicate_provider_send() { + let service = build_phone_service(build_store()); + let input = SendPhoneCodeInput { + country_code: None, + pure_phone_number: "13800138001".to_string(), + scene: PhoneAuthScene::Login, + }; + let now = OffsetDateTime::now_utc(); + + service + .reserve_code_send(&input, now) + .expect("authoritative reservation should be representable locally"); + let duplicate = service + .reserve_code_send(&input, now + Duration::seconds(1)) + .expect_err("a second reservation must observe the local cooldown"); + assert!(matches!(duplicate, PhoneAuthError::SendCoolingDown { .. })); + + service + .send_code_after_authoritative_reservation(input, now) + .await + .expect("provider send should replace the reservation with the real code"); + } + #[tokio::test] async fn phone_send_code_keeps_different_scenes_isolated() { let service = build_phone_service(build_store()); @@ -4012,6 +4577,108 @@ mod tests { ); } + #[tokio::test] + async fn stale_refresh_sessions_are_pruned_from_both_indexes() { + let store = build_store(); + let refresh_service = build_refresh_service(store.clone()); + let user = create_phone_login_user(store.clone(), "13800138008").await; + let now = OffsetDateTime::now_utc(); + + refresh_service + .create_session( + CreateRefreshSessionInput { + user_id: user.id.clone(), + refresh_token_hash: hash_refresh_session_token("stale-revoked"), + issued_by_provider: AuthLoginMethod::Password, + client_info: build_client_info(), + }, + now - Duration::days(2), + ) + .expect("stale session should create"); + store + .revoke_session_by_refresh_token_hash( + &hash_refresh_session_token("stale-revoked"), + now - Duration::days(2), + ) + .expect("stale session should revoke"); + + refresh_service + .create_session( + CreateRefreshSessionInput { + user_id: user.id.clone(), + refresh_token_hash: hash_refresh_session_token("recent-revoked"), + issued_by_provider: AuthLoginMethod::Password, + client_info: build_client_info(), + }, + now, + ) + .expect("recent session should create"); + store + .revoke_session_by_refresh_token_hash( + &hash_refresh_session_token("recent-revoked"), + now, + ) + .expect("recent session should revoke"); + + let projection = store + .export_projection_view(now.unix_timestamp()) + .expect("projection export should prune stale sessions"); + assert_eq!(projection.refresh_sessions.len(), 1); + assert_eq!( + projection.refresh_sessions[0].refresh_token_hash, + hash_refresh_session_token("recent-revoked") + ); + + let stale_error = refresh_service + .rotate_session( + RotateRefreshSessionInput { + refresh_token_hash: hash_refresh_session_token("stale-revoked"), + next_refresh_token_hash: hash_refresh_session_token("stale-next"), + }, + now, + ) + .expect_err("pruned session should no longer be indexed"); + assert_eq!(stale_error, RefreshSessionError::SessionNotFound); + } + + #[test] + fn projection_restore_rejects_too_many_retained_refresh_sessions() { + let client_info_json = + serde_json::to_string(&build_client_info()).expect("client info should serialize"); + let refresh_sessions = (0..=MAX_REFRESH_SESSIONS) + .map(|index| AuthStoreProjectionRefreshSession { + session_id: format!("session-{index}"), + user_id: "user_projection_cap".to_string(), + refresh_token_hash: format!("hash-{index}"), + issued_by_provider: "password".to_string(), + client_info_json: client_info_json.clone(), + expires_at: "2999-01-01T00:00:00Z".to_string(), + revoked_at: None, + created_at: "2026-01-01T00:00:00Z".to_string(), + updated_at: "2026-01-01T00:00:00Z".to_string(), + last_seen_at: "2026-01-01T00:00:00Z".to_string(), + }) + .collect(); + + let error = InMemoryAuthStore::from_projection_view(AuthStoreProjectionView { + base_updated_at_micros: 0, + updated_at_micros: 1, + users: vec![projection_user( + "user_projection_cap", + "projection_cap", + None, + )], + identities: vec![], + refresh_sessions, + phone_codes: vec![], + wechat_states: vec![], + }) + .expect_err("projection restore must enforce the refresh session cap"); + + assert!(error.contains("refresh session")); + assert!(error.contains(&MAX_REFRESH_SESSIONS.to_string())); + } + #[tokio::test] async fn wechat_login_hits_existing_user_by_union_id_before_openid() { let store = build_store(); @@ -4248,6 +4915,7 @@ mod tests { #[tokio::test] async fn bind_wechat_phone_merges_when_existing_phone_restored_from_projection() { let store = InMemoryAuthStore::from_projection_view(AuthStoreProjectionView { + base_updated_at_micros: 0, updated_at_micros: 1, users: vec![projection_user( "user_existing_phone_bind", @@ -4256,6 +4924,8 @@ mod tests { )], identities: vec![], refresh_sessions: vec![], + phone_codes: vec![], + wechat_states: vec![], }) .expect("projection should restore"); let phone_service = build_phone_service(store.clone()); diff --git a/server-rs/crates/module-runtime/src/commands.rs b/server-rs/crates/module-runtime/src/commands.rs index d282cb10e..0fc648db8 100644 --- a/server-rs/crates/module-runtime/src/commands.rs +++ b/server-rs/crates/module-runtime/src/commands.rs @@ -285,6 +285,59 @@ pub fn build_runtime_profile_wallet_adjustment_input_with_metadata( }) } +pub fn build_runtime_profile_wallet_refund_outbox_enqueue_input( + owner_user_id: String, + amount: u64, + refund_ledger_id: String, + created_at_micros: i64, + asset_kind: String, + asset_id: String, + settlement_reason: String, + external_generation_job_id: Option, + external_generation_claim_attempt: Option, +) -> Result { + let adjustment = build_runtime_profile_wallet_adjustment_input( + owner_user_id, + amount, + refund_ledger_id, + created_at_micros, + )?; + let asset_kind = + normalize_required_string(asset_kind).ok_or(RuntimeProfileFieldError::MissingLedgerId)?; + let asset_id = + normalize_required_string(asset_id).ok_or(RuntimeProfileFieldError::MissingLedgerId)?; + let settlement_reason = normalize_required_string(settlement_reason) + .ok_or(RuntimeProfileFieldError::MissingLedgerId)?; + let external_generation_job_id = + external_generation_job_id.and_then(|value| normalize_required_string(value)); + if external_generation_job_id.is_some() != external_generation_claim_attempt.is_some() { + return Err(RuntimeProfileFieldError::InvalidExternalGenerationAttempt); + } + Ok(RuntimeProfileWalletRefundOutboxEnqueueInput { + owner_user_id: adjustment.user_id, + amount: adjustment.amount, + refund_ledger_id: adjustment.ledger_id, + created_at_micros: adjustment.created_at_micros, + asset_kind, + asset_id, + settlement_reason, + external_generation_job_id, + external_generation_claim_attempt, + }) +} + +pub fn build_runtime_profile_wallet_refund_outbox_process_input( + worker_id: String, + limit: u32, +) -> Result { + let worker_id = + normalize_required_string(worker_id).ok_or(RuntimeProfileFieldError::MissingLedgerId)?; + if limit == 0 { + return Err(RuntimeProfileFieldError::InvalidWalletAmount); + } + Ok(RuntimeProfileWalletRefundOutboxProcessInput { worker_id, limit }) +} + pub fn build_runtime_profile_recharge_center_get_input( user_id: String, ) -> Result { diff --git a/server-rs/crates/module-runtime/src/domain.rs b/server-rs/crates/module-runtime/src/domain.rs index d4334fc1f..7a4a8c928 100644 --- a/server-rs/crates/module-runtime/src/domain.rs +++ b/server-rs/crates/module-runtime/src/domain.rs @@ -2077,6 +2077,38 @@ pub struct RuntimeProfileWalletAdjustmentProcedureResult { pub error_message: Option, } +#[cfg_attr(feature = "spacetime-types", derive(SpacetimeType))] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RuntimeProfileWalletRefundOutboxEnqueueInput { + pub owner_user_id: String, + pub amount: u64, + pub refund_ledger_id: String, + pub created_at_micros: i64, + pub asset_kind: String, + pub asset_id: String, + pub settlement_reason: String, + pub external_generation_job_id: Option, + pub external_generation_claim_attempt: Option, +} + +#[cfg_attr(feature = "spacetime-types", derive(SpacetimeType))] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RuntimeProfileWalletRefundOutboxProcessInput { + pub worker_id: String, + pub limit: u32, +} + +#[cfg_attr(feature = "spacetime-types", derive(SpacetimeType))] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RuntimeProfileWalletRefundOutboxProcedureResult { + pub ok: bool, + pub enqueued_count: u32, + pub processed_count: u32, + pub retry_count: u32, + pub failed_count: u32, + pub error_message: Option, +} + #[cfg_attr(feature = "spacetime-types", derive(SpacetimeType))] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct RuntimeProfileWalletLedgerListInput { diff --git a/server-rs/crates/module-runtime/src/errors.rs b/server-rs/crates/module-runtime/src/errors.rs index 74e1924ed..e8f1f0457 100644 --- a/server-rs/crates/module-runtime/src/errors.rs +++ b/server-rs/crates/module-runtime/src/errors.rs @@ -82,6 +82,7 @@ pub enum RuntimeProfileFieldError { TaskNotClaimable, TaskAlreadyClaimed, MissingWorkerId, + InvalidExternalGenerationAttempt, MissingOrderId, MissingProductId, MissingProductTitle, @@ -172,6 +173,9 @@ impl std::fmt::Display for RuntimeProfileFieldError { Self::TaskNotClaimable => f.write_str("任务尚未达成"), Self::TaskAlreadyClaimed => f.write_str("任务奖励已领取"), Self::MissingWorkerId => f.write_str("worker_id 不能为空"), + Self::InvalidExternalGenerationAttempt => { + f.write_str("external_generation_job_id 与 claim_attempt 必须成对提供") + } Self::MissingOrderId => f.write_str("recharge.order_id 不能为空"), Self::MissingProductId => f.write_str("recharge.product_id 不能为空"), Self::MissingProductTitle => f.write_str("recharge.product_title 不能为空"), diff --git a/server-rs/crates/spacetime-client/src/active/mapper.rs b/server-rs/crates/spacetime-client/src/active/mapper.rs index 2d925be39..66845286e 100644 --- a/server-rs/crates/spacetime-client/src/active/mapper.rs +++ b/server-rs/crates/spacetime-client/src/active/mapper.rs @@ -38,6 +38,7 @@ pub use self::ai::{ AiResultReferenceRecord, AiTaskMutationRecord, AiTaskRecord, AiTaskStageRecord, AiTextChunkRecord, }; +pub use self::auth::AuthSessionValidationRecordInput; pub use self::editor_agent::{ EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput, EditorAgentConversationRecord, EditorAgentConversationTouchRecordInput, @@ -93,8 +94,8 @@ pub(crate) use self::assets::{ map_optional_asset_object_procedure_result, map_procedure_result, }; pub(crate) use self::auth::{ - map_auth_store_projection_procedure_result, map_auth_store_projection_sync_procedure_result, - map_auth_store_projection_view_input, + map_auth_session_validation_result, map_auth_store_projection_procedure_result, + map_auth_store_projection_sync_procedure_result, map_auth_store_projection_view_input, }; pub(crate) use self::editor_agent::{ map_editor_agent_conversation_list_procedure_result, diff --git a/server-rs/crates/spacetime-client/src/active/mapper/auth.rs b/server-rs/crates/spacetime-client/src/active/mapper/auth.rs index e0ba264bf..3195f4514 100644 --- a/server-rs/crates/spacetime-client/src/active/mapper/auth.rs +++ b/server-rs/crates/spacetime-client/src/active/mapper/auth.rs @@ -1,5 +1,22 @@ use super::*; +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthSessionValidationRecordInput { + pub user_id: String, + pub session_id: String, + pub token_version: u64, +} + +pub(crate) fn map_auth_session_validation_result( + result: crate::module_bindings::AuthSessionValidationProcedureResult, +) -> Result { + if result.error_message.is_some() { + return Err(SpacetimeClientError::procedure_failed(result.error_message)); + } + + Ok(result.active) +} + pub(crate) fn map_auth_store_projection_procedure_result( result: crate::module_bindings::AuthStoreProjectionProcedureResult, ) -> Result { @@ -35,6 +52,7 @@ pub(crate) fn map_auth_store_projection_view_input( view: module_auth::AuthStoreProjectionView, ) -> crate::module_bindings::AuthStoreProjectionView { crate::module_bindings::AuthStoreProjectionView { + base_updated_at_micros: view.base_updated_at_micros, updated_at_micros: view.updated_at_micros, users: view .users @@ -87,6 +105,39 @@ pub(crate) fn map_auth_store_projection_view_input( }, ) .collect(), + phone_codes: view + .phone_codes + .into_iter() + .map( + |code| crate::module_bindings::AuthStoreProjectionPhoneCode { + phone_number: code.phone_number, + scene: code.scene, + verify_code_hash: code.verify_code_hash, + expires_at: code.expires_at, + last_sent_at: code.last_sent_at, + failed_attempts: code.failed_attempts, + provider_out_id: code.provider_out_id, + }, + ) + .collect(), + wechat_states: view + .wechat_states + .into_iter() + .map( + |state| crate::module_bindings::AuthStoreProjectionWechatState { + wechat_state_id: state.wechat_state_id, + state_token: state.state_token, + redirect_path: state.redirect_path, + scene: state.scene, + request_user_agent: state.request_user_agent, + bind_user_id: state.bind_user_id, + expires_at: state.expires_at, + consumed_at: state.consumed_at, + created_at: state.created_at, + updated_at: state.updated_at, + }, + ) + .collect(), } } @@ -94,6 +145,7 @@ fn map_auth_store_projection_view( view: crate::module_bindings::AuthStoreProjectionView, ) -> module_auth::AuthStoreProjectionView { module_auth::AuthStoreProjectionView { + base_updated_at_micros: view.base_updated_at_micros, updated_at_micros: view.updated_at_micros, users: view .users @@ -142,5 +194,34 @@ fn map_auth_store_projection_view( last_seen_at: session.last_seen_at, }) .collect(), + phone_codes: view + .phone_codes + .into_iter() + .map(|code| module_auth::AuthStoreProjectionPhoneCode { + phone_number: code.phone_number, + scene: code.scene, + verify_code_hash: code.verify_code_hash, + expires_at: code.expires_at, + last_sent_at: code.last_sent_at, + failed_attempts: code.failed_attempts, + provider_out_id: code.provider_out_id, + }) + .collect(), + wechat_states: view + .wechat_states + .into_iter() + .map(|state| module_auth::AuthStoreProjectionWechatState { + wechat_state_id: state.wechat_state_id, + state_token: state.state_token, + redirect_path: state.redirect_path, + scene: state.scene, + request_user_agent: state.request_user_agent, + bind_user_id: state.bind_user_id, + expires_at: state.expires_at, + consumed_at: state.consumed_at, + created_at: state.created_at, + updated_at: state.updated_at, + }) + .collect(), } } diff --git a/server-rs/crates/spacetime-client/src/active/mapper/runtime_profile.rs b/server-rs/crates/spacetime-client/src/active/mapper/runtime_profile.rs index 1077b5b0c..4573466b9 100644 --- a/server-rs/crates/spacetime-client/src/active/mapper/runtime_profile.rs +++ b/server-rs/crates/spacetime-client/src/active/mapper/runtime_profile.rs @@ -117,6 +117,35 @@ impl From } } +impl From + for RuntimeProfileWalletRefundOutboxEnqueueInput +{ + fn from(input: module_runtime::RuntimeProfileWalletRefundOutboxEnqueueInput) -> Self { + Self { + owner_user_id: input.owner_user_id, + amount: input.amount, + refund_ledger_id: input.refund_ledger_id, + created_at_micros: input.created_at_micros, + asset_kind: input.asset_kind, + asset_id: input.asset_id, + settlement_reason: input.settlement_reason, + external_generation_job_id: input.external_generation_job_id, + external_generation_claim_attempt: input.external_generation_claim_attempt, + } + } +} + +impl From + for RuntimeProfileWalletRefundOutboxProcessInput +{ + fn from(input: module_runtime::RuntimeProfileWalletRefundOutboxProcessInput) -> Self { + Self { + worker_id: input.worker_id, + limit: input.limit, + } + } +} + impl From for RuntimeProfileRechargeOrderGetInput { @@ -677,6 +706,24 @@ pub(crate) fn map_runtime_profile_wallet_adjustment_procedure_result( )) } +pub(crate) fn map_runtime_profile_wallet_refund_outbox_procedure_result( + result: RuntimeProfileWalletRefundOutboxProcedureResult, +) -> Result { + if !result.ok { + return Err(SpacetimeClientError::procedure_failed(result.error_message)); + } + Ok( + module_runtime::RuntimeProfileWalletRefundOutboxProcedureResult { + ok: true, + enqueued_count: result.enqueued_count, + processed_count: result.processed_count, + retry_count: result.retry_count, + failed_count: result.failed_count, + error_message: result.error_message, + }, + ) +} + pub(crate) fn map_runtime_profile_recharge_center_procedure_result( result: RuntimeProfileRechargeCenterProcedureResult, ) -> Result { diff --git a/server-rs/crates/spacetime-client/src/active/runtime.rs b/server-rs/crates/spacetime-client/src/active/runtime.rs index f11c2cb89..119393fb1 100644 --- a/server-rs/crates/spacetime-client/src/active/runtime.rs +++ b/server-rs/crates/spacetime-client/src/active/runtime.rs @@ -312,6 +312,66 @@ impl SpacetimeClient { .await } + pub async fn enqueue_profile_wallet_refund_outbox( + &self, + input: module_runtime::RuntimeProfileWalletRefundOutboxEnqueueInput, + ) -> Result + { + let procedure_input: RuntimeProfileWalletRefundOutboxEnqueueInput = input.into(); + self.call_after_connect( + "enqueue_profile_wallet_refund_outbox_and_return", + move |connection, sender| { + connection + .procedures() + .enqueue_profile_wallet_refund_outbox_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then( + map_runtime_profile_wallet_refund_outbox_procedure_result, + ); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + + pub async fn process_profile_wallet_refund_outbox( + &self, + worker_id: String, + limit: u32, + ) -> Result + { + let procedure_input = + module_runtime::build_runtime_profile_wallet_refund_outbox_process_input( + worker_id, limit, + ) + .map_err(SpacetimeClientError::validation_failed)? + .into(); + self.call_after_connect( + "process_profile_wallet_refund_outbox_and_return", + move |connection, sender| { + connection + .procedures() + .process_profile_wallet_refund_outbox_and_return_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then( + map_runtime_profile_wallet_refund_outbox_procedure_result, + ); + send_once(&sender, mapped); + }, + ); + }, + ) + .await + } + pub async fn get_profile_recharge_center( &self, user_id: String, diff --git a/server-rs/crates/spacetime-client/src/auth.rs b/server-rs/crates/spacetime-client/src/auth.rs index 89f14e61a..9d55dac1e 100644 --- a/server-rs/crates/spacetime-client/src/auth.rs +++ b/server-rs/crates/spacetime-client/src/auth.rs @@ -1,6 +1,30 @@ use super::*; impl SpacetimeClient { + pub async fn validate_auth_session( + &self, + input: AuthSessionValidationRecordInput, + ) -> Result { + let procedure_input = crate::module_bindings::AuthSessionValidationInput { + user_id: input.user_id, + session_id: input.session_id, + token_version: input.token_version, + }; + + self.call_after_connect("validate_auth_session", move |connection, sender| { + connection.procedures().validate_auth_session_then( + procedure_input, + move |_, result| { + let mapped = result + .map_err(SpacetimeClientError::from_sdk_error) + .and_then(map_auth_session_validation_result); + send_once(&sender, mapped); + }, + ); + }) + .await + } + pub async fn export_auth_store_projection_from_tables( &self, ) -> Result { diff --git a/server-rs/crates/spacetime-client/src/mapper/auth.rs b/server-rs/crates/spacetime-client/src/mapper/auth.rs index e0ba264bf..0e0c7b5ac 100644 --- a/server-rs/crates/spacetime-client/src/mapper/auth.rs +++ b/server-rs/crates/spacetime-client/src/mapper/auth.rs @@ -35,6 +35,7 @@ pub(crate) fn map_auth_store_projection_view_input( view: module_auth::AuthStoreProjectionView, ) -> crate::module_bindings::AuthStoreProjectionView { crate::module_bindings::AuthStoreProjectionView { + base_updated_at_micros: view.base_updated_at_micros, updated_at_micros: view.updated_at_micros, users: view .users @@ -87,6 +88,35 @@ pub(crate) fn map_auth_store_projection_view_input( }, ) .collect(), + phone_codes: view + .phone_codes + .into_iter() + .map(|code| crate::module_bindings::AuthStoreProjectionPhoneCode { + phone_number: code.phone_number, + scene: code.scene, + verify_code_hash: code.verify_code_hash, + expires_at: code.expires_at, + last_sent_at: code.last_sent_at, + failed_attempts: code.failed_attempts, + provider_out_id: code.provider_out_id, + }) + .collect(), + wechat_states: view + .wechat_states + .into_iter() + .map(|state| crate::module_bindings::AuthStoreProjectionWechatState { + wechat_state_id: state.wechat_state_id, + state_token: state.state_token, + redirect_path: state.redirect_path, + scene: state.scene, + request_user_agent: state.request_user_agent, + bind_user_id: state.bind_user_id, + expires_at: state.expires_at, + consumed_at: state.consumed_at, + created_at: state.created_at, + updated_at: state.updated_at, + }) + .collect(), } } @@ -94,6 +124,7 @@ fn map_auth_store_projection_view( view: crate::module_bindings::AuthStoreProjectionView, ) -> module_auth::AuthStoreProjectionView { module_auth::AuthStoreProjectionView { + base_updated_at_micros: view.base_updated_at_micros, updated_at_micros: view.updated_at_micros, users: view .users @@ -142,5 +173,34 @@ fn map_auth_store_projection_view( last_seen_at: session.last_seen_at, }) .collect(), + phone_codes: view + .phone_codes + .into_iter() + .map(|code| module_auth::AuthStoreProjectionPhoneCode { + phone_number: code.phone_number, + scene: code.scene, + verify_code_hash: code.verify_code_hash, + expires_at: code.expires_at, + last_sent_at: code.last_sent_at, + failed_attempts: code.failed_attempts, + provider_out_id: code.provider_out_id, + }) + .collect(), + wechat_states: view + .wechat_states + .into_iter() + .map(|state| module_auth::AuthStoreProjectionWechatState { + wechat_state_id: state.wechat_state_id, + state_token: state.state_token, + redirect_path: state.redirect_path, + scene: state.scene, + request_user_agent: state.request_user_agent, + bind_user_id: state.bind_user_id, + expires_at: state.expires_at, + consumed_at: state.consumed_at, + created_at: state.created_at, + updated_at: state.updated_at, + }) + .collect(), } } diff --git a/server-rs/crates/spacetime-client/src/module_bindings.rs b/server-rs/crates/spacetime-client/src/module_bindings.rs index 1b24a5403..f6c9c3b9d 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings.rs @@ -115,15 +115,19 @@ pub mod asset_operation_wallet_settlement_type; pub mod attach_ai_result_reference_and_return_procedure; pub mod auth_identity_table; pub mod auth_identity_type; +pub mod auth_session_validation_input_type; +pub mod auth_session_validation_procedure_result_type; pub mod auth_store_projection_identity_type; pub mod auth_store_projection_meta_table; pub mod auth_store_projection_meta_type; +pub mod auth_store_projection_phone_code_type; pub mod auth_store_projection_procedure_result_type; pub mod auth_store_projection_refresh_session_type; pub mod auth_store_projection_sync_procedure_result_type; pub mod auth_store_projection_sync_record_type; pub mod auth_store_projection_user_type; pub mod auth_store_projection_view_type; +pub mod auth_store_projection_wechat_state_type; pub mod authenticate_external_api_key_and_return_procedure; pub mod authorize_database_migration_operator_procedure; pub mod backfill_editor_canvas_layout_and_return_procedure; @@ -379,6 +383,7 @@ pub mod editor_spritesheet_slice_batch_persist_result_type; pub mod editor_spritesheet_slice_persist_item_input_type; pub mod editor_spritesheet_slice_persisted_item_type; pub mod enqueue_external_generation_job_and_return_procedure; +pub mod enqueue_profile_wallet_refund_outbox_and_return_procedure; pub mod ensure_analytics_date_dimension_for_date_reducer; pub mod expire_profile_recharge_order_timer_reducer; pub mod export_auth_store_projection_from_tables_procedure; @@ -409,6 +414,8 @@ pub mod external_generation_job_procedure_result_type; pub mod external_generation_job_renew_lease_input_type; pub mod external_generation_job_result_procedure_result_type; pub mod external_generation_job_result_snapshot_type; +pub mod external_generation_job_retention_input_type; +pub mod external_generation_job_retention_procedure_result_type; pub mod external_generation_job_snapshot_type; pub mod external_generation_job_summary_backfill_input_type; pub mod external_generation_job_summary_backfill_procedure_result_type; @@ -516,6 +523,7 @@ pub mod preflight_editor_generation_target_and_return_procedure; pub mod preflight_editor_pixel_art_result_and_return_procedure; pub mod prepare_profile_recharge_refund_hold_and_return_procedure; pub mod preview_profile_recharge_refund_hold_and_return_procedure; +pub mod process_profile_wallet_refund_outbox_and_return_procedure; pub mod profile_code_operation_table; pub mod profile_code_operation_type; pub mod profile_daily_free_points_table; @@ -570,6 +578,9 @@ pub mod profile_wallet_ledger_table; pub mod profile_wallet_ledger_type; pub mod profile_wallet_manual_restriction_table; pub mod profile_wallet_manual_restriction_type; +pub mod profile_wallet_refund_outbox_table; +pub mod profile_wallet_refund_outbox_type; +pub mod prune_external_generation_job_history_and_return_procedure; pub mod public_work_like_table; pub mod public_work_like_type; pub mod public_work_play_daily_stat_table; @@ -778,6 +789,9 @@ pub mod runtime_profile_wallet_ledger_procedure_result_type; pub mod runtime_profile_wallet_ledger_source_type_type; pub mod runtime_profile_wallet_manual_restriction_snapshot_type; pub mod runtime_profile_wallet_manual_restriction_upsert_input_type; +pub mod runtime_profile_wallet_refund_outbox_enqueue_input_type; +pub mod runtime_profile_wallet_refund_outbox_procedure_result_type; +pub mod runtime_profile_wallet_refund_outbox_process_input_type; pub mod runtime_referral_invite_center_get_input_type; pub mod runtime_referral_invite_center_procedure_result_type; pub mod runtime_referral_invite_center_snapshot_type; @@ -844,6 +858,7 @@ pub mod user_account_table; pub mod user_account_type; pub mod user_browse_history_table; pub mod user_browse_history_type; +pub mod validate_auth_session_procedure; pub mod visual_novel_agent_message_row_type; pub mod visual_novel_agent_message_table; pub mod visual_novel_agent_session_row_type; @@ -974,15 +989,19 @@ pub use asset_operation_wallet_settlement_type::AssetOperationWalletSettlement; pub use attach_ai_result_reference_and_return_procedure::attach_ai_result_reference_and_return; pub use auth_identity_table::*; pub use auth_identity_type::AuthIdentity; +pub use auth_session_validation_input_type::AuthSessionValidationInput; +pub use auth_session_validation_procedure_result_type::AuthSessionValidationProcedureResult; pub use auth_store_projection_identity_type::AuthStoreProjectionIdentity; pub use auth_store_projection_meta_table::*; pub use auth_store_projection_meta_type::AuthStoreProjectionMeta; +pub use auth_store_projection_phone_code_type::AuthStoreProjectionPhoneCode; pub use auth_store_projection_procedure_result_type::AuthStoreProjectionProcedureResult; pub use auth_store_projection_refresh_session_type::AuthStoreProjectionRefreshSession; pub use auth_store_projection_sync_procedure_result_type::AuthStoreProjectionSyncProcedureResult; pub use auth_store_projection_sync_record_type::AuthStoreProjectionSyncRecord; pub use auth_store_projection_user_type::AuthStoreProjectionUser; pub use auth_store_projection_view_type::AuthStoreProjectionView; +pub use auth_store_projection_wechat_state_type::AuthStoreProjectionWechatState; pub use authenticate_external_api_key_and_return_procedure::authenticate_external_api_key_and_return; pub use authorize_database_migration_operator_procedure::authorize_database_migration_operator; pub use backfill_editor_canvas_layout_and_return_procedure::backfill_editor_canvas_layout_and_return; @@ -1238,6 +1257,7 @@ pub use editor_spritesheet_slice_batch_persist_result_type::EditorSpritesheetSli pub use editor_spritesheet_slice_persist_item_input_type::EditorSpritesheetSlicePersistItemInput; pub use editor_spritesheet_slice_persisted_item_type::EditorSpritesheetSlicePersistedItem; pub use enqueue_external_generation_job_and_return_procedure::enqueue_external_generation_job_and_return; +pub use enqueue_profile_wallet_refund_outbox_and_return_procedure::enqueue_profile_wallet_refund_outbox_and_return; pub use ensure_analytics_date_dimension_for_date_reducer::ensure_analytics_date_dimension_for_date; pub use expire_profile_recharge_order_timer_reducer::expire_profile_recharge_order_timer; pub use export_auth_store_projection_from_tables_procedure::export_auth_store_projection_from_tables; @@ -1268,6 +1288,8 @@ pub use external_generation_job_procedure_result_type::ExternalGenerationJobProc pub use external_generation_job_renew_lease_input_type::ExternalGenerationJobRenewLeaseInput; pub use external_generation_job_result_procedure_result_type::ExternalGenerationJobResultProcedureResult; pub use external_generation_job_result_snapshot_type::ExternalGenerationJobResultSnapshot; +pub use external_generation_job_retention_input_type::ExternalGenerationJobRetentionInput; +pub use external_generation_job_retention_procedure_result_type::ExternalGenerationJobRetentionProcedureResult; pub use external_generation_job_snapshot_type::ExternalGenerationJobSnapshot; pub use external_generation_job_summary_backfill_input_type::ExternalGenerationJobSummaryBackfillInput; pub use external_generation_job_summary_backfill_procedure_result_type::ExternalGenerationJobSummaryBackfillProcedureResult; @@ -1375,6 +1397,7 @@ pub use preflight_editor_generation_target_and_return_procedure::preflight_edito pub use preflight_editor_pixel_art_result_and_return_procedure::preflight_editor_pixel_art_result_and_return; pub use prepare_profile_recharge_refund_hold_and_return_procedure::prepare_profile_recharge_refund_hold_and_return; pub use preview_profile_recharge_refund_hold_and_return_procedure::preview_profile_recharge_refund_hold_and_return; +pub use process_profile_wallet_refund_outbox_and_return_procedure::process_profile_wallet_refund_outbox_and_return; pub use profile_code_operation_table::*; pub use profile_code_operation_type::ProfileCodeOperation; pub use profile_daily_free_points_table::*; @@ -1429,6 +1452,9 @@ pub use profile_wallet_ledger_table::*; pub use profile_wallet_ledger_type::ProfileWalletLedger; pub use profile_wallet_manual_restriction_table::*; pub use profile_wallet_manual_restriction_type::ProfileWalletManualRestriction; +pub use profile_wallet_refund_outbox_table::*; +pub use profile_wallet_refund_outbox_type::ProfileWalletRefundOutbox; +pub use prune_external_generation_job_history_and_return_procedure::prune_external_generation_job_history_and_return; pub use public_work_like_table::*; pub use public_work_like_type::PublicWorkLike; pub use public_work_play_daily_stat_table::*; @@ -1637,6 +1663,9 @@ pub use runtime_profile_wallet_ledger_procedure_result_type::RuntimeProfileWalle pub use runtime_profile_wallet_ledger_source_type_type::RuntimeProfileWalletLedgerSourceType; pub use runtime_profile_wallet_manual_restriction_snapshot_type::RuntimeProfileWalletManualRestrictionSnapshot; pub use runtime_profile_wallet_manual_restriction_upsert_input_type::RuntimeProfileWalletManualRestrictionUpsertInput; +pub use runtime_profile_wallet_refund_outbox_enqueue_input_type::RuntimeProfileWalletRefundOutboxEnqueueInput; +pub use runtime_profile_wallet_refund_outbox_procedure_result_type::RuntimeProfileWalletRefundOutboxProcedureResult; +pub use runtime_profile_wallet_refund_outbox_process_input_type::RuntimeProfileWalletRefundOutboxProcessInput; pub use runtime_referral_invite_center_get_input_type::RuntimeReferralInviteCenterGetInput; pub use runtime_referral_invite_center_procedure_result_type::RuntimeReferralInviteCenterProcedureResult; pub use runtime_referral_invite_center_snapshot_type::RuntimeReferralInviteCenterSnapshot; @@ -1703,6 +1732,7 @@ pub use user_account_table::*; pub use user_account_type::UserAccount; pub use user_browse_history_table::*; pub use user_browse_history_type::UserBrowseHistory; +pub use validate_auth_session_procedure::validate_auth_session; pub use visual_novel_agent_message_row_type::VisualNovelAgentMessageRow; pub use visual_novel_agent_message_table::*; pub use visual_novel_agent_session_row_type::VisualNovelAgentSessionRow; @@ -1937,6 +1967,7 @@ pub struct DbUpdate { profile_wallet_consumption_total: __sdk::TableUpdate, profile_wallet_ledger: __sdk::TableUpdate, profile_wallet_manual_restriction: __sdk::TableUpdate, + profile_wallet_refund_outbox: __sdk::TableUpdate, public_work_like: __sdk::TableUpdate, public_work_play_daily_stat: __sdk::TableUpdate, puzzle_agent_message: __sdk::TableUpdate, @@ -2340,6 +2371,9 @@ impl TryFrom<__ws::v2::TransactionUpdate> for DbUpdate { profile_wallet_manual_restriction_table::parse_table_update(table_update)?, ) } + "profile_wallet_refund_outbox" => db_update.profile_wallet_refund_outbox.append( + profile_wallet_refund_outbox_table::parse_table_update(table_update)?, + ), "public_work_like" => db_update .public_work_like .append(public_work_like_table::parse_table_update(table_update)?), @@ -3030,6 +3064,12 @@ impl __sdk::DbUpdate for DbUpdate { &self.profile_wallet_manual_restriction, ) .with_updates_by_pk(|row| &row.user_id); + diff.profile_wallet_refund_outbox = cache + .apply_diff_to_table::( + "profile_wallet_refund_outbox", + &self.profile_wallet_refund_outbox, + ) + .with_updates_by_pk(|row| &row.refund_ledger_id); diff.public_work_like = cache .apply_diff_to_table::("public_work_like", &self.public_work_like) .with_updates_by_pk(|row| &row.like_id); @@ -3525,6 +3565,9 @@ impl __sdk::DbUpdate for DbUpdate { "profile_wallet_manual_restriction" => db_update .profile_wallet_manual_restriction .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), + "profile_wallet_refund_outbox" => db_update + .profile_wallet_refund_outbox + .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), "public_work_like" => db_update .public_work_like .append(__sdk::parse_row_list_as_inserts(table_rows.rows)?), @@ -3955,6 +3998,9 @@ impl __sdk::DbUpdate for DbUpdate { "profile_wallet_manual_restriction" => db_update .profile_wallet_manual_restriction .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), + "profile_wallet_refund_outbox" => db_update + .profile_wallet_refund_outbox + .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), "public_work_like" => db_update .public_work_like .append(__sdk::parse_row_list_as_deletes(table_rows.rows)?), @@ -4196,6 +4242,7 @@ pub struct AppliedDiff<'r> { profile_wallet_consumption_total: __sdk::TableAppliedDiff<'r, ProfileWalletConsumptionTotal>, profile_wallet_ledger: __sdk::TableAppliedDiff<'r, ProfileWalletLedger>, profile_wallet_manual_restriction: __sdk::TableAppliedDiff<'r, ProfileWalletManualRestriction>, + profile_wallet_refund_outbox: __sdk::TableAppliedDiff<'r, ProfileWalletRefundOutbox>, public_work_like: __sdk::TableAppliedDiff<'r, PublicWorkLike>, public_work_play_daily_stat: __sdk::TableAppliedDiff<'r, PublicWorkPlayDailyStat>, puzzle_agent_message: __sdk::TableAppliedDiff<'r, PuzzleAgentMessageRow>, @@ -4737,6 +4784,11 @@ impl<'r> __sdk::AppliedDiff<'r> for AppliedDiff<'r> { &self.profile_wallet_manual_restriction, event, ); + callbacks.invoke_table_row_callbacks::( + "profile_wallet_refund_outbox", + &self.profile_wallet_refund_outbox, + event, + ); callbacks.invoke_table_row_callbacks::( "public_work_like", &self.public_work_like, @@ -5684,6 +5736,7 @@ impl __sdk::SpacetimeModule for RemoteModule { profile_wallet_consumption_total_table::register_table(client_cache); profile_wallet_ledger_table::register_table(client_cache); profile_wallet_manual_restriction_table::register_table(client_cache); + profile_wallet_refund_outbox_table::register_table(client_cache); public_work_like_table::register_table(client_cache); public_work_play_daily_stat_table::register_table(client_cache); puzzle_agent_message_table::register_table(client_cache); @@ -5825,6 +5878,7 @@ impl __sdk::SpacetimeModule for RemoteModule { "profile_wallet_consumption_total", "profile_wallet_ledger", "profile_wallet_manual_restriction", + "profile_wallet_refund_outbox", "public_work_like", "public_work_play_daily_stat", "puzzle_agent_message", diff --git a/server-rs/crates/spacetime-client/src/module_bindings/auth_session_validation_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/auth_session_validation_input_type.rs new file mode 100644 index 000000000..2d1479970 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/auth_session_validation_input_type.rs @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct AuthSessionValidationInput { + pub user_id: String, + pub session_id: String, + pub token_version: u64, +} + +impl __sdk::InModule for AuthSessionValidationInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/auth_session_validation_procedure_result_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/auth_session_validation_procedure_result_type.rs new file mode 100644 index 000000000..5e9afc3d5 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/auth_session_validation_procedure_result_type.rs @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct AuthSessionValidationProcedureResult { + pub active: bool, + pub error_message: Option, +} + +impl __sdk::InModule for AuthSessionValidationProcedureResult { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_meta_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_meta_type.rs index 309dceff7..2e7b2d303 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_meta_type.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_meta_type.rs @@ -9,6 +9,8 @@ use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; pub struct AuthStoreProjectionMeta { pub meta_id: String, pub updated_at: __sdk::Timestamp, + pub phone_codes_json: Option, + pub wechat_states_json: Option, } impl __sdk::InModule for AuthStoreProjectionMeta { @@ -21,6 +23,8 @@ impl __sdk::InModule for AuthStoreProjectionMeta { pub struct AuthStoreProjectionMetaCols { pub meta_id: __sdk::__query_builder::Col, pub updated_at: __sdk::__query_builder::Col, + pub phone_codes_json: __sdk::__query_builder::Col>, + pub wechat_states_json: __sdk::__query_builder::Col>, } impl __sdk::__query_builder::HasCols for AuthStoreProjectionMeta { @@ -29,6 +33,8 @@ impl __sdk::__query_builder::HasCols for AuthStoreProjectionMeta { AuthStoreProjectionMetaCols { meta_id: __sdk::__query_builder::Col::new(table_name, "meta_id"), updated_at: __sdk::__query_builder::Col::new(table_name, "updated_at"), + phone_codes_json: __sdk::__query_builder::Col::new(table_name, "phone_codes_json"), + wechat_states_json: __sdk::__query_builder::Col::new(table_name, "wechat_states_json"), } } } diff --git a/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_phone_code_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_phone_code_type.rs new file mode 100644 index 000000000..10c7bffc2 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_phone_code_type.rs @@ -0,0 +1,21 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct AuthStoreProjectionPhoneCode { + pub phone_number: String, + pub scene: String, + pub verify_code_hash: String, + pub expires_at: String, + pub last_sent_at: String, + pub failed_attempts: u32, + pub provider_out_id: Option, +} + +impl __sdk::InModule for AuthStoreProjectionPhoneCode { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_view_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_view_type.rs index fb99a128c..fa743157a 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_view_type.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_view_type.rs @@ -5,8 +5,10 @@ use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; use super::auth_store_projection_identity_type::AuthStoreProjectionIdentity; +use super::auth_store_projection_phone_code_type::AuthStoreProjectionPhoneCode; use super::auth_store_projection_refresh_session_type::AuthStoreProjectionRefreshSession; use super::auth_store_projection_user_type::AuthStoreProjectionUser; +use super::auth_store_projection_wechat_state_type::AuthStoreProjectionWechatState; #[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] #[sats(crate = __lib)] @@ -15,6 +17,9 @@ pub struct AuthStoreProjectionView { pub users: Vec, pub identities: Vec, pub refresh_sessions: Vec, + pub phone_codes: Vec, + pub wechat_states: Vec, + pub base_updated_at_micros: i64, } impl __sdk::InModule for AuthStoreProjectionView { diff --git a/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_wechat_state_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_wechat_state_type.rs new file mode 100644 index 000000000..7de462fa8 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/auth_store_projection_wechat_state_type.rs @@ -0,0 +1,24 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct AuthStoreProjectionWechatState { + pub wechat_state_id: String, + pub state_token: String, + pub redirect_path: String, + pub scene: String, + pub request_user_agent: Option, + pub bind_user_id: Option, + pub expires_at: String, + pub consumed_at: Option, + pub created_at: String, + pub updated_at: String, +} + +impl __sdk::InModule for AuthStoreProjectionWechatState { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/enqueue_profile_wallet_refund_outbox_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/enqueue_profile_wallet_refund_outbox_and_return_procedure.rs new file mode 100644 index 000000000..4f5f28c68 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/enqueue_profile_wallet_refund_outbox_and_return_procedure.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::runtime_profile_wallet_refund_outbox_enqueue_input_type::RuntimeProfileWalletRefundOutboxEnqueueInput; +use super::runtime_profile_wallet_refund_outbox_procedure_result_type::RuntimeProfileWalletRefundOutboxProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct EnqueueProfileWalletRefundOutboxAndReturnArgs { + pub input: RuntimeProfileWalletRefundOutboxEnqueueInput, +} + +impl __sdk::InModule for EnqueueProfileWalletRefundOutboxAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `enqueue_profile_wallet_refund_outbox_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait enqueue_profile_wallet_refund_outbox_and_return { + fn enqueue_profile_wallet_refund_outbox_and_return( + &self, + input: RuntimeProfileWalletRefundOutboxEnqueueInput, + ) { + self.enqueue_profile_wallet_refund_outbox_and_return_then(input, |_, _| {}); + } + + fn enqueue_profile_wallet_refund_outbox_and_return_then( + &self, + input: RuntimeProfileWalletRefundOutboxEnqueueInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl enqueue_profile_wallet_refund_outbox_and_return for super::RemoteProcedures { + fn enqueue_profile_wallet_refund_outbox_and_return_then( + &self, + input: RuntimeProfileWalletRefundOutboxEnqueueInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, RuntimeProfileWalletRefundOutboxProcedureResult>( + "enqueue_profile_wallet_refund_outbox_and_return", + EnqueueProfileWalletRefundOutboxAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_event_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_event_type.rs index 32c820175..4658052db 100644 --- a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_event_type.rs +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_event_type.rs @@ -56,6 +56,7 @@ impl __sdk::__query_builder::HasCols for ExternalGenerationJobEvent { /// Provides typed access to indexed columns for query building. pub struct ExternalGenerationJobEventIxCols { pub event_id: __sdk::__query_builder::IxCol, + pub job_id: __sdk::__query_builder::IxCol, } impl __sdk::__query_builder::HasIxCols for ExternalGenerationJobEvent { @@ -63,6 +64,7 @@ impl __sdk::__query_builder::HasIxCols for ExternalGenerationJobEvent { fn ix_cols(table_name: &'static str) -> Self::IxCols { ExternalGenerationJobEventIxCols { event_id: __sdk::__query_builder::IxCol::new(table_name, "event_id"), + job_id: __sdk::__query_builder::IxCol::new(table_name, "job_id"), } } } diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_input_type.rs new file mode 100644 index 000000000..cffb3527a --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_input_type.rs @@ -0,0 +1,19 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobRetentionInput { + pub source_module: String, + pub limit: u32, + pub cursor_job_id: Option, + pub completed_before_micros: i64, + pub dry_run: bool, +} + +impl __sdk::InModule for ExternalGenerationJobRetentionInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_procedure_result_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_procedure_result_type.rs new file mode 100644 index 000000000..1f3883749 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/external_generation_job_retention_procedure_result_type.rs @@ -0,0 +1,24 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ExternalGenerationJobRetentionProcedureResult { + pub ok: bool, + pub dry_run: bool, + pub scanned_count: u64, + pub selected_count: u32, + pub deleted_job_count: u32, + pub deleted_summary_count: u32, + pub deleted_event_count: u32, + pub next_cursor_job_id: Option, + pub has_more: bool, + pub error_message: Option, +} + +impl __sdk::InModule for ExternalGenerationJobRetentionProcedureResult { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/process_profile_wallet_refund_outbox_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/process_profile_wallet_refund_outbox_and_return_procedure.rs new file mode 100644 index 000000000..8b6f64ccd --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/process_profile_wallet_refund_outbox_and_return_procedure.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::runtime_profile_wallet_refund_outbox_procedure_result_type::RuntimeProfileWalletRefundOutboxProcedureResult; +use super::runtime_profile_wallet_refund_outbox_process_input_type::RuntimeProfileWalletRefundOutboxProcessInput; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct ProcessProfileWalletRefundOutboxAndReturnArgs { + pub input: RuntimeProfileWalletRefundOutboxProcessInput, +} + +impl __sdk::InModule for ProcessProfileWalletRefundOutboxAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `process_profile_wallet_refund_outbox_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait process_profile_wallet_refund_outbox_and_return { + fn process_profile_wallet_refund_outbox_and_return( + &self, + input: RuntimeProfileWalletRefundOutboxProcessInput, + ) { + self.process_profile_wallet_refund_outbox_and_return_then(input, |_, _| {}); + } + + fn process_profile_wallet_refund_outbox_and_return_then( + &self, + input: RuntimeProfileWalletRefundOutboxProcessInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl process_profile_wallet_refund_outbox_and_return for super::RemoteProcedures { + fn process_profile_wallet_refund_outbox_and_return_then( + &self, + input: RuntimeProfileWalletRefundOutboxProcessInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, RuntimeProfileWalletRefundOutboxProcedureResult>( + "process_profile_wallet_refund_outbox_and_return", + ProcessProfileWalletRefundOutboxAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/profile_wallet_refund_outbox_table.rs b/server-rs/crates/spacetime-client/src/module_bindings/profile_wallet_refund_outbox_table.rs new file mode 100644 index 000000000..6f6233437 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/profile_wallet_refund_outbox_table.rs @@ -0,0 +1,235 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use super::profile_wallet_refund_outbox_type::ProfileWalletRefundOutbox; +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +/// Table handle for the table `profile_wallet_refund_outbox`. +/// +/// Obtain a handle from the [`ProfileWalletRefundOutboxTableAccess::profile_wallet_refund_outbox`] method on [`super::RemoteTables`], +/// like `ctx.db.profile_wallet_refund_outbox()`. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.profile_wallet_refund_outbox().on_insert(...)`. +pub struct ProfileWalletRefundOutboxTableHandle<'ctx> { + imp: __sdk::TableHandle, + ctx: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +/// Lifetime-aware accessor marker for the table `profile_wallet_refund_outbox`. +pub struct ProfileWalletRefundOutboxTableAccessor; + +impl __sdk::TableAccessor for ProfileWalletRefundOutboxTableAccessor { + type Row = ProfileWalletRefundOutbox; + type Handle<'db> = ProfileWalletRefundOutboxTableHandle<'db>; + + fn get<'db>(db: &'db super::RemoteTables) -> Self::Handle<'db> { + db.profile_wallet_refund_outbox() + } +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the table `profile_wallet_refund_outbox`. +/// +/// Implemented for [`super::RemoteTables`]. +pub trait ProfileWalletRefundOutboxTableAccess { + #[allow(non_snake_case)] + /// Obtain a [`ProfileWalletRefundOutboxTableHandle`], which mediates access to the table `profile_wallet_refund_outbox`. + fn profile_wallet_refund_outbox(&self) -> ProfileWalletRefundOutboxTableHandle<'_>; +} + +impl ProfileWalletRefundOutboxTableAccess for super::RemoteTables { + fn profile_wallet_refund_outbox(&self) -> ProfileWalletRefundOutboxTableHandle<'_> { + ProfileWalletRefundOutboxTableHandle { + imp: self + .imp + .get_table::("profile_wallet_refund_outbox"), + ctx: std::marker::PhantomData, + } + } +} + +pub struct ProfileWalletRefundOutboxInsertCallbackId(__sdk::CallbackId); +pub struct ProfileWalletRefundOutboxDeleteCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::TableLike for ProfileWalletRefundOutboxTableHandle<'ctx> { + type Row = ProfileWalletRefundOutbox; + type EventContext = super::EventContext; + + fn count(&self) -> u64 { + self.imp.count() + } + fn iter(&self) -> impl Iterator + '_ { + self.imp.iter() + } +} + +impl<'ctx> __sdk::Table for ProfileWalletRefundOutboxTableHandle<'ctx> { + type Row = ProfileWalletRefundOutbox; + type EventContext = super::EventContext; + + fn count(&self) -> u64 { + self.imp.count() + } + fn iter(&self) -> impl Iterator + '_ { + self.imp.iter() + } + + type InsertCallbackId = ProfileWalletRefundOutboxInsertCallbackId; + + fn on_insert( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> ProfileWalletRefundOutboxInsertCallbackId { + ProfileWalletRefundOutboxInsertCallbackId(self.imp.on_insert(Box::new(callback))) + } + + fn remove_on_insert(&self, callback: ProfileWalletRefundOutboxInsertCallbackId) { + self.imp.remove_on_insert(callback.0) + } + + type DeleteCallbackId = ProfileWalletRefundOutboxDeleteCallbackId; + + fn on_delete( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> ProfileWalletRefundOutboxDeleteCallbackId { + ProfileWalletRefundOutboxDeleteCallbackId(self.imp.on_delete(Box::new(callback))) + } + + fn remove_on_delete(&self, callback: ProfileWalletRefundOutboxDeleteCallbackId) { + self.imp.remove_on_delete(callback.0) + } +} + +impl<'ctx> __sdk::WithInsert for ProfileWalletRefundOutboxTableHandle<'ctx> { + type InsertCallbackId = ProfileWalletRefundOutboxInsertCallbackId; + + fn on_insert( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> ProfileWalletRefundOutboxInsertCallbackId { + ProfileWalletRefundOutboxInsertCallbackId(self.imp.on_insert(Box::new(callback))) + } + + fn remove_on_insert(&self, callback: ProfileWalletRefundOutboxInsertCallbackId) { + self.imp.remove_on_insert(callback.0) + } +} + +impl<'ctx> __sdk::WithDelete for ProfileWalletRefundOutboxTableHandle<'ctx> { + type DeleteCallbackId = ProfileWalletRefundOutboxDeleteCallbackId; + + fn on_delete( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row) + Send + 'static, + ) -> ProfileWalletRefundOutboxDeleteCallbackId { + ProfileWalletRefundOutboxDeleteCallbackId(self.imp.on_delete(Box::new(callback))) + } + + fn remove_on_delete(&self, callback: ProfileWalletRefundOutboxDeleteCallbackId) { + self.imp.remove_on_delete(callback.0) + } +} + +pub struct ProfileWalletRefundOutboxUpdateCallbackId(__sdk::CallbackId); + +impl<'ctx> __sdk::TableWithPrimaryKey for ProfileWalletRefundOutboxTableHandle<'ctx> { + type UpdateCallbackId = ProfileWalletRefundOutboxUpdateCallbackId; + + fn on_update( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row, &Self::Row) + Send + 'static, + ) -> ProfileWalletRefundOutboxUpdateCallbackId { + ProfileWalletRefundOutboxUpdateCallbackId(self.imp.on_update(Box::new(callback))) + } + + fn remove_on_update(&self, callback: ProfileWalletRefundOutboxUpdateCallbackId) { + self.imp.remove_on_update(callback.0) + } +} + +impl<'ctx> __sdk::WithUpdate for ProfileWalletRefundOutboxTableHandle<'ctx> { + type UpdateCallbackId = ProfileWalletRefundOutboxUpdateCallbackId; + + fn on_update( + &self, + callback: impl FnMut(&Self::EventContext, &Self::Row, &Self::Row) + Send + 'static, + ) -> ProfileWalletRefundOutboxUpdateCallbackId { + ProfileWalletRefundOutboxUpdateCallbackId(self.imp.on_update(Box::new(callback))) + } + + fn remove_on_update(&self, callback: ProfileWalletRefundOutboxUpdateCallbackId) { + self.imp.remove_on_update(callback.0) + } +} + +/// Access to the `refund_ledger_id` unique index on the table `profile_wallet_refund_outbox`, +/// which allows point queries on the field of the same name +/// via the [`ProfileWalletRefundOutboxRefundLedgerIdUnique::find`] method. +/// +/// Users are encouraged not to explicitly reference this type, +/// but to directly chain method calls, +/// like `ctx.db.profile_wallet_refund_outbox().refund_ledger_id().find(...)`. +pub struct ProfileWalletRefundOutboxRefundLedgerIdUnique<'ctx> { + imp: __sdk::UniqueConstraintHandle, + phantom: std::marker::PhantomData<&'ctx super::RemoteTables>, +} + +impl<'ctx> ProfileWalletRefundOutboxTableHandle<'ctx> { + /// Get a handle on the `refund_ledger_id` unique index on the table `profile_wallet_refund_outbox`. + pub fn refund_ledger_id(&self) -> ProfileWalletRefundOutboxRefundLedgerIdUnique<'ctx> { + ProfileWalletRefundOutboxRefundLedgerIdUnique { + imp: self.imp.get_unique_constraint::("refund_ledger_id"), + phantom: std::marker::PhantomData, + } + } +} + +impl<'ctx> ProfileWalletRefundOutboxRefundLedgerIdUnique<'ctx> { + /// Find the subscribed row whose `refund_ledger_id` column value is equal to `col_val`, + /// if such a row is present in the client cache. + pub fn find(&self, col_val: &String) -> Option { + self.imp.find(col_val) + } +} + +#[doc(hidden)] +pub(super) fn register_table(client_cache: &mut __sdk::ClientCache) { + let _table = + client_cache.get_or_make_table::("profile_wallet_refund_outbox"); + _table.add_unique_constraint::("refund_ledger_id", |row| &row.refund_ledger_id); +} + +#[doc(hidden)] +pub(super) fn parse_table_update( + raw_updates: __ws::v2::TableUpdate, +) -> __sdk::Result<__sdk::TableUpdate> { + __sdk::TableUpdate::parse_table_update(raw_updates).map_err(|e| { + __sdk::InternalError::failed_parse("TableUpdate", "TableUpdate") + .with_cause(e) + .into() + }) +} + +#[allow(non_camel_case_types)] +/// Extension trait for query builder access to the table `ProfileWalletRefundOutbox`. +/// +/// Implemented for [`__sdk::QueryTableAccessor`]. +pub trait profile_wallet_refund_outboxQueryTableAccess { + #[allow(non_snake_case)] + /// Get a query builder for the table `ProfileWalletRefundOutbox`. + fn profile_wallet_refund_outbox( + &self, + ) -> __sdk::__query_builder::Table; +} + +impl profile_wallet_refund_outboxQueryTableAccess for __sdk::QueryTableAccessor { + fn profile_wallet_refund_outbox( + &self, + ) -> __sdk::__query_builder::Table { + __sdk::__query_builder::Table::new("profile_wallet_refund_outbox") + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/profile_wallet_refund_outbox_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/profile_wallet_refund_outbox_type.rs new file mode 100644 index 000000000..f2c38e984 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/profile_wallet_refund_outbox_type.rs @@ -0,0 +1,103 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct ProfileWalletRefundOutbox { + pub refund_ledger_id: String, + pub consume_ledger_id: String, + pub owner_user_id: String, + pub amount: u64, + pub created_at: __sdk::Timestamp, + pub asset_kind: String, + pub asset_id: String, + pub settlement_reason: String, + pub external_generation_job_id: Option, + pub external_generation_claim_attempt: Option, + pub status: String, + pub available_at: __sdk::Timestamp, + pub attempts: u32, + pub last_error: Option, + pub last_attempted_at: Option<__sdk::Timestamp>, + pub last_worker_id: Option, +} + +impl __sdk::InModule for ProfileWalletRefundOutbox { + type Module = super::RemoteModule; +} + +/// Column accessor struct for the table `ProfileWalletRefundOutbox`. +/// +/// Provides typed access to columns for query building. +pub struct ProfileWalletRefundOutboxCols { + pub refund_ledger_id: __sdk::__query_builder::Col, + pub consume_ledger_id: __sdk::__query_builder::Col, + pub owner_user_id: __sdk::__query_builder::Col, + pub amount: __sdk::__query_builder::Col, + pub created_at: __sdk::__query_builder::Col, + pub asset_kind: __sdk::__query_builder::Col, + pub asset_id: __sdk::__query_builder::Col, + pub settlement_reason: __sdk::__query_builder::Col, + pub external_generation_job_id: + __sdk::__query_builder::Col>, + pub external_generation_claim_attempt: + __sdk::__query_builder::Col>, + pub status: __sdk::__query_builder::Col, + pub available_at: __sdk::__query_builder::Col, + pub attempts: __sdk::__query_builder::Col, + pub last_error: __sdk::__query_builder::Col>, + pub last_attempted_at: + __sdk::__query_builder::Col>, + pub last_worker_id: __sdk::__query_builder::Col>, +} + +impl __sdk::__query_builder::HasCols for ProfileWalletRefundOutbox { + type Cols = ProfileWalletRefundOutboxCols; + fn cols(table_name: &'static str) -> Self::Cols { + ProfileWalletRefundOutboxCols { + refund_ledger_id: __sdk::__query_builder::Col::new(table_name, "refund_ledger_id"), + consume_ledger_id: __sdk::__query_builder::Col::new(table_name, "consume_ledger_id"), + owner_user_id: __sdk::__query_builder::Col::new(table_name, "owner_user_id"), + amount: __sdk::__query_builder::Col::new(table_name, "amount"), + created_at: __sdk::__query_builder::Col::new(table_name, "created_at"), + asset_kind: __sdk::__query_builder::Col::new(table_name, "asset_kind"), + asset_id: __sdk::__query_builder::Col::new(table_name, "asset_id"), + settlement_reason: __sdk::__query_builder::Col::new(table_name, "settlement_reason"), + external_generation_job_id: __sdk::__query_builder::Col::new( + table_name, + "external_generation_job_id", + ), + external_generation_claim_attempt: __sdk::__query_builder::Col::new( + table_name, + "external_generation_claim_attempt", + ), + status: __sdk::__query_builder::Col::new(table_name, "status"), + available_at: __sdk::__query_builder::Col::new(table_name, "available_at"), + attempts: __sdk::__query_builder::Col::new(table_name, "attempts"), + last_error: __sdk::__query_builder::Col::new(table_name, "last_error"), + last_attempted_at: __sdk::__query_builder::Col::new(table_name, "last_attempted_at"), + last_worker_id: __sdk::__query_builder::Col::new(table_name, "last_worker_id"), + } + } +} + +/// Indexed column accessor struct for the table `ProfileWalletRefundOutbox`. +/// +/// Provides typed access to indexed columns for query building. +pub struct ProfileWalletRefundOutboxIxCols { + pub refund_ledger_id: __sdk::__query_builder::IxCol, +} + +impl __sdk::__query_builder::HasIxCols for ProfileWalletRefundOutbox { + type IxCols = ProfileWalletRefundOutboxIxCols; + fn ix_cols(table_name: &'static str) -> Self::IxCols { + ProfileWalletRefundOutboxIxCols { + refund_ledger_id: __sdk::__query_builder::IxCol::new(table_name, "refund_ledger_id"), + } + } +} + +impl __sdk::__query_builder::CanBeLookupTable for ProfileWalletRefundOutbox {} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/prune_external_generation_job_history_and_return_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/prune_external_generation_job_history_and_return_procedure.rs new file mode 100644 index 000000000..877d06380 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/prune_external_generation_job_history_and_return_procedure.rs @@ -0,0 +1,62 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::external_generation_job_retention_input_type::ExternalGenerationJobRetentionInput; +use super::external_generation_job_retention_procedure_result_type::ExternalGenerationJobRetentionProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct PruneExternalGenerationJobHistoryAndReturnArgs { + pub input: ExternalGenerationJobRetentionInput, +} + +impl __sdk::InModule for PruneExternalGenerationJobHistoryAndReturnArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `prune_external_generation_job_history_and_return`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait prune_external_generation_job_history_and_return { + fn prune_external_generation_job_history_and_return( + &self, + input: ExternalGenerationJobRetentionInput, + ) { + self.prune_external_generation_job_history_and_return_then(input, |_, _| {}); + } + + fn prune_external_generation_job_history_and_return_then( + &self, + input: ExternalGenerationJobRetentionInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl prune_external_generation_job_history_and_return for super::RemoteProcedures { + fn prune_external_generation_job_history_and_return_then( + &self, + input: ExternalGenerationJobRetentionInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, ExternalGenerationJobRetentionProcedureResult>( + "prune_external_generation_job_history_and_return", + PruneExternalGenerationJobHistoryAndReturnArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/runtime_profile_wallet_refund_outbox_enqueue_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/runtime_profile_wallet_refund_outbox_enqueue_input_type.rs new file mode 100644 index 000000000..14e090000 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/runtime_profile_wallet_refund_outbox_enqueue_input_type.rs @@ -0,0 +1,23 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct RuntimeProfileWalletRefundOutboxEnqueueInput { + pub owner_user_id: String, + pub amount: u64, + pub refund_ledger_id: String, + pub created_at_micros: i64, + pub asset_kind: String, + pub asset_id: String, + pub settlement_reason: String, + pub external_generation_job_id: Option, + pub external_generation_claim_attempt: Option, +} + +impl __sdk::InModule for RuntimeProfileWalletRefundOutboxEnqueueInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/runtime_profile_wallet_refund_outbox_procedure_result_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/runtime_profile_wallet_refund_outbox_procedure_result_type.rs new file mode 100644 index 000000000..dae26bc0d --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/runtime_profile_wallet_refund_outbox_procedure_result_type.rs @@ -0,0 +1,20 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct RuntimeProfileWalletRefundOutboxProcedureResult { + pub ok: bool, + pub enqueued_count: u32, + pub processed_count: u32, + pub retry_count: u32, + pub failed_count: u32, + pub error_message: Option, +} + +impl __sdk::InModule for RuntimeProfileWalletRefundOutboxProcedureResult { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/runtime_profile_wallet_refund_outbox_process_input_type.rs b/server-rs/crates/spacetime-client/src/module_bindings/runtime_profile_wallet_refund_outbox_process_input_type.rs new file mode 100644 index 000000000..1c3f25e7c --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/runtime_profile_wallet_refund_outbox_process_input_type.rs @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +pub struct RuntimeProfileWalletRefundOutboxProcessInput { + pub worker_id: String, + pub limit: u32, +} + +impl __sdk::InModule for RuntimeProfileWalletRefundOutboxProcessInput { + type Module = super::RemoteModule; +} diff --git a/server-rs/crates/spacetime-client/src/module_bindings/validate_auth_session_procedure.rs b/server-rs/crates/spacetime-client/src/module_bindings/validate_auth_session_procedure.rs new file mode 100644 index 000000000..e96904972 --- /dev/null +++ b/server-rs/crates/spacetime-client/src/module_bindings/validate_auth_session_procedure.rs @@ -0,0 +1,59 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#![allow(unused, clippy::all)] +use spacetimedb_sdk::__codegen::{self as __sdk, __lib, __sats, __ws}; + +use super::auth_session_validation_input_type::AuthSessionValidationInput; +use super::auth_session_validation_procedure_result_type::AuthSessionValidationProcedureResult; + +#[derive(__lib::ser::Serialize, __lib::de::Deserialize, Clone, PartialEq, Debug)] +#[sats(crate = __lib)] +struct ValidateAuthSessionArgs { + pub input: AuthSessionValidationInput, +} + +impl __sdk::InModule for ValidateAuthSessionArgs { + type Module = super::RemoteModule; +} + +#[allow(non_camel_case_types)] +/// Extension trait for access to the procedure `validate_auth_session`. +/// +/// Implemented for [`super::RemoteProcedures`]. +pub trait validate_auth_session { + fn validate_auth_session(&self, input: AuthSessionValidationInput) { + self.validate_auth_session_then(input, |_, _| {}); + } + + fn validate_auth_session_then( + &self, + input: AuthSessionValidationInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ); +} + +impl validate_auth_session for super::RemoteProcedures { + fn validate_auth_session_then( + &self, + input: AuthSessionValidationInput, + + __callback: impl FnOnce( + &super::ProcedureEventContext, + Result, + ) + Send + + 'static, + ) { + self.imp + .invoke_procedure_with_callback::<_, AuthSessionValidationProcedureResult>( + "validate_auth_session", + ValidateAuthSessionArgs { input }, + __callback, + ); + } +} diff --git a/server-rs/crates/spacetime-module/Cargo.toml b/server-rs/crates/spacetime-module/Cargo.toml index 9f264dd64..d2a516e1b 100644 --- a/server-rs/crates/spacetime-module/Cargo.toml +++ b/server-rs/crates/spacetime-module/Cargo.toml @@ -20,3 +20,4 @@ module-runtime = { workspace = true, features = ["spacetime-types"] } sha2 = { workspace = true } spacetimedb = { workspace = true, features = ["unstable"] } spacetimedb-lib = { workspace = true, features = ["serde"] } +time = { workspace = true, features = ["parsing"] } diff --git a/server-rs/crates/spacetime-module/src/ai/snapshots.rs b/server-rs/crates/spacetime-module/src/ai/snapshots.rs index 8ee4c0bec..f76d209e7 100644 --- a/server-rs/crates/spacetime-module/src/ai/snapshots.rs +++ b/server-rs/crates/spacetime-module/src/ai/snapshots.rs @@ -139,17 +139,6 @@ pub(crate) fn build_ai_text_chunk_row_id(snapshot: &AiTextChunkSnapshot) -> Stri ) } -pub(crate) fn build_ai_text_chunk_snapshot_from_row(row: &AiTextChunk) -> AiTextChunkSnapshot { - AiTextChunkSnapshot { - chunk_id: row.chunk_id.clone(), - task_id: row.task_id.clone(), - stage_kind: row.stage_kind, - sequence: row.sequence, - delta_text: row.delta_text.clone(), - created_at_micros: row.created_at.to_micros_since_unix_epoch(), - } -} - pub(crate) fn build_ai_result_reference_row( snapshot: &AiResultReferenceSnapshot, ) -> AiResultReference { diff --git a/server-rs/crates/spacetime-module/src/ai/stages.rs b/server-rs/crates/spacetime-module/src/ai/stages.rs index 8ffea6f22..86bc26bc9 100644 --- a/server-rs/crates/spacetime-module/src/ai/stages.rs +++ b/server-rs/crates/spacetime-module/src/ai/stages.rs @@ -1,9 +1,12 @@ use crate::*; use module_ai::{ - generate_ai_result_ref_id, generate_ai_text_chunk_id, normalize_optional_text, - normalize_string_list, + MAX_AI_TASK_TEXT_CHUNKS_PER_STAGE, MAX_AI_TASK_TEXT_OUTPUT_BYTES, generate_ai_result_ref_id, + generate_ai_text_chunk_id, normalize_optional_text, normalize_string_list, + validate_ai_task_snapshot_memory_limits, }; +const AI_TEXT_CHUNK_DELETE_BATCH_SIZE: usize = 256; + #[spacetimedb::table( accessor = ai_task_stage, index(accessor = by_ai_task_stage_task_id, btree(columns = [task_id])), @@ -178,6 +181,9 @@ pub(crate) fn append_ai_text_chunk_tx( if input.sequence == 0 { return Err("ai_text_chunk.sequence 必须大于 0".to_string()); } + if input.delta_text.len() > MAX_AI_TASK_TEXT_OUTPUT_BYTES { + return Err("AI 任务文本输出超过内存上限".to_string()); + } let mut snapshot = get_ai_task_snapshot_tx(ctx, &input.task_id)?; ensure_ai_task_can_transition(snapshot.status)?; @@ -200,7 +206,7 @@ pub(crate) fn append_ai_text_chunk_tx( .ai_text_chunk() .insert(build_ai_text_chunk_row(&chunk)); - let aggregated_text = collect_ai_stage_text_output(ctx, &chunk.task_id, chunk.stage_kind); + let aggregated_text = collect_ai_stage_text_output(ctx, &chunk.task_id, chunk.stage_kind)?; snapshot.status = AiTaskStatus::Running; if snapshot.started_at_micros.is_none() { @@ -215,6 +221,7 @@ pub(crate) fn append_ai_text_chunk_tx( snapshot.updated_at_micros = input.created_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; emit_ai_task_event( ctx, @@ -252,6 +259,7 @@ pub(crate) fn complete_ai_stage_tx( snapshot.updated_at_micros = input.completed_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; emit_ai_task_event( ctx, @@ -285,26 +293,27 @@ pub(crate) fn attach_ai_result_reference_tx( label: normalize_optional_text(input.label), created_at_micros: input.created_at_micros, }; - ctx.db - .ai_result_reference() - .insert(build_ai_result_reference_row(&reference)); - snapshot.result_references.push(reference); snapshot.updated_at_micros = input.created_at_micros; snapshot.version += 1; - persist_ai_task_snapshot(ctx, &snapshot)?; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; let reference = snapshot .result_references .last() + .cloned() .ok_or_else(|| "ai_result_reference 写入后缺少快照".to_string())?; + ctx.db + .ai_result_reference() + .insert(build_ai_result_reference_row(&reference)); + persist_ai_task_snapshot(ctx, &snapshot)?; emit_ai_task_event( ctx, &snapshot, AiTaskEventKind::ResultReferenceAttached, None, None, - Some(build_ai_result_reference_row_id(reference)), + Some(build_ai_result_reference_row_id(&reference)), input.created_at_micros, ); Ok(snapshot) @@ -333,29 +342,63 @@ pub(crate) fn replace_ai_task_stages( } } +pub(crate) fn delete_ai_text_chunks_for_task(ctx: &ReducerContext, task_id: &str) { + loop { + let chunk_row_ids = ctx + .db + .ai_text_chunk() + .by_ai_text_chunk_task_id() + .filter(task_id) + .take(AI_TEXT_CHUNK_DELETE_BATCH_SIZE) + .map(|row| row.text_chunk_row_id.clone()) + .collect::>(); + if chunk_row_ids.is_empty() { + break; + } + let batch_len = chunk_row_ids.len(); + for row_id in chunk_row_ids { + ctx.db.ai_text_chunk().text_chunk_row_id().delete(&row_id); + } + if batch_len < AI_TEXT_CHUNK_DELETE_BATCH_SIZE { + break; + } + } +} + pub(crate) fn collect_ai_stage_text_output( ctx: &ReducerContext, task_id: &str, stage_kind: AiTaskStageKind, -) -> Option { - let mut chunks = ctx +) -> Result, String> { + let mut chunks = Vec::new(); + let mut chunk_count = 0_usize; + let mut aggregated_bytes = 0_usize; + for row in ctx .db .ai_text_chunk() .by_ai_text_chunk_task_id() .filter(task_id) .filter(|row| row.task_id == task_id && row.stage_kind == stage_kind) - .map(|row| build_ai_text_chunk_snapshot_from_row(&row)) - .collect::>(); - chunks.sort_by_key(|chunk| chunk.sequence); + { + chunk_count = chunk_count.saturating_add(1); + if chunk_count > MAX_AI_TASK_TEXT_CHUNKS_PER_STAGE { + return Err("AI 任务文本 chunk 数量超过内存上限".to_string()); + } + aggregated_bytes = aggregated_bytes.saturating_add(row.delta_text.len()); + if aggregated_bytes > MAX_AI_TASK_TEXT_OUTPUT_BYTES { + return Err("AI 任务文本输出超过内存上限".to_string()); + } + chunks.push((row.sequence, row.delta_text.clone())); + } + chunks.sort_by_key(|(sequence, _)| *sequence); - let aggregated = chunks - .into_iter() - .map(|chunk| chunk.delta_text) - .collect::>() - .join(""); + let mut aggregated = String::with_capacity(aggregated_bytes); + for (_, delta) in chunks { + aggregated.push_str(&delta); + } if aggregated.trim().is_empty() { - None + Ok(None) } else { - Some(aggregated) + Ok(Some(aggregated)) } } diff --git a/server-rs/crates/spacetime-module/src/ai/tasks.rs b/server-rs/crates/spacetime-module/src/ai/tasks.rs index 66c0909d1..b6c646789 100644 --- a/server-rs/crates/spacetime-module/src/ai/tasks.rs +++ b/server-rs/crates/spacetime-module/src/ai/tasks.rs @@ -1,5 +1,8 @@ use crate::*; -use module_ai::{INITIAL_AI_TASK_VERSION, normalize_optional_text, validate_task_create_input}; +use module_ai::{ + INITIAL_AI_TASK_VERSION, normalize_optional_text, validate_ai_task_snapshot_memory_limits, + validate_task_create_input, +}; #[spacetimedb::table( accessor = ai_task, @@ -133,6 +136,7 @@ fn create_ai_task_tx( } let task_snapshot = build_ai_task_snapshot_from_create_input(&input); + validate_ai_task_snapshot_memory_limits(&task_snapshot).map_err(str::to_string)?; ctx.db.ai_task().insert(build_ai_task_row(&task_snapshot)); replace_ai_task_stages(ctx, &task_snapshot.task_id, &task_snapshot.stages); emit_ai_task_event( @@ -187,7 +191,9 @@ fn complete_ai_task_tx( snapshot.updated_at_micros = input.completed_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; + delete_ai_text_chunks_for_task(ctx, &snapshot.task_id); emit_ai_task_event( ctx, &snapshot, @@ -218,7 +224,9 @@ fn fail_ai_task_tx( snapshot.updated_at_micros = input.completed_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; + delete_ai_text_chunks_for_task(ctx, &snapshot.task_id); emit_ai_task_event( ctx, &snapshot, @@ -243,7 +251,9 @@ fn cancel_ai_task_tx( snapshot.updated_at_micros = input.completed_at_micros; snapshot.version += 1; + validate_ai_task_snapshot_memory_limits(&snapshot).map_err(str::to_string)?; persist_ai_task_snapshot(ctx, &snapshot)?; + delete_ai_text_chunks_for_task(ctx, &snapshot.task_id); emit_ai_task_event( ctx, &snapshot, diff --git a/server-rs/crates/spacetime-module/src/auth/procedures.rs b/server-rs/crates/spacetime-module/src/auth/procedures.rs index 5823f5450..9f181c018 100644 --- a/server-rs/crates/spacetime-module/src/auth/procedures.rs +++ b/server-rs/crates/spacetime-module/src/auth/procedures.rs @@ -1,4 +1,7 @@ use crate::{ProcedureContext, ReducerContext, SpacetimeType, Table, Timestamp}; +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; use super::tables::{ AuthIdentity, AuthStoreProjectionMeta, RefreshSession, UserAccount, auth_identity, @@ -13,6 +16,9 @@ pub struct AuthStoreProjectionView { pub users: Vec, pub identities: Vec, pub refresh_sessions: Vec, + pub phone_codes: Vec, + pub wechat_states: Vec, + pub base_updated_at_micros: i64, } #[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] @@ -56,6 +62,31 @@ pub struct AuthStoreProjectionRefreshSession { pub last_seen_at: String, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, SpacetimeType)] +pub struct AuthStoreProjectionPhoneCode { + pub phone_number: String, + pub scene: String, + pub verify_code_hash: String, + pub expires_at: String, + pub last_sent_at: String, + pub failed_attempts: u32, + pub provider_out_id: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, SpacetimeType)] +pub struct AuthStoreProjectionWechatState { + pub wechat_state_id: String, + pub state_token: String, + pub redirect_path: String, + pub scene: String, + pub request_user_agent: Option, + pub bind_user_id: Option, + pub expires_at: String, + pub consumed_at: Option, + pub created_at: String, + pub updated_at: String, +} + #[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] pub struct AuthStoreProjectionSyncRecord { pub imported_user_count: u32, @@ -77,12 +108,82 @@ pub struct AuthStoreProjectionSyncProcedureResult { pub error_message: Option, } +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct AuthSessionValidationInput { + pub user_id: String, + pub session_id: String, + pub token_version: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct AuthSessionValidationProcedureResult { + pub active: bool, + pub error_message: Option, +} + +#[spacetimedb::procedure] +pub fn validate_auth_session( + ctx: &mut ProcedureContext, + input: AuthSessionValidationInput, +) -> AuthSessionValidationProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + require_auth_service_identity(tx, caller)?; + validate_auth_session_tx(tx, input.clone()) + }) { + Ok(active) => AuthSessionValidationProcedureResult { + active, + error_message: None, + }, + Err(message) => AuthSessionValidationProcedureResult { + active: false, + error_message: Some(message), + }, + } +} + +fn validate_auth_session_tx( + ctx: &ReducerContext, + input: AuthSessionValidationInput, +) -> Result { + let Some(user) = ctx.db.user_account().user_id().find(&input.user_id) else { + return Ok(false); + }; + if user.token_version != input.token_version { + return Ok(false); + } + + let Some(session) = ctx + .db + .refresh_session() + .session_id() + .find(&input.session_id) + else { + return Ok(false); + }; + if session.user_id != input.user_id || session.revoked_at.is_some() { + return Ok(false); + } + + let expires_at = OffsetDateTime::parse(&session.expires_at, &Rfc3339) + .map_err(|_| "refresh session 过期时间格式非法".to_string())?; + let now = OffsetDateTime::from_unix_timestamp_nanos( + i128::from(ctx.timestamp.to_micros_since_unix_epoch()) * 1_000, + ) + .map_err(|_| "SpacetimeDB 当前时间超出认证时间范围".to_string())?; + Ok(expires_at > now) +} + #[spacetimedb::procedure] pub fn sync_auth_store_projection( ctx: &mut ProcedureContext, input: AuthStoreProjectionView, ) -> AuthStoreProjectionSyncProcedureResult { - match ctx.try_with_tx(|tx| sync_auth_store_projection_tx(tx, input.clone())) { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + require_auth_service_identity(tx, caller)?; + sync_auth_store_projection_tx(tx, input.clone()) + }) { Ok(record) => AuthStoreProjectionSyncProcedureResult { ok: true, record: Some(record), @@ -100,7 +201,11 @@ pub fn sync_auth_store_projection( pub fn export_auth_store_projection_from_tables( ctx: &mut ProcedureContext, ) -> AuthStoreProjectionProcedureResult { - match ctx.try_with_tx(|tx| export_auth_store_projection_from_tables_tx(tx)) { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + require_auth_service_identity(tx, caller)?; + export_auth_store_projection_from_tables_tx(tx) + }) { Ok(record) => AuthStoreProjectionProcedureResult { ok: true, record: Some(record), @@ -114,10 +219,34 @@ pub fn export_auth_store_projection_from_tables( } } +fn require_auth_service_identity( + ctx: &ReducerContext, + caller: crate::Identity, +) -> Result<(), String> { + crate::editor_project_storage::require_editor_generation_runtime_service_identity(ctx, caller) + .map_err(|_| "当前 identity 无权调用认证服务".to_string()) +} + fn sync_auth_store_projection_tx( ctx: &ReducerContext, input: AuthStoreProjectionView, ) -> Result { + let current_updated_at_micros = ctx + .db + .auth_store_projection_meta() + .meta_id() + .find(&AUTH_STORE_PROJECTION_META_ID.to_string()) + .map(|row| row.updated_at.to_micros_since_unix_epoch()); + ensure_auth_projection_base_version( + input.base_updated_at_micros, + current_updated_at_micros.unwrap_or_default(), + )?; + ensure_newer_auth_projection_version(current_updated_at_micros, input.updated_at_micros)?; + let phone_codes_json = serde_json::to_string(&input.phone_codes) + .map_err(|error| format!("序列化短信验证码投影失败：{error}"))?; + let wechat_states_json = serde_json::to_string(&input.wechat_states) + .map_err(|error| format!("序列化微信授权 state 投影失败：{error}"))?; + let user_ids = input .users .iter() @@ -246,7 +375,12 @@ fn sync_auth_store_projection_tx( imported_refresh_session_count += 1; } - upsert_auth_projection_meta(ctx, input.updated_at_micros); + upsert_auth_projection_meta( + ctx, + input.updated_at_micros, + phone_codes_json, + wechat_states_json, + ); Ok(AuthStoreProjectionSyncRecord { imported_user_count, @@ -255,16 +389,66 @@ fn sync_auth_store_projection_tx( }) } +/// Full projections are emitted by API-local auth worksets. The base version +/// is read immediately before the write and checked in this same transaction, +/// so a stale API node cannot replace data written by another node meanwhile. +/// The timestamp remains a diagnostic/monotonic watermark for accepted writes. +fn ensure_auth_projection_base_version( + expected_updated_at_micros: i64, + current_updated_at_micros: i64, +) -> Result<(), String> { + if expected_updated_at_micros != current_updated_at_micros { + return Err(format!( + "认证投影基线版本冲突：请求基线 {expected_updated_at_micros} 不等于当前版本 {current_updated_at_micros}" + )); + } + + Ok(()) +} + +fn ensure_newer_auth_projection_version( + current_updated_at_micros: Option, + incoming_updated_at_micros: i64, +) -> Result<(), String> { + if let Some(current_updated_at_micros) = current_updated_at_micros { + if incoming_updated_at_micros <= current_updated_at_micros { + return Err(format!( + "认证投影版本冲突：请求版本 {incoming_updated_at_micros} 不晚于当前版本 {current_updated_at_micros}" + )); + } + } + + Ok(()) +} + fn export_auth_store_projection_from_tables_tx( ctx: &ReducerContext, ) -> Result { - let updated_at_micros = ctx + let meta = ctx .db .auth_store_projection_meta() .meta_id() - .find(&AUTH_STORE_PROJECTION_META_ID.to_string()) + .find(&AUTH_STORE_PROJECTION_META_ID.to_string()); + let updated_at_micros = meta + .as_ref() .map(|row| row.updated_at.to_micros_since_unix_epoch()) .unwrap_or(0); + let phone_codes = meta + .as_ref() + .and_then(|row| row.phone_codes_json.as_deref()) + .filter(|value| !value.trim().is_empty()) + .map(serde_json::from_str) + .transpose() + .map_err(|error| format!("解析短信验证码投影失败：{error}"))? + .unwrap_or_default(); + let wechat_states = meta + .as_ref() + .and_then(|row| row.wechat_states_json.as_deref()) + .filter(|value| !value.trim().is_empty()) + .map(serde_json::from_str) + .transpose() + .map_err(|error| format!("解析微信授权 state 投影失败：{error}"))? + .unwrap_or_default(); let users = ctx .db .user_account() @@ -317,10 +501,13 @@ fn export_auth_store_projection_from_tables_tx( .collect(); Ok(AuthStoreProjectionView { + base_updated_at_micros: updated_at_micros, updated_at_micros, users, identities, refresh_sessions, + phone_codes, + wechat_states, }) } @@ -363,7 +550,12 @@ fn delete_missing_refresh_sessions( } } -fn upsert_auth_projection_meta(ctx: &ReducerContext, updated_at_micros: i64) { +fn upsert_auth_projection_meta( + ctx: &ReducerContext, + updated_at_micros: i64, + phone_codes_json: String, + wechat_states_json: String, +) { let meta_id = AUTH_STORE_PROJECTION_META_ID.to_string(); if ctx .db @@ -382,6 +574,8 @@ fn upsert_auth_projection_meta(ctx: &ReducerContext, updated_at_micros: i64) { .insert(AuthStoreProjectionMeta { meta_id, updated_at: Timestamp::from_micros_since_unix_epoch(updated_at_micros), + phone_codes_json: Some(phone_codes_json), + wechat_states_json: Some(wechat_states_json), }); } @@ -465,4 +659,19 @@ mod tests { ); assert_eq!(created_at, "2026-07-01T00:00:00Z"); } + + #[test] + fn auth_projection_version_must_advance_monotonically() { + assert!(ensure_newer_auth_projection_version(None, 1).is_ok()); + assert!(ensure_newer_auth_projection_version(Some(10), 11).is_ok()); + assert!(ensure_newer_auth_projection_version(Some(10), 10).is_err()); + assert!(ensure_newer_auth_projection_version(Some(10), 9).is_err()); + } + + #[test] + fn auth_projection_base_version_must_match_current_version() { + assert!(ensure_auth_projection_base_version(0, 0).is_ok()); + assert!(ensure_auth_projection_base_version(10, 10).is_ok()); + assert!(ensure_auth_projection_base_version(9, 10).is_err()); + } } diff --git a/server-rs/crates/spacetime-module/src/auth/tables.rs b/server-rs/crates/spacetime-module/src/auth/tables.rs index e9f6e6c11..49251e549 100644 --- a/server-rs/crates/spacetime-module/src/auth/tables.rs +++ b/server-rs/crates/spacetime-module/src/auth/tables.rs @@ -5,6 +5,10 @@ pub struct AuthStoreProjectionMeta { #[primary_key] pub(crate) meta_id: String, pub(crate) updated_at: Timestamp, + #[default(None::)] + pub(crate) phone_codes_json: Option, + #[default(None::)] + pub(crate) wechat_states_json: Option, } #[spacetimedb::table( diff --git a/server-rs/crates/spacetime-module/src/external_generation.rs b/server-rs/crates/spacetime-module/src/external_generation.rs index 9cfa4fa13..5e93b162c 100644 --- a/server-rs/crates/spacetime-module/src/external_generation.rs +++ b/server-rs/crates/spacetime-module/src/external_generation.rs @@ -23,6 +23,7 @@ const MAX_EXTERNAL_GENERATION_REQUEST_PROMPT_CHARS: usize = 2_048; const MAX_EXTERNAL_GENERATION_ERROR_MESSAGE_CHARS: usize = 2_048; const MAX_EXTERNAL_GENERATION_WARNING_MESSAGE_CHARS: usize = 2_048; const MAX_EXTERNAL_GENERATION_MAINTENANCE_BATCH_SIZE: u32 = 25; +const EXTERNAL_GENERATION_EVENT_DELETE_BATCH_SIZE: usize = 256; const INLINE_MEDIA_REMOVED_PLACEHOLDER: &str = "[inline-media-removed]"; const INLINE_MEDIA_ERROR_REDACTED_MESSAGE: &str = "外部生成失败（错误详情含内联媒体引用，已省略）"; const INLINE_MEDIA_WARNING_REDACTED_MESSAGE: &str = @@ -97,6 +98,10 @@ pub struct ExternalGenerationJob { accessor = by_external_generation_job_event_job_id, btree(columns = [job_id, created_at]) ), + index( + accessor = by_external_generation_job_event_job_id_only, + btree(columns = [job_id]) + ), index( accessor = by_external_generation_job_event_owner, btree(columns = [owner_user_id, created_at]) @@ -378,6 +383,29 @@ pub struct ExternalGenerationJobPayloadCompactionProcedureResult { pub error_message: Option, } +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobRetentionInput { + pub source_module: String, + pub limit: u32, + pub cursor_job_id: Option, + pub completed_before_micros: i64, + pub dry_run: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +pub struct ExternalGenerationJobRetentionProcedureResult { + pub ok: bool, + pub dry_run: bool, + pub scanned_count: u64, + pub selected_count: u32, + pub deleted_job_count: u32, + pub deleted_summary_count: u32, + pub deleted_event_count: u32, + pub next_cursor_job_id: Option, + pub has_more: bool, + pub error_message: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] pub struct ExternalGenerationQueueStatsSnapshot { pub pending_count: u32, @@ -676,6 +704,21 @@ pub fn compact_external_generation_job_payloads_and_return( } } +#[spacetimedb::procedure] +pub fn prune_external_generation_job_history_and_return( + ctx: &mut ProcedureContext, + input: ExternalGenerationJobRetentionInput, +) -> ExternalGenerationJobRetentionProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + crate::migration::require_migration_operator(tx, caller)?; + prune_external_generation_job_history_tx(tx, input.clone()) + }) { + Ok(result) => result, + Err(message) => failed_external_generation_job_retention_result(input.dry_run, message), + } +} + #[spacetimedb::procedure] pub fn get_external_generation_queue_stats_and_return( ctx: &mut ProcedureContext, @@ -888,6 +931,8 @@ fn finalize_external_generation_job_after_lease_exhaustion( row.price_mud_points, failed_at, "final_attempt_lease_expired", + &row.job_kind, + &row.source_entity_id, )?; let row = mark_external_generation_job_lease_exhausted(row, failed_at, refund_ledger_id); persist_external_generation_job_row(ctx, row.clone()); @@ -1367,6 +1412,124 @@ fn compact_external_generation_job_payloads_tx( }) } +fn prune_external_generation_job_history_tx( + ctx: &ReducerContext, + input: ExternalGenerationJobRetentionInput, +) -> Result { + let source_module = input.source_module.trim().to_string(); + validate_required("external_generation_job.source_module", &source_module)?; + let now_micros = ctx.timestamp.to_micros_since_unix_epoch(); + if input.completed_before_micros > now_micros { + return Err( + "external_generation_job.completed_before_micros 不能晚于数据库当前时间".to_string(), + ); + } + + let cursor_job_id = input + .cursor_job_id + .as_deref() + .and_then(normalize_optional_text); + let limit = input + .limit + .clamp(1, MAX_EXTERNAL_GENERATION_MAINTENANCE_BATCH_SIZE) as usize; + let cursor_range = external_generation_job_maintenance_cursor_range(cursor_job_id.as_deref()); + let cursor_to_skip = cursor_job_id.clone(); + // 若 cursor 对应的任务仍存在，说明上一事务只删完了事件的一部分，或 dry-run + // 尚未执行 apply；下一次必须包含该任务继续清理，成功删除后它会自然消失。 + let include_existing_cursor = cursor_job_id.as_deref().is_some_and(|cursor| { + ctx.db + .external_generation_job() + .job_id() + .find(&cursor.to_string()) + .is_some() + }); + let rows = ctx + .db + .external_generation_job() + .by_external_generation_job_source_cursor() + .filter((source_module.as_str(), cursor_range)) + .filter(move |row| { + cursor_to_skip + .as_deref() + .is_none_or(|cursor| row.job_id != cursor || include_existing_cursor) + }); + let (job_ids, next_cursor_job_id, has_more, scanned_count) = + select_external_generation_job_ids_for_maintenance(rows, limit, |row| { + ctx.db + .external_generation_job_summary() + .job_id() + .find(&row.job_id) + .is_some_and(|summary| { + is_external_generation_job_retention_candidate( + row, + &summary, + &source_module, + input.completed_before_micros, + ) + }) + }); + + let mut deleted_job_count = 0u32; + let mut deleted_summary_count = 0u32; + let mut deleted_event_count = 0u32; + let mut pending_event_cursor_job_id = None; + if !input.dry_run { + for job_id in &job_ids { + let Some(row) = ctx.db.external_generation_job().job_id().find(job_id) else { + continue; + }; + let Some(summary) = ctx + .db + .external_generation_job_summary() + .job_id() + .find(job_id) + else { + continue; + }; + if !is_external_generation_job_retention_candidate( + &row, + &summary, + &source_module, + input.completed_before_micros, + ) { + continue; + } + + let (deleted_for_job, has_more_events) = + delete_external_generation_job_events_for_job(ctx, job_id); + deleted_event_count = deleted_event_count.saturating_add(deleted_for_job); + if has_more_events { + pending_event_cursor_job_id = Some(job_id.clone()); + break; + } + ctx.db + .external_generation_job_summary() + .job_id() + .delete(job_id); + deleted_summary_count = deleted_summary_count.saturating_add(1); + ctx.db.external_generation_job().job_id().delete(job_id); + deleted_job_count = deleted_job_count.saturating_add(1); + } + } + + let (next_cursor_job_id, has_more) = pending_event_cursor_job_id + .map(|job_id| (Some(job_id), true)) + .unwrap_or((next_cursor_job_id, has_more)); + + Ok(ExternalGenerationJobRetentionProcedureResult { + ok: true, + dry_run: input.dry_run, + scanned_count, + selected_count: job_ids.len() as u32, + deleted_job_count, + deleted_summary_count, + deleted_event_count, + next_cursor_job_id, + has_more, + error_message: None, + }) +} + fn renew_external_generation_job_lease_tx( ctx: &ReducerContext, input: ExternalGenerationJobRenewLeaseInput, @@ -1437,6 +1600,8 @@ fn fail_external_generation_job_tx( row.price_mud_points, failed_at, "worker_attempt_failed", + &row.job_kind, + &row.source_entity_id, )?; let requested_refund_ledger_id = input .refund_ledger_id @@ -1797,6 +1962,25 @@ fn should_compact_external_generation_job_payloads( }) } +fn is_external_generation_job_retention_candidate( + row: &ExternalGenerationJob, + summary: &ExternalGenerationJobSummary, + source_module: &str, + completed_before_micros: i64, +) -> bool { + row.source_module.trim() == source_module.trim() + && summary.job_id == row.job_id + && summary.status == row.status + && is_external_generation_job_terminal(row) + && is_external_generation_job_summary_terminal(summary) + && summary.notification_acknowledged_at.is_some() + && row + .completed_at + .unwrap_or(row.updated_at) + .to_micros_since_unix_epoch() + <= completed_before_micros +} + fn external_generation_job_maintenance_cursor_range( cursor_job_id: Option<&str>, ) -> RangeFrom<&str> { @@ -1832,6 +2016,36 @@ fn select_external_generation_job_ids_for_maintenance( ) } +fn delete_external_generation_job_events_for_job( + ctx: &ReducerContext, + job_id: &str, +) -> (u32, bool) { + // 每次 procedure 最多删除一个固定批次；若仍有事件，保留 job/summary，调用方 + // 通过同一个 job cursor 重试，避免单个任务把整段审计历史塞进一个事务写集。 + let event_ids = ctx + .db + .external_generation_job_event() + .by_external_generation_job_event_job_id_only() + .filter(job_id) + .take(EXTERNAL_GENERATION_EVENT_DELETE_BATCH_SIZE + 1) + .map(|event| event.event_id.clone()) + .collect::>(); + let has_more = event_ids.len() > EXTERNAL_GENERATION_EVENT_DELETE_BATCH_SIZE; + let deleted_count = event_ids + .len() + .min(EXTERNAL_GENERATION_EVENT_DELETE_BATCH_SIZE) as u32; + for event_id in event_ids + .into_iter() + .take(EXTERNAL_GENERATION_EVENT_DELETE_BATCH_SIZE) + { + ctx.db + .external_generation_job_event() + .event_id() + .delete(&event_id); + } + (deleted_count, has_more) +} + fn count_external_generation_job_summaries_for_owner( ctx: &ReducerContext, owner_user_id: &str, @@ -2544,6 +2758,24 @@ fn failed_external_generation_job_payload_compaction_result( } } +fn failed_external_generation_job_retention_result( + dry_run: bool, + message: String, +) -> ExternalGenerationJobRetentionProcedureResult { + ExternalGenerationJobRetentionProcedureResult { + ok: false, + dry_run, + scanned_count: 0, + selected_count: 0, + deleted_job_count: 0, + deleted_summary_count: 0, + deleted_event_count: 0, + next_cursor_job_id: None, + has_more: false, + error_message: Some(message), + } +} + fn validate_required(field: &str, value: &str) -> Result<(), String> { if value.trim().is_empty() { return Err(format!("{field} 不能为空")); @@ -3434,6 +3666,86 @@ mod tests { )); } + #[test] + fn retention_only_selects_acknowledged_terminal_rows_before_cutoff() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_COMPLETED); + row.source_module = EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE.to_string(); + row.completed_at = Some(micros(1_000)); + row.updated_at = micros(1_000); + let mut summary = build_external_generation_job_summary_row(&row, None); + summary.notification_acknowledged_at = Some(micros(2_000)); + + assert!(is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + summary.notification_acknowledged_at = None; + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + summary.notification_acknowledged_at = Some(micros(2_000)); + row.status = EXTERNAL_GENERATION_STATUS_RUNNING.to_string(); + summary.status = EXTERNAL_GENERATION_STATUS_RUNNING.to_string(); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + row.status = EXTERNAL_GENERATION_STATUS_COMPLETED.to_string(); + summary.status = EXTERNAL_GENERATION_STATUS_COMPLETED.to_string(); + row.completed_at = Some(micros(1_001)); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + row.completed_at = Some(micros(1_000)); + row.source_module = "puzzle".to_string(); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + } + + #[test] + fn retention_rejects_mismatched_summary_identity_or_status() { + let mut row = external_generation_job_fixture(EXTERNAL_GENERATION_STATUS_FAILED); + row.source_module = EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE.to_string(); + row.completed_at = Some(micros(1_000)); + let mut summary = build_external_generation_job_summary_row(&row, None); + summary.notification_acknowledged_at = Some(micros(2_000)); + + summary.job_id = "different-job".to_string(); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + + summary.job_id = row.job_id.clone(); + summary.status = EXTERNAL_GENERATION_STATUS_CANCELLED.to_string(); + assert!(!is_external_generation_job_retention_candidate( + &row, + &summary, + EXTERNAL_GENERATION_EDITOR_SOURCE_MODULE, + 1_000, + )); + } + #[test] fn maintenance_selector_bounds_scanned_rows_and_advances_by_last_scanned_job() { let rows = (1..=4).map(|index| { diff --git a/server-rs/crates/spacetime-module/src/migration.rs b/server-rs/crates/spacetime-module/src/migration.rs index 4b6993bf7..32e02063c 100644 --- a/server-rs/crates/spacetime-module/src/migration.rs +++ b/server-rs/crates/spacetime-module/src/migration.rs @@ -199,6 +199,7 @@ macro_rules! migration_tables { profile_wallet_ledger, profile_wallet_consumption_total, asset_operation_wallet_settlement, + profile_wallet_refund_outbox, profile_wallet_config, analytics_date_dimension, tracking_event, diff --git a/server-rs/crates/spacetime-module/src/runtime/active/profile.rs b/server-rs/crates/spacetime-module/src/runtime/active/profile.rs index e53d8e6b9..4fd3e8818 100644 --- a/server-rs/crates/spacetime-module/src/runtime/active/profile.rs +++ b/server-rs/crates/spacetime-module/src/runtime/active/profile.rs @@ -19,6 +19,8 @@ const PROFILE_RECHARGE_ORDER_EXPIRATION_CHECK_LIMIT_DEFAULT: u32 = 50; const PROFILE_RECHARGE_ORDER_EXPIRATION_CHECK_LIMIT_MAX: u32 = 200; const ASSET_OPERATION_CONSUME_LEDGER_PREFIX: &str = "asset_operation_consume:"; const ASSET_OPERATION_REFUND_LEDGER_PREFIX: &str = "asset_operation_refund:"; +const PROFILE_WALLET_REFUND_OUTBOX_STATUS_PENDING: &str = "pending"; +const PROFILE_WALLET_REFUND_OUTBOX_MAX_BATCH_SIZE: u32 = 100; #[spacetimedb::table(accessor = profile_dashboard_state)] pub struct ProfileDashboardState { @@ -82,6 +84,34 @@ pub struct AssetOperationWalletSettlement { pub(crate) settled_at: Timestamp, } +#[spacetimedb::table( + accessor = profile_wallet_refund_outbox, + index( + accessor = by_profile_wallet_refund_outbox_status_available, + btree(columns = [status, available_at]) + ) +)] +#[derive(Clone)] +pub struct ProfileWalletRefundOutbox { + #[primary_key] + pub(crate) refund_ledger_id: String, + pub(crate) consume_ledger_id: String, + pub(crate) owner_user_id: String, + pub(crate) amount: u64, + pub(crate) created_at: Timestamp, + pub(crate) asset_kind: String, + pub(crate) asset_id: String, + pub(crate) settlement_reason: String, + pub(crate) external_generation_job_id: Option, + pub(crate) external_generation_claim_attempt: Option, + pub(crate) status: String, + pub(crate) available_at: Timestamp, + pub(crate) attempts: u32, + pub(crate) last_error: Option, + pub(crate) last_attempted_at: Option, + pub(crate) last_worker_id: Option, +} + #[spacetimedb::table(accessor = profile_wallet_config)] #[derive(Clone)] pub struct ProfileWalletConfig { @@ -1159,6 +1189,82 @@ pub fn refund_profile_wallet_points_and_return( } } +#[spacetimedb::procedure] +pub fn enqueue_profile_wallet_refund_outbox_and_return( + ctx: &mut ProcedureContext, + input: RuntimeProfileWalletRefundOutboxEnqueueInput, +) -> RuntimeProfileWalletRefundOutboxProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + crate::editor_project_storage::require_editor_generation_runtime_service_identity( + tx, caller, + )?; + enqueue_profile_wallet_refund_outbox_tx( + tx, + input.owner_user_id.clone(), + input.amount, + input.refund_ledger_id.clone(), + input.created_at_micros, + input.asset_kind.clone(), + input.asset_id.clone(), + input.settlement_reason.clone(), + input.external_generation_job_id.clone(), + input.external_generation_claim_attempt, + ) + .map(|enqueued| u32::from(enqueued)) + }) { + Ok(enqueued_count) => RuntimeProfileWalletRefundOutboxProcedureResult { + ok: true, + enqueued_count, + processed_count: 0, + retry_count: 0, + failed_count: 0, + error_message: None, + }, + Err(message) => RuntimeProfileWalletRefundOutboxProcedureResult { + ok: false, + enqueued_count: 0, + processed_count: 0, + retry_count: 0, + failed_count: 0, + error_message: Some(message), + }, + } +} + +#[spacetimedb::procedure] +pub fn process_profile_wallet_refund_outbox_and_return( + ctx: &mut ProcedureContext, + input: RuntimeProfileWalletRefundOutboxProcessInput, +) -> RuntimeProfileWalletRefundOutboxProcedureResult { + let caller = ctx.sender(); + match ctx.try_with_tx(|tx| { + crate::editor_project_storage::require_editor_generation_runtime_service_identity( + tx, caller, + )?; + process_profile_wallet_refund_outbox_tx(tx, input.clone()) + }) { + Ok((processed_count, retry_count, failed_count)) => { + RuntimeProfileWalletRefundOutboxProcedureResult { + ok: true, + enqueued_count: 0, + processed_count, + retry_count, + failed_count, + error_message: None, + } + } + Err(message) => RuntimeProfileWalletRefundOutboxProcedureResult { + ok: false, + enqueued_count: 0, + processed_count: 0, + retry_count: 0, + failed_count: 0, + error_message: Some(message), + }, + } +} + // play stats 与 dashboard 共用 dashboard projection 的 total_play_time / updated_at，避免 Axum 侧拼装。 #[cfg(any())] #[spacetimedb::procedure] @@ -2827,6 +2933,45 @@ mod tests { ); } + #[test] + fn refund_outbox_idempotency_allows_recovery_reason_to_change() { + let row = ProfileWalletRefundOutbox { + refund_ledger_id: "asset_operation_refund:external_generation_job:job-1:attempt:1" + .to_string(), + consume_ledger_id: "asset_operation_consume:external_generation_job:job-1:attempt:1" + .to_string(), + owner_user_id: "user-1".to_string(), + amount: 37, + created_at: Timestamp::from_micros_since_unix_epoch(1), + asset_kind: "editor-image".to_string(), + asset_id: "asset-1".to_string(), + settlement_reason: "worker_attempt_failed".to_string(), + external_generation_job_id: Some("job-1".to_string()), + external_generation_claim_attempt: Some(1), + status: PROFILE_WALLET_REFUND_OUTBOX_STATUS_PENDING.to_string(), + available_at: Timestamp::from_micros_since_unix_epoch(1), + attempts: 0, + last_error: None, + last_attempted_at: None, + last_worker_id: None, + }; + + assert!( + validate_profile_wallet_refund_outbox_fact( + &row, + &row.refund_ledger_id, + &row.consume_ledger_id, + &row.owner_user_id, + row.amount, + &row.asset_kind, + &row.asset_id, + Some("job-1"), + Some(1), + ) + .is_ok() + ); + } + #[test] fn wallet_idempotent_replay_requires_matching_user_amount_and_source() { let existing = asset_operation_wallet_ledger( @@ -9125,6 +9270,250 @@ pub(crate) fn grant_profile_wallet_points_with_metadata( ) } +pub(crate) fn enqueue_profile_wallet_refund_outbox_tx( + ctx: &ReducerContext, + owner_user_id: String, + amount: u64, + refund_ledger_id: String, + created_at_micros: i64, + asset_kind: String, + asset_id: String, + settlement_reason: String, + external_generation_job_id: Option, + external_generation_claim_attempt: Option, +) -> Result { + let owner_user_id = owner_user_id.trim().to_string(); + if owner_user_id.is_empty() { + return Err("资产操作退款用户不能为空".to_string()); + } + if amount == 0 { + return Err("资产操作退款金额必须大于 0".to_string()); + } + let refund_ledger_id = refund_ledger_id.trim().to_string(); + let consume_ledger_id = asset_operation_consume_ledger_id(&refund_ledger_id)?; + let asset_kind = asset_kind.trim().to_string(); + if asset_kind.is_empty() { + return Err("资产操作退款 asset_kind 不能为空".to_string()); + } + let asset_id = asset_id.trim().to_string(); + if asset_id.is_empty() { + return Err("资产操作退款 asset_id 不能为空".to_string()); + } + let settlement_reason = settlement_reason.trim().to_string(); + if settlement_reason.is_empty() { + return Err("资产操作退款 settlement_reason 不能为空".to_string()); + } + let external_generation_job_id = normalize_optional_text(external_generation_job_id); + if external_generation_job_id.is_some() != external_generation_claim_attempt.is_some() { + return Err("资产操作退款 outbox 任务与 attempt 必须成对提供".to_string()); + } + let created_at = Timestamp::from_micros_since_unix_epoch(created_at_micros); + let disposition = resolve_asset_operation_refund_disposition_from_ledger( + ctx, + owner_user_id.as_str(), + i64::try_from(amount).map_err(|_| "资产操作退款金额超出范围".to_string())?, + refund_ledger_id.as_str(), + )?; + + if matches!(disposition, AssetOperationRefundDisposition::Noop) { + if let Some(existing) = ctx + .db + .profile_wallet_refund_outbox() + .refund_ledger_id() + .find(&refund_ledger_id) + { + validate_profile_wallet_refund_outbox_fact( + &existing, + &refund_ledger_id, + &consume_ledger_id, + &owner_user_id, + amount, + &asset_kind, + &asset_id, + external_generation_job_id.as_deref(), + external_generation_claim_attempt, + )?; + ctx.db + .profile_wallet_refund_outbox() + .refund_ledger_id() + .delete(&refund_ledger_id); + } + return Ok(false); + } + if matches!(disposition, AssetOperationRefundDisposition::RecordIntent) { + record_asset_operation_wallet_settlement( + ctx, + owner_user_id.as_str(), + i64::try_from(amount).map_err(|_| "资产操作退款金额超出范围".to_string())?, + refund_ledger_id.as_str(), + created_at, + )?; + // consume 尚不可见时，settlement 是取消 intent；不应再创建会被 worker + // 误认为可退款的 outbox 行。迟到 consume 会在同一 resolver 中被拒绝。 + return Ok(false); + } + + let existing = ctx + .db + .profile_wallet_refund_outbox() + .refund_ledger_id() + .find(&refund_ledger_id); + if let Some(existing) = existing { + validate_profile_wallet_refund_outbox_fact( + &existing, + &refund_ledger_id, + &consume_ledger_id, + &owner_user_id, + amount, + &asset_kind, + &asset_id, + external_generation_job_id.as_deref(), + external_generation_claim_attempt, + )?; + let mut existing = existing; + existing.status = PROFILE_WALLET_REFUND_OUTBOX_STATUS_PENDING.to_string(); + existing.available_at = ctx.timestamp; + existing.last_error = None; + ctx.db + .profile_wallet_refund_outbox() + .refund_ledger_id() + .update(existing); + return Ok(false); + } + + ctx.db + .profile_wallet_refund_outbox() + .insert(ProfileWalletRefundOutbox { + refund_ledger_id, + consume_ledger_id, + owner_user_id, + amount, + created_at, + asset_kind, + asset_id, + settlement_reason, + external_generation_job_id, + external_generation_claim_attempt, + status: PROFILE_WALLET_REFUND_OUTBOX_STATUS_PENDING.to_string(), + available_at: ctx.timestamp, + attempts: 0, + last_error: None, + last_attempted_at: None, + last_worker_id: None, + }); + Ok(true) +} + +fn validate_profile_wallet_refund_outbox_fact( + row: &ProfileWalletRefundOutbox, + expected_refund_ledger_id: &str, + expected_consume_ledger_id: &str, + expected_owner_user_id: &str, + expected_amount: u64, + expected_asset_kind: &str, + expected_asset_id: &str, + expected_external_generation_job_id: Option<&str>, + expected_external_generation_claim_attempt: Option, +) -> Result<(), String> { + if row.refund_ledger_id != expected_refund_ledger_id + || row.consume_ledger_id != expected_consume_ledger_id + || row.owner_user_id != expected_owner_user_id + || row.amount != expected_amount + { + return Err("资产操作退款 outbox 金融事实不匹配".to_string()); + } + if row.asset_kind != expected_asset_kind || row.asset_id != expected_asset_id { + return Err("资产操作退款 outbox 资源事实不匹配".to_string()); + } + // settlement_reason is diagnostic context, not an idempotency fact. A failed + // attempt can be observed first by its failure transaction and later by stale + // attempt recovery, which legitimately use different reason labels. + if row.external_generation_job_id.as_deref() != expected_external_generation_job_id + || row.external_generation_claim_attempt != expected_external_generation_claim_attempt + { + return Err("资产操作退款 outbox 任务 attempt 不匹配".to_string()); + } + Ok(()) +} + +fn process_profile_wallet_refund_outbox_tx( + ctx: &ReducerContext, + input: RuntimeProfileWalletRefundOutboxProcessInput, +) -> Result<(u32, u32, u32), String> { + let worker_id = input.worker_id.trim(); + if worker_id.is_empty() { + return Err("退款 outbox worker_id 不能为空".to_string()); + } + let limit = input + .limit + .clamp(1, PROFILE_WALLET_REFUND_OUTBOX_MAX_BATCH_SIZE); + let now = ctx.timestamp; + let mut rows = ctx + .db + .profile_wallet_refund_outbox() + .by_profile_wallet_refund_outbox_status_available() + .filter(&PROFILE_WALLET_REFUND_OUTBOX_STATUS_PENDING.to_string()) + .filter(|row| row.available_at <= now) + .collect::>(); + rows.sort_by(|left, right| { + left.available_at + .cmp(&right.available_at) + .then_with(|| left.created_at.cmp(&right.created_at)) + .then_with(|| left.refund_ledger_id.cmp(&right.refund_ledger_id)) + }); + + let mut processed_count: u32 = 0; + let mut retry_count: u32 = 0; + let mut failed_count: u32 = 0; + for mut row in rows.into_iter().take(limit as usize) { + row.attempts = row.attempts.saturating_add(1); + row.last_attempted_at = Some(now); + row.last_worker_id = Some(worker_id.to_string()); + let metadata_json = serde_json::json!({ + "externalGenerationJobId": &row.external_generation_job_id, + "externalGenerationClaimAttempt": row.external_generation_claim_attempt, + "assetKind": &row.asset_kind, + "assetId": &row.asset_id, + "settlementReason": &row.settlement_reason, + "refundOutboxWorkerId": worker_id, + }) + .to_string(); + let result = apply_profile_wallet_adjustment( + ctx, + RuntimeProfileWalletAdjustmentInput { + user_id: row.owner_user_id.clone(), + amount: row.amount, + ledger_id: row.refund_ledger_id.clone(), + created_at_micros: row.created_at.to_micros_since_unix_epoch(), + metadata_json, + }, + RuntimeProfileWalletLedgerSourceType::AssetOperationRefund, + false, + ); + match result { + Ok(_) => { + ctx.db + .profile_wallet_refund_outbox() + .refund_ledger_id() + .delete(&row.refund_ledger_id); + processed_count = processed_count.saturating_add(1); + } + Err(error) => { + row.available_at = + now + std::time::Duration::from_secs(2u64.saturating_pow(row.attempts.min(10))); + row.last_error = Some(error); + ctx.db + .profile_wallet_refund_outbox() + .refund_ledger_id() + .update(row); + retry_count = retry_count.saturating_add(1); + failed_count = failed_count.saturating_add(1); + } + } + } + Ok((processed_count, retry_count, failed_count)) +} + fn apply_profile_wallet_adjustment( ctx: &ReducerContext, input: RuntimeProfileWalletAdjustmentInput, @@ -9242,6 +9631,8 @@ pub(crate) fn settle_external_generation_attempt_refund( amount: u64, settled_at: Timestamp, settlement_reason: &str, + asset_kind: &str, + asset_id: &str, ) -> Result, String> { if amount == 0 { return Ok(None); @@ -9250,22 +9641,17 @@ pub(crate) fn settle_external_generation_attempt_refund( "{ASSET_OPERATION_REFUND_LEDGER_PREFIX}external_generation_job:{}:attempt:{attempt}", job_id.trim() ); - apply_profile_wallet_adjustment( + enqueue_profile_wallet_refund_outbox_tx( ctx, - RuntimeProfileWalletAdjustmentInput { - user_id: user_id.to_string(), - amount, - ledger_id: refund_ledger_id.clone(), - created_at_micros: settled_at.to_micros_since_unix_epoch(), - metadata_json: serde_json::json!({ - "externalGenerationJobId": job_id.trim(), - "claimAttempt": attempt, - "settlementReason": settlement_reason, - }) - .to_string(), - }, - RuntimeProfileWalletLedgerSourceType::AssetOperationRefund, - false, + user_id.to_string(), + amount, + refund_ledger_id.clone(), + settled_at.to_micros_since_unix_epoch(), + asset_kind.to_string(), + asset_id.to_string(), + settlement_reason.to_string(), + Some(job_id.trim().to_string()), + Some(attempt), )?; Ok(Some(refund_ledger_id)) }