From 9ae3f539899aa3f79b90a2c5bb927a52ee07b816 Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 16 Jul 2026 18:00:06 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E5=96=84SpacetimeDB=E9=80=90=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=A2=9E=E9=87=8F=E5=A4=87=E4=BB=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增data-dir逐文件CAS基线、history归档与安全清理 发布并校验OSS latest pointer，支持异机自动恢复 接入dev和release可选的files-history定时备份profile 补齐备份测试、生产门禁、环境示例与运维文档 --- deploy/env/api-server.env.example | 6 + ...rrative-database-backup-files-history.conf | 3 + .../shared-memory/decision-log.md | 13 + docs/project-memory/shared-memory/pitfalls.md | 8 + ...发运维】本地开发验证与生产运维-2026-05-15.md | 56 +- .../Jenkinsfile.production-server-provision | 10 + scripts/check-database-backup-to-oss.mjs | 895 +++++++- scripts/check-production-ops-guardrails.mjs | 71 + scripts/database-backup-to-oss.mjs | 1992 ++++++++++++++++- scripts/jenkins-server-provision.sh | 71 +- 10 files changed, 3052 insertions(+), 73 deletions(-) create mode 100644 deploy/systemd/genarrative-database-backup-files-history.conf diff --git a/deploy/env/api-server.env.example b/deploy/env/api-server.env.example index 617553225..0b0b38798 100644 --- a/deploy/env/api-server.env.example +++ b/deploy/env/api-server.env.example @@ -157,8 +157,14 @@ GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET= GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT=oss-cn-shanghai.aliyuncs.com GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX=database-backups GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL=false +# 可选：files 为逐文件 CAS + catalog，不生成 tar.gz；archive 保留旧全量压缩包兼容行为。 +GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT=archive # 可选：显式要求备份工作目录所在文件系统至少保留的可用空间；为空时按数据目录大小 + 安全余量估算。 GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES= +# archive history 模式持久化已验真 full baseline 与追加批次；files 模式改用 work-dir 下的 files state。 +GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE=/var/lib/genarrative/database-backups/genarrative-prod-history-state.json +# 仅 archive history 首次从 uploadStatus=uploaded 的 full manifest 初始化 state 时设置；初始化后可留空。 +GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST= # 可选：定时 / publish 前备份使用独立最小权限 AccessKey；为空时回退 ALIYUN_OSS_ACCESS_KEY_*。 GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID= GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET= diff --git a/deploy/systemd/genarrative-database-backup-files-history.conf b/deploy/systemd/genarrative-database-backup-files-history.conf new file mode 100644 index 000000000..512898b80 --- /dev/null +++ b/deploy/systemd/genarrative-database-backup-files-history.conf @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode history --work-dir /var/lib/genarrative/database-backups/files-history diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 826c4db62..df7cfa4bb 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -16,6 +16,19 @@ --- +## 2026-07-16 SpacetimeDB 备份采用逐文件基线、CAS 增量与安全历史清理 + +- 背景：SpacetimeDB standalone 2.6.0 不自动删除已被 snapshot 覆盖的历史 commitlog 与旧 snapshot；反复压缩整个 `/stdb` 会重复占用磁盘、停机和 OSS 带宽。上游 issue #5542 的 contributor 明确说明，不触碰最新 snapshot 与重启所需 commitlog suffix 时，可在运行中移动或删除这些历史文件。 +- 决策：统一脚本新增 `--storage-format files`，完整基线递归保留目录与文件相对路径，普通文件按 SHA-256 上传为不可变 CAS 对象，catalog 记录目录、路径、长度、SHA 与对象 key；相同内容不重复 PUT，后续 full 扫描只上传新增或变化内容，不再生成 tar.gz。旧 `archive` 路径保留兼容。full 必须从停库目录或已验证的冻结副本生成，不能把在线跨文件扫描称为一致时点备份。 +- history 继续按 replica 计算安全边界：只接受完整、未锁定且含同 offset `.snapshot_bsatn` 的 snapshot，保留跨越最新 snapshot 的边界 segment 及全部后缀。旧 segment 对和旧 snapshot 被递归映射为单文件 CAS 对象；对象、history catalog、full baseline catalog、候选 fingerprint 与当前边界全部验真后才删除源文件。同库执行用 work-dir PID lock 互斥。 +- OSS 固定恢复入口为 `//latest.json`。CAS 文件和 full/history catalog 保持不可变；latest pointer 只保存最新 full catalog 与已发布 history catalog 的 object key、长度和 SHA，不包含主机绝对路径或文件内容。每次 state 变化先验真全部引用 catalog，再覆盖上传并 HEAD 验真 latest pointer，成功后才落本地 state；history 还必须在 pointer 成功后才允许删除源文件。全新机器可仅凭 bucket、database、prefix 与 OSS 凭据自动下载 pointer 和 full catalog。 +- dev 带宽不足时，允许把已冻结的 dev 基线经 `10.2.0.10 -> 10.2.4.16` 内网 rsync 到 release 独立 staging，再用 release 出口上传 dev bucket；staging 不得指向 release `/stdb`，不得停止或修改 release 服务，传输凭据必须临时创建并在演练后移除。catalog 不记录 staging 绝对路径，files state 可回传 dev 继续 history。 +- 恢复边界：恢复时默认从 OSS `latest.json` 自动定位 full catalog，创建目录并按相对路径下载每个对象、逐文件校验长度与 SHA；本地 state 只用于备份续跑，不再是异机恢复前置条件。远程 dev 已完成真实 OSS、清理、重启和异机隔离恢复演练；release timer 与 publish 前备份继续保持原行为。 +- systemd 接线：主 service 保持 `archive-full`。Server-Provision 新增默认值为 `archive-full` 的 `DATABASE_BACKUP_PROFILE`；dev 或 release 显式选择 `files-history` 时，必须为各自主机指定独立 work-dir，并先用 current release 脚本执行 history dry-run，确认已有 full state 后才安装仓库托管 drop-in，并删除现场手写旧 drop-in。切回默认 profile 必须删除所有 history 覆盖。 +- 影响范围：`scripts/database-backup-to-oss.mjs`、备份门禁、生产 env 示例、systemd 模板、Server-Provision、SpacetimeDB 运维与恢复流程；release timer 可在独立 baseline 验证后显式选择 profile，publish 前备份是否切换仍需单独决策。 +- 验证方式：`npm run check:database-backup`、`npm run check:production-ops`、`npm run check:encoding`、`git diff --check`；dev 现场必须完成逐文件 full catalog、重复 full 零 PUT、history dry-run、上传后清理、STDB 重启和按 catalog 隔离恢复 roundtrip。 +- 关联：。 + ## 2026-07-14 后台账号采用 owner 引导账号与一级 Tab 实时授权 - 背景：后台此前只支持一组环境变量管理员，所有 `/admin/api/*` 共用统一 admin 门禁，无法给运营、审核等人员分配独立账号和页面范围。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 5d2a3d099..0f7605295 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -3117,3 +3117,11 @@ - 原因：`.admin-info-list div` 会命中列表内所有后代 `div`，把字段值内部的 `.admin-inline-identity` 和昵称容器也覆盖成双列 grid；陶泥号又允许任意位置换行，最终只剩单字符宽度。素材详情布局若始终固定为 `220px + 信息列`，移动端也没有足够空间。 - 处理：信息列表的行布局只使用直接子选择器 `.admin-info-list > div`；作者昵称与陶泥号在身份组件内分行，陶泥号保持单行并在真正不足时省略。`560px` 以下的素材详情改为单列，缩略图居中；素材查询与精选审核共用该规则。 - 验证：在桌面、560px、390px 和 320px 浏览器宽度打开素材详情，确认 `.admin-inline-identity` 的 computed `display` 为 `flex`、陶泥号横向显示、详情字段不溢出页面。 + +## SpacetimeDB 历史归档不能按文件名小于 snapshot 就全部删除 + +- 现象：看到最新 `N.snapshot_dir` 后，把所有起始 offset 小于 `N` 的 `.stdb.log` 删除，或者只把旧日志上传 OSS 就宣称已有完整增量灾备。 +- 原因：segment 文件名只表示该段最早事务；起始 offset 小于等于最新 snapshot 的最后一个 segment 可能跨越 snapshot 边界，重启仍需要它。历史归档也不会及时覆盖 control-db、program bytes、最新 snapshot 和 active segment。 +- 处理：latest snapshot 必须是未锁定且存在同 offset `.snapshot_bsatn` 的完整目录，空目录或同名 `.lock` 存在时忽略。每个 replica 独立保留 `max(segment_start <= latest_snapshot)` 及全部后缀，只处理更早 segment 对；旧 snapshot 只保留最新一个。`--storage-format files` 必须先发布完整 full catalog；history 对每个候选文件 CAS 对象、history catalog 和 full catalog 执行 HEAD 长度/SHA 验真，再复算边界与 stat fingerprint，最后发布并验真固定 `latest.json`；pointer 失败时不得推进 state 或删除源文件。不要把在线逐文件 full 扫描当成跨文件一致备份，基线必须来自停库目录或已验证冻结副本。SSH 或工具超时后先检查 work-dir PID lock 与原进程，不要直接并发重跑；不要在 history 模式传 `--stop-service`。定时任务通过 Server-Provision 的显式 profile 和仓库 drop-in 管理，启用前 dry-run 验证 baseline，切回 archive 时同时移除托管与现场遗留 drop-in；不要在 `/etc/systemd/system` 长期保留手写覆盖，release 必须建立和验证自己的 full baseline 与 work-dir，不能直接复用 dev 的本地 state。 +- 验证：dry-run 输出 replica 的 `latestSnapshot`、`boundarySegment` 和候选清单；从另一台机器仅凭 OSS `latest.json` 自动定位 full catalog，创建目录、下载文件并逐项校验长度/SHA，启动隔离 data-dir 验证 `/v1/ping`、snapshot restore、commitlog replay、module launch、代表性 SQL 与 reducer。 +- 关联：`scripts/database-backup-to-oss.mjs`、`scripts/check-database-backup-to-oss.mjs`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 0699549eb..e97d3ae0a 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -303,7 +303,7 @@ UI 相关修改要重点验证： ### SpacetimeDB 数据目录 OSS 备份 -数据库备份不放进 `spacetime-module` reducer / procedure：备份属于文件系统与 OSS 外部副作用，必须由运维脚本在 SpacetimeDB 宿主外执行。当前统一脚本为 `scripts/database-backup-to-oss.mjs`（npm 命令 `npm run database:backup:oss`）；生产 provision 还会安装 `genarrative-database-backup.timer`，每天 `03:20` 左右自动执行一次 OSS 冷备份： +数据库备份不放进 `spacetime-module` reducer / procedure：备份属于文件系统与 OSS 外部副作用，必须由运维脚本在 SpacetimeDB 宿主外执行。当前统一脚本为 `scripts/database-backup-to-oss.mjs`（npm 命令 `npm run database:backup:oss`）。默认 `--storage-format archive --mode full` 保持原有全量压缩包冷备行为；`--storage-format files` 不生成 tar.gz，而是把目录树映射成逐文件 CAS 对象与 catalog，full 重跑只上传新增或内容变化的文件，history 只处理已被最新 snapshot 完全覆盖的历史 commitlog 与旧 snapshot。`Genarrative-Server-Provision` 的 `DATABASE_BACKUP_PROFILE` 默认是 `archive-full`，继续安装每天 `03:20` 左右执行的全量冷备主 service；development 和 release 都可以显式选择 `files-history`，但指定 work-dir 必须已经有与本机 database/bucket 匹配且已发布的 full baseline state： ```bash npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service @@ -326,13 +326,67 @@ GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET= GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT=oss-cn-shanghai.aliyuncs.com GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX=database-backups GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL=false +GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT=archive GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES= +GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE=/var/lib/genarrative/database-backups/genarrative-prod-history-state.json +# 仅 archive history 首次从一份 uploadStatus=uploaded 的全量 manifest 初始化 state 时设置或传 --baseline-manifest。 +GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST= GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID= GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET= ``` `GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET` 为空时会回退 `ALIYUN_OSS_BUCKET`；AccessKey 默认复用 `ALIYUN_OSS_ACCESS_KEY_ID` / `ALIYUN_OSS_ACCESS_KEY_SECRET`，也可用 `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID` / `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET` 为备份 bucket 单独配置最小权限账号。冷备脚本会在停止 SpacetimeDB 前检查 `GENARRATIVE_DATABASE_BACKUP_WORK_DIR` 所在文件系统剩余空间；未设置 `GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES` 时，按数据目录大小加安全余量估算，空间不足会在停库前失败，避免写满根分区。即使打包或上传前步骤失败，只要脚本已经停过 SpacetimeDB，也会先恢复 SpacetimeDB 并执行 `--restart-service-after` 指定的 API / worker / controller，再带着原始备份错误退出。`Genarrative-Server-Provision` 会创建 `/var/lib/genarrative/database-backups` 并归属 `genarrative:genarrative`，同时安装并启用 `genarrative-database-backup.timer`。手动检查定时器：`systemctl list-timers genarrative-database-backup.timer`；手动触发一次：`systemctl start genarrative-database-backup.service`。如果 timer 显示 `enabled` 但 `inactive/dead` 且 `NEXT` / `Trigger` 为空，先写入当前 stamp 避免 `Persistent=true` 在白天立刻补跑冷备份：`touch /var/lib/systemd/timers/stamp-genarrative-database-backup.timer && systemctl daemon-reload && systemctl start genarrative-database-backup.timer`，随后确认下一次触发时间约为次日 `03:20`。 +`files-history` 使用仓库模板 `deploy/systemd/genarrative-database-backup-files-history.conf` 覆盖主 service 的 `ExecStart`，从 `/etc/genarrative/api-server.env` 读取 data-dir、database、bucket、prefix 与 OSS 凭据，不在 unit 写死环境目标，也不传 `--stop-service`。Server-Provision 在改动 drop-in 前，先用 current release 的同一脚本、同一 env 和 `DATABASE_BACKUP_FILES_HISTORY_WORK_DIR` 执行一次 history `--dry-run`；缺少已发布 full catalog 的 files state、current 脚本过旧或配置不匹配都会在安装 drop-in 和 `daemon-reload` 前失败。选择 `archive-full` 会主动删除仓库托管的 `10-files-history.conf` 与 dev 试点遗留的 `10-dev-files.conf`，防止 systemd 继续合并旧覆盖。dev 可继续指定已有 `/var/lib/genarrative/database-backups/dev-files`，release 建议先在 `/var/lib/genarrative/database-backups/release-files` 建立自己的 full baseline；两台机器不得复用或互传本地 state 目录冒充本机基线。启用时通过 Server-Provision Job 选择目标、`DATABASE_BACKUP_PROFILE=files-history` 和对应 work-dir，先保持 `DRY_RUN=true` 核对，再以同参数正式 provision。不要直接在 `/etc/systemd/system` 手写第二份 drop-in。 + +files full 会递归扫描 data-dir，保留空目录和每个普通文件的相对路径；文件按 SHA-256 上传到不可变对象 key，catalog 记录目录、路径、长度、SHA 和对象 key，不写 staging 主机的绝对路径。相同 catalog 重跑不重复 PUT；新增或变化文件先 HEAD CAS 对象，存在且长度/SHA 元数据一致就复用，否则上传。full 基线必须来自停库后的 data-dir 或已通过恢复验证的冻结副本；源文件上传前后 stat 虽会复核，但在线扫描不能保证 2083 个文件属于同一跨文件一致时点。catalog 验真后，脚本把最新 full/history 引用发布到固定 `//latest.json`，全新机器不需要本地 state 即可自动发现恢复入口。 + +history 的安全边界按每个 replica 独立计算。设最新完整且未锁定的 snapshot offset 为 `S`；数字更大但缺少同 offset `.snapshot_bsatn`、仍存在同名 `.lock` 的目录不能参与边界计算。脚本必须保留起始 offset 小于等于 `S` 的最后一个 commitlog segment，以及它之后的全部 segment；只处理更早的 `.stdb.log` / `.stdb.ofs`，snapshot 只处理最新目录之前的旧目录。files history 会递归展开候选目录，逐对象复用或上传，随后依次验真候选对象、history catalog 与 full baseline catalog，再重新扫描边界和 stat fingerprint，最后覆盖发布并验真 `latest.json`；任何一步失败都不推进 state 或删除源文件。脚本在 work-dir 使用 PID lock 拒绝同库并发上传，SSH 超时后必须先检查原进程，不能直接重跑。 + +```bash +# 从停库目录或已验证冻结副本建立逐文件完整基线；相同 work-dir 重跑只传变化内容。 +node -- scripts/database-backup-to-oss.mjs \ + --storage-format files \ + --mode full \ + --data-dir /path/to/frozen/stdb \ + --work-dir /var/lib/genarrative/database-backups/dev-files \ + --database genarrative-prod \ + --env-file /etc/genarrative/api-server.env + +# 把完整 work-dir/state 放回 dev 后，先只读查看可清理历史候选。 +node -- scripts/database-backup-to-oss.mjs \ + --storage-format files \ + --mode history \ + --data-dir /stdb \ + --work-dir /var/lib/genarrative/database-backups/dev-files \ + --database genarrative-prod \ + --env-file /etc/genarrative/api-server.env \ + --dry-run \ + --result-file /var/lib/genarrative/database-backups/history-dry-run.json + +# 核对 dry-run 后执行真实归档与清理；history 在线处理不可变历史文件，不传 --stop-service。 +node -- scripts/database-backup-to-oss.mjs \ + --storage-format files \ + --mode history \ + --data-dir /stdb \ + --work-dir /var/lib/genarrative/database-backups/dev-files \ + --database genarrative-prod \ + --env-file /etc/genarrative/api-server.env +``` + +dev 出口过慢时，可以把冻结基线经内网 rsync 到 release 独立 staging，再由 release 上传 dev bucket。staging 必须位于 `/var/lib/genarrative/dev-database-backup-staging/` 一类隔离目录，命令显式传 staging `--data-dir`、独立 `--work-dir`、dev `--bucket`，且不得传 `--stop-service`；禁止指向或修改 release `/stdb`。中转 key 只为本次传输临时授权，结束后从 dev 私钥和 release `authorized_keys` 同时移除。上传完成后把整个 files work-dir/state 回传 dev，history 才能延续同一 baseline catalog。 + +完整恢复默认从 OSS 固定 `latest.json` 读取最新 full catalog：先创建 `directories`，再把每个 `files[].objectKey` 下载到 `/` 并逐项核对 `sizeBytes` / `sha256`；history catalog 用于证明已清理历史仍有 OSS 对象，不需要把已被 full baseline 覆盖的旧文件叠回当前恢复目录。本地 state 仍可作为兼容入口，但不再是异机恢复的前置条件。随后用隔离 data-dir 启动同版本 standalone，验证 `/v1/ping`、日志中的 snapshot restore / commitlog replay / module launch、代表性 SQL 和 reducer。dev 已完成这轮 OSS-only 异机恢复与重启演练；当前 live release 仍保持 `archive-full`，需要切换时先为 release 建立并恢复验证独立 full baseline，再通过 Server-Provision 显式选择 `files-history`，无需修改代码或解除额外硬门禁。 + +```bash +node -- scripts/database-backup-to-oss.mjs \ + --env-file /etc/genarrative/api-server.env \ + --database genarrative-prod \ + --restore-files-latest \ + --restore-dir /var/lib/genarrative/database-backup-restore/stdb \ + --result-file /var/lib/genarrative/database-backup-restore/restore-result.json +``` + 冷备份后必须做一次只读验收，不要只看 `genarrative-database-backup.service` 是否成功退出： ```bash diff --git a/jenkins/Jenkinsfile.production-server-provision b/jenkins/Jenkinsfile.production-server-provision index e59a82fa9..2d2e550ba 100644 --- a/jenkins/Jenkinsfile.production-server-provision +++ b/jenkins/Jenkinsfile.production-server-provision @@ -33,6 +33,8 @@ pipeline { string(name: 'WEB_LINK', defaultValue: '/srv/genarrative/web', description: 'Nginx 静态站点目录或软链接') string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'api-server 环境文件') string(name: 'API_PORT', defaultValue: '8082', description: 'api-server 本机监听端口') + choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profile；默认 archive-full，files-history 仅在指定 work-dir 已有完整 full baseline 后启用') + string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'files-history 的本地 state/catalog 目录；dev/release 必须使用各自已建立 full baseline 的独立目录') choice(name: 'NGINX_CONFIG_MODE', choices: ['none', 'production-https', 'development-http'], description: 'Nginx 配置模式；开发服无域名时选 development-http，release 正式入口选 production-https') booleanParam(name: 'ENABLE_SERVICES', defaultValue: true, description: '启用并启动 spacetimedb 与 api-server systemd 服务') booleanParam(name: 'ENABLE_OTELCOL', defaultValue: true, description: '安装并启用本机 OpenTelemetry Collector；api-server 模板默认开启 OTLP，如需关闭请在 API_ENV_FILE 中将 GENARRATIVE_OTEL_ENABLED 改为 false') @@ -103,6 +105,14 @@ pipeline { if (params.DEPLOY_TARGET == 'release' && nginxMode == 'development-http') { error('release 目标禁止安装 development-http Nginx 配置；无证书初始化请使用 NGINX_CONFIG_MODE=none。') } + def databaseBackupProfile = params.DATABASE_BACKUP_PROFILE?.trim() + if (!(databaseBackupProfile in ['archive-full', 'files-history'])) { + error("DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${params.DATABASE_BACKUP_PROFILE}") + } + def databaseBackupFilesHistoryWorkDir = params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR?.trim() + if (!(databaseBackupFilesHistoryWorkDir ==~ /^\/var\/lib\/genarrative\/database-backups\/[A-Za-z0-9._\/-]+$/) || databaseBackupFilesHistoryWorkDir.contains('..')) { + error("DATABASE_BACKUP_FILES_HISTORY_WORK_DIR 必须是 /var/lib/genarrative/database-backups/ 下不含连续点号的绝对路径，当前值: ${params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}") + } if (!params.DRY_RUN && nginxMode == 'production-https' && params.SERVER_NAME?.trim() == 'genarrative.example.com') { error('真实初始化安装 Nginx 配置时必须把 SERVER_NAME 改成真实域名，不能使用 genarrative.example.com 占位值。证书未准备好时请先保持 NGINX_CONFIG_MODE=none。') } diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index bbc8b0fb2..0905febc2 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -1,11 +1,25 @@ #!/usr/bin/env node import {spawnSync} from 'node:child_process'; -import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {createHash} from 'node:crypto'; +import {chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import path from 'node:path'; -import {buildAuthorization, buildCanonicalQuery, uploadArchive} from './database-backup-to-oss.mjs'; +import { + buildAuthorization, + buildCanonicalQuery, + cleanupHistoryCandidates, + collectDirectFileEntries, + discoverHistoryPlan, + restoreDirectFilesBackup, + restoreDirectFilesLatest, + resumeUploadedHistoryBatch, + runDirectFilesBackup, + uploadArchive, + uploadHistoryArchiveWithCleanup, + uploadManifestFile, +} from './database-backup-to-oss.mjs'; const BACKUP_SCRIPT = path.resolve('scripts/database-backup-to-oss.mjs'); const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-database-backup-check-')); @@ -35,6 +49,296 @@ async function main() { await assertMissingPartEtagAbortsMultipartUpload(); await assertCompleteResponseAmbiguityUsesHeadVerification(); await assertHeadLengthMismatchAbortsMultipartUpload(); + await assertHeadShaMismatchAbortsMultipartUpload(); + await assertManifestUploadUsesShaAndHeadVerification(); + assertHistoryDiscoversDevAndProductionLayoutsWithMultipleReplicas(); + assertHistoryRequiresBaselineAndProducesDeterministicDeferredBatch(); + assertHistoryBackupLockRejectsLiveAndStaleOwners(); + assertHistorySkipsReplicaWithoutSnapshotAndRejectsMalformedNames(); + assertHistoryStatDriftPreventsAnyCleanup(); + await assertHistoryUploadFailureDoesNotDeleteSources(); + await assertHistorySuccessfulUploadCleansAndIsIdempotent(); + await assertHistoryResumeReverifiesArchiveAndManifest(); + await assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload(); + await assertDirectHistoryPublishesCatalogBeforeCleanup(); + await assertDirectHistoryWithoutCandidatesPublishesLatest(); + await assertDirectFilesRestoreDownloadsCatalogAndObjects(); +} + +function createDirectOssHarness() { + const objects = new Map(); + const uploadedKeys = []; + const verifiedKeys = []; + const uploadFn = async ({archivePath, objectKey, archiveSha256}) => { + const body = readFileSync(archivePath); + const sha256 = createHash('sha256').update(body).digest('hex'); + assertEqual(sha256, archiveSha256, `direct file ${objectKey} 的上传 SHA 必须来自实际内容。`); + objects.set(objectKey, {body, contentLength: body.length, sha256}); + uploadedKeys.push(objectKey); + return {objectKey, contentLength: body.length, archiveSha256: sha256, verifiedAt: '2026-07-16T01:00:00.000Z'}; + }; + const uploadManifestFn = async ({manifestPath, objectKey}) => { + const body = readFileSync(manifestPath); + const sha256 = createHash('sha256').update(body).digest('hex'); + objects.set(objectKey, {body, contentLength: body.length, sha256}); + uploadedKeys.push(objectKey); + return {objectKey, contentLength: body.length, archiveSha256: sha256, verifiedAt: '2026-07-16T01:00:01.000Z'}; + }; + const verifyFn = async ({objectKey, contentLength, archiveSha256}) => { + verifiedKeys.push(objectKey); + const object = objects.get(objectKey); + if (!object) { + const error = new Error(`missing ${objectKey}`); + error.status = 404; + throw error; + } + if (object.contentLength !== contentLength || object.sha256 !== archiveSha256) { + throw new Error(`mismatch ${objectKey}`); + } + return {verifiedAt: '2026-07-16T01:00:02.000Z'}; + }; + return {objects, uploadedKeys, verifiedKeys, uploadFn, uploadManifestFn, verifyFn}; +} + +async function assertDirectFilesPreservePathsAndIncrementWithoutDuplicateUpload() { + const root = path.join(tmpRoot, 'direct-files-incremental'); + const dataDir = path.join(root, 'stdb'); + const workDir = path.join(root, 'work'); + mkdirSync(path.join(dataDir, 'replicas', '1', 'snapshots', '00000000000000000010.snapshot_dir', 'objects'), {recursive: true}); + mkdirSync(path.join(dataDir, 'empty-directory'), {recursive: true}); + writeFileSync(path.join(dataDir, 'control-db'), 'control'); + writeFileSync( + path.join(dataDir, 'replicas', '1', 'snapshots', '00000000000000000010.snapshot_dir', 'objects', 'object.bin'), + 'snapshot object', + ); + const harness = createDirectOssHarness(); + const options = { + mode: 'full', dataDir, workDir, database: 'test-db', bucket: 'backup-bucket', objectPrefix: 'database-backups', + uploadOptions: {}, uploadFn: harness.uploadFn, uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn, + }; + const collected = await collectDirectFileEntries({dataDir, database: 'test-db', objectPrefix: 'database-backups'}); + assertTrue( + collected.files.some(({path: filePath}) => filePath === 'replicas/1/snapshots/00000000000000000010.snapshot_dir/objects/object.bin'), + 'files catalog 必须原样保留 snapshot 内文件的相对路径。', + ); + assertTrue(collected.directories.includes('empty-directory'), 'files catalog 必须保留空目录。'); + + const first = await runDirectFilesBackup(options); + assertEqual(first.uploadedCount, 2, '首次 files full 应上传全部普通文件。'); + assertTrue(!Object.hasOwn(first.catalog, 'dataDir'), '远端 files catalog 不得绑定 staging 主机的绝对 data-dir。'); + const latestObjectKey = 'database-backups/test-db/latest.json'; + const latest = JSON.parse(harness.objects.get(latestObjectKey).body.toString('utf8')); + assertEqual(latest.latestFullCatalog.catalogId, first.catalogId, 'latest pointer 必须指向已验真的最新 full catalog。'); + assertTrue(!Object.hasOwn(latest.latestFullCatalog, 'files'), 'latest full ref 不得嵌入 files 数组。'); + assertTrue(latest.historyCatalogs.every((catalog) => !Object.hasOwn(catalog, 'files')), 'latest history ref 不得嵌入 files 数组。'); + const immutableUploadsAfterFirst = harness.uploadedKeys.filter((objectKey) => objectKey !== latestObjectKey).length; + const repeated = await runDirectFilesBackup(options); + assertEqual(repeated.uploadedCount, 0, '相同目录重复运行不得重复上传文件。'); + assertEqual( + harness.uploadedKeys.filter((objectKey) => objectKey !== latestObjectKey).length, + immutableUploadsAfterFirst, + '相同 catalog 重跑不得重复 PUT 文件或 catalog，但应覆盖验真 latest pointer。', + ); + + writeFileSync(path.join(dataDir, 'control-db'), 'control changed'); + writeFileSync(path.join(dataDir, 'new-program.bin'), 'new program'); + const incremental = await runDirectFilesBackup(options); + assertEqual(incremental.uploadedCount, 2, '增量 files full 只应上传新增和变化文件。'); + assertEqual(incremental.reusedCount, 1, '增量 files full 应复用未变化 snapshot 文件。'); +} + +async function assertDirectHistoryPublishesCatalogBeforeCleanup() { + const fixture = createHistoryFixture('direct-files-history-cleanup', {nestedData: false}); + createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const harness = createDirectOssHarness(); + const common = { + dataDir: fixture.dataDir, + workDir: fixture.workDir, + database: 'test-db', + bucket: 'backup-bucket', + objectPrefix: 'database-backups', + uploadOptions: {}, + uploadFn: harness.uploadFn, + verifyFn: harness.verifyFn, + }; + await runDirectFilesBackup({...common, mode: 'full', uploadManifestFn: harness.uploadManifestFn}); + const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); + let failure = null; + try { + await runDirectFilesBackup({ + ...common, + mode: 'history', + uploadManifestFn: async () => { + throw new Error('synthetic direct catalog failure'); + }, + }); + } catch (error) { + failure = error; + } + assertIncludes(failure?.message ?? '', 'synthetic direct catalog failure', 'direct history catalog 发布失败必须向上返回。'); + for (const candidate of plan.candidates) { + assertTrue(existsSync(path.join(fixture.dataDir, candidate.path)), `direct history catalog 发布失败不得删除: ${candidate.path}`); + } + + let pointerFailure = null; + try { + await runDirectFilesBackup({ + ...common, + mode: 'history', + uploadManifestFn: harness.uploadManifestFn, + verifyFn: async (options) => { + if (options.objectKey.endsWith('/latest.json')) { + throw new Error('synthetic latest pointer HEAD failure'); + } + return harness.verifyFn(options); + }, + }); + } catch (error) { + pointerFailure = error; + } + assertIncludes(pointerFailure?.message ?? '', 'synthetic latest pointer HEAD failure', 'latest pointer HEAD 验真失败必须向上返回。'); + for (const candidate of plan.candidates) { + assertTrue(existsSync(path.join(fixture.dataDir, candidate.path)), `latest pointer 发布失败不得删除: ${candidate.path}`); + } + + const success = await runDirectFilesBackup({...common, mode: 'history', uploadManifestFn: harness.uploadManifestFn}); + assertEqual(success.uploadedCount, 0, 'history 文件已在 full CAS baseline 时不应重复上传内容。'); + for (const file of success.catalog.files) { + assertTrue( + harness.verifiedKeys.includes(file.objectKey), + `history 清理前必须逐个验真 baseline 复用对象: ${file.path}`, + ); + } + assertEqual(success.cleanup?.deletedCount, plan.candidates.length, 'catalog 和 baseline 验真后才应清理全部安全候选。'); + + const state = JSON.parse(readFileSync(success.statePath, 'utf8')); + const historyCatalogObjectKey = state.historyCatalogs[0].objectKey; + harness.objects.delete(historyCatalogObjectKey); + let brokenHistoryFailure = null; + try { + await runDirectFilesBackup({...common, mode: 'history', uploadManifestFn: harness.uploadManifestFn}); + } catch (error) { + brokenHistoryFailure = error; + } + assertIncludes( + brokenHistoryFailure?.message ?? '', + `missing ${historyCatalogObjectKey}`, + 'latest pointer 发布前必须重新验真所有 history catalog 引用。', + ); +} + +async function assertDirectHistoryWithoutCandidatesPublishesLatest() { + const fixture = createHistoryFixture('direct-files-history-empty', {nestedData: false}); + createReplicaHistory(fixture.replicasDir, '1', {snapshots: [10], segments: [0]}); + const harness = createDirectOssHarness(); + const common = { + dataDir: fixture.dataDir, + workDir: fixture.workDir, + database: 'test-db', + bucket: 'backup-bucket', + objectPrefix: 'database-backups', + uploadOptions: {}, + uploadFn: harness.uploadFn, + uploadManifestFn: harness.uploadManifestFn, + verifyFn: harness.verifyFn, + }; + await runDirectFilesBackup({...common, mode: 'full'}); + const latestObjectKey = 'database-backups/test-db/latest.json'; + harness.objects.delete(latestObjectKey); + const result = await runDirectFilesBackup({...common, mode: 'history'}); + assertEqual(result.candidateCount, 0, 'fixture 应没有可归档 history 候选。'); + assertTrue(harness.objects.has(latestObjectKey), 'history 无候选时仍必须从现有 state 发布 latest pointer。'); + assertTrue(harness.verifiedKeys.includes(latestObjectKey), 'history 无候选时 latest pointer 仍必须 HEAD 验真。'); +} + +async function assertDirectFilesRestoreDownloadsCatalogAndObjects() { + const root = path.join(tmpRoot, 'direct-files-restore'); + const dataDir = path.join(root, 'stdb'); + const workDir = path.join(root, 'work'); + const restoreDir = path.join(root, 'restore'); + mkdirSync(path.join(dataDir, 'empty-directory'), {recursive: true}); + mkdirSync(path.join(dataDir, 'config'), {recursive: true}); + const keyPath = path.join(dataDir, 'config', 'id_ecdsa'); + writeFileSync(keyPath, 'private key fixture'); + chmodSync(keyPath, 0o640); + const harness = createDirectOssHarness(); + await runDirectFilesBackup({ + mode: 'full', dataDir, workDir, database: 'test-db', bucket: 'backup-bucket', objectPrefix: 'database-backups', + uploadOptions: {}, uploadFn: harness.uploadFn, uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn, + }); + writeFileSync(keyPath, 'updated private key fixture'); + chmodSync(keyPath, 0o640); + const latestFull = await runDirectFilesBackup({ + mode: 'full', dataDir, workDir, database: 'test-db', bucket: 'backup-bucket', objectPrefix: 'database-backups', + uploadOptions: {}, uploadFn: harness.uploadFn, uploadManifestFn: harness.uploadManifestFn, verifyFn: harness.verifyFn, + }); + const restored = await restoreDirectFilesBackup({ + statePath: latestFull.statePath, + restoreDir, + database: 'test-db', + bucket: 'backup-bucket', + uploadOptions: {}, + downloadBufferFn: async ({objectKey}) => Buffer.from(harness.objects.get(objectKey)?.body ?? ''), + downloadFileFn: async ({objectKey, destinationPath}) => { + const object = harness.objects.get(objectKey); + if (!object) { + throw new Error(`missing ${objectKey}`); + } + writeFileSync(destinationPath, object.body); + }, + }); + assertEqual(restored.downloadedCount, 1, 'files restore 必须从对象存储下载 catalog 中的普通文件。'); + assertEqual(readFileSync(path.join(restoreDir, 'config', 'id_ecdsa'), 'utf8'), 'updated private key fixture', 'files restore 必须按最新 full catalog 的原相对路径恢复内容。'); + assertTrue(existsSync(path.join(restoreDir, 'empty-directory')), 'files restore 必须重建空目录。'); + assertEqual(statSync(path.join(restoreDir, 'config', 'id_ecdsa')).mode & 0o7777, 0o640, 'files restore 必须恢复文件权限。'); + + rmSync(restoreDir, {recursive: true, force: true}); + const downloadBufferFn = async ({objectKey}) => { + const object = harness.objects.get(objectKey); + if (!object) { + throw new Error(`missing ${objectKey}`); + } + return Buffer.from(object.body); + }; + let objectDownloadCount = 0; + const downloadFileFn = async ({objectKey, destinationPath}) => { + objectDownloadCount += 1; + const object = harness.objects.get(objectKey); + if (!object) { + throw new Error(`missing ${objectKey}`); + } + writeFileSync(destinationPath, object.body); + }; + const dryRun = await restoreDirectFilesLatest({ + restoreDir, + database: 'test-db', + bucket: 'backup-bucket', + objectPrefix: 'database-backups', + uploadOptions: {}, + dryRun: true, + downloadBufferFn, + downloadFileFn, + verifyFn: harness.verifyFn, + }); + assertEqual(dryRun.catalogId, latestFull.catalogId, 'OSS-only dry-run 必须选择 latestFullCatalog。'); + assertEqual(dryRun.fileCount, 1, 'OSS-only dry-run 应返回 full catalog 文件数。'); + assertEqual(dryRun.totalSizeBytes, String(Buffer.byteLength('updated private key fixture')), 'OSS-only dry-run 应返回总字节数。'); + assertEqual(objectDownloadCount, 0, 'OSS-only dry-run 不得下载数据对象。'); + assertTrue(!existsSync(restoreDir), 'OSS-only dry-run 不得创建恢复目录。'); + + const latestRestored = await restoreDirectFilesLatest({ + restoreDir, + database: 'test-db', + bucket: 'backup-bucket', + objectPrefix: 'database-backups', + uploadOptions: {}, + downloadBufferFn, + downloadFileFn, + verifyFn: harness.verifyFn, + }); + assertEqual(latestRestored.catalogId, latestFull.catalogId, 'OSS-only restore 必须选择 latestFullCatalog。'); + assertEqual(latestRestored.downloadedCount, 1, 'OSS-only restore 应下载 latest full catalog 的数据对象。'); + assertEqual(readFileSync(path.join(restoreDir, 'config', 'id_ecdsa'), 'utf8'), 'updated private key fixture', 'OSS-only restore 应还原最新 full 内容。'); } function assertCanonicalQueryAndAuthorizationIncludeMultipartParameters() { @@ -138,6 +442,7 @@ async function assertMultipartUploadRetriesAndVerifiesRemoteLength() { Buffer.alloc(partSizeBytes, 'b'), Buffer.alloc(17, 'c'), ]); + const payloadSha256 = createHash('sha256').update(payload).digest('hex'); mkdirSync(root, {recursive: true}); writeFileSync(archivePath, payload); @@ -167,7 +472,10 @@ async function assertMultipartUploadRetriesAndVerifiesRemoteLength() { return new Response('', {status: 200, headers: {etag: '"complete-etag"'}}); } if (options.method === 'HEAD') { - return new Response(null, {status: 200, headers: {'content-length': String(payload.length)}}); + return new Response(null, {status: 200, headers: { + 'content-length': String(payload.length), + 'x-oss-meta-archive-sha256': payloadSha256, + }}); } throw new Error(`unexpected request: ${options.method} ${url}`); }; @@ -199,6 +507,7 @@ async function assertMultipartUploadRetriesAndVerifiesRemoteLength() { const initiateRequest = requests[0]; assertTrue(initiateRequest.url.endsWith('?uploads'), 'InitiateMultipartUpload URL 必须使用裸 uploads 参数。'); assertTrue(!initiateRequest.url.endsWith('?uploads='), 'InitiateMultipartUpload URL 不能把裸参数写成 uploads=。'); + assertEqual(initiateRequest.headers['x-oss-meta-archive-sha256'], payloadSha256, 'multipart 对象必须保存本地归档 SHA-256 元数据。'); const firstPartRequests = requests.filter(({method, url}) => method === 'PUT' && new URL(url).searchParams.get('partNumber') === '1'); assertEqual(firstPartRequests.length, 2, '第一段应产生原请求和一次重试。'); assertBufferEqual(firstPartRequests[0].body, payload.subarray(0, partSizeBytes), '第一段原请求内容必须完整。'); @@ -222,6 +531,7 @@ async function assertHeadLengthMismatchAbortsMultipartUpload() { const archivePath = path.join(root, 'backup.tar.gz'); const partSizeBytes = 100 * 1024; const payload = Buffer.alloc(partSizeBytes + 1, 'x'); + const payloadSha256 = createHash('sha256').update(payload).digest('hex'); mkdirSync(root, {recursive: true}); writeFileSync(archivePath, payload); @@ -240,7 +550,10 @@ async function assertHeadLengthMismatchAbortsMultipartUpload() { return new Response('', {status: 200, headers: {etag: '"complete-etag"'}}); } if (options.method === 'HEAD') { - return new Response(null, {status: 200, headers: {'content-length': String(payload.length - 1)}}); + return new Response(null, {status: 200, headers: { + 'content-length': String(payload.length - 1), + 'x-oss-meta-archive-sha256': payloadSha256, + }}); } if (options.method === 'DELETE') { return new Response(null, {status: 204}); @@ -277,6 +590,107 @@ async function assertHeadLengthMismatchAbortsMultipartUpload() { assertTrue(abortRequest?.url.endsWith('?uploadId=mismatch-upload'), 'AbortMultipartUpload 必须携带同一 uploadId。'); } +async function assertHeadShaMismatchAbortsMultipartUpload() { + const root = path.join(tmpRoot, 'multipart-head-sha-mismatch'); + const archivePath = path.join(root, 'backup.tar.gz'); + const payload = Buffer.alloc(100 * 1024, 's'); + mkdirSync(root, {recursive: true}); + writeFileSync(archivePath, payload); + const requests = []; + const fetchImpl = async (url, options) => { + await readRequestBody(options.body); + requests.push({url, method: options.method}); + const parsedUrl = new URL(url); + if (options.method === 'POST' && parsedUrl.search === '?uploads') { + return new Response('sha-mismatch-upload', {status: 200}); + } + if (options.method === 'PUT') { + return new Response('', {status: 200, headers: {etag: '"part-etag"'}}); + } + if (options.method === 'POST') { + return new Response('', {status: 200}); + } + if (options.method === 'HEAD') { + return new Response(null, {status: 200, headers: { + 'content-length': String(payload.length), + 'x-oss-meta-archive-sha256': '0'.repeat(64), + }}); + } + if (options.method === 'DELETE') { + return new Response(null, {status: 204}); + } + throw new Error(`unexpected request: ${options.method} ${url}`); + }; + let uploadError = null; + try { + await uploadArchive({ + archivePath, + bucket: 'genarrative-test', + endpoint: 'oss-cn-shanghai.aliyuncs.com', + objectKey: 'database-backups/test/sha-mismatch.tar.gz', + accessKeyId: 'test-access-key', + accessKeySecret: 'test-access-secret', + partSizeBytes: 100 * 1024, + maxAttempts: 1, + fetchImpl, + nowFn: () => new Date('2026-07-13T10:20:30.000Z'), + sleepImpl: async () => {}, + randomFn: () => 0, + }); + } catch (error) { + uploadError = error; + } + assertIncludes(uploadError?.message ?? '', 'SHA-256 不一致', 'HEAD SHA-256 不一致时上传必须失败。'); + assertTrue( + requests.some(({method, url}) => method === 'DELETE' && url.endsWith('?uploadId=sha-mismatch-upload')), + 'HEAD SHA-256 不一致后必须 best-effort AbortMultipartUpload。', + ); +} + +async function assertManifestUploadUsesShaAndHeadVerification() { + const root = path.join(tmpRoot, 'manifest-upload'); + const manifestPath = path.join(root, 'backup.manifest.json'); + const body = Buffer.from('{"uploadStatus":"uploaded"}\n'); + const bodySha256 = createHash('sha256').update(body).digest('hex'); + mkdirSync(root, {recursive: true}); + writeFileSync(manifestPath, body); + const requests = []; + const result = await uploadManifestFile({ + manifestPath, + bucket: 'genarrative-test', + endpoint: 'oss-cn-shanghai.aliyuncs.com', + objectKey: 'database-backups/test/backup.tar.gz.manifest.json', + accessKeyId: 'test-access-key', + accessKeySecret: 'test-access-secret', + maxAttempts: 1, + nowFn: () => new Date('2026-07-13T10:20:30.000Z'), + sleepImpl: async () => {}, + randomFn: () => 0, + fetchImpl: async (url, options) => { + const requestBody = await readRequestBody(options.body); + requests.push({url, method: options.method, headers: options.headers, body: requestBody}); + if (options.method === 'PUT') { + return new Response(null, {status: 200}); + } + if (options.method === 'HEAD') { + return new Response(null, {status: 200, headers: { + 'x-oss-meta-file-size': String(body.length), + 'x-oss-meta-archive-sha256': bodySha256, + }}); + } + throw new Error(`unexpected request: ${options.method} ${url}`); + }, + }); + assertEqual(result.archiveSha256, bodySha256, 'manifest 上传结果必须记录本地 SHA-256。'); + assertBufferEqual(requests.find(({method}) => method === 'PUT')?.body, body, 'manifest PUT 必须上传完整 JSON。'); + assertTrue(requests.some(({method}) => method === 'HEAD'), 'manifest PUT 后必须执行 HEAD 验真。'); + assertEqual( + requests.find(({method}) => method === 'PUT')?.headers['x-oss-meta-file-size'], + String(body.length), + 'manifest PUT 必须记录原始字节数，供动态压缩 HEAD 缺少 content-length 时验真。', + ); +} + async function assertMissingPartEtagAbortsMultipartUpload() { const root = path.join(tmpRoot, 'multipart-missing-etag'); const archivePath = path.join(root, 'backup.tar.gz'); @@ -331,6 +745,7 @@ async function assertCompleteResponseAmbiguityUsesHeadVerification() { const root = path.join(tmpRoot, 'multipart-complete-ambiguity'); const archivePath = path.join(root, 'backup.tar.gz'); const payload = Buffer.alloc(100 * 1024, 'c'); + const payloadSha256 = createHash('sha256').update(payload).digest('hex'); mkdirSync(root, {recursive: true}); writeFileSync(archivePath, payload); @@ -354,7 +769,10 @@ async function assertCompleteResponseAmbiguityUsesHeadVerification() { return new Response('NoSuchUpload', {status: 404}); } if (options.method === 'HEAD') { - return new Response(null, {status: 200, headers: {'content-length': String(payload.length)}}); + return new Response(null, {status: 200, headers: { + 'content-length': String(payload.length), + 'x-oss-meta-archive-sha256': payloadSha256, + }}); } if (options.method === 'DELETE') { return new Response(null, {status: 204}); @@ -385,6 +803,459 @@ async function assertCompleteResponseAmbiguityUsesHeadVerification() { assertTrue(!requests.some(({method}) => method === 'DELETE'), 'HEAD 已证实对象完整时不得 Abort 已完成上传。'); } +function assertHistoryDiscoversDevAndProductionLayoutsWithMultipleReplicas() { + const dev = createHistoryFixture('history-dev-layout', {nestedData: false}); + createReplicaHistory(dev.replicasDir, '1', { + snapshots: [0, 187, 279], + segments: [0, 188, 280], + }); + createReplicaHistory(dev.replicasDir, '2', { + snapshots: [50, 99], + segments: [0, 51, 100], + }); + const devPlan = discoverHistoryPlan({dataDir: dev.dataDir}); + assertEqual(devPlan.replicas.length, 2, 'history 应逐 replica 计算安全边界。'); + assertEqual(devPlan.candidates.length, 7, '多 replica history 候选数量必须符合 snapshot/segment 边界。'); + assertTrue( + devPlan.candidates.some(({path}) => path === 'replicas/1/clog/00000000000000000000.stdb.log'), + 'dev 布局应识别边界 segment 之前的 commitlog。', + ); + assertTrue( + !devPlan.candidates.some(({path}) => path.includes('00000000000000000188.stdb.log')), + '跨越 latest snapshot 的边界 segment 必须保留。', + ); + assertTrue( + !devPlan.candidates.some(({path}) => path.includes('00000000000000000279.snapshot_dir')), + '每个 replica 的 latest snapshot 必须保留。', + ); + + const production = createHistoryFixture('history-production-layout', {nestedData: true}); + createReplicaHistory(production.replicasDir, '7', { + snapshots: [10, 20], + segments: [0, 11, 21], + }); + const productionPlan = discoverHistoryPlan({dataDir: production.dataDir}); + assertEqual(productionPlan.replicasDir, 'data/replicas', 'history 必须兼容 /stdb/data/replicas 布局。'); + assertEqual(productionPlan.candidates.length, 3, 'production 布局应识别一个旧 snapshot 与一对旧 commitlog 文件。'); + + const importResult = runHistoryDryRun(dev); + assertStatus(importResult, 0, 'history dry-run 应能从已有 uploaded baseline manifest 导入 state。'); + assertTrue(existsSync(dev.statePath), 'history dry-run 应持久化导入后的 baseline state。'); + assertIncludes(importResult.stdout, 'history dry-run', 'history dry-run 应明确说明不会上传或删除。'); + for (const candidate of devPlan.candidates) { + assertTrue(existsSync(path.join(dev.dataDir, candidate.path)), `history dry-run 不得删除候选: ${candidate.path}`); + } +} + +function assertHistorySkipsReplicaWithoutSnapshotAndRejectsMalformedNames() { + const noSnapshot = createHistoryFixture('history-no-snapshot', {nestedData: false}); + createReplicaHistory(noSnapshot.replicasDir, '1', {snapshots: [], segments: [0]}); + const plan = discoverHistoryPlan({dataDir: noSnapshot.dataDir}); + assertEqual(plan.candidates.length, 0, '没有 snapshot 的 replica 不得产生可删除候选。'); + assertEqual(plan.replicas[0]?.reason, 'no-snapshot', '没有 snapshot 时应记录明确跳过原因。'); + + const incompleteSnapshot = createHistoryFixture('history-incomplete-snapshot', {nestedData: false}); + const incompleteReplica = createReplicaHistory(incompleteSnapshot.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + mkdirSync(path.join(incompleteReplica.snapshotsDir, '00000000000000000020.snapshot_dir')); + mkdirSync(path.join(incompleteReplica.snapshotsDir, '00000000000000000030.snapshot_dir')); + writeFileSync(path.join(incompleteReplica.snapshotsDir, '00000000000000000030.snapshot_dir', '00000000000000000030.snapshot_bsatn'), 'locked'); + writeFileSync(path.join(incompleteReplica.snapshotsDir, '00000000000000000030.lock'), `${process.pid}\n`); + const incompletePlan = discoverHistoryPlan({dataDir: incompleteSnapshot.dataDir}); + assertEqual(incompletePlan.replicas[0]?.latestSnapshot, '10', '缺少 snapshot_bsatn 或仍有 lockfile 的目录不得成为 latest snapshot。'); + assertTrue( + !incompletePlan.candidates.some(({path}) => path.includes('00000000000000000010.snapshot_dir')), + '最后一个完整且未锁定的 snapshot 必须保留。', + ); + + const malformedLog = createHistoryFixture('history-malformed-log', {nestedData: false}); + const malformedLogReplica = createReplicaHistory(malformedLog.replicasDir, '1', {snapshots: [10], segments: [0, 11]}); + writeFileSync(path.join(malformedLogReplica.clogDir, 'broken.stdb.log'), 'broken'); + assertThrows( + () => discoverHistoryPlan({dataDir: malformedLog.dataDir}), + 'commitlog 文件名不符合预期', + '异常 commitlog 名称必须阻断整个清理计划。', + ); +} + +function assertHistoryRequiresBaselineAndProducesDeterministicDeferredBatch() { + const missingBaseline = createHistoryFixture('history-missing-baseline', {nestedData: false}); + createReplicaHistory(missingBaseline.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + rmSync(missingBaseline.baselineManifestPath, {force: true}); + const missingResult = runHistoryCommand(missingBaseline, ['--dry-run'], {includeBaselineManifest: false}); + assertStatus(missingResult, 1, 'history 没有 baseline state 或 imported manifest 时必须失败。'); + assertIncludes(missingResult.stderr, '缺少已验真 baseline state', 'baseline 门禁失败应给出明确错误。'); + + const wrongKind = createHistoryFixture('history-wrong-baseline-kind', {nestedData: false}); + createReplicaHistory(wrongKind.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const wrongKindManifest = JSON.parse(readFileSync(wrongKind.baselineManifestPath, 'utf8')); + wrongKindManifest.backupKind = 'spacetimedb-history'; + writeFileSync(wrongKind.baselineManifestPath, `${JSON.stringify(wrongKindManifest)}\n`); + const wrongKindResult = runHistoryDryRun(wrongKind); + assertStatus(wrongKindResult, 1, 'history archive manifest 不得被导入为 full baseline。'); + assertIncludes(wrongKindResult.stderr, 'backupKind 必须是 spacetimedb-data-dir', 'baseline 类型不匹配应失败关闭。'); + + const deterministic = createHistoryFixture('history-deterministic-batch', {nestedData: false}); + createReplicaHistory(deterministic.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + importHistoryState(deterministic); + const firstResultFile = path.join(deterministic.workDir, 'defer-first.json'); + const secondResultFile = path.join(deterministic.workDir, 'defer-second.json'); + const first = runHistoryCommand(deterministic, ['--defer-upload', '--result-file', firstResultFile]); + const second = runHistoryCommand(deterministic, ['--defer-upload', '--result-file', secondResultFile]); + assertStatus(first, 0, '第一次 history defer 应成功生成归档。'); + assertStatus(second, 0, '相同候选重复 history defer 应幂等复用 batch identity。'); + const firstPayload = JSON.parse(readFileSync(firstResultFile, 'utf8')); + const secondPayload = JSON.parse(readFileSync(secondResultFile, 'utf8')); + assertEqual(firstPayload.batchId, secondPayload.batchId, '相同 baseline 与候选必须生成确定性 batchId。'); + assertEqual(firstPayload.objectKey, secondPayload.objectKey, '相同 batch 重跑不得制造新的 OSS object key。'); + const archiveListing = spawnSync('tar', ['-tzf', firstPayload.archivePath], {encoding: 'utf8'}); + assertStatus(archiveListing, 0, 'history 归档应可被 tar 正常读取。'); + assertIncludes(archiveListing.stdout, path.basename(firstPayload.manifestPath), 'history 归档内部必须携带安全候选 manifest。'); + + const dryRunPending = createHistoryFixture('history-dry-run-pending-cleanup', {nestedData: false}); + createReplicaHistory(dryRunPending.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const dryRunState = importHistoryState(dryRunPending); + const dryRunPlan = discoverHistoryPlan({dataDir: dryRunPending.dataDir}); + dryRunState.batches.push({ + batchId: 'pending-cleanup', + objectKey: 'database-backups/test-db/history/pending-cleanup.tar.gz', + contentLength: 10, + verifiedAt: '2026-07-16T00:20:00.000Z', + uploadedAt: '2026-07-16T00:20:00.000Z', + status: 'uploaded', + candidates: dryRunPlan.candidates, + }); + writeFileSync(dryRunPending.statePath, `${JSON.stringify(dryRunState)}\n`); + const pendingDryRunResult = runHistoryDryRun(dryRunPending); + assertStatus(pendingDryRunResult, 0, '存在待清理 uploaded batch 时 history dry-run 仍应只读成功。'); + for (const candidate of dryRunPlan.candidates) { + assertTrue(existsSync(path.join(dryRunPending.dataDir, candidate.path)), `history dry-run 不得恢复执行待清理 batch: ${candidate.path}`); + } +} + +function assertHistoryBackupLockRejectsLiveAndStaleOwners() { + const liveOwner = createHistoryFixture('history-live-lock', {nestedData: false}); + createReplicaHistory(liveOwner.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + importHistoryState(liveOwner); + const liveLockPath = path.join(liveOwner.workDir, 'test-db.backup.lock'); + writeFileSync(liveLockPath, `${process.pid}\n`); + const liveResult = runHistoryCommand(liveOwner, ['--defer-upload']); + assertStatus(liveResult, 1, '仍存活进程持有 backup lock 时必须拒绝并发备份。'); + assertIncludes(liveResult.stderr, '已有数据库备份进程持有锁', '并发备份失败应报告 lock owner pid。'); + + const staleOwner = createHistoryFixture('history-stale-lock', {nestedData: false}); + createReplicaHistory(staleOwner.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + importHistoryState(staleOwner); + const staleLockPath = path.join(staleOwner.workDir, 'test-db.backup.lock'); + writeFileSync(staleLockPath, '2147483647\n'); + const staleResult = runHistoryCommand(staleOwner, ['--defer-upload']); + assertStatus(staleResult, 1, '失效 owner pid 的 backup lock 也必须失败关闭，避免并发抢锁。'); + assertIncludes(staleResult.stderr, '拒绝自动抢锁', '失效 backup lock 应要求人工核对 multipart 与进程。'); + assertTrue(existsSync(staleLockPath), '失效 backup lock 未经人工核对不得自动删除。'); +} + +function assertHistoryStatDriftPreventsAnyCleanup() { + const fixture = createHistoryFixture('history-stat-drift', {nestedData: false}); + createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); + const driftCandidate = plan.candidates.find(({kind}) => kind === 'commitlog'); + const untouchedCandidate = plan.candidates.find(({kind}) => kind === 'snapshot'); + writeFileSync(path.join(fixture.dataDir, driftCandidate.path), 'changed-after-plan'); + assertThrows( + () => cleanupHistoryCandidates({dataDir: fixture.dataDir, candidates: plan.candidates}), + 'stat 漂移', + '任一候选 stat 漂移时必须在删除任何文件前失败。', + ); + assertTrue(existsSync(path.join(fixture.dataDir, untouchedCandidate.path)), 'stat 漂移失败时不得删除其他候选。'); +} + +async function assertHistoryUploadFailureDoesNotDeleteSources() { + const fixture = createHistoryFixture('history-upload-failure', {nestedData: false}); + createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const state = importHistoryState(fixture); + const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); + const archivePath = path.join(fixture.workDir, 'history.tar.gz'); + const manifestPath = `${archivePath}.manifest.json`; + writeFileSync(archivePath, 'history archive'); + const manifest = createHistoryManifest({fixture, state, plan, archivePath}); + writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`); + + let uploadError = null; + try { + await uploadHistoryArchiveWithCleanup({ + archivePath, + manifestPath, + manifest, + statePath: fixture.statePath, + uploadOptions: {}, + uploadFn: async () => { + throw new Error('synthetic upload failure'); + }, + manifestUploadFn: async () => { + throw new Error('manifest upload must not run after archive failure'); + }, + }); + } catch (error) { + uploadError = error; + } + assertIncludes(uploadError?.message ?? '', 'synthetic upload failure', 'history 应保留上传失败原因。'); + for (const candidate of plan.candidates) { + assertTrue(existsSync(path.join(fixture.dataDir, candidate.path)), `上传失败不得删除 history 源文件: ${candidate.path}`); + } + const stateAfterFailure = JSON.parse(readFileSync(fixture.statePath, 'utf8')); + assertEqual(stateAfterFailure.batches.length, 0, '上传失败不得把 batch 标记为 uploaded。'); + + const manifestFailure = createHistoryFixture('history-manifest-upload-failure', {nestedData: false}); + createReplicaHistory(manifestFailure.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const manifestFailureState = importHistoryState(manifestFailure); + const manifestFailurePlan = discoverHistoryPlan({dataDir: manifestFailure.dataDir}); + const manifestFailureArchive = path.join(manifestFailure.workDir, 'history.tar.gz'); + const manifestFailurePath = `${manifestFailureArchive}.manifest.json`; + writeFileSync(manifestFailureArchive, 'history archive'); + const manifestFailurePayload = createHistoryManifest({ + fixture: manifestFailure, + state: manifestFailureState, + plan: manifestFailurePlan, + archivePath: manifestFailureArchive, + }); + writeFileSync(manifestFailurePath, `${JSON.stringify(manifestFailurePayload)}\n`); + let manifestUploadError = null; + try { + await uploadHistoryArchiveWithCleanup({ + archivePath: manifestFailureArchive, + manifestPath: manifestFailurePath, + manifest: manifestFailurePayload, + statePath: manifestFailure.statePath, + uploadOptions: {}, + uploadFn: async () => ({ + bucket: 'backup-bucket', + objectKey: manifestFailurePayload.objectKey, + contentLength: 15, + archiveSha256: 'c'.repeat(64), + verifiedAt: '2026-07-16T00:05:00.000Z', + }), + manifestUploadFn: async () => { + throw new Error('synthetic manifest upload failure'); + }, + }); + } catch (error) { + manifestUploadError = error; + } + assertIncludes(manifestUploadError?.message ?? '', 'synthetic manifest upload failure', 'sidecar manifest 上传失败应阻断清理。'); + for (const candidate of manifestFailurePlan.candidates) { + assertTrue(existsSync(path.join(manifestFailure.dataDir, candidate.path)), `manifest 上传失败不得删除源文件: ${candidate.path}`); + } +} + +async function assertHistorySuccessfulUploadCleansAndIsIdempotent() { + const fixture = createHistoryFixture('history-upload-success', {nestedData: false}); + createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const state = importHistoryState(fixture); + const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); + const archivePath = path.join(fixture.workDir, 'history.tar.gz'); + const manifestPath = `${archivePath}.manifest.json`; + writeFileSync(archivePath, 'history archive'); + const manifest = createHistoryManifest({fixture, state, plan, archivePath}); + writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`); + let baselineVerifyCount = 0; + const result = await uploadHistoryArchiveWithCleanup({ + archivePath, + manifestPath, + manifest, + statePath: fixture.statePath, + uploadOptions: {}, + uploadFn: async () => ({ + bucket: 'backup-bucket', + objectKey: manifest.objectKey, + contentLength: 15, + archiveSha256: 'b'.repeat(64), + etag: 'test-etag', + uploadMode: 'multipart', + partCount: 1, + partSizeBytes: 102400, + verifiedAt: '2026-07-16T00:10:00.000Z', + }), + manifestUploadFn: async ({objectKey}) => ({ + objectKey, + contentLength: 512, + archiveSha256: 'd'.repeat(64), + verifiedAt: '2026-07-16T00:10:01.000Z', + }), + verifyFn: async () => { + baselineVerifyCount += 1; + return {verifiedAt: '2026-07-16T00:10:02.000Z'}; + }, + }); + assertEqual(baselineVerifyCount, 2, 'history 删除源文件前必须重新验真 full baseline 与 sidecar。'); + assertEqual(result.cleanup.deletedCount, plan.candidates.length, '验真上传成功后应删除全部安全候选。'); + for (const candidate of plan.candidates) { + assertTrue(!existsSync(path.join(fixture.dataDir, candidate.path)), `验真成功后应删除 history 源文件: ${candidate.path}`); + } + const repeatedCleanup = cleanupHistoryCandidates({dataDir: fixture.dataDir, candidates: plan.candidates}); + assertEqual(repeatedCleanup.alreadyMissingCount, plan.candidates.length, '重复清理同一 uploaded batch 应幂等。'); +} + +async function assertHistoryResumeReverifiesArchiveAndManifest() { + const fixture = createHistoryFixture('history-resume-verification', {nestedData: false}); + createReplicaHistory(fixture.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const state = importHistoryState(fixture); + const plan = discoverHistoryPlan({dataDir: fixture.dataDir}); + state.batches.push({ + batchId: 'resume-batch', + objectKey: 'database-backups/test-db/history/resume.tar.gz', + contentLength: 100, + archiveSha256: 'e'.repeat(64), + verifiedAt: '2026-07-16T00:30:00.000Z', + manifestObjectKey: 'database-backups/test-db/history/resume.tar.gz.manifest.json', + manifestContentLength: 200, + manifestArchiveSha256: 'f'.repeat(64), + manifestVerifiedAt: '2026-07-16T00:30:01.000Z', + uploadedAt: '2026-07-16T00:30:00.000Z', + status: 'uploaded', + candidates: plan.candidates, + }); + writeFileSync(fixture.statePath, `${JSON.stringify(state)}\n`); + const verifiedKeys = []; + await resumeUploadedHistoryBatch({ + statePath: fixture.statePath, + state, + dataDir: fixture.dataDir, + verificationOptions: {}, + verifyFn: async ({objectKey}) => { + verifiedKeys.push(objectKey); + return {verifiedAt: '2026-07-16T00:31:00.000Z'}; + }, + }); + assertEqual(verifiedKeys.length, 2, '续清理前必须重新验真 history archive 与 sidecar manifest。'); + for (const candidate of plan.candidates) { + assertTrue(!existsSync(path.join(fixture.dataDir, candidate.path)), `续清理验真后应删除候选: ${candidate.path}`); + } + + const failure = createHistoryFixture('history-resume-verification-failure', {nestedData: false}); + createReplicaHistory(failure.replicasDir, '1', {snapshots: [0, 10], segments: [0, 1, 11]}); + const failureState = importHistoryState(failure); + const failurePlan = discoverHistoryPlan({dataDir: failure.dataDir}); + failureState.batches.push({...state.batches[0], candidates: failurePlan.candidates}); + writeFileSync(failure.statePath, `${JSON.stringify(failureState)}\n`); + let resumeError = null; + try { + await resumeUploadedHistoryBatch({ + statePath: failure.statePath, + state: failureState, + dataDir: failure.dataDir, + verificationOptions: {}, + verifyFn: async () => { + throw new Error('synthetic resume HEAD failure'); + }, + }); + } catch (error) { + resumeError = error; + } + assertIncludes(resumeError?.message ?? '', 'synthetic resume HEAD failure', '续清理 OSS 复核失败应保留错误。'); + for (const candidate of failurePlan.candidates) { + assertTrue(existsSync(path.join(failure.dataDir, candidate.path)), `续清理验真失败不得删除候选: ${candidate.path}`); + } +} + +function createHistoryFixture(name, {nestedData}) { + const root = path.join(tmpRoot, name); + const dataDir = path.join(root, 'stdb'); + const replicasDir = nestedData ? path.join(dataDir, 'data', 'replicas') : path.join(dataDir, 'replicas'); + const workDir = path.join(root, 'work'); + const statePath = path.join(workDir, 'history-state.json'); + const baselineManifestPath = path.join(workDir, 'baseline.manifest.json'); + mkdirSync(replicasDir, {recursive: true}); + mkdirSync(workDir, {recursive: true}); + writeFileSync(baselineManifestPath, `${JSON.stringify({ + backupKind: 'spacetimedb-data-dir', + uploadStatus: 'uploaded', + database: 'test-db', + dataDir, + bucket: 'backup-bucket', + objectKey: 'database-backups/test-db/baseline.tar.gz', + manifestObjectKey: 'database-backups/test-db/baseline.tar.gz.manifest.json', + contentLength: 1234, + archiveSha256: 'a'.repeat(64), + manifestContentLength: 512, + manifestArchiveSha256: '9'.repeat(64), + manifestVerifiedAt: '2026-07-16T00:00:00.500Z', + verifiedAt: '2026-07-16T00:00:00.000Z', + uploadedAt: '2026-07-16T00:00:01.000Z', + }, null, 2)}\n`); + return {root, dataDir, replicasDir, workDir, statePath, baselineManifestPath}; +} + +function createReplicaHistory(replicasDir, replicaId, {snapshots, segments}) { + const replicaDir = path.join(replicasDir, replicaId); + const snapshotsDir = path.join(replicaDir, 'snapshots'); + const clogDir = path.join(replicaDir, 'clog'); + mkdirSync(snapshotsDir, {recursive: true}); + mkdirSync(clogDir, {recursive: true}); + for (const transaction of snapshots) { + const name = `${String(transaction).padStart(20, '0')}.snapshot_dir`; + const snapshotDir = path.join(snapshotsDir, name); + mkdirSync(path.join(snapshotDir, 'objects'), {recursive: true}); + writeFileSync(path.join(snapshotDir, `${String(transaction).padStart(20, '0')}.snapshot_bsatn`), `snapshot-${transaction}`); + writeFileSync(path.join(snapshotDir, 'objects', 'object.bin'), `object-${transaction}`); + } + for (const transaction of segments) { + const prefix = String(transaction).padStart(20, '0'); + writeFileSync(path.join(clogDir, `${prefix}.stdb.log`), `log-${transaction}`); + writeFileSync(path.join(clogDir, `${prefix}.stdb.ofs`), `ofs-${transaction}`); + } + return {replicaDir, snapshotsDir, clogDir}; +} + +function runHistoryDryRun(fixture) { + const resultFile = path.join(fixture.workDir, 'dry-run-result.json'); + return runHistoryCommand(fixture, [ + '--result-file', resultFile, + '--dry-run', + ]); +} + +function runHistoryCommand(fixture, extraArgs = [], {includeBaselineManifest = true} = {}) { + const baselineManifestArgs = includeBaselineManifest + ? ['--baseline-manifest', fixture.baselineManifestPath] + : []; + return spawnSync(process.execPath, [ + BACKUP_SCRIPT, + '--mode', 'history', + '--data-dir', fixture.dataDir, + '--work-dir', fixture.workDir, + '--database', 'test-db', + '--bucket', 'backup-bucket', + '--endpoint', 'oss-cn-shanghai.aliyuncs.com', + '--access-key-id', 'test-access-key', + '--access-key-secret', 'test-access-secret', + '--baseline-state', fixture.statePath, + ...baselineManifestArgs, + ...extraArgs, + ], {encoding: 'utf8'}); +} + +function importHistoryState(fixture) { + const result = runHistoryDryRun(fixture); + assertStatus(result, 0, '测试 fixture 应能导入 baseline state。'); + return JSON.parse(readFileSync(fixture.statePath, 'utf8')); +} + +function createHistoryManifest({fixture, state, plan, archivePath}) { + return { + schemaVersion: 1, + backupKind: 'spacetimedb-history', + database: 'test-db', + dataDir: fixture.dataDir, + bucket: 'backup-bucket', + objectKey: `database-backups/test-db/history/${state.baseline.id}/test-batch.tar.gz`, + archivePath, + baselineId: state.baseline.id, + baselineStatePath: fixture.statePath, + batchId: 'test-batch', + candidates: plan.candidates, + uploadStatus: 'pending', + }; +} + async function readRequestBody(body) { if (body === undefined || body === null) { return Buffer.alloc(0); @@ -502,6 +1373,20 @@ function assertTrue(condition, reason) { } } +function assertThrows(callback, expectedMessage, reason) { + let thrown = null; + try { + callback(); + } catch (error) { + thrown = error; + } + if (!(thrown instanceof Error)) { + failures.push(`${reason} 预期抛出错误。`); + return; + } + assertIncludes(thrown.message, expectedMessage, reason); +} + function assertBufferEqual(actual, expected, reason) { if (!Buffer.isBuffer(actual) || !actual.equals(expected)) { failures.push(`${reason} 预期 ${expected.length} bytes，实际 ${actual?.length ?? ''} bytes。`); diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index d46621969..22843943d 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -514,6 +514,76 @@ const checks = [ reason: '生产冷备份 service 必须用 node -- 分隔脚本参数，避免 Node 22 抢占业务 --env-file。', }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + excludes: '--storage-format files', + reason: '生产数据库备份主 service 必须继续保持 archive full 默认行为。', + }, + { + file: 'deploy/systemd/genarrative-database-backup-files-history.conf', + includes: 'ExecStart=\nExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file', + reason: 'files-history drop-in 必须先清空主 service 的 ExecStart，并使用 current release 脚本。', + }, + { + file: 'deploy/systemd/genarrative-database-backup-files-history.conf', + includes: '--storage-format files --mode history --work-dir /var/lib/genarrative/database-backups/files-history', + reason: 'files-history drop-in 必须只执行逐文件历史归档，并复用已建立基线的独立 work-dir。', + }, + { + file: 'deploy/systemd/genarrative-database-backup-files-history.conf', + excludes: '--stop-service', + reason: 'files-history 在线归档不可停止 SpacetimeDB。', + }, + { + file: 'deploy/systemd/genarrative-database-backup-files-history.conf', + excludes: '--database', + reason: 'files-history drop-in 不得写死数据库名，必须从环境文件读取。', + }, + { + file: 'deploy/systemd/genarrative-database-backup-files-history.conf', + excludes: '--bucket', + reason: 'files-history drop-in 不得写死 OSS bucket，必须从环境文件读取。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'DATABASE_BACKUP_PROFILE="${DATABASE_BACKUP_PROFILE:-archive-full}"', + reason: 'Server-Provision 的数据库备份 profile 必须默认保持 archive-full。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'archive-full|files-history)', + reason: 'Server-Provision 必须拒绝未知数据库备份 profile。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: '--storage-format files --mode history --work-dir', + reason: 'Server-Provision 启用 files-history 前必须用真实 current release 脚本执行只读 baseline 预检。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"', + reason: 'Server-Provision 切回 archive-full 时必须移除托管及现场遗留的 history drop-in。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" 0644', + reason: 'Server-Provision 必须从仓库模板安装托管的 files-history drop-in。', + }, + { + file: 'jenkins/Jenkinsfile.production-server-provision', + includes: "choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history']", + reason: 'Server-Provision Job 必须显式暴露 archive-first 的数据库备份 profile。', + }, + { + file: 'jenkins/Jenkinsfile.production-server-provision', + includes: "string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history'", + reason: 'Server-Provision Job 必须允许 dev/release 为 files-history 选择各自的 baseline state 目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-server-provision', + excludes: "params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history'", + reason: 'release 必须能在显式选择 profile 且 baseline 预检通过后启用 files-history。', + }, { file: 'scripts/database-backup-to-oss.mjs', includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})', @@ -6837,6 +6907,7 @@ const checks = [ const nodeEnvFileCommandFiles = [ 'package.json', 'deploy/systemd/genarrative-database-backup.service', + 'deploy/systemd/genarrative-database-backup-files-history.conf', 'scripts/deploy/production-stdb-publish.sh', 'scripts/deploy/pingora-direct-enable.sh', 'scripts/deploy/pingora-direct-rollback.sh', diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index d38a28598..c5abd6a3d 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -1,8 +1,27 @@ #!/usr/bin/env node import {spawnSync} from 'node:child_process'; import {createHash, createHmac} from 'node:crypto'; -import {createReadStream, existsSync, mkdirSync, readFileSync, realpathSync, rmSync, statSync, statfsSync, writeFileSync} from 'node:fs'; -import {basename, dirname, isAbsolute, resolve} from 'node:path'; +import { + chmodSync, + closeSync, + createReadStream, + createWriteStream, + existsSync, + lstatSync, + mkdirSync, + openSync, + readdirSync, + readFileSync, + realpathSync, + renameSync, + rmSync, + statfsSync, + statSync, + writeFileSync, +} from 'node:fs'; +import {basename, dirname, isAbsolute, join, relative, resolve, sep} from 'node:path'; +import {Readable} from 'node:stream'; +import {pipeline} from 'node:stream/promises'; import {setTimeout as sleep} from 'node:timers/promises'; import {fileURLToPath} from 'node:url'; @@ -27,15 +46,26 @@ const DEFAULT_OSS_REQUEST_MAX_ATTEMPTS = 5; const DEFAULT_OSS_RETRY_BASE_DELAY_MS = 1_000; const DEFAULT_OSS_RETRY_MAX_DELAY_MS = 30_000; const RETRYABLE_OSS_HTTP_STATUSES = new Set([408, 429, 500, 502, 503, 504]); +const HISTORY_STATE_SCHEMA_VERSION = 1; +const HISTORY_MANIFEST_SCHEMA_VERSION = 1; +const DIRECT_FILES_STATE_SCHEMA_VERSION = 1; +const DIRECT_FILES_CATALOG_SCHEMA_VERSION = 1; +const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1; function usage() { console.log(`用法: - npm run database:backup:oss -- [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] + npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] node -- scripts/database-backup-to-oss.mjs --upload-archive + node -- scripts/database-backup-to-oss.mjs --publish-manifest + node -- scripts/database-backup-to-oss.mjs --restore-files-state --restore-dir + node -- scripts/database-backup-to-oss.mjs --restore-files-latest --restore-dir [--dry-run] 说明: - 将 SpacetimeDB 数据目录打包成 .tar.gz，并上传到阿里云 OSS 指定 bucket。 + 将 SpacetimeDB 数据目录以 .tar.gz 或逐文件 catalog 形式上传到阿里云 OSS 指定 bucket。 + 默认 full 模式保持原有全量冷备行为；history 模式只归档已被最新 snapshot 覆盖的历史 commitlog 与旧 snapshot。 + --storage-format files 不打包：按原相对路径建立 catalog，文件内容以 SHA-256 不可变对象上传；重复运行只上传新增或变化内容。 + archive history 必须有已验真的 full baseline state；files history 必须复用同一 work-dir 中已发布的 full catalog state。 --defer-upload 只生成本地冷备份和 manifest，不上传；后续用 --upload-archive 异步上传。 默认读取 .env / .env.local / .env.secrets.local；生产服务可传 --env-file /etc/genarrative/api-server.env。 shell 环境变量优先级最高，不会被 env 文件覆盖。 @@ -46,8 +76,11 @@ function usage() { GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET 备份 bucket；未设置时回退 ALIYUN_OSS_BUCKET GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX 对象前缀，默认 database-backups GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT OSS endpoint；未设置时回退 ALIYUN_OSS_ENDPOINT + GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT archive（默认）或 files GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES 备份前要求 work dir 所在文件系统至少有这些可用字节；未设置时按数据目录大小估算 + GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE history 使用的 full baseline 与追加批次状态文件 + GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST 首次初始化 history state 的 uploaded full manifest ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET `); } @@ -121,6 +154,14 @@ function parseArgs(argv) { objectKey: '', resultFile: '', minFreeBytes: '', + mode: 'full', + baselineState: '', + baselineManifest: '', + publishManifest: '', + storageFormat: '', + restoreFilesState: '', + restoreFilesLatest: false, + restoreDir: '', }; for (let index = 0; index < argv.length; index += 1) { @@ -198,6 +239,30 @@ function parseArgs(argv) { case '--min-free-bytes': options.minFreeBytes = readValue(); break; + case '--mode': + options.mode = readValue(); + break; + case '--baseline-state': + options.baselineState = readValue(); + break; + case '--baseline-manifest': + options.baselineManifest = readValue(); + break; + case '--publish-manifest': + options.publishManifest = readValue(); + break; + case '--storage-format': + options.storageFormat = readValue(); + break; + case '--restore-files-state': + options.restoreFilesState = readValue(); + break; + case '--restore-files-latest': + options.restoreFilesLatest = true; + break; + case '--restore-dir': + options.restoreDir = readValue(); + break; default: throw new Error(`未知参数: ${arg}`); } @@ -251,6 +316,356 @@ function buildBackupNames({database, dataDir, objectPrefix}) { return {fileName, objectKey}; } +function atomicWriteJson(filePath, payload) { + mkdirSync(dirname(filePath), {recursive: true}); + const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`; + writeFileSync(tempPath, `${JSON.stringify(payload, null, 2)}\n`, {encoding: 'utf8', mode: 0o600}); + chmodSync(tempPath, 0o600); + renameSync(tempPath, filePath); +} + +function processIsAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code === 'EPERM'; + } +} + +function acquireBackupLock({workDir, database}) { + mkdirSync(workDir, {recursive: true}); + const lockPath = join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}.backup.lock`); + try { + const fd = openSync(lockPath, 'wx', 0o600); + writeFileSync(fd, `${process.pid}\n`, 'utf8'); + closeSync(fd); + const release = () => { + try { + const ownerPid = Number(String(readFileSync(lockPath, 'utf8')).trim()); + if (ownerPid === process.pid) { + rmSync(lockPath, {force: true}); + } + } catch { + // The lock may already have been removed by the normal exit path. + } + }; + process.once('exit', release); + for (const signal of ['SIGINT', 'SIGTERM']) { + process.once(signal, () => { + release(); + process.exit(signal === 'SIGINT' ? 130 : 143); + }); + } + return lockPath; + } catch (error) { + if (error?.code !== 'EEXIST') { + throw error; + } + } + const ownerPid = Number(String(readFileSync(lockPath, 'utf8')).trim()); + if (Number.isSafeInteger(ownerPid) && ownerPid > 0 && processIsAlive(ownerPid)) { + throw new Error(`已有数据库备份进程持有锁: ${lockPath} pid=${ownerPid}`); + } + throw new Error(`发现失效数据库备份锁，拒绝自动抢锁；请核对 OSS multipart 与进程后手工删除: ${lockPath} pid=${ownerPid || ''}`); +} + +function historyStatePath({args, env, workDir, database}) { + return resolvePath(firstNonEmpty( + args.baselineState, + env.GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE, + join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-history-state.json`), + )); +} + +function baselineIdFor(baseline) { + return sha256Hex([ + baseline.bucket, + baseline.objectKey, + baseline.verifiedAt, + baseline.contentLength, + baseline.archiveSha256, + ].join('\0')).slice(0, 24); +} + +function normalizeUploadedBaselineManifest(manifest, {database, dataDir}) { + if (manifest.uploadStatus !== 'uploaded') { + throw new Error(`baseline manifest 必须是 uploaded，实际: ${manifest.uploadStatus ?? ''}`); + } + if (manifest.backupKind !== 'spacetimedb-data-dir') { + throw new Error(`baseline manifest backupKind 必须是 spacetimedb-data-dir，实际: ${manifest.backupKind ?? ''}`); + } + const baseline = { + backupKind: 'spacetimedb-data-dir', + database: firstNonEmpty(manifest.database, database), + dataDir: resolvePath(firstNonEmpty(manifest.dataDir, dataDir)), + bucket: String(manifest.bucket ?? '').trim(), + objectKey: String(manifest.objectKey ?? '').trim(), + verifiedAt: String(manifest.verifiedAt ?? '').trim(), + uploadedAt: String(manifest.uploadedAt ?? '').trim(), + contentLength: Number(manifest.contentLength), + archiveSha256: String(manifest.archiveSha256 ?? '').trim().toLowerCase(), + manifestObjectKey: String(manifest.manifestObjectKey ?? '').trim(), + manifestContentLength: Number(manifest.manifestContentLength), + manifestArchiveSha256: String(manifest.manifestArchiveSha256 ?? '').trim().toLowerCase(), + manifestVerifiedAt: String(manifest.manifestVerifiedAt ?? '').trim(), + }; + if ( + !baseline.bucket + || !baseline.objectKey + || !baseline.verifiedAt + || !Number.isSafeInteger(baseline.contentLength) + || baseline.contentLength <= 0 + || !/^[a-f0-9]{64}$/u.test(baseline.archiveSha256) + || !baseline.manifestObjectKey + || !Number.isSafeInteger(baseline.manifestContentLength) + || baseline.manifestContentLength <= 0 + || !/^[a-f0-9]{64}$/u.test(baseline.manifestArchiveSha256) + || !baseline.manifestVerifiedAt + ) { + throw new Error('baseline manifest 缺少已验真 OSS 归档或 sidecar 信息。'); + } + baseline.id = baselineIdFor(baseline); + return baseline; +} + +function validateHistoryState(state, {database, dataDir}) { + if (state.schemaVersion !== HISTORY_STATE_SCHEMA_VERSION || !state.baseline) { + throw new Error('history state schemaVersion 或 baseline 无效。'); + } + const baseline = normalizeUploadedBaselineManifest( + {...state.baseline, uploadStatus: 'uploaded'}, + {database, dataDir}, + ); + if (baseline.database !== database) { + throw new Error(`history state database 不匹配: expected=${database}, actual=${baseline.database}`); + } + if (resolvePath(baseline.dataDir) !== resolvePath(dataDir)) { + throw new Error(`history state dataDir 不匹配: expected=${resolvePath(dataDir)}, actual=${resolvePath(baseline.dataDir)}`); + } + return { + ...state, + baseline, + batches: Array.isArray(state.batches) ? state.batches : [], + }; +} + +function writeBaselineState({statePath, baseline, previousState = null}) { + const state = { + schemaVersion: HISTORY_STATE_SCHEMA_VERSION, + updatedAt: new Date().toISOString(), + baseline, + batches: previousState?.baseline?.id === baseline.id && Array.isArray(previousState.batches) + ? previousState.batches + : [], + }; + atomicWriteJson(statePath, state); + return state; +} + +function loadOrImportHistoryState({args, env, statePath, database, dataDir}) { + if (existsSync(statePath)) { + return validateHistoryState(readManifest(statePath), {database, dataDir}); + } + const importPath = firstNonEmpty(args.baselineManifest, env.GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST); + if (!importPath) { + throw new Error(`history 模式缺少已验真 baseline state: ${statePath}；可用 --baseline-manifest 导入已有 uploaded baseline manifest。`); + } + const baseline = normalizeUploadedBaselineManifest(readManifest(resolvePath(importPath)), {database, dataDir}); + return writeBaselineState({statePath, baseline}); +} + +function assertSafeRelativePath(dataDir, absolutePath) { + const relativePath = relative(resolvePath(dataDir), resolvePath(absolutePath)); + if (!relativePath || relativePath === '..' || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) { + throw new Error(`history 候选路径越界或等于数据目录: ${absolutePath}`); + } + return relativePath.split(sep).join('/'); +} + +function statFingerprint(absolutePath, rootPath = absolutePath) { + const entries = []; + let totalSize = 0n; + const visit = (currentPath) => { + const stat = lstatSync(currentPath, {bigint: true}); + if (stat.isSymbolicLink()) { + throw new Error(`history 候选不得包含符号链接: ${currentPath}`); + } + const entryPath = currentPath === rootPath ? '.' : relative(rootPath, currentPath).split(sep).join('/'); + const kind = stat.isDirectory() ? 'directory' : stat.isFile() ? 'file' : 'other'; + if (kind === 'other') { + throw new Error(`history 候选只允许普通文件或目录: ${currentPath}`); + } + entries.push([ + entryPath, + kind, + stat.dev.toString(), + stat.ino.toString(), + stat.mode.toString(), + stat.size.toString(), + stat.mtimeNs.toString(), + ].join('\0')); + if (stat.isFile()) { + totalSize += stat.size; + } else { + for (const name of readdirSync(currentPath).sort()) { + visit(join(currentPath, name)); + } + } + }; + visit(rootPath); + return { + fingerprint: sha256Hex(entries.join('\n')), + sizeBytes: totalSize.toString(), + entryCount: entries.length, + }; +} + +function findReplicasDir(dataDir) { + const candidates = [resolve(dataDir, 'replicas'), resolve(dataDir, 'data', 'replicas')] + .filter((candidate) => existsSync(candidate) && lstatSync(candidate).isDirectory()); + if (candidates.length !== 1) { + throw new Error(`无法唯一确定 replicas 目录: ${candidates.length === 0 ? '' : candidates.join(', ')}`); + } + return candidates[0]; +} + +function historyCandidate({dataDir, absolutePath, kind, replicaId, transaction}) { + const stat = statFingerprint(absolutePath); + return { + path: assertSafeRelativePath(dataDir, absolutePath), + kind, + replicaId, + transaction: transaction.toString(), + ...stat, + }; +} + +export function discoverHistoryPlan({dataDir}) { + const resolvedDataDir = resolvePath(dataDir); + const replicasDir = findReplicasDir(resolvedDataDir); + const replicaEntries = readdirSync(replicasDir, {withFileTypes: true}); + const replicas = []; + const candidates = []; + + for (const replicaEntry of replicaEntries.sort((left, right) => left.name.localeCompare(right.name))) { + if (!replicaEntry.isDirectory()) { + continue; + } + if (!/^\d+$/u.test(replicaEntry.name)) { + throw new Error(`replica 目录名不符合预期: ${replicaEntry.name}`); + } + const replicaId = replicaEntry.name; + const replicaDir = join(replicasDir, replicaId); + const snapshotsDir = join(replicaDir, 'snapshots'); + const clogDir = join(replicaDir, 'clog'); + if (!existsSync(snapshotsDir) || !lstatSync(snapshotsDir).isDirectory()) { + replicas.push({replicaId, status: 'skipped', reason: 'no-snapshots-directory'}); + continue; + } + const snapshotEntries = readdirSync(snapshotsDir, {withFileTypes: true}); + const snapshots = snapshotEntries.flatMap((entry) => { + const match = /^(\d{20})\.snapshot_dir$/u.exec(entry.name); + if (!match) { + return []; + } + const transaction = BigInt(match[1]); + if (transaction > 0xffff_ffff_ffff_ffffn) { + throw new Error(`snapshot transaction 超出 u64: ${entry.name}`); + } + if (!entry.isDirectory()) { + throw new Error(`snapshot 候选必须是目录: ${join(snapshotsDir, entry.name)}`); + } + const snapshotDir = join(snapshotsDir, entry.name); + const lockPath = join(snapshotsDir, `${match[1]}.lock`); + const snapshotFile = join(snapshotDir, `${match[1]}.snapshot_bsatn`); + if (existsSync(lockPath) || !existsSync(snapshotFile) || !lstatSync(snapshotFile).isFile()) { + return []; + } + return [{name: entry.name, transaction}]; + }).sort((left, right) => left.transaction < right.transaction ? -1 : left.transaction > right.transaction ? 1 : 0); + if (snapshots.length === 0) { + replicas.push({replicaId, status: 'skipped', reason: 'no-snapshot'}); + continue; + } + if (!existsSync(clogDir) || !lstatSync(clogDir).isDirectory()) { + throw new Error(`replica ${replicaId} 缺少 clog 目录。`); + } + const segmentFiles = new Map(); + for (const entry of readdirSync(clogDir, {withFileTypes: true})) { + const match = /^(\d{20})\.stdb\.(log|ofs)$/u.exec(entry.name); + if (!match) { + throw new Error(`commitlog 文件名不符合预期: ${entry.name}`); + } + if (!entry.isFile()) { + throw new Error(`commitlog 候选必须是普通文件: ${join(clogDir, entry.name)}`); + } + const transaction = BigInt(match[1]); + if (transaction > 0xffff_ffff_ffff_ffffn) { + throw new Error(`commitlog transaction 超出 u64: ${entry.name}`); + } + const key = transaction.toString(); + const group = segmentFiles.get(key) ?? {transaction}; + group[match[2]] = entry.name; + segmentFiles.set(key, group); + } + for (const group of segmentFiles.values()) { + if (group.ofs && !group.log) { + throw new Error(`commitlog offset 缺少对应 log: replica=${replicaId}, transaction=${group.transaction}`); + } + } + const segments = [...segmentFiles.values()] + .filter((group) => group.log) + .sort((left, right) => left.transaction < right.transaction ? -1 : left.transaction > right.transaction ? 1 : 0); + const latestSnapshot = snapshots.at(-1).transaction; + const boundarySegment = segments.filter((segment) => segment.transaction <= latestSnapshot).at(-1); + if (!boundarySegment) { + throw new Error(`replica ${replicaId} 无法找到覆盖 latest snapshot ${latestSnapshot} 的 commitlog 边界。`); + } + for (const snapshot of snapshots.slice(0, -1)) { + candidates.push(historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(snapshotsDir, snapshot.name), + kind: 'snapshot', + replicaId, + transaction: snapshot.transaction, + })); + } + for (const segment of segments.filter((item) => item.transaction < boundarySegment.transaction)) { + candidates.push(historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.log), + kind: 'commitlog', + replicaId, + transaction: segment.transaction, + })); + if (segment.ofs) { + candidates.push(historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.ofs), + kind: 'commitlog-offset', + replicaId, + transaction: segment.transaction, + })); + } + } + replicas.push({ + replicaId, + status: 'ready', + latestSnapshot: latestSnapshot.toString(), + boundarySegment: boundarySegment.transaction.toString(), + }); + } + candidates.sort((left, right) => left.path.localeCompare(right.path)); + return { + dataDir: resolvedDataDir, + replicasDir: assertSafeRelativePath(resolvedDataDir, replicasDir), + replicas, + candidates, + totalSizeBytes: candidates.reduce((sum, item) => sum + BigInt(item.sizeBytes), 0n).toString(), + }; +} + function runCommand(command, args, options = {}) { const result = spawnSync(command, args, { cwd: options.cwd ?? REPO_ROOT, @@ -371,6 +786,18 @@ function assertSufficientWorkDirSpace({dataDir, workDir, args, env}) { } } +function assertSufficientHistoryWorkDirSpace({historySizeBytes, workDir, args, env}) { + mkdirSync(workDir, {recursive: true}); + const availableBytes = getAvailableBytes(workDir); + const requiredFreeBytes = calculateRequiredFreeBytes({dataSizeBytes: BigInt(historySizeBytes), args, env}); + console.log( + `[database-backup] history 空间预检: candidates=${formatBytes(historySizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, + ); + if (availableBytes < requiredFreeBytes) { + throw new Error(`history 工作目录剩余空间不足: available=${formatBytes(availableBytes)}；required=${formatBytes(requiredFreeBytes)}`); + } +} + function collectRestartServicesAfterBackup({args, env}) { const serviceNames = [ ...String(env.GENARRATIVE_DATABASE_BACKUP_RESTART_SERVICE_AFTER ?? '') @@ -448,9 +875,119 @@ function createArchive({dataDir, workDir, fileName}) { const entryName = basename(dataDir); console.log(`[database-backup] 打包: ${dataDir} -> ${archivePath}`); runCommand('tar', ['-czf', archivePath, '-C', parentDir, entryName], {stdio: 'inherit'}); + verifyArchive(archivePath); return archivePath; } +function verifyArchive(archivePath) { + console.log(`[database-backup] 校验归档: ${archivePath}`); + runCommand('tar', ['-tzf', archivePath], {stdio: 'ignore'}); +} + +function historyBatchId({baselineId, plan}) { + const identity = plan.candidates.map((candidate) => [ + candidate.path, + candidate.kind, + candidate.transaction, + candidate.fingerprint, + ].join('\0')).join('\n'); + return sha256Hex(`${baselineId}\0${identity}`).slice(0, 32); +} + +function buildHistoryNames({database, objectPrefix, baselineId, batchId}) { + const databasePart = sanitizeObjectPart(database, 'spacetimedb'); + const prefix = String(objectPrefix || 'database-backups') + .trim() + .replace(/^\/+|\/+$/gu, '') + .split('/') + .filter(Boolean) + .map((part) => sanitizeObjectPart(part, 'backup')) + .join('/'); + const fileName = `${databasePart}-history-${batchId}.tar.gz`; + return { + fileName, + objectKey: [prefix, databasePart, 'history', baselineId, fileName].filter(Boolean).join('/'), + }; +} + +function createHistoryArchive({dataDir, workDir, fileName, manifestPath, candidates}) { + mkdirSync(workDir, {recursive: true}); + const archivePath = resolve(workDir, fileName); + const candidatePaths = candidates.map((candidate) => candidate.path); + console.log(`[database-backup] 打包 history: ${candidatePaths.length} 个候选 -> ${archivePath}`); + runCommand('tar', [ + '-czf', + archivePath, + '-C', + dataDir, + ...candidatePaths, + '-C', + dirname(manifestPath), + basename(manifestPath), + ], {stdio: 'inherit'}); + verifyArchive(archivePath); + return archivePath; +} + +function recordHistoryBatch({statePath, state, manifest, uploadResult, manifestUpload, status, cleanedAt = ''}) { + const batch = { + batchId: manifest.batchId, + objectKey: uploadResult.objectKey, + contentLength: uploadResult.contentLength, + archiveSha256: uploadResult.archiveSha256, + verifiedAt: uploadResult.verifiedAt, + manifestObjectKey: manifestUpload.objectKey, + manifestContentLength: manifestUpload.contentLength, + manifestArchiveSha256: manifestUpload.archiveSha256, + manifestVerifiedAt: manifestUpload.verifiedAt, + uploadedAt: manifest.uploadedAt, + status, + cleanedAt, + candidates: manifest.candidates, + }; + const batches = state.batches.filter((item) => item.batchId !== batch.batchId); + batches.push(batch); + const nextState = {...state, updatedAt: new Date().toISOString(), batches}; + atomicWriteJson(statePath, nextState); + return nextState; +} + +function candidateKey(candidate) { + return `${candidate.kind}\0${candidate.path}`; +} + +export function cleanupHistoryCandidates({dataDir, candidates}) { + const currentPlan = discoverHistoryPlan({dataDir}); + const eligible = new Map(currentPlan.candidates.map((candidate) => [candidateKey(candidate), candidate])); + const existing = []; + for (const candidate of candidates) { + const absolutePath = resolve(dataDir, candidate.path); + assertSafeRelativePath(dataDir, absolutePath); + if (!existsSync(absolutePath)) { + continue; + } + const current = eligible.get(candidateKey(candidate)); + if (!current) { + throw new Error(`history 候选已不在当前安全边界内，拒绝删除: ${candidate.path}`); + } + const currentStat = statFingerprint(absolutePath); + if (currentStat.fingerprint !== candidate.fingerprint || currentStat.sizeBytes !== candidate.sizeBytes) { + throw new Error(`history 候选 stat 漂移，拒绝删除: ${candidate.path}`); + } + existing.push({candidate, absolutePath}); + } + existing.sort((left, right) => { + const priority = {'commitlog-offset': 0, commitlog: 1, snapshot: 2}; + return (priority[left.candidate.kind] ?? 3) - (priority[right.candidate.kind] ?? 3) + || left.candidate.path.localeCompare(right.candidate.path); + }); + for (const {candidate, absolutePath} of existing) { + rmSync(absolutePath, {recursive: candidate.kind === 'snapshot', force: false}); + console.log(`[database-backup] 已清理 history 源文件: ${candidate.path}`); + } + return {deletedCount: existing.length, alreadyMissingCount: candidates.length - existing.length}; +} + function writeManifest({manifestPath, payload}) { writeFileSync(manifestPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); } @@ -470,6 +1007,724 @@ function sha256Hex(content) { return createHash('sha256').update(content).digest('hex'); } +async function sha256FileHex(filePath) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(filePath)) { + hash.update(chunk); + } + return hash.digest('hex'); +} + +function directFilesStatePath({workDir, database}) { + return join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-files-state.json`); +} + +function normalizeObjectPrefix(objectPrefix, database) { + const prefix = String(objectPrefix || 'database-backups') + .trim() + .replace(/^\/+|\/+$/gu, '') + .split('/') + .filter(Boolean) + .map((part) => sanitizeObjectPart(part, 'backup')) + .join('/'); + return [prefix, sanitizeObjectPart(database, 'spacetimedb')].filter(Boolean).join('/'); +} + +function directFileIdentity(filePath) { + const stat = lstatSync(filePath, {bigint: true}); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error(`files 模式只允许普通文件: ${filePath}`); + } + return { + dev: stat.dev.toString(), + ino: stat.ino.toString(), + size: stat.size.toString(), + mtimeNs: stat.mtimeNs.toString(), + mode: Number(stat.mode & 0o7777n), + }; +} + +function sameDirectFileIdentity(left, right) { + return left.dev === right.dev + && left.ino === right.ino + && left.size === right.size + && left.mtimeNs === right.mtimeNs + && left.mode === right.mode; +} + +export async function collectDirectFileEntries({dataDir, candidates = null, objectPrefix, database}) { + const resolvedDataDir = resolvePath(dataDir); + if (!existsSync(resolvedDataDir) || !lstatSync(resolvedDataDir).isDirectory()) { + throw new Error(`files 数据目录不存在或不是目录: ${resolvedDataDir}`); + } + const files = new Map(); + const directories = new Set(['.']); + const roots = candidates === null + ? [{absolutePath: resolvedDataDir, relativePath: '.'}] + : candidates.map((candidate) => ({ + absolutePath: resolve(resolvedDataDir, candidate.path), + relativePath: assertSafeRelativePath(resolvedDataDir, resolve(resolvedDataDir, candidate.path)), + })); + + const visit = async (absolutePath, relativePath) => { + const stat = lstatSync(absolutePath); + if (stat.isSymbolicLink()) { + throw new Error(`files 模式拒绝符号链接: ${absolutePath}`); + } + if (stat.isDirectory()) { + directories.add(relativePath); + for (const name of readdirSync(absolutePath).sort()) { + const childRelative = relativePath === '.' ? name : `${relativePath}/${name}`; + await visit(join(absolutePath, name), childRelative); + } + return; + } + if (!stat.isFile()) { + throw new Error(`files 模式只允许普通文件或目录: ${absolutePath}`); + } + const before = directFileIdentity(absolutePath); + const sha256 = await sha256FileHex(absolutePath); + const after = directFileIdentity(absolutePath); + if (!sameDirectFileIdentity(before, after)) { + throw new Error(`files 扫描期间源文件发生变化: ${relativePath}`); + } + const basePrefix = normalizeObjectPrefix(objectPrefix, database); + files.set(relativePath, { + path: relativePath, + sizeBytes: Number(after.size), + sha256, + mode: after.mode, + objectKey: `${basePrefix}/files/sha256/${sha256.slice(0, 2)}/${sha256}`, + sourceStat: after, + }); + }; + + for (const root of roots.sort((left, right) => left.relativePath.localeCompare(right.relativePath))) { + if (!existsSync(root.absolutePath)) { + throw new Error(`files 候选在扫描前消失: ${root.relativePath}`); + } + await visit(root.absolutePath, root.relativePath); + } + return { + directories: [...directories].sort(), + files: [...files.values()].sort((left, right) => left.path.localeCompare(right.path)), + }; +} + +function directCatalogIdentity({mode, baselineCatalogId, rootName, directories, files}) { + return sha256Hex(JSON.stringify({ + mode, + baselineCatalogId: baselineCatalogId || '', + rootName, + directories, + files: files.map(({path, sizeBytes, sha256, mode, objectKey}) => ({path, sizeBytes, sha256, mode, objectKey})), + })); +} + +function readDirectFilesState(statePath, {database, bucket}) { + if (!existsSync(statePath)) { + return null; + } + const state = readManifest(statePath); + if ( + state.schemaVersion !== DIRECT_FILES_STATE_SCHEMA_VERSION + || state.backupKind !== 'spacetimedb-direct-files-state' + || state.database !== database + || state.bucket !== bucket + ) { + throw new Error(`files state 与本次数据源或 bucket 不匹配: ${statePath}`); + } + return state; +} + +async function ensureDirectObject({ + file, + dataDir, + uploadOptions, + previousFile, + verifyCatalogReuse = false, + uploadFn, + verifyFn, +}) { + const absolutePath = resolve(dataDir, file.path); + assertSafeRelativePath(dataDir, absolutePath); + if (previousFile?.sha256 === file.sha256 + && previousFile?.sizeBytes === file.sizeBytes + && previousFile?.objectKey === file.objectKey) { + if (verifyCatalogReuse) { + await verifyFn({ + ...uploadOptions, + objectKey: file.objectKey, + contentLength: file.sizeBytes, + archiveSha256: file.sha256, + }); + } + return { + status: verifyCatalogReuse ? 'catalog-reused-verified' : 'catalog-reused', + objectKey: file.objectKey, + }; + } + try { + await verifyFn({ + ...uploadOptions, + objectKey: file.objectKey, + contentLength: file.sizeBytes, + archiveSha256: file.sha256, + }); + return {status: 'oss-reused', objectKey: file.objectKey}; + } catch (error) { + if (error?.status !== 404) { + throw error; + } + } + const beforeUpload = directFileIdentity(absolutePath); + if (!sameDirectFileIdentity(beforeUpload, file.sourceStat)) { + throw new Error(`files 上传前源文件 stat 漂移: ${file.path}`); + } + await uploadFn({ + archivePath: absolutePath, + ...uploadOptions, + objectKey: file.objectKey, + archiveSha256: file.sha256, + backupKind: 'spacetimedb-direct-file', + contentType: 'application/octet-stream', + allowEmpty: true, + }); + const afterUpload = directFileIdentity(absolutePath); + if (!sameDirectFileIdentity(afterUpload, file.sourceStat)) { + throw new Error(`files 上传期间源文件 stat 漂移: ${file.path}`); + } + return {status: 'uploaded', objectKey: file.objectKey}; +} + +async function ensureDirectManifest({manifestPath, objectKey, uploadOptions, uploadManifestFn, verifyFn}) { + const body = readFileSync(manifestPath); + const archiveSha256 = sha256Hex(body); + try { + const verification = await verifyFn({ + ...uploadOptions, + objectKey, + contentLength: body.length, + archiveSha256, + }); + return {objectKey, contentLength: body.length, archiveSha256, verifiedAt: verification.verifiedAt, reused: true}; + } catch (error) { + if (error?.status !== 404) { + throw error; + } + } + return uploadManifestFn({manifestPath, ...uploadOptions, objectKey}); +} + +function directCatalogRef(catalog) { + return { + mode: catalog.mode, + catalogId: catalog.catalogId, + objectKey: catalog.objectKey, + contentLength: catalog.contentLength, + sha256: catalog.sha256, + verifiedAt: catalog.verifiedAt, + }; +} + +function assertDirectCatalogRef(catalog, expectedMode, label) { + if ( + !catalog + || catalog.mode !== expectedMode + || !/^[a-f0-9]{64}$/u.test(catalog.catalogId) + || typeof catalog.objectKey !== 'string' + || !catalog.objectKey + || !Number.isSafeInteger(catalog.contentLength) + || catalog.contentLength <= 0 + || !/^[a-f0-9]{64}$/u.test(catalog.sha256) + || typeof catalog.verifiedAt !== 'string' + || !catalog.verifiedAt + ) { + throw new Error(`files ${label} catalog ref 无效。`); + } + return directCatalogRef(catalog); +} + +function buildDirectFilesLatest({database, bucket, state}) { + const latestFullCatalog = assertDirectCatalogRef(state?.latestCatalog, 'full', 'latest full'); + const historyCatalogs = (state?.historyCatalogs ?? []).map((catalog) => ( + assertDirectCatalogRef(catalog, 'history', 'history') + )); + return { + schemaVersion: DIRECT_FILES_LATEST_SCHEMA_VERSION, + backupKind: 'spacetimedb-direct-files-latest', + database, + bucket, + updatedAt: new Date().toISOString(), + latestFullCatalog, + historyCatalogs, + }; +} + +function validateDirectFilesLatest(latest, {database, bucket}) { + if ( + latest?.schemaVersion !== DIRECT_FILES_LATEST_SCHEMA_VERSION + || latest.backupKind !== 'spacetimedb-direct-files-latest' + || latest.database !== database + || latest.bucket !== bucket + || !Array.isArray(latest.historyCatalogs) + ) { + throw new Error('files latest pointer 契约无效。'); + } + return { + ...latest, + latestFullCatalog: assertDirectCatalogRef(latest.latestFullCatalog, 'full', 'latest full'), + historyCatalogs: latest.historyCatalogs.map((catalog) => assertDirectCatalogRef(catalog, 'history', 'history')), + }; +} + +async function publishDirectFilesLatest({ + workDir, + database, + bucket, + objectPrefix, + state, + uploadOptions, + uploadManifestFn, + verifyFn, +}) { + const latest = buildDirectFilesLatest({database, bucket, state}); + for (const catalogRef of [latest.latestFullCatalog, ...latest.historyCatalogs]) { + await verifyFn({ + ...uploadOptions, + objectKey: catalogRef.objectKey, + contentLength: catalogRef.contentLength, + archiveSha256: catalogRef.sha256, + }); + } + const latestPath = join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-latest.json`); + const latestObjectKey = `${normalizeObjectPrefix(objectPrefix, database)}/latest.json`; + writeManifest({manifestPath: latestPath, payload: latest}); + const uploaded = await uploadManifestFn({manifestPath: latestPath, ...uploadOptions, objectKey: latestObjectKey}); + const verification = await verifyFn({ + ...uploadOptions, + objectKey: latestObjectKey, + contentLength: uploaded.contentLength, + archiveSha256: uploaded.archiveSha256, + }); + return { + latest, + latestPath, + latestObjectKey, + contentLength: uploaded.contentLength, + sha256: uploaded.archiveSha256, + verifiedAt: verification.verifiedAt, + }; +} + +export async function runDirectFilesBackup({ + mode, + dataDir, + workDir, + database, + bucket, + objectPrefix, + dryRun = false, + resultFile = '', + uploadOptions, + uploadFn = uploadArchive, + uploadManifestFn = uploadManifestFile, + verifyFn = verifyOssObject, +}) { + mkdirSync(workDir, {recursive: true}); + const statePath = directFilesStatePath({workDir, database}); + const state = readDirectFilesState(statePath, {database, bucket}); + if (mode === 'history' && (!state?.baselineCatalog || state?.latestCatalog?.mode !== 'full')) { + throw new Error(`files history 模式缺少已发布 full baseline catalog: ${statePath}`); + } + const plan = mode === 'history' ? discoverHistoryPlan({dataDir}) : null; + const collected = await collectDirectFileEntries({ + dataDir, + candidates: plan?.candidates ?? null, + objectPrefix, + database, + }); + const baselineCatalogId = mode === 'history' ? (state?.baselineCatalog?.catalogId ?? '') : ''; + const rootName = basename(dataDir); + const catalogId = directCatalogIdentity({mode, baselineCatalogId, rootName, ...collected}); + const basePrefix = normalizeObjectPrefix(objectPrefix, database); + const catalogObjectKey = `${basePrefix}/catalogs/${mode}/${catalogId}.json`; + const catalogPath = join(workDir, `${sanitizeObjectPart(database, 'spacetimedb')}-${mode}-${catalogId}.catalog.json`); + const catalog = { + schemaVersion: DIRECT_FILES_CATALOG_SCHEMA_VERSION, + backupKind: mode === 'full' ? 'spacetimedb-data-dir-files' : 'spacetimedb-history-files', + database, + bucket, + mode, + catalogId, + catalogObjectKey, + baselineCatalogId, + rootName, + directories: collected.directories, + files: collected.files.map(({sourceStat: _sourceStat, ...file}) => file), + }; + writeManifest({manifestPath: catalogPath, payload: catalog}); + const summary = { + statePath, + catalogPath, + catalogObjectKey, + catalogId, + fileCount: collected.files.length, + totalSizeBytes: collected.files.reduce((sum, file) => sum + BigInt(file.sizeBytes), 0n).toString(), + candidateCount: plan?.candidates.length ?? 0, + }; + if (resultFile) { + atomicWriteJson(resolvePath(resultFile), {...summary, dryRun}); + } + console.log(`[database-backup] files ${mode}: files=${summary.fileCount}, size=${formatBytes(summary.totalSizeBytes)}, catalog=${catalogId}`); + if (dryRun) { + console.log('[database-backup] files dry-run，仅扫描并生成本地 catalog，不上传或删除。'); + return {...summary, catalog, uploadedCount: 0, reusedCount: 0}; + } + if (mode === 'history' && plan.candidates.length === 0) { + await verifyFn({ + ...uploadOptions, + objectKey: state.latestCatalog.objectKey, + contentLength: state.latestCatalog.contentLength, + archiveSha256: state.latestCatalog.sha256, + }); + const latestPointer = await publishDirectFilesLatest({ + workDir, + database, + bucket, + objectPrefix, + state, + uploadOptions, + uploadManifestFn, + verifyFn, + }); + console.log('[database-backup] files history 没有可归档候选。'); + const emptyResult = {...summary, catalog, latestPointer, uploadedCount: 0, reusedCount: 0, cleanup: null}; + if (resultFile) { + atomicWriteJson(resolvePath(resultFile), emptyResult); + } + return emptyResult; + } + + if (state?.latestCatalog?.catalogId === catalogId && state.latestCatalog.mode === mode) { + await verifyFn({...uploadOptions, objectKey: state.latestCatalog.objectKey, contentLength: state.latestCatalog.contentLength, archiveSha256: state.latestCatalog.sha256}); + if (mode === 'full') { + const latestPointer = await publishDirectFilesLatest({ + workDir, + database, + bucket, + objectPrefix, + state, + uploadOptions, + uploadManifestFn, + verifyFn, + }); + console.log('[database-backup] files catalog 未变化，无文件需要上传。'); + return {...summary, catalog, latestPointer, uploadedCount: 0, reusedCount: collected.files.length, unchanged: true}; + } + } + + if (state?.latestCatalog) { + await verifyFn({ + ...uploadOptions, + objectKey: state.latestCatalog.objectKey, + contentLength: state.latestCatalog.contentLength, + archiveSha256: state.latestCatalog.sha256, + }); + } + + const previousFiles = new Map((state?.latestCatalog?.files ?? []).map((file) => [file.path, file])); + let uploadedCount = 0; + let reusedCount = 0; + for (const [index, file] of collected.files.entries()) { + const result = await ensureDirectObject({ + file, + dataDir, + uploadOptions, + previousFile: previousFiles.get(file.path), + verifyCatalogReuse: mode === 'history', + uploadFn, + verifyFn, + }); + if (result.status === 'uploaded') { + uploadedCount += 1; + } else { + reusedCount += 1; + } + console.log(`[database-backup] files 进度: ${index + 1}/${collected.files.length} (${result.status}) ${file.path}`); + } + const catalogUpload = await ensureDirectManifest({ + manifestPath: catalogPath, + objectKey: catalogObjectKey, + uploadOptions, + uploadManifestFn, + verifyFn, + }); + await verifyFn({...uploadOptions, objectKey: catalogObjectKey, contentLength: catalogUpload.contentLength, archiveSha256: catalogUpload.archiveSha256}); + + if (mode === 'history') { + await verifyFn({ + ...uploadOptions, + objectKey: state.baselineCatalog.objectKey, + contentLength: state.baselineCatalog.contentLength, + archiveSha256: state.baselineCatalog.sha256, + }); + } + const catalogRef = { + mode, + catalogId, + objectKey: catalogObjectKey, + contentLength: catalogUpload.contentLength, + sha256: catalogUpload.archiveSha256, + verifiedAt: catalogUpload.verifiedAt, + files: catalog.files, + }; + const nextState = { + schemaVersion: DIRECT_FILES_STATE_SCHEMA_VERSION, + backupKind: 'spacetimedb-direct-files-state', + database, + dataDir, + bucket, + updatedAt: new Date().toISOString(), + baselineCatalog: state?.baselineCatalog ?? catalogRef, + latestCatalog: mode === 'full' ? catalogRef : state.latestCatalog, + historyCatalogs: mode === 'history' + ? [...(state.historyCatalogs ?? []).filter((item) => item.catalogId !== catalogId), catalogRef] + : (state?.historyCatalogs ?? []), + }; + const latestPointer = await publishDirectFilesLatest({ + workDir, + database, + bucket, + objectPrefix, + state: nextState, + uploadOptions, + uploadManifestFn, + verifyFn, + }); + atomicWriteJson(statePath, nextState); + let cleanup = null; + if (mode === 'history') { + cleanup = cleanupHistoryCandidates({dataDir, candidates: plan.candidates}); + } + const finalResult = {...summary, catalog, latestPointer, uploadedCount, reusedCount, cleanup}; + if (resultFile) { + atomicWriteJson(resolvePath(resultFile), finalResult); + } + return finalResult; +} + +async function downloadOssBuffer({objectKey, uploadOptions}) { + const response = await signedOssRequest({ + ...ossRequestDefaults(uploadOptions), + method: 'GET', + objectKey, + operation: '下载对象', + }); + return Buffer.from(await response.arrayBuffer()); +} + +async function downloadOssFile({objectKey, destinationPath, uploadOptions}) { + const response = await signedOssRequest({ + ...ossRequestDefaults(uploadOptions), + method: 'GET', + objectKey, + operation: '下载对象', + }); + const tempPath = `${destinationPath}.partial-${process.pid}`; + rmSync(tempPath, {force: true}); + try { + if (response.body) { + await pipeline(Readable.fromWeb(response.body), createWriteStream(tempPath, {mode: 0o600})); + } else { + writeFileSync(tempPath, Buffer.alloc(0), {mode: 0o600}); + } + renameSync(tempPath, destinationPath); + } catch (error) { + rmSync(tempPath, {force: true}); + throw error; + } +} + +async function loadDirectFilesCatalog({catalogRef, database, bucket, uploadOptions, downloadBufferFn}) { + const catalogBody = await downloadBufferFn({objectKey: catalogRef.objectKey, uploadOptions}); + if (catalogBody.length !== catalogRef.contentLength || sha256Hex(catalogBody) !== catalogRef.sha256) { + throw new Error(`files restore catalog 长度或 SHA-256 不一致: ${catalogRef.objectKey}`); + } + const catalog = JSON.parse(catalogBody.toString('utf8')); + if ( + catalog.schemaVersion !== DIRECT_FILES_CATALOG_SCHEMA_VERSION + || catalog.backupKind !== 'spacetimedb-data-dir-files' + || catalog.database !== database + || catalog.bucket !== bucket + || catalog.catalogId !== catalogRef.catalogId + || !Array.isArray(catalog.directories) + || !Array.isArray(catalog.files) + ) { + throw new Error(`files restore catalog 契约无效: ${catalogRef.objectKey}`); + } + return catalog; +} + +function assertDirectCatalogFile(file, index) { + if ( + !file + || typeof file.path !== 'string' + || !Number.isSafeInteger(file.sizeBytes) + || file.sizeBytes < 0 + || !/^[a-f0-9]{64}$/u.test(file.sha256) + || typeof file.objectKey !== 'string' + || !file.objectKey + || !Number.isSafeInteger(file.mode) + ) { + throw new Error(`files restore catalog 文件项无效: index=${index}`); + } +} + +async function restoreDirectFilesCatalog({ + catalog, + restoreDir, + uploadOptions, + resultFile = '', + dryRun = false, + downloadFileFn = downloadOssFile, +}) { + const resolvedRestoreDir = resolvePath(restoreDir); + catalog.files.forEach(assertDirectCatalogFile); + const totalSizeBytes = catalog.files.reduce((sum, file) => sum + BigInt(file.sizeBytes), 0n).toString(); + if (dryRun) { + const result = { + restoreDir: resolvedRestoreDir, + catalogId: catalog.catalogId, + fileCount: catalog.files.length, + totalSizeBytes, + downloadedCount: 0, + reusedCount: 0, + dryRun: true, + }; + if (resultFile) { + atomicWriteJson(resolvePath(resultFile), result); + } + return result; + } + mkdirSync(resolvedRestoreDir, {recursive: true, mode: 0o700}); + for (const directoryPath of catalog.directories) { + if (directoryPath === '.') { + continue; + } + const absolutePath = resolve(resolvedRestoreDir, directoryPath); + assertSafeRelativePath(resolvedRestoreDir, absolutePath); + mkdirSync(absolutePath, {recursive: true}); + } + + let downloadedCount = 0; + let reusedCount = 0; + for (const [index, file] of catalog.files.entries()) { + const destinationPath = resolve(resolvedRestoreDir, file.path); + assertSafeRelativePath(resolvedRestoreDir, destinationPath); + mkdirSync(dirname(destinationPath), {recursive: true}); + let reusable = false; + if (existsSync(destinationPath) && lstatSync(destinationPath).isFile()) { + const stat = statSync(destinationPath); + reusable = stat.size === file.sizeBytes && await sha256FileHex(destinationPath) === file.sha256; + } + if (reusable) { + reusedCount += 1; + } else { + rmSync(destinationPath, {force: true}); + await downloadFileFn({objectKey: file.objectKey, destinationPath, uploadOptions}); + const stat = statSync(destinationPath); + const sha256 = await sha256FileHex(destinationPath); + if (stat.size !== file.sizeBytes || sha256 !== file.sha256) { + rmSync(destinationPath, {force: true}); + throw new Error(`files restore 对象长度或 SHA-256 不一致: ${file.path}`); + } + downloadedCount += 1; + } + chmodSync(destinationPath, file.mode & 0o7777); + console.log(`[database-backup] files restore: ${index + 1}/${catalog.files.length} (${reusable ? 'reused' : 'downloaded'}) ${file.path}`); + } + const result = { + restoreDir: resolvedRestoreDir, + catalogId: catalog.catalogId, + fileCount: catalog.files.length, + totalSizeBytes, + downloadedCount, + reusedCount, + }; + if (resultFile) { + atomicWriteJson(resolvePath(resultFile), result); + } + return result; +} + +export async function restoreDirectFilesBackup({ + statePath, + restoreDir, + database, + bucket, + uploadOptions, + resultFile = '', + dryRun = false, + downloadBufferFn = downloadOssBuffer, + downloadFileFn = downloadOssFile, +}) { + const state = readDirectFilesState(resolvePath(statePath), {database, bucket}); + if (!state?.latestCatalog || state.latestCatalog.mode !== 'full') { + throw new Error(`files restore 缺少 full baseline catalog: ${statePath}`); + } + const catalogRef = assertDirectCatalogRef(state.latestCatalog, 'full', 'latest full'); + const catalog = await loadDirectFilesCatalog({catalogRef, database, bucket, uploadOptions, downloadBufferFn}); + return restoreDirectFilesCatalog({ + catalog, + restoreDir, + uploadOptions, + resultFile, + dryRun, + downloadFileFn, + }); +} + +export async function restoreDirectFilesLatest({ + restoreDir, + database, + bucket, + objectPrefix, + uploadOptions, + resultFile = '', + dryRun = false, + downloadBufferFn = downloadOssBuffer, + downloadFileFn = downloadOssFile, + verifyFn = verifyOssObject, +}) { + const latestObjectKey = `${normalizeObjectPrefix(objectPrefix, database)}/latest.json`; + const latestBody = await downloadBufferFn({objectKey: latestObjectKey, uploadOptions}); + const latestSha256 = sha256Hex(latestBody); + await verifyFn({ + ...uploadOptions, + objectKey: latestObjectKey, + contentLength: latestBody.length, + archiveSha256: latestSha256, + }); + const latest = validateDirectFilesLatest(JSON.parse(latestBody.toString('utf8')), {database, bucket}); + const catalogRef = latest.latestFullCatalog; + await verifyFn({ + ...uploadOptions, + objectKey: catalogRef.objectKey, + contentLength: catalogRef.contentLength, + archiveSha256: catalogRef.sha256, + }); + const catalog = await loadDirectFilesCatalog({catalogRef, database, bucket, uploadOptions, downloadBufferFn}); + return restoreDirectFilesCatalog({ + catalog, + restoreDir, + uploadOptions, + resultFile, + dryRun, + downloadFileFn, + }); +} + function regionFromEndpoint(endpoint) { const match = /^oss-([a-z0-9-]+)\./u.exec(endpoint); if (!match) { @@ -563,6 +1818,34 @@ function retryDelayMs({attempt, baseDelayMs, maxDelayMs, randomFn}) { return Math.floor(randomFn() * ceiling); } +function ossRequestDefaults({ + bucket, + endpoint, + accessKeyId, + accessKeySecret, + fetchImpl = globalThis.fetch, + nowFn = () => new Date(), + sleepImpl = sleep, + randomFn = Math.random, + maxAttempts = DEFAULT_OSS_REQUEST_MAX_ATTEMPTS, + retryBaseDelayMs = DEFAULT_OSS_RETRY_BASE_DELAY_MS, + retryMaxDelayMs = DEFAULT_OSS_RETRY_MAX_DELAY_MS, +}) { + return { + bucket, + endpoint, + accessKeyId, + accessKeySecret, + fetchImpl, + nowFn, + sleepImpl, + randomFn, + maxAttempts, + retryBaseDelayMs, + retryMaxDelayMs, + }; +} + async function signedOssRequest({ method, bucket, @@ -694,21 +1977,70 @@ function resolveMultipartPartSize(fileSize, configuredPartSize) { return partSize; } -async function verifyUploadedObject({requestOptions, expectedContentLength}) { +async function verifyUploadedObject({requestOptions, expectedContentLength, expectedArchiveSha256}) { const response = await signedOssRequest({ ...requestOptions, method: 'HEAD', operation: 'HEAD 验证', }); const contentLengthHeader = response.headers.get('content-length'); - if (!contentLengthHeader || !/^\d+$/u.test(contentLengthHeader)) { - throw new Error(`OSS HEAD 验证缺少有效 content-length: ${contentLengthHeader ?? ''}`); + const metadataLengthHeader = response.headers.get('x-oss-meta-file-size'); + const effectiveLengthHeader = /^\d+$/u.test(contentLengthHeader ?? '') + ? contentLengthHeader + : metadataLengthHeader; + if (!effectiveLengthHeader || !/^\d+$/u.test(effectiveLengthHeader)) { + throw new Error( + `OSS HEAD 验证缺少有效 content-length/file-size: content-length=${contentLengthHeader ?? ''}, file-size=${metadataLengthHeader ?? ''}`, + ); } - const remoteContentLength = Number(contentLengthHeader); + const remoteContentLength = Number(effectiveLengthHeader); if (remoteContentLength !== expectedContentLength) { throw new Error(`OSS HEAD 验证长度不一致: local=${expectedContentLength}, remote=${remoteContentLength}`); } - return {verifiedAt: new Date().toISOString(), remoteContentLength}; + const remoteArchiveSha256 = String( + response.headers.get('x-oss-meta-file-sha256') + ?? response.headers.get('x-oss-meta-archive-sha256') + ?? '', + ).trim().toLowerCase(); + if (remoteArchiveSha256 !== expectedArchiveSha256) { + throw new Error(`OSS HEAD 验证 SHA-256 不一致: local=${expectedArchiveSha256}, remote=${remoteArchiveSha256 || ''}`); + } + return {verifiedAt: new Date().toISOString(), remoteContentLength, remoteArchiveSha256}; +} + +export async function verifyOssObject({ + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + contentLength, + archiveSha256, + fetchImpl = globalThis.fetch, + nowFn = () => new Date(), + sleepImpl = sleep, + randomFn = Math.random, + maxAttempts = DEFAULT_OSS_REQUEST_MAX_ATTEMPTS, +}) { + const requestOptions = { + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + fetchImpl, + nowFn, + sleepImpl, + randomFn, + maxAttempts, + retryBaseDelayMs: DEFAULT_OSS_RETRY_BASE_DELAY_MS, + retryMaxDelayMs: DEFAULT_OSS_RETRY_MAX_DELAY_MS, + }; + return verifyUploadedObject({ + requestOptions, + expectedContentLength: Number(contentLength), + expectedArchiveSha256: String(archiveSha256 ?? '').trim().toLowerCase(), + }); } async function abortMultipartUpload({requestOptions, uploadId}) { @@ -741,13 +2073,19 @@ export async function uploadArchive({ nowFn = () => new Date(), sleepImpl = sleep, randomFn = Math.random, + backupKind = 'spacetimedb-data-dir', + archiveSha256 = '', + contentType = 'application/gzip', + allowEmpty = false, }) { const fileStat = statSync(archivePath); - if (!fileStat.isFile() || fileStat.size <= 0) { - throw new Error(`待上传备份必须是非空文件: ${archivePath}`); + if (!fileStat.isFile() || (!allowEmpty && fileStat.size <= 0)) { + throw new Error(`待上传备份必须是${allowEmpty ? '' : '非空'}普通文件: ${archivePath}`); + } + const verifiedArchiveSha256 = archiveSha256 || await sha256FileHex(archivePath); + if (!/^[a-f0-9]{64}$/u.test(verifiedArchiveSha256)) { + throw new Error(`归档 SHA-256 无效: ${verifiedArchiveSha256}`); } - const partSize = resolveMultipartPartSize(fileStat.size, partSizeBytes); - const partCount = Math.ceil(fileStat.size / partSize); const requestOptions = { bucket, endpoint, @@ -762,6 +2100,26 @@ export async function uploadArchive({ retryBaseDelayMs, retryMaxDelayMs, }; + if (fileStat.size === 0) { + await signedOssRequest({ + ...requestOptions, + method: 'PUT', + headers: { + 'content-type': contentType, + 'x-oss-meta-archive-sha256': verifiedArchiveSha256, + 'x-oss-meta-file-sha256': verifiedArchiveSha256, + 'x-oss-meta-file-size': '0', + 'x-oss-meta-backup-kind': backupKind, + }, + contentLength: 0, + bodyFactory: () => Buffer.alloc(0), + operation: '上传空文件', + }); + const verification = await verifyUploadedObject({requestOptions, expectedContentLength: 0, expectedArchiveSha256: verifiedArchiveSha256}); + return {bucket, objectKey, contentLength: 0, archiveSha256: verifiedArchiveSha256, etag: '', uploadMode: 'single', partCount: 1, partSizeBytes: 0, verifiedAt: verification.verifiedAt}; + } + const partSize = resolveMultipartPartSize(fileStat.size, partSizeBytes); + const partCount = Math.ceil(fileStat.size / partSize); let uploadId = ''; let uploadCompleted = false; @@ -772,8 +2130,11 @@ export async function uploadArchive({ method: 'POST', queries: {uploads: null}, headers: { - 'content-type': 'application/gzip', - 'x-oss-meta-backup-kind': 'spacetimedb-data-dir', + 'content-type': contentType, + 'x-oss-meta-archive-sha256': verifiedArchiveSha256, + 'x-oss-meta-file-sha256': verifiedArchiveSha256, + 'x-oss-meta-file-size': String(fileStat.size), + 'x-oss-meta-backup-kind': backupKind, }, operation: 'InitiateMultipartUpload', }); @@ -822,19 +2183,24 @@ export async function uploadArchive({ } } catch (completeError) { try { - await verifyUploadedObject({requestOptions, expectedContentLength: fileStat.size}); + await verifyUploadedObject({requestOptions, expectedContentLength: fileStat.size, expectedArchiveSha256: verifiedArchiveSha256}); completeResponse = null; } catch { throw completeError; } } - const verification = await verifyUploadedObject({requestOptions, expectedContentLength: fileStat.size}); + const verification = await verifyUploadedObject({ + requestOptions, + expectedContentLength: fileStat.size, + expectedArchiveSha256: verifiedArchiveSha256, + }); uploadCompleted = true; return { bucket, objectKey, contentLength: fileStat.size, + archiveSha256: verifiedArchiveSha256, etag: completeResponse?.headers.get('etag')?.replace(/^"|"$/gu, '') ?? '', uploadMode: 'multipart', partCount, @@ -849,6 +2215,149 @@ export async function uploadArchive({ } } +export async function uploadManifestFile({ + manifestPath, + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + fetchImpl = globalThis.fetch, + nowFn = () => new Date(), + sleepImpl = sleep, + randomFn = Math.random, + maxAttempts = DEFAULT_OSS_REQUEST_MAX_ATTEMPTS, +}) { + const body = readFileSync(manifestPath); + if (body.length === 0) { + throw new Error(`待上传 manifest 不能为空: ${manifestPath}`); + } + const archiveSha256 = sha256Hex(body); + const requestOptions = { + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + fetchImpl, + nowFn, + sleepImpl, + randomFn, + maxAttempts, + retryBaseDelayMs: DEFAULT_OSS_RETRY_BASE_DELAY_MS, + retryMaxDelayMs: DEFAULT_OSS_RETRY_MAX_DELAY_MS, + }; + await signedOssRequest({ + ...requestOptions, + method: 'PUT', + headers: { + 'content-type': 'application/json', + 'x-oss-meta-archive-sha256': archiveSha256, + 'x-oss-meta-file-size': String(body.length), + 'x-oss-meta-backup-kind': 'spacetimedb-backup-manifest', + }, + contentLength: body.length, + bodyFactory: () => body, + operation: '上传 manifest', + }); + const verification = await verifyUploadedObject({ + requestOptions, + expectedContentLength: body.length, + expectedArchiveSha256: archiveSha256, + }); + return {objectKey, contentLength: body.length, archiveSha256, verifiedAt: verification.verifiedAt}; +} + +function uploadedManifestPayload({manifest, database, result}) { + return { + ...manifest, + database, + bucket: result.bucket, + objectKey: result.objectKey, + manifestObjectKey: `${result.objectKey}.manifest.json`, + contentLength: result.contentLength, + archiveSha256: result.archiveSha256, + etag: result.etag, + uploadMode: result.uploadMode, + partCount: result.partCount, + partSizeBytes: result.partSizeBytes, + verifiedAt: result.verifiedAt, + uploadedAt: new Date().toISOString(), + uploadStatus: 'uploaded', + }; +} + +export async function uploadHistoryArchiveWithCleanup({ + archivePath, + manifestPath, + manifest, + statePath, + uploadOptions, + uploadFn = uploadArchive, + manifestUploadFn = uploadManifestFile, + verifyFn = verifyOssObject, +}) { + const result = await uploadFn({ + archivePath, + ...uploadOptions, + backupKind: 'spacetimedb-history', + }); + const uploadedManifest = uploadedManifestPayload({manifest, database: manifest.database, result}); + writeManifest({manifestPath, payload: uploadedManifest}); + const manifestUpload = await manifestUploadFn({ + manifestPath, + ...uploadOptions, + objectKey: uploadedManifest.manifestObjectKey, + }); + uploadedManifest.manifestVerifiedAt = manifestUpload.verifiedAt; + uploadedManifest.manifestContentLength = manifestUpload.contentLength; + uploadedManifest.manifestArchiveSha256 = manifestUpload.archiveSha256; + writeManifest({manifestPath, payload: uploadedManifest}); + let state = validateHistoryState(readManifest(statePath), { + database: uploadedManifest.database, + dataDir: uploadedManifest.dataDir, + }); + if (state.baseline.id !== uploadedManifest.baselineId) { + throw new Error(`history manifest baselineId 与 state 不匹配: manifest=${uploadedManifest.baselineId}, state=${state.baseline.id}`); + } + await verifyFn({ + ...uploadOptions, + bucket: state.baseline.bucket, + objectKey: state.baseline.objectKey, + contentLength: state.baseline.contentLength, + archiveSha256: state.baseline.archiveSha256, + }); + await verifyFn({ + ...uploadOptions, + bucket: state.baseline.bucket, + objectKey: state.baseline.manifestObjectKey, + contentLength: state.baseline.manifestContentLength, + archiveSha256: state.baseline.manifestArchiveSha256, + }); + state = recordHistoryBatch({ + statePath, + state, + manifest: uploadedManifest, + uploadResult: result, + manifestUpload, + status: 'uploaded', + }); + const cleanup = cleanupHistoryCandidates({ + dataDir: uploadedManifest.dataDir, + candidates: uploadedManifest.candidates, + }); + state = recordHistoryBatch({ + statePath, + state, + manifest: uploadedManifest, + uploadResult: result, + manifestUpload, + status: 'cleaned', + cleanedAt: new Date().toISOString(), + }); + return {result, uploadedManifest, cleanup, state}; +} + async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix}) { const archivePath = resolvePath(args.uploadArchive); if (!existsSync(archivePath)) { @@ -860,6 +2369,13 @@ async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, const dataDir = firstNonEmpty(manifest.dataDir, env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, DEFAULT_PRODUCTION_DATA_DIR); const database = firstNonEmpty(args.database, manifest.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir)); const objectKey = firstNonEmpty(args.objectKey, manifest.objectKey, buildBackupNames({database, dataDir, objectPrefix}).objectKey); + if (manifest.backupKind !== 'spacetimedb-history') { + manifest.backupKind = 'spacetimedb-data-dir'; + manifest.baselineStatePath = firstNonEmpty( + manifest.baselineStatePath, + historyStatePath({args, env, workDir: dirname(archivePath), database}), + ); + } console.log(`[database-backup] 上传已有备份: ${archivePath}`); console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`); @@ -869,30 +2385,51 @@ async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, return; } - const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}); + const statePath = resolvePath(firstNonEmpty( + manifest.baselineStatePath, + historyStatePath({args, env, workDir: dirname(archivePath), database}), + )); + let result; + let uploadedAt; + if (manifest.backupKind === 'spacetimedb-history') { + const historyResult = await uploadHistoryArchiveWithCleanup({ + archivePath, + manifestPath, + manifest, + statePath, + uploadOptions: {bucket, endpoint, objectKey, accessKeyId, accessKeySecret}, + }); + result = historyResult.result; + uploadedAt = historyResult.uploadedManifest.uploadedAt; + console.log(`[database-backup] history 上传并清理完成: ${JSON.stringify(historyResult.cleanup)}`); + } else { + result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}); + const uploadedManifest = uploadedManifestPayload({manifest, database, result}); + uploadedAt = uploadedManifest.uploadedAt; + writeManifest({manifestPath, payload: uploadedManifest}); + const manifestUpload = await uploadManifestFile({ + manifestPath, + bucket, + endpoint, + objectKey: uploadedManifest.manifestObjectKey, + accessKeyId, + accessKeySecret, + }); + uploadedManifest.manifestVerifiedAt = manifestUpload.verifiedAt; + uploadedManifest.manifestContentLength = manifestUpload.contentLength; + uploadedManifest.manifestArchiveSha256 = manifestUpload.archiveSha256; + writeManifest({manifestPath, payload: uploadedManifest}); + const previousState = existsSync(statePath) + ? validateHistoryState(readManifest(statePath), {database, dataDir}) + : null; + const baseline = normalizeUploadedBaselineManifest(uploadedManifest, {database, dataDir}); + writeBaselineState({statePath, baseline, previousState}); + console.log(`[database-backup] 已写入 baseline state: ${statePath}`); + } console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`); - const uploadedAt = new Date().toISOString(); - writeManifest({ - manifestPath, - payload: { - ...manifest, - database, - bucket: result.bucket, - objectKey: result.objectKey, - contentLength: result.contentLength, - etag: result.etag, - uploadMode: result.uploadMode, - partCount: result.partCount, - partSizeBytes: result.partSizeBytes, - verifiedAt: result.verifiedAt, - uploadedAt, - uploadStatus: 'uploaded', - }, - }); - if (args.resultFile) { - writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, ...result, uploadedAt}, null, 2)}\n`, 'utf8'); + writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, statePath, ...result, uploadedAt}, null, 2)}\n`, 'utf8'); } const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true'; @@ -906,6 +2443,204 @@ async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, } } +async function publishExistingManifest({args, bucket, endpoint, accessKeyId, accessKeySecret}) { + const manifestPath = resolvePath(args.publishManifest); + const manifest = readManifest(manifestPath); + if (manifest.uploadStatus !== 'uploaded' || !manifest.objectKey) { + throw new Error('只允许发布 uploadStatus=uploaded 且包含 objectKey 的备份 manifest。'); + } + manifest.manifestObjectKey = manifest.manifestObjectKey || `${manifest.objectKey}.manifest.json`; + writeManifest({manifestPath, payload: manifest}); + const result = await uploadManifestFile({ + manifestPath, + bucket, + endpoint, + objectKey: manifest.manifestObjectKey, + accessKeyId, + accessKeySecret, + }); + manifest.manifestVerifiedAt = result.verifiedAt; + manifest.manifestContentLength = result.contentLength; + manifest.manifestArchiveSha256 = result.archiveSha256; + writeManifest({manifestPath, payload: manifest}); + console.log(`[database-backup] manifest 上传并验真完成: ${JSON.stringify(result)}`); +} + +export async function resumeUploadedHistoryBatch({statePath, state, dataDir, verificationOptions, verifyFn = verifyOssObject}) { + const pendingBatch = state.batches.find((batch) => batch.status === 'uploaded'); + if (!pendingBatch) { + return state; + } + console.log(`[database-backup] 重试已上传 history 批次的本地清理: ${pendingBatch.batchId}`); + await verifyFn({ + ...verificationOptions, + objectKey: pendingBatch.objectKey, + contentLength: pendingBatch.contentLength, + archiveSha256: pendingBatch.archiveSha256, + }); + await verifyFn({ + ...verificationOptions, + objectKey: pendingBatch.manifestObjectKey, + contentLength: pendingBatch.manifestContentLength, + archiveSha256: pendingBatch.manifestArchiveSha256, + }); + const cleanup = cleanupHistoryCandidates({dataDir, candidates: pendingBatch.candidates}); + const manifest = { + batchId: pendingBatch.batchId, + uploadedAt: pendingBatch.uploadedAt, + candidates: pendingBatch.candidates, + }; + const uploadResult = { + objectKey: pendingBatch.objectKey, + contentLength: pendingBatch.contentLength, + archiveSha256: pendingBatch.archiveSha256, + verifiedAt: pendingBatch.verifiedAt, + }; + const manifestUpload = { + objectKey: pendingBatch.manifestObjectKey, + contentLength: pendingBatch.manifestContentLength, + archiveSha256: pendingBatch.manifestArchiveSha256, + verifiedAt: pendingBatch.manifestVerifiedAt, + }; + const nextState = recordHistoryBatch({ + statePath, + state, + manifest, + uploadResult, + manifestUpload, + status: 'cleaned', + cleanedAt: new Date().toISOString(), + }); + console.log(`[database-backup] 已完成 history 清理重试: ${JSON.stringify(cleanup)}`); + return nextState; +} + +async function runHistoryBackup({ + args, + env, + dataDir, + workDir, + database, + bucket, + endpoint, + accessKeyId, + accessKeySecret, + objectPrefix, + keepLocal, +}) { + const statePath = historyStatePath({args, env, workDir, database}); + let state = loadOrImportHistoryState({args, env, statePath, database, dataDir}); + if (!args.dryRun && !args.deferUpload) { + console.log(`[database-backup] 重新验真 full baseline: oss://${state.baseline.bucket}/${state.baseline.objectKey}`); + await verifyOssObject({ + bucket: state.baseline.bucket, + endpoint, + objectKey: state.baseline.objectKey, + accessKeyId, + accessKeySecret, + contentLength: state.baseline.contentLength, + archiveSha256: state.baseline.archiveSha256, + }); + await verifyOssObject({ + bucket: state.baseline.bucket, + endpoint, + objectKey: state.baseline.manifestObjectKey, + accessKeyId, + accessKeySecret, + contentLength: state.baseline.manifestContentLength, + archiveSha256: state.baseline.manifestArchiveSha256, + }); + state = await resumeUploadedHistoryBatch({ + statePath, + state, + dataDir, + verificationOptions: {bucket, endpoint, accessKeyId, accessKeySecret}, + }); + } + const plan = discoverHistoryPlan({dataDir}); + console.log(`[database-backup] history replicas: ${JSON.stringify(plan.replicas)}`); + console.log(`[database-backup] history 候选: count=${plan.candidates.length}, size=${formatBytes(plan.totalSizeBytes)}`); + if (args.resultFile) { + writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({statePath, baseline: state.baseline, ...plan}, null, 2)}\n`, 'utf8'); + } + if (args.dryRun) { + console.log('[database-backup] history dry-run，仅输出安全候选，不打包、上传或删除。'); + return; + } + if (plan.candidates.length === 0) { + console.log('[database-backup] 没有可归档的 history 候选。'); + return; + } + + assertSufficientHistoryWorkDirSpace({historySizeBytes: plan.totalSizeBytes, workDir, args, env}); + const batchId = historyBatchId({baselineId: state.baseline.id, plan}); + const {fileName, objectKey} = buildHistoryNames({ + database, + objectPrefix, + baselineId: state.baseline.id, + batchId, + }); + const archivePath = resolve(workDir, fileName); + const manifestPath = `${archivePath}.manifest.json`; + const manifest = { + schemaVersion: HISTORY_MANIFEST_SCHEMA_VERSION, + backupKind: 'spacetimedb-history', + createdAt: new Date().toISOString(), + database, + dataDir, + bucket, + objectKey, + archivePath, + baselineId: state.baseline.id, + baselineStatePath: statePath, + batchId, + replicas: plan.replicas, + candidates: plan.candidates, + totalSizeBytes: plan.totalSizeBytes, + uploadStatus: args.deferUpload ? 'deferred' : 'pending', + }; + writeManifest({manifestPath, payload: manifest}); + createHistoryArchive({ + dataDir, + workDir, + fileName, + manifestPath, + candidates: plan.candidates, + }); + + if (args.deferUpload) { + console.log(`[database-backup] 已生成 history 归档，延后上传且未清理源文件: ${archivePath}`); + if (args.resultFile) { + writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, statePath, bucket, objectKey, batchId}, null, 2)}\n`, 'utf8'); + } + return; + } + + const historyResult = await uploadHistoryArchiveWithCleanup({ + archivePath, + manifestPath, + manifest, + statePath, + uploadOptions: {bucket, endpoint, objectKey, accessKeyId, accessKeySecret}, + }); + console.log(`[database-backup] history 上传并清理完成: ${JSON.stringify(historyResult.cleanup)}`); + if (args.resultFile) { + writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({ + archivePath, + manifestPath, + statePath, + batchId, + ...historyResult.result, + uploadedAt: historyResult.uploadedManifest.uploadedAt, + }, null, 2)}\n`, 'utf8'); + } + if (!keepLocal) { + rmSync(archivePath, {force: true}); + rmSync(manifestPath, {force: true}); + console.log('[database-backup] 已删除本地 history 临时归档和清单。'); + } +} + async function main() { const args = parseArgs(process.argv.slice(2)); const env = loadEffectiveEnv(args.envFiles); @@ -927,6 +2662,14 @@ async function main() { const objectPrefix = firstNonEmpty(args.objectPrefix, env.GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX, 'database-backups'); const database = firstNonEmpty(args.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir)); const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true'; + const storageFormat = firstNonEmpty(args.storageFormat, env.GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT, 'archive'); + + if (!['full', 'history'].includes(args.mode)) { + throw new Error(`--mode 只能是 full 或 history，实际: ${args.mode}`); + } + if (!['archive', 'files'].includes(storageFormat)) { + throw new Error(`--storage-format 只能是 archive 或 files，实际: ${storageFormat}`); + } for (const [label, value] of Object.entries({bucket, endpoint, accessKeyId, accessKeySecret})) { if (!value) { @@ -934,11 +2677,124 @@ async function main() { } } + if (args.restoreFilesState && args.restoreFilesLatest) { + throw new Error('--restore-files-state 与 --restore-files-latest 不能同时使用。'); + } + if (args.restoreFilesState) { + if (!args.restoreDir) { + throw new Error('--restore-files-state 必须同时传 --restore-dir。'); + } + await restoreDirectFilesBackup({ + statePath: args.restoreFilesState, + restoreDir: args.restoreDir, + database, + bucket, + uploadOptions: {bucket, endpoint, accessKeyId, accessKeySecret}, + resultFile: args.resultFile, + dryRun: args.dryRun, + }); + return; + } + if (args.restoreFilesLatest) { + if (!args.restoreDir) { + throw new Error('--restore-files-latest 必须同时传 --restore-dir。'); + } + await restoreDirectFilesLatest({ + restoreDir: args.restoreDir, + database, + bucket, + objectPrefix, + uploadOptions: {bucket, endpoint, accessKeyId, accessKeySecret}, + resultFile: args.resultFile, + dryRun: args.dryRun, + }); + return; + } + if (args.restoreDir) { + throw new Error('--restore-dir 只能与 --restore-files-state 或 --restore-files-latest 一起使用。'); + } + + if (!args.dryRun) { + const lockPath = acquireBackupLock({workDir, database}); + console.log(`[database-backup] 已获取进程锁: ${lockPath}`); + } + + if (args.publishManifest) { + await publishExistingManifest({args, bucket, endpoint, accessKeyId, accessKeySecret}); + return; + } + if (args.uploadArchive) { await uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix}); return; } + if (storageFormat === 'files') { + if (args.deferUpload) { + throw new Error('files 模式无需本地归档且不支持 --defer-upload；失败后使用同一 work-dir 重跑即可续传。'); + } + const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE); + const restartServicesAfter = collectRestartServicesAfterBackup({args, env}); + let serviceStopped = false; + let backupError = null; + let restoreError = null; + try { + if (args.mode === 'full' && !args.dryRun) { + serviceStopped = stopServiceIfNeeded(stopService); + } + await runDirectFilesBackup({ + mode: args.mode, + dataDir, + workDir, + database, + bucket, + objectPrefix, + dryRun: args.dryRun, + resultFile: args.resultFile, + uploadOptions: {bucket, endpoint, accessKeyId, accessKeySecret}, + }); + } catch (error) { + backupError = error; + } finally { + try { + if (serviceStopped) { + restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}); + } else if (!backupError && args.mode === 'full' && !args.dryRun) { + restartServicesAfterBackup(restartServicesAfter); + } + } catch (error) { + restoreError = error; + } + } + if (backupError && restoreError) { + throw new AggregateError([backupError, restoreError], `files 备份失败，且恢复依赖服务时也失败: ${backupError.message}; ${restoreError.message}`); + } + if (backupError) { + throw backupError; + } + if (restoreError) { + throw restoreError; + } + return; + } + + if (args.mode === 'history') { + await runHistoryBackup({ + args, + env, + dataDir, + workDir, + database, + bucket, + endpoint, + accessKeyId, + accessKeySecret, + objectPrefix, + keepLocal, + }); + return; + } + const {fileName, objectKey} = buildBackupNames({database, dataDir, objectPrefix}); console.log(`[database-backup] 数据目录: ${dataDir}`); console.log(`[database-backup] 本地临时目录: ${workDir}`); @@ -983,50 +2839,54 @@ async function main() { } const manifestPath = `${archivePath}.manifest.json`; + const baselineStatePath = historyStatePath({args, env, workDir, database}); + const fullManifest = { + backupKind: 'spacetimedb-data-dir', + createdAt: new Date().toISOString(), + database, + dataDir, + bucket, + objectKey, + archivePath, + baselineStatePath, + uploadStatus: args.deferUpload ? 'deferred' : 'pending', + }; writeManifest({ manifestPath, - payload: { - createdAt: new Date().toISOString(), - database, - dataDir, - bucket, - objectKey, - archivePath, - uploadStatus: args.deferUpload ? 'deferred' : 'pending', - }, + payload: fullManifest, }); if (args.deferUpload) { console.log(`[database-backup] 已生成本地冷备份，延后上传: ${archivePath}`); console.log(`[database-backup] 已写入备份清单: ${manifestPath}`); if (args.resultFile) { - writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, bucket, objectKey}, null, 2)}\n`, 'utf8'); + writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, baselineStatePath, bucket, objectKey}, null, 2)}\n`, 'utf8'); } return; } const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}); console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`); - - writeManifest({ + const uploadedManifest = uploadedManifestPayload({manifest: fullManifest, database, result}); + writeManifest({manifestPath, payload: uploadedManifest}); + const manifestUpload = await uploadManifestFile({ manifestPath, - payload: { - createdAt: new Date().toISOString(), - database, - dataDir, - bucket: result.bucket, - objectKey: result.objectKey, - archivePath, - contentLength: result.contentLength, - etag: result.etag, - uploadMode: result.uploadMode, - partCount: result.partCount, - partSizeBytes: result.partSizeBytes, - verifiedAt: result.verifiedAt, - uploadedAt: new Date().toISOString(), - uploadStatus: 'uploaded', - }, + bucket, + endpoint, + objectKey: uploadedManifest.manifestObjectKey, + accessKeyId, + accessKeySecret, }); + uploadedManifest.manifestVerifiedAt = manifestUpload.verifiedAt; + uploadedManifest.manifestContentLength = manifestUpload.contentLength; + uploadedManifest.manifestArchiveSha256 = manifestUpload.archiveSha256; + writeManifest({manifestPath, payload: uploadedManifest}); + const previousState = existsSync(baselineStatePath) + ? validateHistoryState(readManifest(baselineStatePath), {database, dataDir}) + : null; + const baseline = normalizeUploadedBaselineManifest(uploadedManifest, {database, dataDir}); + writeBaselineState({statePath: baselineStatePath, baseline, previousState}); + console.log(`[database-backup] 已写入 baseline state: ${baselineStatePath}`); if (!keepLocal) { rmSync(archivePath, {force: true}); diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 3a49739be..89d1b6cdc 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -10,6 +10,11 @@ GENARRATIVE_OPENSSL_VERSION="${GENARRATIVE_OPENSSL_VERSION:-3.2.0}" GENARRATIVE_OPENSSL_PREFIX="${GENARRATIVE_OPENSSL_PREFIX:-/opt/genarrative/openssl-3.2.0}" GENARRATIVE_OPENSSL_SOURCE_URL="${GENARRATIVE_OPENSSL_SOURCE_URL:-https://github.com/openssl/openssl/releases/download/openssl-${GENARRATIVE_OPENSSL_VERSION}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz}" GENARRATIVE_OPENSSL_SOURCE_SHA256="${GENARRATIVE_OPENSSL_SOURCE_SHA256:-14c826f07c7e433706fb5c69fa9e25dab95684844b4c962a2cf1bf183eb4690e}" +DATABASE_BACKUP_PROFILE="${DATABASE_BACKUP_PROFILE:-archive-full}" +DATABASE_BACKUP_FILES_HISTORY_WORK_DIR="${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR:-/var/lib/genarrative/database-backups/files-history}" +DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR="/etc/systemd/system/genarrative-database-backup.service.d" +DATABASE_BACKUP_FILES_HISTORY_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-history.conf" +DATABASE_BACKUP_LEGACY_DEV_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-dev-files.conf" require_non_root_relative_path() { local label="$1" @@ -63,6 +68,21 @@ validate_server_names() { done } +validate_database_backup_profile() { + case "${DATABASE_BACKUP_PROFILE}" in + archive-full|files-history) + ;; + *) + echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${DATABASE_BACKUP_PROFILE}" >&2 + exit 1 + ;; + esac + if [[ ! "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" =~ ^/var/lib/genarrative/database-backups/[A-Za-z0-9._/-]+$ || "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" == *..* ]]; then + echo "[server-provision] DATABASE_BACKUP_FILES_HISTORY_WORK_DIR 必须是 /var/lib/genarrative/database-backups/ 下不含连续点号的绝对路径，当前值: ${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" >&2 + exit 1 + fi +} + run_cmd() { echo "+ $*" if [[ "${DRY_RUN}" != "true" ]]; then @@ -917,6 +937,52 @@ render_database_backup_service() { deploy/systemd/genarrative-database-backup.service } +render_database_backup_files_history_drop_in() { + local current_escaped env_escaped work_dir_escaped + current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" + env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")" + work_dir_escaped="$(escape_sed_replacement "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}")" + sed \ + -e "s|/opt/genarrative/current|${current_escaped}|g" \ + -e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \ + -e "s|/var/lib/genarrative/database-backups/files-history|${work_dir_escaped}|g" \ + deploy/systemd/genarrative-database-backup-files-history.conf +} + +configure_database_backup_profile() { + local rendered_drop_in + + if [[ "${DATABASE_BACKUP_PROFILE}" == "archive-full" ]]; then + echo "[server-provision] 数据库备份 profile=archive-full，保留主 service 的全量冷备行为。" + run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + return + fi + + echo "[server-provision] 数据库备份 profile=files-history，先只读验证 full baseline state。" + if [[ "${DRY_RUN}" == "true" ]]; then + echo "+ /usr/bin/node -- ${CURRENT_LINK}/scripts/database-backup-to-oss.mjs --env-file ${API_ENV_FILE} --storage-format files --mode history --work-dir ${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR} --dry-run" + else + if [[ ! -f "${CURRENT_LINK}/scripts/database-backup-to-oss.mjs" ]]; then + echo "[server-provision] current release 缺少数据库备份脚本: ${CURRENT_LINK}/scripts/database-backup-to-oss.mjs" >&2 + exit 1 + fi + /usr/bin/node -- "${CURRENT_LINK}/scripts/database-backup-to-oss.mjs" \ + --env-file "${API_ENV_FILE}" \ + --storage-format files \ + --mode history \ + --work-dir "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" \ + --dry-run + fi + + run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" + run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}" + run_cmd rm -f "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + rendered_drop_in="$(mktemp)" + render_database_backup_files_history_drop_in >"${rendered_drop_in}" + install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" 0644 + rm -f "${rendered_drop_in}" +} + render_health_patrol_service() { local current_escaped current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" @@ -930,6 +996,7 @@ require_path deploy/systemd/genarrative-api.service require_path deploy/systemd/genarrative-external-generation-worker@.service require_path deploy/systemd/genarrative-external-generation-controller.service require_path deploy/systemd/genarrative-database-backup.service +require_path deploy/systemd/genarrative-database-backup-files-history.conf require_path deploy/systemd/genarrative-database-backup.timer require_path deploy/systemd/genarrative-health-patrol.service require_path deploy/systemd/genarrative-health-patrol.timer @@ -951,9 +1018,10 @@ require_path scripts/deploy/maintenance-off.sh require_path scripts/deploy/maintenance-status.sh validate_server_names +validate_database_backup_profile require_non_root_relative_path "PROVISION_TOOLS_DIR" "${PROVISION_TOOLS_DIR}" -echo "[server-provision] target=${DEPLOY_TARGET}, dry_run=${DRY_RUN}, nginx_config_mode=${NGINX_CONFIG_MODE}, source_commit=$(cat .jenkins-source-commit)" +echo "[server-provision] target=${DEPLOY_TARGET}, dry_run=${DRY_RUN}, nginx_config_mode=${NGINX_CONFIG_MODE}, database_backup_profile=${DATABASE_BACKUP_PROFILE}, source_commit=$(cat .jenkins-source-commit)" run_cmd id require_root_for_real_provision @@ -1033,6 +1101,7 @@ else echo "[server-provision] 已存在环境文件，保留不覆盖: ${API_ENV_FILE}" fi ensure_api_runtime_env_defaults +configure_database_backup_profile if [[ ! -f "${WORKER_ENV_FILE}" ]]; then echo "+ create ${WORKER_ENV_FILE} from example"