From 2f248ea0c330d5b6c29ecab75d2d31ec99d75014 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 11:33:39 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=20minimal=20=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=8F=A3=E5=BE=84=E5=B9=B6=E5=88=87=E6=8D=A2=20releas?= =?UTF-8?q?e=20=E5=AE=9A=E6=97=B6=E5=A4=87=E4=BB=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - database-backup-to-oss.mjs 新增 --minimal/--retain-snapshots：按上游 retention 语义只备份最近 N 份 snapshot、覆盖最老保留 snapshot 的 commitlog 边界段及其后全部段，外加身份/配置/控制库/模块字节，且不停止任何服务 - 备份检查脚本补充 minimal 计划用例（保留次新+最新 snapshot、丢弃更早 snapshot 与 clog 段、状态路径齐全、保留数校验） - 发布前备份默认走 minimal 热备（GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=0 可回到冷备），不再需要 44.7GiB 冷备空间 - Server-Provision 新增 DATABASE_BACKUP_PROFILE=files-minimal 与 systemd drop-in，release 推荐使用并继续禁止 files-history - 运维文档与共享记忆记录上游 retention 依据、实测 40G→3.6G 与不停服热备口径 --- ...rrative-database-backup-files-minimal.conf | 5 + docs/project-memory/shared-memory/pitfalls.md | 2 + ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- .../Jenkinsfile.production-server-provision | 8 +- scripts/check-database-backup-to-oss.mjs | 102 +++++++ scripts/check-production-ops-guardrails.mjs | 24 +- scripts/database-backup-to-oss.mjs | 256 +++++++++++++++++- scripts/deploy/production-stdb-publish.sh | 35 ++- scripts/jenkins-server-provision.sh | 41 ++- 9 files changed, 452 insertions(+), 23 deletions(-) create mode 100644 deploy/systemd/genarrative-database-backup-files-minimal.conf diff --git a/deploy/systemd/genarrative-database-backup-files-minimal.conf b/deploy/systemd/genarrative-database-backup-files-minimal.conf new file mode 100644 index 000000000..2124eb345 --- /dev/null +++ b/deploy/systemd/genarrative-database-backup-files-minimal.conf @@ -0,0 +1,5 @@ +[Service] +Environment=GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=4 +Environment=GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND=20971520 +ExecStart= +ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode full --minimal --retain-snapshots 2 --data-dir /stdb --work-dir /var/lib/genarrative/database-backups/files-minimal diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index a82811675..5912a73b2 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -4,6 +4,8 @@ `Genarrative-Stdb-Module-Publish` 先进入维护模式、停掉 API/controller/worker，再执行发布前冷备份；archive 口径要求 `data × 1.1`（40.6GiB 数据 → 44.7GiB，加上生产根盘只剩 13.5GiB），于是 2026-09-21 的 release 发布在停服后失败并保持维护态，站点 503 直到人工恢复。规则：空间预检必须先于 `maintenance-on` 与停服；archive 不足且未显式禁用降级时改用 files（`max(data × 0.05, 2GiB)`，不落地本地归档，但 files 不支持 `--defer-upload`，会从 async 收敛为 sync）；只有真正开始 `spacetime publish` 之后的失败才允许保持维护态。另：定时备份的失效锁在当前仓库版本会自动清理，但生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若是旧版会拒绝抢锁并要求人工删锁，需随 provision 更新。 +按上游 [#5555](https://github.com/clockworklabs/SpacetimeDB/pull/5555) 的 retention 语义，只有最近 `retain-snapshots`（默认 2）份 snapshot 与覆盖它之后的 commitlog 段是重启所需，其余历史可丢；因此备份改为 `files + full + --minimal --retain-snapshots 2`（release 实测 40G → 3.6G，热备不停服），不再做增量差异计算，也不需要 44.7G 冷备空间。 + ## copyArtifacts 报「Unable to find project for artifact copy」的用户触发构建差异 Copy Artifact 插件在**非 SYSTEM 认证**下按「认证用户」判权：只有当被复制 Job 的 `CopyArtifactPermissionProperty`（仓库里由 Declarative 的 `copyArtifactPermission(...)` 维护）显式列出当前消费者，或者该 Job 对认证用户开放 Item.Read 时才放行；`ACL.SYSTEM2` 的定时构建会短路通过。因此会出现「定时调度一路成功、手动发布必挂」的现象（2026-09-21 手动发布 #6/#7 与同期的用户触发探测全部命中，定时调度 #104+ 正常）。`Genarrative-Agc-Global-Version-Issue` 生产权限模式的授权名单必须同时包含 `Genarrative-Scheduled-Revision-Trigger` 与 `Genarrative-Manual-Build-And-Deploy`；改完 `copyArtifactPermission` 后要先跑一次发号 Job 把 Job property 写回 Jenkins，只改仓库文件不生效。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 0dca6f394..801261b85 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -449,7 +449,7 @@ UI 相关修改要重点验证： npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service ``` -脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 `2026-09-21` 又命中一次同类事故：release 冷备按 `archive` 口径要求 `data × 1.1`（40.6GiB → 44.7GiB），生产根盘只剩 13.5GiB，备份在停服后才失败，生产被留在维护态。现在 `production-stdb-publish.sh` 在进入维护模式与停服务之前先执行 `database-backup-to-oss.mjs --check-space-only`：archive 口径为 `max(data × 1.1, data + 512MiB)`，files 口径为 `max(data × 0.05, 2GiB)`，空间不足以退出码 `3` 返回。archive 空间不足且未显式设置 `GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=0` 时，脚本自动降级为 `--storage-format files`（不落地本地归档；files 不支持 `--defer-upload`，因此 `DATABASE_BACKUP_MODE=async` 会收敛为 `sync`，窗口内直传 OSS）；也可用 `GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=files` 直接指定。任何「尚未开始 `spacetime publish`」的失败（空间预检、备份执行等）都会自动把本次停掉的 API/controller/worker 拉回并退出维护，只有真正开始发布之后的失败才保持维护态等待人工；该自动恢复可用 `GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=0` 关闭。定时备份的失效锁：当前仓库版本在持有者进程退出后会自动清理并重试，而生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若仍是旧版（2026-07-16 前）会拒绝抢锁并需要人工删锁，必须随 provision 一起更新。 +脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 `2026-09-21` 又命中一次同类事故：release 冷备按 `archive` 口径要求 `data × 1.1`（40.6GiB → 44.7GiB），生产根盘只剩 13.5GiB，备份在停服后才失败，生产被留在维护态。现在 `production-stdb-publish.sh` 在进入维护模式与停服务之前先执行 `database-backup-to-oss.mjs --check-space-only`：archive 口径为 `max(data × 1.1, data + 512MiB)`，files 口径为 `max(data × 0.05, 2GiB)`，空间不足以退出码 `3` 返回。archive 空间不足且未显式设置 `GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=0` 时，脚本自动降级为 `--storage-format files`（不落地本地归档；files 不支持 `--defer-upload`，因此 `DATABASE_BACKUP_MODE=async` 会收敛为 `sync`，窗口内直传 OSS）；也可用 `GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=files` 直接指定。任何「尚未开始 `spacetime publish`」的失败（空间预检、备份执行等）都会自动把本次停掉的 API/controller/worker 拉回并退出维护，只有真正开始发布之后的失败才保持维护态等待人工；该自动恢复可用 `GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=0` 关闭。定时备份的失效锁：当前仓库版本在持有者进程退出后会自动清理并重试，而生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若仍是旧版（2026-07-16 前）会拒绝抢锁并需要人工删锁，必须随 provision 一起更新。 上游确认（[#5542](https://github.com/clockworklabs/SpacetimeDB/issues/5542) / [#5555](https://github.com/clockworklabs/SpacetimeDB/pull/5555)）只保留最近 `retain-snapshots`（默认 2）份 snapshot 与覆盖它之后的 commitlog 段就足以重启数据库，更早的 snapshot/commitlog 属于可丢弃历史；因此备份不再需要"保留每一版 + 计算增量"，改为 `database-backup-to-oss.mjs --storage-format files --mode full --minimal --retain-snapshots 2`：每个 replica 取最近 2 份 snapshot、覆盖最老保留 snapshot 的 commitlog 边界段及其后全部段，外加身份/`data/config.toml`/`data/metadata.toml`/`data/control-db`/`data/program-bytes`，其余历史不备份。实测 release `replicas/2` 为 55 个 snapshot(18G)+108 个 clog 文件(22G)，minimal 口径只需约 3.6GiB（16 个候选），因此定时任务与发布前备份都可以在**不停服**的前提下完成，也不再需要 44.7GiB 的冷备空间；Server-Provision 新增 `DATABASE_BACKUP_PROFILE=files-minimal`（release 推荐，`files-history` 继续禁止），发布前备份默认走 minimal（`GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=0` 可回到冷备）。本地磁盘上的历史数据仍由 SpacetimeDB 自己保留（PR #5555 尚未合并），升级到含该特性的版本后可配置 `[retention] policy = "delete"` 回收。 生产环境变量模板在 `deploy/env/api-server.env.example`： diff --git a/jenkins/Jenkinsfile.production-server-provision b/jenkins/Jenkinsfile.production-server-provision index d2104e68c..28a940332 100644 --- a/jenkins/Jenkinsfile.production-server-provision +++ b/jenkins/Jenkinsfile.production-server-provision @@ -34,7 +34,7 @@ pipeline { string(name: 'WEB_LINK', defaultValue: '/srv/genarrative/web', description: 'Nginx 静态站点目录或软链接') string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'api-server 环境文件') string(name: 'API_PORT', defaultValue: '8082', description: 'api-server 本机监听端口') - choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profile；release 仅允许 archive-full，files-history 仅供 development 在指定 work-dir 已有完整 full baseline 后启用') + choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history'], description: '数据库定时备份 profile；release 用 files-minimal（最近 snapshot + 其后 commitlog 的热备）或 archive-full，files-history 因内存峰值仅 development 使用') string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'development files-history 的本地 state/catalog 目录；必须使用已建立 full baseline 的独立目录') choice(name: 'NGINX_CONFIG_MODE', choices: ['none', 'production-https', 'development-http'], description: 'Nginx 配置模式；开发服无域名时选 development-http，release 正式入口选 production-https') booleanParam(name: 'ENABLE_SERVICES', defaultValue: true, description: '启用并启动 spacetimedb 与 api-server systemd 服务') @@ -110,11 +110,11 @@ pipeline { error('release 目标禁止安装 development-http Nginx 配置；无证书初始化请使用 NGINX_CONFIG_MODE=none。') } def databaseBackupProfile = params.DATABASE_BACKUP_PROFILE?.trim() - if (!(databaseBackupProfile in ['archive-full', 'files-history'])) { - error("DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${params.DATABASE_BACKUP_PROFILE}") + if (!(databaseBackupProfile in ['archive-full', 'files-minimal', 'files-history'])) { + error("DATABASE_BACKUP_PROFILE 只能是 archive-full、files-minimal 或 files-history，当前值: ${params.DATABASE_BACKUP_PROFILE}") } if (params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history') { - error('release 仅允许 archive-full；files-history 会把整棵历史目录加载到 Node 内存，需先完成流式 catalog 改造后才能重新启用。') + error('release 不允许 files-history；files-history 会把整棵历史目录加载到 Node 内存。release 请选择 files-minimal（最近 snapshot + 其后 commitlog）或 archive-full。') } def databaseBackupFilesHistoryWorkDir = params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR?.trim() if (!(databaseBackupFilesHistoryWorkDir ==~ /^\/var\/lib\/genarrative\/database-backups\/[A-Za-z0-9._\/-]+$/) || databaseBackupFilesHistoryWorkDir.contains('..')) { diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index f097b8e37..049d7d851 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -29,6 +29,7 @@ import { describeBackupSpaceRequirement, discoverDeferredArchiveUploads, discoverHistoryPlan, + discoverMinimalPlan, restoreDirectFilesBackup, restoreDirectFilesLatest, resumeUploadedHistoryBatch, @@ -66,6 +67,7 @@ async function main() { assertCanonicalQueryAndAuthorizationIncludeMultipartParameters(); assertInsufficientSpaceStopsBeforeServiceChanges(); assertCheckSpaceOnlyUsesFormatSpecificRequirement(); + assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog(); assertStopFailureRetainsRecoveryMarker(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); @@ -2562,6 +2564,106 @@ async function assertHistoryResumeReverifiesArchiveAndManifest() { } } +function assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog() { + const fixture = createHistoryFixture('minimal-plan', { nestedData: true }); + const replicaDir = path.join(fixture.replicasDir, '2'); + const snapshotsDir = path.join(replicaDir, 'snapshots'); + const clogDir = path.join(replicaDir, 'clog'); + mkdirSync(snapshotsDir, { recursive: true }); + mkdirSync(clogDir, { recursive: true }); + for (const transaction of ['100', '200', '300']) { + const padded = transaction.padStart(20, '0'); + const snapshotDir = path.join(snapshotsDir, `${padded}.snapshot_dir`); + mkdirSync(snapshotDir, { recursive: true }); + writeFileSync( + path.join(snapshotDir, `${padded}.snapshot_bsatn`), + 'snapshot', + ); + } + for (const transaction of ['50', '150', '250', '350']) { + const padded = transaction.padStart(20, '0'); + writeFileSync(path.join(clogDir, `${padded}.stdb.log`), 'log'); + writeFileSync(path.join(clogDir, `${padded}.stdb.ofs`), 'ofs'); + } + for (const relativeDir of ['config', 'data/control-db', 'data/program-bytes']) { + const directory = path.join(fixture.dataDir, relativeDir); + mkdirSync(directory, { recursive: true }); + writeFileSync(path.join(directory, 'state.bin'), 'state'); + } + writeFileSync(path.join(fixture.dataDir, 'data/config.toml'), 'config'); + writeFileSync(path.join(fixture.dataDir, 'data/metadata.toml'), 'metadata'); + + const plan = discoverMinimalPlan({ dataDir: fixture.dataDir }); + const paths = plan.candidates.map((item) => item.path); + const expect = (condition, reason) => { + if (!condition) { + failures.push(reason); + } + }; + + expect( + plan.retainSnapshots === 2, + `minimal 默认应保留 2 份 snapshot，实际 ${plan.retainSnapshots}`, + ); + expect( + paths.includes( + 'data/replicas/2/snapshots/00000000000000000200.snapshot_dir', + ), + 'minimal 必须保留次新 snapshot。', + ); + expect( + paths.includes( + 'data/replicas/2/snapshots/00000000000000000300.snapshot_dir', + ), + 'minimal 必须保留最新 snapshot。', + ); + expect( + !paths.includes( + 'data/replicas/2/snapshots/00000000000000000100.snapshot_dir', + ), + 'minimal 不得备份更早的 snapshot。', + ); + expect( + paths.includes('data/replicas/2/clog/00000000000000000150.stdb.log'), + 'minimal 必须保留覆盖最老保留 snapshot 的边界 commitlog 段。', + ); + expect( + paths.includes('data/replicas/2/clog/00000000000000000350.stdb.log'), + 'minimal 必须保留最新 commitlog 段。', + ); + expect( + !paths.includes('data/replicas/2/clog/00000000000000000050.stdb.log'), + 'minimal 不得备份更早的 commitlog 段。', + ); + for (const staticPath of [ + 'config', + 'data/config.toml', + 'data/metadata.toml', + 'data/control-db', + 'data/program-bytes', + ]) { + expect( + paths.includes(staticPath), + `minimal 必须保留状态路径 ${staticPath}。`, + ); + } + const replica = plan.replicas.find((item) => item.replicaId === '2'); + expect( + replica?.retainedSnapshots === 2 && replica?.droppedSnapshots === 1, + 'minimal 必须报告保留/丢弃的 snapshot 数量。', + ); + expect( + replica?.droppedSegments === 1, + `minimal 必须报告丢弃的 commitlog 段数量，实际 ${replica?.droppedSegments}`, + ); + + assertThrows( + () => discoverMinimalPlan({ dataDir: fixture.dataDir, retainSnapshots: 0 }), + '--retain-snapshots 必须是 >= 1 的整数', + 'minimal 必须校验保留 snapshot 数量。', + ); +} + function createHistoryFixture(name, { nestedData }) { const root = path.join(tmpRoot, name); const dataDir = path.join(root, 'stdb'); diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index bfb4bdbe9..4c6e65d85 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -938,7 +938,7 @@ const checks = [ }, { file: 'scripts/jenkins-server-provision.sh', - includes: 'archive-full|files-history)', + includes: 'archive-full|files-history|files-minimal)', reason: 'Server-Provision 必须拒绝未知数据库备份 profile。', }, { @@ -963,7 +963,7 @@ const checks = [ { file: 'jenkins/Jenkinsfile.production-server-provision', includes: - "choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history']", + "choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history']", reason: 'Server-Provision Job 必须显式暴露 archive-first 的数据库备份 profile。', }, @@ -1016,10 +1016,28 @@ const checks = [ }, { file: 'jenkins/Jenkinsfile.production-server-provision', - includes: 'release 仅允许 archive-full；files-history', + includes: 'release 不允许 files-history', reason: 'release 必须拒绝 files-history，避免逐文件 catalog 扫描再次触发生产内存峰值。', }, + { + file: 'jenkins/Jenkinsfile.production-server-provision', + includes: 'files-minimal', + reason: + 'release 定时备份必须提供 files-minimal profile：只备最近 snapshot 与其后 commitlog，热备不停服。', + }, + { + file: 'deploy/systemd/genarrative-database-backup-files-minimal.conf', + includes: '--mode full --minimal --retain-snapshots 2', + reason: + 'files-minimal 定时备份必须使用 minimal 口径并保留上游默认的最近 2 份 snapshot。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: '--minimal --retain-snapshots', + reason: + '发布前备份默认使用 minimal 热备，避免 40G 级冷备空间门槛与停服。', + }, { file: 'scripts/database-backup-to-oss.mjs', includes: diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index 2202017df..ba6e236fd 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -56,6 +56,16 @@ const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join( const DEFAULT_SPACE_SAFETY_RATIO = 1.1; const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024; // files 模式不落地本地归档（只写 catalog/state），所需空间远小于数据目录本身。 +// minimal 备份保留的最近 snapshot 数（与上游 retention 默认值一致）。 +const DEFAULT_RETAIN_SNAPSHOTS = 2; +// minimal 备份额外保留的小体积状态：身份、standalone 配置、控制库与模块字节。 +const MINIMAL_STATIC_PATHS = [ + 'config', + 'data/config.toml', + 'data/metadata.toml', + 'data/control-db', + 'data/program-bytes', +]; const DEFAULT_FILES_SPACE_SAFETY_RATIO = 0.05; const DEFAULT_FILES_EXTRA_FREE_BYTES = 2 * 1024 * 1024 * 1024; // 空间不足用独立退出码，调用方可据此决定是否降级存储格式。 @@ -84,7 +94,7 @@ const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1; function usage() { console.log(`用法: - npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] + npm run database:backup:oss -- [--mode full|history] [--minimal] [--retain-snapshots 2] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] npm run database:backup:oss -- --check-space-only [--storage-format archive|files] [--data-dir ] [--work-dir ] node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] node -- scripts/database-backup-to-oss.mjs --upload-archive @@ -97,6 +107,7 @@ function usage() { 将 SpacetimeDB 数据目录以 .tar.gz 或逐文件 catalog 形式上传到阿里云 OSS 指定 bucket。 默认 full 模式保持原有全量冷备行为；history 模式只归档已被最新 snapshot 覆盖的历史 commitlog 与旧 snapshot。 --storage-format files 不打包：按原相对路径建立 catalog，文件内容以 SHA-256 不可变对象上传；重复运行只上传新增或变化内容。 + --minimal 只备份上游 retention 语义下仍需要的部分（最近 --retain-snapshots 份 snapshot + 其后 commitlog 段 + 身份/配置/控制库/模块字节）；只支持 files 格式，且不停止任何服务。 archive history 必须有已验真的 full baseline state；files history 必须复用同一 work-dir 中已发布的 full catalog state。 --defer-upload 只生成本地冷备份和 manifest，不上传；后续用 --upload-archive 异步上传。 --upload-deferred-dir 串行收敛目录内 deferred/pending 及已上传未清理归档；只有 OSS 上传与验真完成后才按 keep-local 规则删除。 @@ -192,6 +203,8 @@ function parseArgs(argv) { resultFile: '', minFreeBytes: '', mode: 'full', + minimal: false, + retainSnapshots: DEFAULT_RETAIN_SNAPSHOTS, baselineState: '', baselineManifest: '', publishManifest: '', @@ -282,6 +295,12 @@ function parseArgs(argv) { case '--mode': options.mode = readValue(); break; + case '--minimal': + options.minimal = true; + break; + case '--retain-snapshots': + options.retainSnapshots = readValue(); + break; case '--baseline-state': options.baselineState = readValue(); break; @@ -1383,6 +1402,217 @@ function candidateKey(candidate) { return `${candidate.kind}\0${candidate.path}`; } +/** + * minimal 备份只保留上游 retention 语义下仍然需要的部分： + * 每个 replica 最近 retainSnapshots 份 snapshot、覆盖最老保留 snapshot 的 + * commitlog 边界段及其后的全部 commitlog（段边界与上游 pruner 一致）， + * 外加身份、standalone 配置、控制库与模块字节等小体积状态。 + * 历史 snapshot 与更早的 commitlog 段不再备份，因此也不需要计算增量差异。 + */ +export function discoverMinimalPlan({ + dataDir, + retainSnapshots = DEFAULT_RETAIN_SNAPSHOTS, +}) { + const retainCount = Number(retainSnapshots); + if (!Number.isInteger(retainCount) || retainCount < 1) { + throw new Error( + `--retain-snapshots 必须是 >= 1 的整数，实际: ${retainSnapshots}`, + ); + } + const resolvedDataDir = resolvePath(dataDir); + const replicasDir = findReplicasDir(resolvedDataDir); + const replicas = []; + const candidates = []; + + const replicaIds = readdirSync(replicasDir, { withFileTypes: true }) + .sort((left, right) => left.name.localeCompare(right.name)) + .flatMap((entry) => { + if (!entry.isDirectory()) { + return []; + } + if (!/^\d+$/u.test(entry.name)) { + throw new Error(`replica 目录名不符合预期: ${entry.name}`); + } + return [entry.name]; + }); + + for (const replicaId of replicaIds) { + const replicaDir = join(replicasDir, replicaId); + const snapshotsDir = join(replicaDir, 'snapshots'); + const clogDir = join(replicaDir, 'clog'); + if (!existsSync(snapshotsDir) || !lstatSync(snapshotsDir).isDirectory()) { + replicas.push({ + replicaId, + status: 'skipped', + reason: 'no-snapshots-directory', + }); + continue; + } + const snapshots = readdirSync(snapshotsDir, { withFileTypes: true }) + .flatMap((entry) => { + const match = /^(\d{20})[.]snapshot_dir$/u.exec(entry.name); + if (!match) { + return []; + } + const transaction = BigInt(match[1]); + if (transaction > 0xffff_ffff_ffff_ffffn) { + throw new Error(`snapshot transaction 超出 u64: ${entry.name}`); + } + if (!entry.isDirectory()) { + throw new Error( + `snapshot 候选必须是目录: ${join(snapshotsDir, entry.name)}`, + ); + } + const snapshotDir = join(snapshotsDir, entry.name); + const lockPath = join(snapshotsDir, `${match[1]}.lock`); + const snapshotFile = join(snapshotDir, `${match[1]}.snapshot_bsatn`); + if ( + existsSync(lockPath) || + !existsSync(snapshotFile) || + !lstatSync(snapshotFile).isFile() + ) { + return []; + } + return [{ name: entry.name, transaction }]; + }) + .sort((left, right) => + left.transaction < right.transaction + ? -1 + : left.transaction > right.transaction + ? 1 + : 0, + ); + if (snapshots.length === 0) { + replicas.push({ replicaId, status: 'skipped', reason: 'no-snapshot' }); + continue; + } + if (!existsSync(clogDir) || !lstatSync(clogDir).isDirectory()) { + throw new Error(`replica ${replicaId} 缺少 clog 目录。`); + } + const segmentFiles = new Map(); + for (const entry of readdirSync(clogDir, { withFileTypes: true })) { + const match = /^(\d{20})[.]stdb[.](log|ofs)$/u.exec(entry.name); + if (!match) { + throw new Error(`commitlog 文件名不符合预期: ${entry.name}`); + } + if (!entry.isFile()) { + throw new Error( + `commitlog 候选必须是普通文件: ${join(clogDir, entry.name)}`, + ); + } + const transaction = BigInt(match[1]); + if (transaction > 0xffff_ffff_ffff_ffffn) { + throw new Error(`commitlog transaction 超出 u64: ${entry.name}`); + } + const key = transaction.toString(); + const group = segmentFiles.get(key) ?? { transaction }; + group[match[2]] = entry.name; + segmentFiles.set(key, group); + } + for (const group of segmentFiles.values()) { + if (group.ofs && !group.log) { + throw new Error( + `commitlog offset 缺少对应 log: replica=${replicaId}, transaction=${group.transaction}`, + ); + } + } + const segments = [...segmentFiles.values()] + .filter((group) => group.log) + .sort((left, right) => + left.transaction < right.transaction + ? -1 + : left.transaction > right.transaction + ? 1 + : 0, + ); + + const retainedSnapshots = snapshots.slice(-retainCount); + const oldestRetainedSnapshot = retainedSnapshots[0].transaction; + const boundarySegment = segments + .filter((segment) => segment.transaction <= oldestRetainedSnapshot) + .at(-1); + if (!boundarySegment) { + throw new Error( + `replica ${replicaId} 无法找到覆盖最老保留 snapshot ${oldestRetainedSnapshot} 的 commitlog 边界。`, + ); + } + const retainedSegments = segments.filter( + (segment) => segment.transaction >= boundarySegment.transaction, + ); + + for (const snapshot of retainedSnapshots) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(snapshotsDir, snapshot.name), + kind: 'snapshot', + replicaId, + transaction: snapshot.transaction, + }), + ); + } + for (const segment of retainedSegments) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.log), + kind: 'commitlog', + replicaId, + transaction: segment.transaction, + }), + ); + if (segment.ofs) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.ofs), + kind: 'commitlog-offset', + replicaId, + transaction: segment.transaction, + }), + ); + } + } + replicas.push({ + replicaId, + status: 'ready', + retainedSnapshots: retainedSnapshots.length, + droppedSnapshots: snapshots.length - retainedSnapshots.length, + oldestRetainedSnapshot: oldestRetainedSnapshot.toString(), + boundarySegment: boundarySegment.transaction.toString(), + droppedSegments: segments.length - retainedSegments.length, + }); + } + + for (const relativePath of MINIMAL_STATIC_PATHS) { + const absolutePath = resolve(resolvedDataDir, relativePath); + if (!existsSync(absolutePath)) { + continue; + } + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath, + kind: 'state', + replicaId: 'state', + transaction: 0n, + }), + ); + } + + candidates.sort((left, right) => left.path.localeCompare(right.path)); + return { + dataDir: resolvedDataDir, + replicasDir: assertSafeRelativePath(resolvedDataDir, replicasDir), + replicas, + retainSnapshots: retainCount, + candidates, + totalSizeBytes: candidates + .reduce((sum, item) => sum + BigInt(item.sizeBytes), 0n) + .toString(), + }; +} + export function cleanupHistoryCandidates({ dataDir, candidates }) { const currentPlan = discoverHistoryPlan({ dataDir }); const eligible = new Map( @@ -2114,6 +2344,8 @@ async function publishDirectFilesLatest({ export async function runDirectFilesBackup({ mode, + minimal = false, + retainSnapshots = DEFAULT_RETAIN_SNAPSHOTS, dataDir, workDir, database, @@ -2139,7 +2371,17 @@ export async function runDirectFilesBackup({ `files history 模式缺少已发布 full baseline catalog: ${statePath}`, ); } - const plan = mode === 'history' ? discoverHistoryPlan({ dataDir }) : null; + const plan = + mode === 'history' + ? discoverHistoryPlan({ dataDir }) + : minimal + ? discoverMinimalPlan({ dataDir, retainSnapshots }) + : null; + if (minimal) { + console.log( + `[database-backup] files minimal: retainSnapshots=${plan.retainSnapshots}, candidates=${plan.candidates.length}, size=${plan.totalSizeBytes}`, + ); + } const collected = await collectDirectFileEntries({ dataDir, candidates: plan?.candidates ?? null, @@ -4217,6 +4459,12 @@ async function main() { `--storage-format 只能是 archive 或 files，实际: ${storageFormat}`, ); } + if (args.minimal && storageFormat !== 'files') { + throw new Error('--minimal 只支持 --storage-format files。'); + } + if (args.minimal && args.mode === 'history') { + throw new Error('--minimal 只支持 --mode full：minimal 自身就是自包含快照。'); + } if (args.checkSpaceOnly) { assertSufficientWorkDirSpace({ @@ -4351,11 +4599,13 @@ async function main() { let backupError = null; let restoreError = null; try { - if (args.mode === 'full' && !args.dryRun) { + if (args.mode === 'full' && !args.minimal && !args.dryRun) { serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); } await runDirectFilesBackup({ mode: args.mode, + minimal: args.minimal, + retainSnapshots: args.retainSnapshots, dataDir, workDir, database, diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index b44eccf9c..3b16188f8 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -24,6 +24,8 @@ usage() { 环境变量： GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files（默认 archive） GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0（默认 1：archive 空间不足自动降级 files） + GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=1|0（默认 1：发布前备份只保留最近 N 份 snapshot + 其后 commitlog，热备不停服） + GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS=N（默认 2，仅在 minimal 模式下生效） GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0（默认 1：尚未开始 publish 的失败自动恢复服务并退出维护） migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。 如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。 @@ -65,6 +67,8 @@ BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}" BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}" AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}" AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}" +BACKUP_MINIMAL="${GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL:-1}" +BACKUP_RETAIN_SNAPSHOTS="${GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS:-2}" DEPLOY_COMPLETED=0 PUBLISH_STARTED=0 MAINTENANCE_ENTERED=0 @@ -334,6 +338,14 @@ precheck_backup_space_before_maintenance() { echo "[production-stdb-publish] 已跳过发布前备份空间预检（--backup-mode skip）" return 0 fi + if [[ "${BACKUP_MINIMAL}" == "1" ]]; then + # minimal 备份是热备：只保留最近 N 份 snapshot + 其后 commitlog，不落地归档也不停服务。 + BACKUP_STORAGE_FORMAT="files" + if [[ "${BACKUP_MODE}" == "async" ]]; then + echo "[production-stdb-publish] minimal 备份为同步热备（无本地归档），备份模式由 async 调整为 sync。" >&2 + BACKUP_MODE="sync" + fi + fi local status=0 run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$? @@ -836,14 +848,21 @@ case "${BACKUP_MODE}" in SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service) fi - echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" - node -- "${BACKUP_SCRIPT}" \ - --env-file /etc/genarrative/api-server.env \ - --data-dir "${SPACETIME_ROOT_DIR}" \ - --database "${DATABASE}" \ - --storage-format "${BACKUP_STORAGE_FORMAT}" \ - --stop-service spacetimedb.service \ - "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}" + SYNC_BACKUP_ARGS=( + --env-file /etc/genarrative/api-server.env + --data-dir "${SPACETIME_ROOT_DIR}" + --database "${DATABASE}" + --storage-format "${BACKUP_STORAGE_FORMAT}" + ) + if [[ "${BACKUP_MINIMAL}" == "1" ]]; then + echo "[production-stdb-publish] publish 前执行 minimal 热备（最近 ${BACKUP_RETAIN_SNAPSHOTS} 份 snapshot + 其后 commitlog），不停服务" + SYNC_BACKUP_ARGS+=(--mode full --minimal --retain-snapshots "${BACKUP_RETAIN_SNAPSHOTS}") + else + echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" + SYNC_BACKUP_ARGS+=(--stop-service spacetimedb.service) + SYNC_BACKUP_ARGS+=("${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}") + fi + node -- "${BACKUP_SCRIPT}" "${SYNC_BACKUP_ARGS[@]}" ;; skip) echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份" diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 259b6f6c8..9d2bda38a 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -15,6 +15,8 @@ DATABASE_BACKUP_PROFILE="${DATABASE_BACKUP_PROFILE:-archive-full}" DATABASE_BACKUP_FILES_HISTORY_WORK_DIR="${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR:-/var/lib/genarrative/database-backups/files-history}" DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR="/etc/systemd/system/genarrative-database-backup.service.d" DATABASE_BACKUP_FILES_HISTORY_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-history.conf" +DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR="${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR:-/var/lib/genarrative/database-backups/files-minimal}" +DATABASE_BACKUP_FILES_MINIMAL_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-minimal.conf" DATABASE_BACKUP_LEGACY_DEV_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-dev-files.conf" require_non_root_relative_path() { @@ -71,10 +73,10 @@ validate_server_names() { validate_database_backup_profile() { case "${DATABASE_BACKUP_PROFILE}" in - archive-full|files-history) + archive-full|files-history|files-minimal) ;; *) - echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${DATABASE_BACKUP_PROFILE}" >&2 + echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full、files-history 或 files-minimal，当前值: ${DATABASE_BACKUP_PROFILE}" >&2 exit 1 ;; esac @@ -1323,12 +1325,29 @@ render_database_backup_files_history_drop_in() { deploy/systemd/genarrative-database-backup-files-history.conf } +render_database_backup_files_minimal_drop_in() { + local current_escaped env_escaped work_dir_escaped + current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" + env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")" + work_dir_escaped="$(escape_sed_replacement "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}")" + sed \ + -e "s|/opt/genarrative/current|${current_escaped}|g" \ + -e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \ + -e "s|/var/lib/genarrative/database-backups/files-minimal|${work_dir_escaped}|g" \ + deploy/systemd/genarrative-database-backup-files-minimal.conf +} + configure_database_backup_profile() { local rendered_drop_in if [[ "${DATABASE_BACKUP_PROFILE}" == "archive-full" ]]; then echo "[server-provision] 数据库备份 profile=archive-full，保留主 service 的全量冷备行为。" - run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + return + fi + + if [[ "${DATABASE_BACKUP_PROFILE}" == "files-minimal" ]]; then + configure_database_backup_files_minimal_profile return fi @@ -1350,13 +1369,26 @@ configure_database_backup_profile() { run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}" - run_cmd rm -f "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + run_cmd rm -f "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" rendered_drop_in="$(mktemp)" render_database_backup_files_history_drop_in >"${rendered_drop_in}" install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" 0644 rm -f "${rendered_drop_in}" } +configure_database_backup_files_minimal_profile() { + local rendered_drop_in + + echo "[server-provision] 数据库备份 profile=files-minimal，只保留最近 snapshot 与其后 commitlog（热备、不停服）。" + run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}" + run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}" + run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + rendered_drop_in="$(mktemp)" + render_database_backup_files_minimal_drop_in >"${rendered_drop_in}" + install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" 0644 + rm -f "${rendered_drop_in}" +} + render_health_patrol_service() { local current_escaped current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" @@ -1372,6 +1404,7 @@ require_path deploy/systemd/genarrative-external-generation-controller.service require_path deploy/systemd/genarrative-bgfilter-worker.service require_path deploy/systemd/genarrative-database-backup.service require_path deploy/systemd/genarrative-database-backup-files-history.conf +require_path deploy/systemd/genarrative-database-backup-files-minimal.conf require_path deploy/systemd/genarrative-database-backup.timer require_path deploy/systemd/genarrative-health-patrol.service require_path deploy/systemd/genarrative-health-patrol.timer