From 117d482f93e9bd4f48491cdc0cf94792ccad71c2 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 10:56:25 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8F=91=E5=B8=83=E5=89=8D=E5=A4=87=E4=BB=BD?= =?UTF-8?q?=E7=A9=BA=E9=97=B4=E9=A2=84=E6=A3=80=E5=89=8D=E7=BD=AE=E5=B9=B6?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E8=87=AA=E5=8A=A8=E9=99=8D=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - database-backup-to-oss.mjs 新增 --check-space-only：archive 按 data×1.1、files 按 max(data×0.05, 2GiB) 计算门槛，空间不足以退出码 3 返回 - production-stdb-publish.sh 在进入维护与停服务之前先做空间预检，archive 不足时自动降级 files（files 不支持 defer，async 收敛为 sync），可用环境变量关闭 - 尚未开始 publish 的失败自动恢复本次停掉的 API/controller/worker 并退出维护，只有真正开始发布之后的失败才保持维护态 - 备份检查脚本与生产运维门禁补上新口径与执行顺序断言，运维文档与共享记忆记录事故与规则 --- docs/project-memory/shared-memory/pitfalls.md | 4 + ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- scripts/check-database-backup-to-oss.mjs | 67 ++++++- scripts/check-production-ops-guardrails.mjs | 41 ++++- scripts/database-backup-to-oss.mjs | 98 ++++++++-- scripts/deploy/production-stdb-publish.sh | 170 ++++++++++++++++-- 6 files changed, 353 insertions(+), 29 deletions(-) diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 51ed5f002..a82811675 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -1,5 +1,9 @@ # 踩坑与排障记录 +## release 冷备空间不足会把生产留在维护态 + +`Genarrative-Stdb-Module-Publish` 先进入维护模式、停掉 API/controller/worker，再执行发布前冷备份；archive 口径要求 `data × 1.1`（40.6GiB 数据 → 44.7GiB，加上生产根盘只剩 13.5GiB），于是 2026-09-21 的 release 发布在停服后失败并保持维护态，站点 503 直到人工恢复。规则：空间预检必须先于 `maintenance-on` 与停服；archive 不足且未显式禁用降级时改用 files（`max(data × 0.05, 2GiB)`，不落地本地归档，但 files 不支持 `--defer-upload`，会从 async 收敛为 sync）；只有真正开始 `spacetime publish` 之后的失败才允许保持维护态。另：定时备份的失效锁在当前仓库版本会自动清理，但生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若是旧版会拒绝抢锁并要求人工删锁，需随 provision 更新。 + ## copyArtifacts 报「Unable to find project for artifact copy」的用户触发构建差异 Copy Artifact 插件在**非 SYSTEM 认证**下按「认证用户」判权：只有当被复制 Job 的 `CopyArtifactPermissionProperty`（仓库里由 Declarative 的 `copyArtifactPermission(...)` 维护）显式列出当前消费者，或者该 Job 对认证用户开放 Item.Read 时才放行；`ACL.SYSTEM2` 的定时构建会短路通过。因此会出现「定时调度一路成功、手动发布必挂」的现象（2026-09-21 手动发布 #6/#7 与同期的用户触发探测全部命中，定时调度 #104+ 正常）。`Genarrative-Agc-Global-Version-Issue` 生产权限模式的授权名单必须同时包含 `Genarrative-Scheduled-Revision-Trigger` 与 `Genarrative-Manual-Build-And-Deploy`；改完 `copyArtifactPermission` 后要先跑一次发号 Job 把 Job property 写回 Jenkins，只改仓库文件不生效。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 0596085fc..0dca6f394 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -449,7 +449,7 @@ UI 相关修改要重点验证： npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service ``` -脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 +脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 `2026-09-21` 又命中一次同类事故：release 冷备按 `archive` 口径要求 `data × 1.1`（40.6GiB → 44.7GiB），生产根盘只剩 13.5GiB，备份在停服后才失败，生产被留在维护态。现在 `production-stdb-publish.sh` 在进入维护模式与停服务之前先执行 `database-backup-to-oss.mjs --check-space-only`：archive 口径为 `max(data × 1.1, data + 512MiB)`，files 口径为 `max(data × 0.05, 2GiB)`，空间不足以退出码 `3` 返回。archive 空间不足且未显式设置 `GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=0` 时，脚本自动降级为 `--storage-format files`（不落地本地归档；files 不支持 `--defer-upload`，因此 `DATABASE_BACKUP_MODE=async` 会收敛为 `sync`，窗口内直传 OSS）；也可用 `GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=files` 直接指定。任何「尚未开始 `spacetime publish`」的失败（空间预检、备份执行等）都会自动把本次停掉的 API/controller/worker 拉回并退出维护，只有真正开始发布之后的失败才保持维护态等待人工；该自动恢复可用 `GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=0` 关闭。定时备份的失效锁：当前仓库版本在持有者进程退出后会自动清理并重试，而生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若仍是旧版（2026-07-16 前）会拒绝抢锁并需要人工删锁，必须随 provision 一起更新。 生产环境变量模板在 `deploy/env/api-server.env.example`： diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index 44060f394..f097b8e37 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -26,6 +26,7 @@ import { cleanupHistoryCandidates, collectDirectFileEntries, createUploadBandwidthLimiter, + describeBackupSpaceRequirement, discoverDeferredArchiveUploads, discoverHistoryPlan, restoreDirectFilesBackup, @@ -64,6 +65,7 @@ async function main() { assertDeferredArchiveDiscoveryIsBoundedAndDeterministic(); assertCanonicalQueryAndAuthorizationIncludeMultipartParameters(); assertInsufficientSpaceStopsBeforeServiceChanges(); + assertCheckSpaceOnlyUsesFormatSpecificRequirement(); assertStopFailureRetainsRecoveryMarker(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); @@ -1198,7 +1200,7 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() { '999999999999999999', ]); - assertStatus(result, 1, '空间不足时必须失败。'); + assertStatus(result, 3, '空间不足必须用独立退出码 3 失败。'); assertIncludes( result.stdout, '备份空间预检', @@ -1213,6 +1215,66 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() { assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。'); } +function assertCheckSpaceOnlyUsesFormatSpecificRequirement() { + const fixture = createFixture('check-space-only'); + + const filesOk = runBackup( + fixture, + ['--check-space-only', '--storage-format', 'files'], + { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1M' }, + ); + assertStatus(filesOk, 0, 'files 模式空间预检应在阈值满足时通过。'); + assertIncludes( + filesOk.stdout, + '备份空间预检(files)', + 'files 预检必须打印 files 口径的预检结果。', + ); + assertIncludes( + filesOk.stdout, + '空间预检通过(check-space-only)', + 'check-space-only 通过时必须给出显式成功标记。', + ); + assertFileMissing(fixture.systemctlLog, '空间预检不得调用 systemctl。'); + assertFileMissing(fixture.tarLog, '空间预检不得调用 tar。'); + + const filesInsufficient = runBackup( + fixture, + ['--check-space-only', '--storage-format', 'files'], + { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1000T' }, + ); + assertStatus(filesInsufficient, 3, 'files 空间不足同样使用退出码 3。'); + + const archiveInsufficient = runBackup( + fixture, + ['--check-space-only', '--storage-format', 'archive'], + { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1000T' }, + ); + assertStatus(archiveInsufficient, 3, 'archive 空间不足必须使用退出码 3。'); + + const archive = describeBackupSpaceRequirement({ + dataDir: fixture.dataDir, + workDir: fixture.workDir, + storageFormat: 'archive', + args: {}, + env: { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1G' }, + }); + const files = describeBackupSpaceRequirement({ + dataDir: fixture.dataDir, + workDir: fixture.workDir, + storageFormat: 'files', + args: {}, + env: { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1G' }, + }); + if (archive.requiredFreeBytes !== 1024n ** 3n || files.requiredFreeBytes !== 1024n ** 3n) { + failures.push( + `空间口径覆盖参数应生效：archive=${archive.requiredFreeBytes} files=${files.requiredFreeBytes}`, + ); + } + if (archive.storageFormat !== 'archive' || files.storageFormat !== 'files') { + failures.push('空间预检结果必须回显实际 storage-format。'); + } +} + function assertStopFailureRetainsRecoveryMarker() { const fixture = createFixture('stop-failure-marker'); writeExecutable( @@ -2695,7 +2757,7 @@ exit 2 return { root, binDir, dataDir, workDir, systemctlLog, tarLog }; } -function runBackup(fixture, extraArgs = []) { +function runBackup(fixture, extraArgs = [], envOverrides = {}) { return spawnSync( process.execPath, [ @@ -2720,6 +2782,7 @@ function runBackup(fixture, extraArgs = []) { env: { ...process.env, PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`, + ...envOverrides, }, }, ); diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 323a17e6a..bfb4bdbe9 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -258,6 +258,44 @@ const checks = [ reason: 'Stdb 先于 API 发布时必须先补齐 api-server env 的 FILE 路径，保证首次 rollout 重启即可读取 secret。', }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: + 'precheck_backup_space_before_maintenance\n\n"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"', + reason: + '备份空间预检必须先于进入维护模式执行：磁盘不足时不得停服务或把生产留在维护态。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: '--check-space-only', + reason: + '生产 Stdb publish 必须复用备份脚本的 --check-space-only 预检，保证空间口径与真实备份一致。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: 'GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK', + reason: + 'archive 冷备份空间不足时必须能自动降级为 files 存储格式（不落地本地归档）。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: 'restore_runtime_services_before_publish', + reason: + '尚未开始 publish 的失败必须自动恢复运行时服务并退出维护，避免生产停在维护态等人工。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: + 'if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then', + reason: + '只有尚未开始 publish 的失败才允许自动恢复并退出维护，半发布状态必须保持维护。', + }, + { + file: 'scripts/database-backup-to-oss.mjs', + includes: 'SPACE_INSUFFICIENT_EXIT_CODE', + reason: + '备份空间不足必须使用独立退出码，调用方据此决定降级存储格式而不是一律失败。', + }, { file: 'scripts/deploy/production-stdb-publish.sh', includes: 'stop_runtime_services_for_rollout_gate', @@ -984,7 +1022,8 @@ const checks = [ }, { file: 'scripts/database-backup-to-oss.mjs', - includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})', + includes: + "assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' })", normalizeWhitespace: true, reason: '生产冷备份必须先做工作目录剩余空间预检，避免停库后写满磁盘。', }, diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index bc215509f..2202017df 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -55,6 +55,11 @@ const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join( ); const DEFAULT_SPACE_SAFETY_RATIO = 1.1; const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024; +// files 模式不落地本地归档（只写 catalog/state），所需空间远小于数据目录本身。 +const DEFAULT_FILES_SPACE_SAFETY_RATIO = 0.05; +const DEFAULT_FILES_EXTRA_FREE_BYTES = 2 * 1024 * 1024 * 1024; +// 空间不足用独立退出码，调用方可据此决定是否降级存储格式。 +const SPACE_INSUFFICIENT_EXIT_CODE = 3; const OSS_ALGORITHM = 'OSS4-HMAC-SHA256'; const OSS_SERVICE = 'oss'; const OSS_REQUEST = 'aliyun_v4_request'; @@ -80,6 +85,7 @@ const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1; function usage() { console.log(`用法: npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] + npm run database:backup:oss -- --check-space-only [--storage-format archive|files] [--data-dir ] [--work-dir ] node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] node -- scripts/database-backup-to-oss.mjs --upload-archive node -- scripts/database-backup-to-oss.mjs --upload-deferred-dir @@ -106,6 +112,8 @@ function usage() { GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT archive（默认）或 files GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES 备份前要求 work dir 所在文件系统至少有这些可用字节；未设置时按数据目录大小估算 + GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES files 模式的最小可用字节（默认 2GiB） + GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO files 模式按数据目录比例要求的可用空间（默认 0.05） GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE history 使用的 full baseline 与追加批次状态文件 GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST 首次初始化 history state 的 uploaded full manifest ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET @@ -175,6 +183,7 @@ function parseArgs(argv) { restartServicesAfter: [], database: '', dryRun: false, + checkSpaceOnly: false, deferUpload: false, uploadArchive: '', uploadDeferredDir: '', @@ -282,6 +291,9 @@ function parseArgs(argv) { case '--publish-manifest': options.publishManifest = readValue(); break; + case '--check-space-only': + options.checkSpaceOnly = true; + break; case '--storage-format': options.storageFormat = readValue(); break; @@ -1013,6 +1025,20 @@ function parseSafetyRatio(rawValue) { return ratio; } +function parseFilesSpaceSafetyRatio(rawValue) { + const value = String(rawValue ?? '').trim(); + if (!value) { + return DEFAULT_FILES_SPACE_SAFETY_RATIO; + } + const ratio = Number(value); + if (!Number.isFinite(ratio) || ratio <= 0) { + throw new Error( + `GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO 必须是 > 0 的数字，实际: ${rawValue}`, + ); + } + return ratio; +} + function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) { const explicitMinFreeBytes = parseByteSize( firstNonEmpty( @@ -1042,22 +1068,57 @@ function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) { : extraRequirement; } -function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) { +function calculateFilesRequiredFreeBytes({ dataSizeBytes, env }) { + const explicitMinFreeBytes = parseByteSize( + env.GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES, + 'GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES', + ); + if (explicitMinFreeBytes !== null) { + return explicitMinFreeBytes; + } + const ratio = parseFilesSpaceSafetyRatio( + env.GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO, + ); + const ratioBasisPoints = BigInt(Math.ceil(ratio * 10000)); + const ratioRequirement = (dataSizeBytes * ratioBasisPoints + 9999n) / 10000n; + const floorRequirement = BigInt(DEFAULT_FILES_EXTRA_FREE_BYTES); + return ratioRequirement > floorRequirement ? ratioRequirement : floorRequirement; +} + +export function describeBackupSpaceRequirement({ + dataDir, + workDir, + storageFormat = 'archive', + args = {}, + env = {}, +}) { mkdirSync(workDir, { recursive: true }); const dataSizeBytes = getDirectorySizeBytes(dataDir); const availableBytes = getAvailableBytes(workDir); - const requiredFreeBytes = calculateRequiredFreeBytes({ - dataSizeBytes, - args, - env, - }); + const requiredFreeBytes = + storageFormat === 'files' + ? calculateFilesRequiredFreeBytes({ dataSizeBytes, env }) + : calculateRequiredFreeBytes({ dataSizeBytes, args, env }); + return { storageFormat, dataSizeBytes, availableBytes, requiredFreeBytes }; +} + +function assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat = 'archive' }) { + const { dataSizeBytes, availableBytes, requiredFreeBytes } = + describeBackupSpaceRequirement({ + dataDir, + workDir, + storageFormat, + args, + env, + }); console.log( - `[database-backup] 备份空间预检: data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, + `[database-backup] 备份空间预检(${storageFormat}): data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, ); if (availableBytes < requiredFreeBytes) { - throw new Error( + const error = new Error( [ `备份工作目录所在文件系统剩余空间不足: ${workDir}`, + `storage-format=${storageFormat}`, `available=${formatBytes(availableBytes)}`, `required=${formatBytes(requiredFreeBytes)}`, `dataDir=${dataDir}`, @@ -1065,7 +1126,10 @@ function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) { '为避免停库后写满磁盘，本次备份已在停止服务前中止。', ].join('；'), ); + error.spaceInsufficient = true; + throw error; } + return { dataSizeBytes, availableBytes, requiredFreeBytes }; } function assertSufficientHistoryWorkDirSpace({ @@ -4154,6 +4218,20 @@ async function main() { ); } + if (args.checkSpaceOnly) { + assertSufficientWorkDirSpace({ + dataDir, + workDir, + args, + env, + storageFormat, + }); + console.log( + `[database-backup] 空间预检通过(check-space-only): storage-format=${storageFormat}, workDir=${workDir}`, + ); + return; + } + for (const [label, value] of Object.entries({ bucket, endpoint, @@ -4369,7 +4447,7 @@ async function main() { const restartServicesAfter = collectRestartServicesAfterBackup({ args, env }); const stopMarkerPath = databaseBackupStopMarkerPath(workDir); try { - assertSufficientWorkDirSpace({ dataDir, workDir, args, env }); + assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' }); serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); archivePath = createArchive({ dataDir, workDir, fileName }); } catch (error) { @@ -4558,6 +4636,6 @@ if ( } catch { // 参数解析或状态文件写入失败时，不能掩盖原始备份错误。 } - process.exit(1); + process.exit(error?.spaceInsufficient ? SPACE_INSUFFICIENT_EXIT_CODE : 1); }); } diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index e96210ecb..b44eccf9c 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -18,9 +18,16 @@ usage() { 如需强制等待备份完成并在失败时阻断 publish，传入 --backup-mode sync。 发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径，再重启并验活重启前 active 的服务。 --keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker，并在成功后保持维护态，交由后续 API deploy 恢复服务。 + 发布前先做备份空间预检（不进入维护、不停服务）；archive 空间不足且未显式关闭自动降级时， + 自动改用 files 存储格式（不落地本地归档，改为同步直传 OSS），避免磁盘不足把生产留在维护态。 + +环境变量： + GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files（默认 archive） + GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0（默认 1：archive 空间不足自动降级 files） + GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0（默认 1：尚未开始 publish 的失败自动恢复服务并退出维护） migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。 如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。 - 失败时保留维护模式。 + 失败时：尚未开始 publish 的失败会自动恢复运行时服务并退出维护；真正开始 publish 之后的失败保留维护模式。 EOF } @@ -55,7 +62,13 @@ API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-serv WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}" KEEP_MAINTENANCE_MODE=0 BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}" +BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}" +AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}" +AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}" DEPLOY_COMPLETED=0 +PUBLISH_STARTED=0 +MAINTENANCE_ENTERED=0 +STOPPED_RUNTIME_SERVICES=() PUBLISH_TMP_DIR="" ASYNC_BACKUP_STATUS_FILE="" ASYNC_BACKUP_SCRIPT="" @@ -287,6 +300,106 @@ restart_runtime_services_after_bootstrap_secret_install() { fi } +backup_script_path() { + local candidate="" + for candidate in \ + "${SCRIPT_DIR}/../database-backup-to-oss.mjs" \ + "${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"; do + if [[ -f "${candidate}" ]]; then + printf '%s\n' "${candidate}" + return 0 + fi + done + return 1 +} + +run_backup_space_precheck() { + local storage_format="$1" + local backup_script="" + if ! backup_script="$(backup_script_path)"; then + echo "[production-stdb-publish] 缺少数据库备份脚本，无法做备份空间预检" >&2 + return 1 + fi + node -- "${backup_script}" \ + --env-file /etc/genarrative/api-server.env \ + --data-dir "${SPACETIME_ROOT_DIR}" \ + --database "${DATABASE}" \ + --storage-format "${storage_format}" \ + --check-space-only +} + +# 空间预检必须发生在进入维护模式与停服务之前：磁盘不够时不允许再动生产。 +precheck_backup_space_before_maintenance() { + if [[ "${BACKUP_MODE}" == "skip" ]]; then + echo "[production-stdb-publish] 已跳过发布前备份空间预检（--backup-mode skip）" + return 0 + fi + + local status=0 + run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$? + if [[ "${status}" -eq 0 ]]; then + echo "[production-stdb-publish] 发布前备份空间预检通过: storage-format=${BACKUP_STORAGE_FORMAT}（尚未进入维护模式、未停服务）" + return 0 + fi + if [[ "${status}" -ne 3 ]]; then + echo "[production-stdb-publish] 发布前备份空间预检失败（非空间原因），中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + if [[ "${BACKUP_STORAGE_FORMAT}" == "files" ]]; then + echo "[production-stdb-publish] files 模式备份空间仍不足，中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + if [[ "${AUTO_FILES_FALLBACK}" != "1" ]]; then + echo "[production-stdb-publish] archive 备份空间不足且已禁用自动降级（GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=${AUTO_FILES_FALLBACK}），中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + + echo "[production-stdb-publish] archive 冷备份空间不足：自动降级为 files 存储格式（不落地本地归档，改为文件级 catalog 直传 OSS）。" >&2 + BACKUP_STORAGE_FORMAT="files" + if [[ "${BACKUP_MODE}" == "async" ]]; then + echo "[production-stdb-publish] files 模式不支持 --defer-upload，本次备份改为同步执行。" >&2 + BACKUP_MODE="sync" + fi + status=0 + run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$? + if [[ "${status}" -ne 0 ]]; then + echo "[production-stdb-publish] 降级为 files 后空间预检仍失败，中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + echo "[production-stdb-publish] 已降级为 files 存储格式且空间预检通过。" +} + +# 仅在「尚未开始 publish」的失败路径调用：把本次停掉的运行时服务拉回来。 +restore_runtime_services_before_publish() { + if [[ "${#STOPPED_RUNTIME_SERVICES[@]}" -eq 0 ]]; then + return 0 + fi + local service="" + local state="" + local attempt=0 + echo "[production-stdb-publish] 发布尚未开始，恢复本次停掉的运行时服务: ${STOPPED_RUNTIME_SERVICES[*]}" + if ! run_privileged systemctl start "${STOPPED_RUNTIME_SERVICES[@]}"; then + echo "[production-stdb-publish] 启动运行时服务失败: ${STOPPED_RUNTIME_SERVICES[*]}" >&2 + return 1 + fi + for service in "${STOPPED_RUNTIME_SERVICES[@]}"; do + state="" + for attempt in $(seq 1 15); do + state="$(get_runtime_service_active_state "${service}" 2>/dev/null || true)" + if [[ "${state}" == "active" ]]; then + break + fi + sleep 1 + done + if [[ "${state}" != "active" ]]; then + echo "[production-stdb-publish] 运行时服务未恢复 active: ${service}, state=${state}" >&2 + return 1 + fi + echo "[production-stdb-publish] 运行时服务已恢复 active: ${service}" + done + return 0 +} + stop_runtime_services_for_rollout_gate() { local api_state="" local controller_state="" @@ -326,6 +439,7 @@ stop_runtime_services_for_rollout_gate() { fi echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}" + STOPPED_RUNTIME_SERVICES=("${services_to_stop[@]}") run_privileged systemctl stop "${services_to_stop[@]}" for worker_service in "${services_to_stop[@]}"; do if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then @@ -402,6 +516,10 @@ while [[ $# -gt 0 ]]; do BACKUP_MODE="${2:?缺少 --backup-mode 的值}" shift 2 ;; + --backup-storage-format) + BACKUP_STORAGE_FORMAT="${2:?缺少 --backup-storage-format 的值}" + shift 2 + ;; *) echo "[production-stdb-publish] 未知参数: $1" >&2 usage >&2 @@ -431,6 +549,14 @@ for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do fi done +if [[ ! "${BACKUP_STORAGE_FORMAT}" =~ ^(archive|files)$ ]]; then + echo "[production-stdb-publish] --backup-storage-format 只能是 archive 或 files: ${BACKUP_STORAGE_FORMAT}" >&2 + exit 1 +fi +if [[ "${BACKUP_STORAGE_FORMAT}" == "files" && "${BACKUP_MODE}" == "async" ]]; then + echo "[production-stdb-publish] files 存储格式不支持 --defer-upload，备份模式由 async 调整为 sync" >&2 + BACKUP_MODE="sync" +fi if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2 exit 1 @@ -497,7 +623,22 @@ on_exit() { rm -rf "${PUBLISH_TMP_DIR}" fi if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then - echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2 + if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then + # 尚未开始 publish 就失败（例如备份空间/备份执行失败）：本次没有任何发布变更， + # 必须把停掉的运行时服务拉回来并退出维护，避免生产停在维护态等人工救。 + if restore_runtime_services_before_publish; then + if [[ "${MAINTENANCE_ENTERED}" -eq 1 ]]; then + if ! "${SCRIPT_DIR}/maintenance-off.sh"; then + echo "[production-stdb-publish] 自动退出维护模式失败，请手工执行 maintenance-off.sh。" >&2 + fi + fi + echo "[production-stdb-publish] 发布尚未开始即失败，已自动恢复运行时服务并退出维护模式。" + else + echo "[production-stdb-publish] 自动恢复运行时服务失败，保持维护模式，请手工处理。" >&2 + fi + else + echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2 + fi fi exit "${exit_code}" } @@ -506,12 +647,8 @@ trap on_exit EXIT prepare_async_backup() { local -a restart_service_args=() - ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs" - if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then - ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" - fi - if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then - echo "[production-stdb-publish] 缺少数据库备份脚本: ${ASYNC_BACKUP_SCRIPT}" >&2 + if ! ASYNC_BACKUP_SCRIPT="$(backup_script_path)"; then + echo "[production-stdb-publish] 缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 exit 1 fi @@ -527,6 +664,7 @@ prepare_async_backup() { --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ + --storage-format "${BACKUP_STORAGE_FORMAT}" \ --stop-service spacetimedb.service \ "${restart_service_args[@]}" \ --defer-upload \ @@ -676,7 +814,10 @@ wait_for_api_healthz_ready() { return 1 } +precheck_backup_space_before_maintenance + "${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}" +MAINTENANCE_ENTERED=1 if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then stop_runtime_services_for_rollout_gate fi @@ -687,23 +828,20 @@ case "${BACKUP_MODE}" in ;; sync) SYNC_BACKUP_RESTART_SERVICE_ARGS=() - BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs" - if [[ ! -f "${BACKUP_SCRIPT}" ]]; then - BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" - fi - if [[ ! -f "${BACKUP_SCRIPT}" ]]; then - echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2 + if ! BACKUP_SCRIPT="$(backup_script_path)"; then + echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 exit 1 fi if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service) fi - echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布" + echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" node -- "${BACKUP_SCRIPT}" \ --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ + --storage-format "${BACKUP_STORAGE_FORMAT}" \ --stop-service spacetimedb.service \ "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}" ;; @@ -768,8 +906,10 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then else PUBLISH_ARGS+=(--server "${SERVER_ALIAS}") fi + PUBLISH_STARTED=1 runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}" else + PUBLISH_STARTED=1 spacetime "${PUBLISH_ARGS[@]}" fi