From 6d7b3e74a615a06f04b0f406b091494bb45c61d0 Mon Sep 17 00:00:00 2001 From: kdletters Date: Tue, 15 Sep 2026 23:23:06 +0800 Subject: [PATCH] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=B8=85=E7=90=86=E5=A4=B1?= =?UTF-8?q?=E6=95=88=E6=95=B0=E6=8D=AE=E5=BA=93=E5=A4=87=E4=BB=BD=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 失效进程锁自动清理后重试获取 仍存活进程持锁时继续阻断并发备份 补充锁生命周期文档与回归测试 --- ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- scripts/check-database-backup-to-oss.mjs | 18 ++-- scripts/database-backup-to-oss.mjs | 82 ++++++++++++------- 3 files changed, 62 insertions(+), 40 deletions(-) diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 066bc0a5d..1ca7a142f 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -431,7 +431,7 @@ UI 相关修改要重点验证： npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service ``` -脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 +脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 生产环境变量模板在 `deploy/env/api-server.env.example`： diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index b42f5dbfc..44060f394 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -76,7 +76,7 @@ async function main() { await assertManifestUploadUsesShaAndHeadVerification(); assertHistoryDiscoversDevAndProductionLayoutsWithMultipleReplicas(); assertHistoryRequiresBaselineAndProducesDeterministicDeferredBatch(); - assertHistoryBackupLockRejectsLiveAndStaleOwners(); + assertHistoryBackupLockRejectsLiveAndCleansStaleOwners(); assertHistorySkipsReplicaWithoutSnapshotAndRejectsMalformedNames(); assertHistoryStatDriftPreventsAnyCleanup(); await assertHistoryUploadFailureDoesNotDeleteSources(); @@ -2140,7 +2140,7 @@ function assertHistoryRequiresBaselineAndProducesDeterministicDeferredBatch() { } } -function assertHistoryBackupLockRejectsLiveAndStaleOwners() { +function assertHistoryBackupLockRejectsLiveAndCleansStaleOwners() { const liveOwner = createHistoryFixture('history-live-lock', { nestedData: false, }); @@ -2176,17 +2176,17 @@ function assertHistoryBackupLockRejectsLiveAndStaleOwners() { const staleResult = runHistoryCommand(staleOwner, ['--defer-upload']); assertStatus( staleResult, - 1, - '失效 owner pid 的 backup lock 也必须失败关闭，避免并发抢锁。', + 0, + '失效 owner pid 的 backup lock 应自动清理并成功获取新锁。', ); assertIncludes( - staleResult.stderr, - '拒绝自动抢锁', - '失效 backup lock 应要求人工核对 multipart 与进程。', + `${staleResult.stdout}\n${staleResult.stderr}`, + '失效数据库备份锁', + '自动清理失效 backup lock 时应输出可审计日志。', ); assertTrue( - existsSync(staleLockPath), - '失效 backup lock 未经人工核对不得自动删除。', + !existsSync(staleLockPath), + '成功获取并释放新锁后，失效 backup lock 不得残留。', ); } diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index 148267ac7..4b9031410 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -456,43 +456,65 @@ function acquireBackupLock({ workDir, database }) { workDir, `${sanitizeObjectPart(database, 'spacetimedb')}.backup.lock`, ); - try { - const fd = openSync(lockPath, 'wx', 0o600); - writeFileSync(fd, `${process.pid}\n`, 'utf8'); - closeSync(fd); - const release = () => { - try { - const ownerPid = Number(String(readFileSync(lockPath, 'utf8')).trim()); - if (ownerPid === process.pid) { - rmSync(lockPath, { force: true }); + for (let attempt = 0; attempt < 5; attempt += 1) { + try { + const fd = openSync(lockPath, 'wx', 0o600); + writeFileSync(fd, `${process.pid}\n`, 'utf8'); + closeSync(fd); + const release = () => { + try { + const ownerPid = Number( + String(readFileSync(lockPath, 'utf8')).trim(), + ); + if (ownerPid === process.pid) { + rmSync(lockPath, { force: true }); + } + } catch { + // The lock may already have been removed by the normal exit path. } - } catch { - // The lock may already have been removed by the normal exit path. + }; + process.once('exit', release); + for (const signal of ['SIGINT', 'SIGTERM']) { + process.once(signal, () => { + release(); + process.exit(signal === 'SIGINT' ? 130 : 143); + }); + } + return lockPath; + } catch (error) { + if (error?.code !== 'EEXIST') { + throw error; } - }; - process.once('exit', release); - for (const signal of ['SIGINT', 'SIGTERM']) { - process.once(signal, () => { - release(); - process.exit(signal === 'SIGINT' ? 130 : 143); - }); } - return lockPath; - } catch (error) { - if (error?.code !== 'EEXIST') { + let ownerPid = 0; + try { + ownerPid = Number(String(readFileSync(lockPath, 'utf8')).trim()); + } catch (error) { + if (error?.code === 'ENOENT') { + continue; + } throw error; } - } - const ownerPid = Number(String(readFileSync(lockPath, 'utf8')).trim()); - if ( - Number.isSafeInteger(ownerPid) && - ownerPid > 0 && - processIsAlive(ownerPid) - ) { - throw new Error(`已有数据库备份进程持有锁: ${lockPath} pid=${ownerPid}`); + if ( + Number.isSafeInteger(ownerPid) && + ownerPid > 0 && + processIsAlive(ownerPid) + ) { + throw new Error(`已有数据库备份进程持有锁: ${lockPath} pid=${ownerPid}`); + } + try { + rmSync(lockPath); + console.log( + `[database-backup] 已清理失效数据库备份锁，准备重新获取: ${lockPath} pid=${ownerPid || ''}`, + ); + } catch (error) { + if (error?.code !== 'ENOENT') { + throw error; + } + } } throw new Error( - `发现失效数据库备份锁，拒绝自动抢锁；请核对 OSS multipart 与进程后手工删除: ${lockPath} pid=${ownerPid || ''}`, + `数据库备份锁在清理后仍无法获取，可能存在并发进程: ${lockPath}`, ); }