From 9f15c1c572aceca1b9fe7ee00319b099bd651e3b Mon Sep 17 00:00:00 2001 From: suzmii Date: Mon, 21 Sep 2026 10:35:43 +0800 Subject: [PATCH 01/14] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=9D=9EWindows?= =?UTF-8?q?=E4=B8=BB=E6=9C=BA=E4=B8=8AAGC=E5=8F=91=E5=B8=83=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E6=9C=AAstub=20Node=E8=BF=90=E8=A1=8C=E6=97=B6staging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build-release.test.mjs的渠道注入用例漏注入stageRuntime，真实staging会用宿主平台与默认Windows目标做一致性校验 该缺陷在macOS主机上必然失败，卡住macOS渠道构建的测试阶段 补上stageRuntime stub，本机（darwin/arm64）验证37项全通过 --- apps/ai-game-creator-shell/scripts/build-release.test.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/ai-game-creator-shell/scripts/build-release.test.mjs b/apps/ai-game-creator-shell/scripts/build-release.test.mjs index 732302eb1..59e1a878b 100644 --- a/apps/ai-game-creator-shell/scripts/build-release.test.mjs +++ b/apps/ai-game-creator-shell/scripts/build-release.test.mjs @@ -211,6 +211,9 @@ test('packaged renderer receives the same channel as the updater manifest', () = }); let spawnOptions; runTauriBuild([], context, { + // 必须 stub：真实 staging 会用宿主平台（如 macOS 的 darwin/arm64）去对默认的 + // Windows 目标做一致性校验，在非 Windows 主机上直接失败——本用例只关心渠道注入。 + stageRuntime: () => {}, spawn: (_binary, _args, options) => { spawnOptions = options; return { status: 0 }; From 117d482f93e9bd4f48491cdc0cf94792ccad71c2 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 10:56:25 +0800 Subject: [PATCH 02/14] =?UTF-8?q?=E5=8F=91=E5=B8=83=E5=89=8D=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E7=A9=BA=E9=97=B4=E9=A2=84=E6=A3=80=E5=89=8D=E7=BD=AE?= =?UTF-8?q?=E5=B9=B6=E6=94=AF=E6=8C=81=E8=87=AA=E5=8A=A8=E9=99=8D=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - database-backup-to-oss.mjs 新增 --check-space-only：archive 按 data×1.1、files 按 max(data×0.05, 2GiB) 计算门槛，空间不足以退出码 3 返回 - production-stdb-publish.sh 在进入维护与停服务之前先做空间预检，archive 不足时自动降级 files（files 不支持 defer，async 收敛为 sync），可用环境变量关闭 - 尚未开始 publish 的失败自动恢复本次停掉的 API/controller/worker 并退出维护，只有真正开始发布之后的失败才保持维护态 - 备份检查脚本与生产运维门禁补上新口径与执行顺序断言，运维文档与共享记忆记录事故与规则 --- docs/project-memory/shared-memory/pitfalls.md | 4 + ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- scripts/check-database-backup-to-oss.mjs | 67 ++++++- scripts/check-production-ops-guardrails.mjs | 41 ++++- scripts/database-backup-to-oss.mjs | 98 ++++++++-- scripts/deploy/production-stdb-publish.sh | 170 ++++++++++++++++-- 6 files changed, 353 insertions(+), 29 deletions(-) diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 51ed5f002..a82811675 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -1,5 +1,9 @@ # 踩坑与排障记录 +## release 冷备空间不足会把生产留在维护态 + +`Genarrative-Stdb-Module-Publish` 先进入维护模式、停掉 API/controller/worker，再执行发布前冷备份；archive 口径要求 `data × 1.1`（40.6GiB 数据 → 44.7GiB，加上生产根盘只剩 13.5GiB），于是 2026-09-21 的 release 发布在停服后失败并保持维护态，站点 503 直到人工恢复。规则：空间预检必须先于 `maintenance-on` 与停服；archive 不足且未显式禁用降级时改用 files（`max(data × 0.05, 2GiB)`，不落地本地归档，但 files 不支持 `--defer-upload`，会从 async 收敛为 sync）；只有真正开始 `spacetime publish` 之后的失败才允许保持维护态。另：定时备份的失效锁在当前仓库版本会自动清理，但生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若是旧版会拒绝抢锁并要求人工删锁，需随 provision 更新。 + ## copyArtifacts 报「Unable to find project for artifact copy」的用户触发构建差异 Copy Artifact 插件在**非 SYSTEM 认证**下按「认证用户」判权：只有当被复制 Job 的 `CopyArtifactPermissionProperty`（仓库里由 Declarative 的 `copyArtifactPermission(...)` 维护）显式列出当前消费者，或者该 Job 对认证用户开放 Item.Read 时才放行；`ACL.SYSTEM2` 的定时构建会短路通过。因此会出现「定时调度一路成功、手动发布必挂」的现象（2026-09-21 手动发布 #6/#7 与同期的用户触发探测全部命中，定时调度 #104+ 正常）。`Genarrative-Agc-Global-Version-Issue` 生产权限模式的授权名单必须同时包含 `Genarrative-Scheduled-Revision-Trigger` 与 `Genarrative-Manual-Build-And-Deploy`；改完 `copyArtifactPermission` 后要先跑一次发号 Job 把 Job property 写回 Jenkins，只改仓库文件不生效。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 0596085fc..0dca6f394 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -449,7 +449,7 @@ UI 相关修改要重点验证： npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service ``` -脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 +脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 `2026-09-21` 又命中一次同类事故：release 冷备按 `archive` 口径要求 `data × 1.1`（40.6GiB → 44.7GiB），生产根盘只剩 13.5GiB，备份在停服后才失败，生产被留在维护态。现在 `production-stdb-publish.sh` 在进入维护模式与停服务之前先执行 `database-backup-to-oss.mjs --check-space-only`：archive 口径为 `max(data × 1.1, data + 512MiB)`，files 口径为 `max(data × 0.05, 2GiB)`，空间不足以退出码 `3` 返回。archive 空间不足且未显式设置 `GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=0` 时，脚本自动降级为 `--storage-format files`（不落地本地归档；files 不支持 `--defer-upload`，因此 `DATABASE_BACKUP_MODE=async` 会收敛为 `sync`，窗口内直传 OSS）；也可用 `GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=files` 直接指定。任何「尚未开始 `spacetime publish`」的失败（空间预检、备份执行等）都会自动把本次停掉的 API/controller/worker 拉回并退出维护，只有真正开始发布之后的失败才保持维护态等待人工；该自动恢复可用 `GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=0` 关闭。定时备份的失效锁：当前仓库版本在持有者进程退出后会自动清理并重试，而生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若仍是旧版（2026-07-16 前）会拒绝抢锁并需要人工删锁，必须随 provision 一起更新。 生产环境变量模板在 `deploy/env/api-server.env.example`： diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index 44060f394..f097b8e37 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -26,6 +26,7 @@ import { cleanupHistoryCandidates, collectDirectFileEntries, createUploadBandwidthLimiter, + describeBackupSpaceRequirement, discoverDeferredArchiveUploads, discoverHistoryPlan, restoreDirectFilesBackup, @@ -64,6 +65,7 @@ async function main() { assertDeferredArchiveDiscoveryIsBoundedAndDeterministic(); assertCanonicalQueryAndAuthorizationIncludeMultipartParameters(); assertInsufficientSpaceStopsBeforeServiceChanges(); + assertCheckSpaceOnlyUsesFormatSpecificRequirement(); assertStopFailureRetainsRecoveryMarker(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); @@ -1198,7 +1200,7 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() { '999999999999999999', ]); - assertStatus(result, 1, '空间不足时必须失败。'); + assertStatus(result, 3, '空间不足必须用独立退出码 3 失败。'); assertIncludes( result.stdout, '备份空间预检', @@ -1213,6 +1215,66 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() { assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。'); } +function assertCheckSpaceOnlyUsesFormatSpecificRequirement() { + const fixture = createFixture('check-space-only'); + + const filesOk = runBackup( + fixture, + ['--check-space-only', '--storage-format', 'files'], + { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1M' }, + ); + assertStatus(filesOk, 0, 'files 模式空间预检应在阈值满足时通过。'); + assertIncludes( + filesOk.stdout, + '备份空间预检(files)', + 'files 预检必须打印 files 口径的预检结果。', + ); + assertIncludes( + filesOk.stdout, + '空间预检通过(check-space-only)', + 'check-space-only 通过时必须给出显式成功标记。', + ); + assertFileMissing(fixture.systemctlLog, '空间预检不得调用 systemctl。'); + assertFileMissing(fixture.tarLog, '空间预检不得调用 tar。'); + + const filesInsufficient = runBackup( + fixture, + ['--check-space-only', '--storage-format', 'files'], + { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1000T' }, + ); + assertStatus(filesInsufficient, 3, 'files 空间不足同样使用退出码 3。'); + + const archiveInsufficient = runBackup( + fixture, + ['--check-space-only', '--storage-format', 'archive'], + { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1000T' }, + ); + assertStatus(archiveInsufficient, 3, 'archive 空间不足必须使用退出码 3。'); + + const archive = describeBackupSpaceRequirement({ + dataDir: fixture.dataDir, + workDir: fixture.workDir, + storageFormat: 'archive', + args: {}, + env: { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1G' }, + }); + const files = describeBackupSpaceRequirement({ + dataDir: fixture.dataDir, + workDir: fixture.workDir, + storageFormat: 'files', + args: {}, + env: { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1G' }, + }); + if (archive.requiredFreeBytes !== 1024n ** 3n || files.requiredFreeBytes !== 1024n ** 3n) { + failures.push( + `空间口径覆盖参数应生效：archive=${archive.requiredFreeBytes} files=${files.requiredFreeBytes}`, + ); + } + if (archive.storageFormat !== 'archive' || files.storageFormat !== 'files') { + failures.push('空间预检结果必须回显实际 storage-format。'); + } +} + function assertStopFailureRetainsRecoveryMarker() { const fixture = createFixture('stop-failure-marker'); writeExecutable( @@ -2695,7 +2757,7 @@ exit 2 return { root, binDir, dataDir, workDir, systemctlLog, tarLog }; } -function runBackup(fixture, extraArgs = []) { +function runBackup(fixture, extraArgs = [], envOverrides = {}) { return spawnSync( process.execPath, [ @@ -2720,6 +2782,7 @@ function runBackup(fixture, extraArgs = []) { env: { ...process.env, PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`, + ...envOverrides, }, }, ); diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 323a17e6a..bfb4bdbe9 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -258,6 +258,44 @@ const checks = [ reason: 'Stdb 先于 API 发布时必须先补齐 api-server env 的 FILE 路径，保证首次 rollout 重启即可读取 secret。', }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: + 'precheck_backup_space_before_maintenance\n\n"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"', + reason: + '备份空间预检必须先于进入维护模式执行：磁盘不足时不得停服务或把生产留在维护态。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: '--check-space-only', + reason: + '生产 Stdb publish 必须复用备份脚本的 --check-space-only 预检，保证空间口径与真实备份一致。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: 'GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK', + reason: + 'archive 冷备份空间不足时必须能自动降级为 files 存储格式（不落地本地归档）。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: 'restore_runtime_services_before_publish', + reason: + '尚未开始 publish 的失败必须自动恢复运行时服务并退出维护，避免生产停在维护态等人工。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: + 'if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then', + reason: + '只有尚未开始 publish 的失败才允许自动恢复并退出维护，半发布状态必须保持维护。', + }, + { + file: 'scripts/database-backup-to-oss.mjs', + includes: 'SPACE_INSUFFICIENT_EXIT_CODE', + reason: + '备份空间不足必须使用独立退出码，调用方据此决定降级存储格式而不是一律失败。', + }, { file: 'scripts/deploy/production-stdb-publish.sh', includes: 'stop_runtime_services_for_rollout_gate', @@ -984,7 +1022,8 @@ const checks = [ }, { file: 'scripts/database-backup-to-oss.mjs', - includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})', + includes: + "assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' })", normalizeWhitespace: true, reason: '生产冷备份必须先做工作目录剩余空间预检，避免停库后写满磁盘。', }, diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index bc215509f..2202017df 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -55,6 +55,11 @@ const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join( ); const DEFAULT_SPACE_SAFETY_RATIO = 1.1; const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024; +// files 模式不落地本地归档（只写 catalog/state），所需空间远小于数据目录本身。 +const DEFAULT_FILES_SPACE_SAFETY_RATIO = 0.05; +const DEFAULT_FILES_EXTRA_FREE_BYTES = 2 * 1024 * 1024 * 1024; +// 空间不足用独立退出码，调用方可据此决定是否降级存储格式。 +const SPACE_INSUFFICIENT_EXIT_CODE = 3; const OSS_ALGORITHM = 'OSS4-HMAC-SHA256'; const OSS_SERVICE = 'oss'; const OSS_REQUEST = 'aliyun_v4_request'; @@ -80,6 +85,7 @@ const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1; function usage() { console.log(`用法: npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] + npm run database:backup:oss -- --check-space-only [--storage-format archive|files] [--data-dir ] [--work-dir ] node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] node -- scripts/database-backup-to-oss.mjs --upload-archive node -- scripts/database-backup-to-oss.mjs --upload-deferred-dir @@ -106,6 +112,8 @@ function usage() { GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT archive（默认）或 files GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES 备份前要求 work dir 所在文件系统至少有这些可用字节；未设置时按数据目录大小估算 + GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES files 模式的最小可用字节（默认 2GiB） + GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO files 模式按数据目录比例要求的可用空间（默认 0.05） GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE history 使用的 full baseline 与追加批次状态文件 GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST 首次初始化 history state 的 uploaded full manifest ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET @@ -175,6 +183,7 @@ function parseArgs(argv) { restartServicesAfter: [], database: '', dryRun: false, + checkSpaceOnly: false, deferUpload: false, uploadArchive: '', uploadDeferredDir: '', @@ -282,6 +291,9 @@ function parseArgs(argv) { case '--publish-manifest': options.publishManifest = readValue(); break; + case '--check-space-only': + options.checkSpaceOnly = true; + break; case '--storage-format': options.storageFormat = readValue(); break; @@ -1013,6 +1025,20 @@ function parseSafetyRatio(rawValue) { return ratio; } +function parseFilesSpaceSafetyRatio(rawValue) { + const value = String(rawValue ?? '').trim(); + if (!value) { + return DEFAULT_FILES_SPACE_SAFETY_RATIO; + } + const ratio = Number(value); + if (!Number.isFinite(ratio) || ratio <= 0) { + throw new Error( + `GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO 必须是 > 0 的数字，实际: ${rawValue}`, + ); + } + return ratio; +} + function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) { const explicitMinFreeBytes = parseByteSize( firstNonEmpty( @@ -1042,22 +1068,57 @@ function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) { : extraRequirement; } -function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) { +function calculateFilesRequiredFreeBytes({ dataSizeBytes, env }) { + const explicitMinFreeBytes = parseByteSize( + env.GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES, + 'GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES', + ); + if (explicitMinFreeBytes !== null) { + return explicitMinFreeBytes; + } + const ratio = parseFilesSpaceSafetyRatio( + env.GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO, + ); + const ratioBasisPoints = BigInt(Math.ceil(ratio * 10000)); + const ratioRequirement = (dataSizeBytes * ratioBasisPoints + 9999n) / 10000n; + const floorRequirement = BigInt(DEFAULT_FILES_EXTRA_FREE_BYTES); + return ratioRequirement > floorRequirement ? ratioRequirement : floorRequirement; +} + +export function describeBackupSpaceRequirement({ + dataDir, + workDir, + storageFormat = 'archive', + args = {}, + env = {}, +}) { mkdirSync(workDir, { recursive: true }); const dataSizeBytes = getDirectorySizeBytes(dataDir); const availableBytes = getAvailableBytes(workDir); - const requiredFreeBytes = calculateRequiredFreeBytes({ - dataSizeBytes, - args, - env, - }); + const requiredFreeBytes = + storageFormat === 'files' + ? calculateFilesRequiredFreeBytes({ dataSizeBytes, env }) + : calculateRequiredFreeBytes({ dataSizeBytes, args, env }); + return { storageFormat, dataSizeBytes, availableBytes, requiredFreeBytes }; +} + +function assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat = 'archive' }) { + const { dataSizeBytes, availableBytes, requiredFreeBytes } = + describeBackupSpaceRequirement({ + dataDir, + workDir, + storageFormat, + args, + env, + }); console.log( - `[database-backup] 备份空间预检: data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, + `[database-backup] 备份空间预检(${storageFormat}): data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, ); if (availableBytes < requiredFreeBytes) { - throw new Error( + const error = new Error( [ `备份工作目录所在文件系统剩余空间不足: ${workDir}`, + `storage-format=${storageFormat}`, `available=${formatBytes(availableBytes)}`, `required=${formatBytes(requiredFreeBytes)}`, `dataDir=${dataDir}`, @@ -1065,7 +1126,10 @@ function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) { '为避免停库后写满磁盘，本次备份已在停止服务前中止。', ].join('；'), ); + error.spaceInsufficient = true; + throw error; } + return { dataSizeBytes, availableBytes, requiredFreeBytes }; } function assertSufficientHistoryWorkDirSpace({ @@ -4154,6 +4218,20 @@ async function main() { ); } + if (args.checkSpaceOnly) { + assertSufficientWorkDirSpace({ + dataDir, + workDir, + args, + env, + storageFormat, + }); + console.log( + `[database-backup] 空间预检通过(check-space-only): storage-format=${storageFormat}, workDir=${workDir}`, + ); + return; + } + for (const [label, value] of Object.entries({ bucket, endpoint, @@ -4369,7 +4447,7 @@ async function main() { const restartServicesAfter = collectRestartServicesAfterBackup({ args, env }); const stopMarkerPath = databaseBackupStopMarkerPath(workDir); try { - assertSufficientWorkDirSpace({ dataDir, workDir, args, env }); + assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' }); serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); archivePath = createArchive({ dataDir, workDir, fileName }); } catch (error) { @@ -4558,6 +4636,6 @@ if ( } catch { // 参数解析或状态文件写入失败时，不能掩盖原始备份错误。 } - process.exit(1); + process.exit(error?.spaceInsufficient ? SPACE_INSUFFICIENT_EXIT_CODE : 1); }); } diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index e96210ecb..b44eccf9c 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -18,9 +18,16 @@ usage() { 如需强制等待备份完成并在失败时阻断 publish，传入 --backup-mode sync。 发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径，再重启并验活重启前 active 的服务。 --keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker，并在成功后保持维护态，交由后续 API deploy 恢复服务。 + 发布前先做备份空间预检（不进入维护、不停服务）；archive 空间不足且未显式关闭自动降级时， + 自动改用 files 存储格式（不落地本地归档，改为同步直传 OSS），避免磁盘不足把生产留在维护态。 + +环境变量： + GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files（默认 archive） + GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0（默认 1：archive 空间不足自动降级 files） + GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0（默认 1：尚未开始 publish 的失败自动恢复服务并退出维护） migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。 如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。 - 失败时保留维护模式。 + 失败时：尚未开始 publish 的失败会自动恢复运行时服务并退出维护；真正开始 publish 之后的失败保留维护模式。 EOF } @@ -55,7 +62,13 @@ API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-serv WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}" KEEP_MAINTENANCE_MODE=0 BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}" +BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}" +AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}" +AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}" DEPLOY_COMPLETED=0 +PUBLISH_STARTED=0 +MAINTENANCE_ENTERED=0 +STOPPED_RUNTIME_SERVICES=() PUBLISH_TMP_DIR="" ASYNC_BACKUP_STATUS_FILE="" ASYNC_BACKUP_SCRIPT="" @@ -287,6 +300,106 @@ restart_runtime_services_after_bootstrap_secret_install() { fi } +backup_script_path() { + local candidate="" + for candidate in \ + "${SCRIPT_DIR}/../database-backup-to-oss.mjs" \ + "${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"; do + if [[ -f "${candidate}" ]]; then + printf '%s\n' "${candidate}" + return 0 + fi + done + return 1 +} + +run_backup_space_precheck() { + local storage_format="$1" + local backup_script="" + if ! backup_script="$(backup_script_path)"; then + echo "[production-stdb-publish] 缺少数据库备份脚本，无法做备份空间预检" >&2 + return 1 + fi + node -- "${backup_script}" \ + --env-file /etc/genarrative/api-server.env \ + --data-dir "${SPACETIME_ROOT_DIR}" \ + --database "${DATABASE}" \ + --storage-format "${storage_format}" \ + --check-space-only +} + +# 空间预检必须发生在进入维护模式与停服务之前：磁盘不够时不允许再动生产。 +precheck_backup_space_before_maintenance() { + if [[ "${BACKUP_MODE}" == "skip" ]]; then + echo "[production-stdb-publish] 已跳过发布前备份空间预检（--backup-mode skip）" + return 0 + fi + + local status=0 + run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$? + if [[ "${status}" -eq 0 ]]; then + echo "[production-stdb-publish] 发布前备份空间预检通过: storage-format=${BACKUP_STORAGE_FORMAT}（尚未进入维护模式、未停服务）" + return 0 + fi + if [[ "${status}" -ne 3 ]]; then + echo "[production-stdb-publish] 发布前备份空间预检失败（非空间原因），中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + if [[ "${BACKUP_STORAGE_FORMAT}" == "files" ]]; then + echo "[production-stdb-publish] files 模式备份空间仍不足，中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + if [[ "${AUTO_FILES_FALLBACK}" != "1" ]]; then + echo "[production-stdb-publish] archive 备份空间不足且已禁用自动降级（GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=${AUTO_FILES_FALLBACK}），中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + + echo "[production-stdb-publish] archive 冷备份空间不足：自动降级为 files 存储格式（不落地本地归档，改为文件级 catalog 直传 OSS）。" >&2 + BACKUP_STORAGE_FORMAT="files" + if [[ "${BACKUP_MODE}" == "async" ]]; then + echo "[production-stdb-publish] files 模式不支持 --defer-upload，本次备份改为同步执行。" >&2 + BACKUP_MODE="sync" + fi + status=0 + run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$? + if [[ "${status}" -ne 0 ]]; then + echo "[production-stdb-publish] 降级为 files 后空间预检仍失败，中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + echo "[production-stdb-publish] 已降级为 files 存储格式且空间预检通过。" +} + +# 仅在「尚未开始 publish」的失败路径调用：把本次停掉的运行时服务拉回来。 +restore_runtime_services_before_publish() { + if [[ "${#STOPPED_RUNTIME_SERVICES[@]}" -eq 0 ]]; then + return 0 + fi + local service="" + local state="" + local attempt=0 + echo "[production-stdb-publish] 发布尚未开始，恢复本次停掉的运行时服务: ${STOPPED_RUNTIME_SERVICES[*]}" + if ! run_privileged systemctl start "${STOPPED_RUNTIME_SERVICES[@]}"; then + echo "[production-stdb-publish] 启动运行时服务失败: ${STOPPED_RUNTIME_SERVICES[*]}" >&2 + return 1 + fi + for service in "${STOPPED_RUNTIME_SERVICES[@]}"; do + state="" + for attempt in $(seq 1 15); do + state="$(get_runtime_service_active_state "${service}" 2>/dev/null || true)" + if [[ "${state}" == "active" ]]; then + break + fi + sleep 1 + done + if [[ "${state}" != "active" ]]; then + echo "[production-stdb-publish] 运行时服务未恢复 active: ${service}, state=${state}" >&2 + return 1 + fi + echo "[production-stdb-publish] 运行时服务已恢复 active: ${service}" + done + return 0 +} + stop_runtime_services_for_rollout_gate() { local api_state="" local controller_state="" @@ -326,6 +439,7 @@ stop_runtime_services_for_rollout_gate() { fi echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}" + STOPPED_RUNTIME_SERVICES=("${services_to_stop[@]}") run_privileged systemctl stop "${services_to_stop[@]}" for worker_service in "${services_to_stop[@]}"; do if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then @@ -402,6 +516,10 @@ while [[ $# -gt 0 ]]; do BACKUP_MODE="${2:?缺少 --backup-mode 的值}" shift 2 ;; + --backup-storage-format) + BACKUP_STORAGE_FORMAT="${2:?缺少 --backup-storage-format 的值}" + shift 2 + ;; *) echo "[production-stdb-publish] 未知参数: $1" >&2 usage >&2 @@ -431,6 +549,14 @@ for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do fi done +if [[ ! "${BACKUP_STORAGE_FORMAT}" =~ ^(archive|files)$ ]]; then + echo "[production-stdb-publish] --backup-storage-format 只能是 archive 或 files: ${BACKUP_STORAGE_FORMAT}" >&2 + exit 1 +fi +if [[ "${BACKUP_STORAGE_FORMAT}" == "files" && "${BACKUP_MODE}" == "async" ]]; then + echo "[production-stdb-publish] files 存储格式不支持 --defer-upload，备份模式由 async 调整为 sync" >&2 + BACKUP_MODE="sync" +fi if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2 exit 1 @@ -497,7 +623,22 @@ on_exit() { rm -rf "${PUBLISH_TMP_DIR}" fi if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then - echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2 + if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then + # 尚未开始 publish 就失败（例如备份空间/备份执行失败）：本次没有任何发布变更， + # 必须把停掉的运行时服务拉回来并退出维护，避免生产停在维护态等人工救。 + if restore_runtime_services_before_publish; then + if [[ "${MAINTENANCE_ENTERED}" -eq 1 ]]; then + if ! "${SCRIPT_DIR}/maintenance-off.sh"; then + echo "[production-stdb-publish] 自动退出维护模式失败，请手工执行 maintenance-off.sh。" >&2 + fi + fi + echo "[production-stdb-publish] 发布尚未开始即失败，已自动恢复运行时服务并退出维护模式。" + else + echo "[production-stdb-publish] 自动恢复运行时服务失败，保持维护模式，请手工处理。" >&2 + fi + else + echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2 + fi fi exit "${exit_code}" } @@ -506,12 +647,8 @@ trap on_exit EXIT prepare_async_backup() { local -a restart_service_args=() - ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs" - if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then - ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" - fi - if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then - echo "[production-stdb-publish] 缺少数据库备份脚本: ${ASYNC_BACKUP_SCRIPT}" >&2 + if ! ASYNC_BACKUP_SCRIPT="$(backup_script_path)"; then + echo "[production-stdb-publish] 缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 exit 1 fi @@ -527,6 +664,7 @@ prepare_async_backup() { --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ + --storage-format "${BACKUP_STORAGE_FORMAT}" \ --stop-service spacetimedb.service \ "${restart_service_args[@]}" \ --defer-upload \ @@ -676,7 +814,10 @@ wait_for_api_healthz_ready() { return 1 } +precheck_backup_space_before_maintenance + "${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}" +MAINTENANCE_ENTERED=1 if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then stop_runtime_services_for_rollout_gate fi @@ -687,23 +828,20 @@ case "${BACKUP_MODE}" in ;; sync) SYNC_BACKUP_RESTART_SERVICE_ARGS=() - BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs" - if [[ ! -f "${BACKUP_SCRIPT}" ]]; then - BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" - fi - if [[ ! -f "${BACKUP_SCRIPT}" ]]; then - echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2 + if ! BACKUP_SCRIPT="$(backup_script_path)"; then + echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 exit 1 fi if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service) fi - echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布" + echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" node -- "${BACKUP_SCRIPT}" \ --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ + --storage-format "${BACKUP_STORAGE_FORMAT}" \ --stop-service spacetimedb.service \ "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}" ;; @@ -768,8 +906,10 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then else PUBLISH_ARGS+=(--server "${SERVER_ALIAS}") fi + PUBLISH_STARTED=1 runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}" else + PUBLISH_STARTED=1 spacetime "${PUBLISH_ARGS[@]}" fi From 2f248ea0c330d5b6c29ecab75d2d31ec99d75014 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 11:33:39 +0800 Subject: [PATCH 03/14] =?UTF-8?q?=E6=96=B0=E5=A2=9E=20minimal=20=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=8F=A3=E5=BE=84=E5=B9=B6=E5=88=87=E6=8D=A2=20releas?= =?UTF-8?q?e=20=E5=AE=9A=E6=97=B6=E5=A4=87=E4=BB=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - database-backup-to-oss.mjs 新增 --minimal/--retain-snapshots：按上游 retention 语义只备份最近 N 份 snapshot、覆盖最老保留 snapshot 的 commitlog 边界段及其后全部段，外加身份/配置/控制库/模块字节，且不停止任何服务 - 备份检查脚本补充 minimal 计划用例（保留次新+最新 snapshot、丢弃更早 snapshot 与 clog 段、状态路径齐全、保留数校验） - 发布前备份默认走 minimal 热备（GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=0 可回到冷备），不再需要 44.7GiB 冷备空间 - Server-Provision 新增 DATABASE_BACKUP_PROFILE=files-minimal 与 systemd drop-in，release 推荐使用并继续禁止 files-history - 运维文档与共享记忆记录上游 retention 依据、实测 40G→3.6G 与不停服热备口径 --- ...rrative-database-backup-files-minimal.conf | 5 + docs/project-memory/shared-memory/pitfalls.md | 2 + ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- .../Jenkinsfile.production-server-provision | 8 +- scripts/check-database-backup-to-oss.mjs | 102 +++++++ scripts/check-production-ops-guardrails.mjs | 24 +- scripts/database-backup-to-oss.mjs | 256 +++++++++++++++++- scripts/deploy/production-stdb-publish.sh | 35 ++- scripts/jenkins-server-provision.sh | 41 ++- 9 files changed, 452 insertions(+), 23 deletions(-) create mode 100644 deploy/systemd/genarrative-database-backup-files-minimal.conf diff --git a/deploy/systemd/genarrative-database-backup-files-minimal.conf b/deploy/systemd/genarrative-database-backup-files-minimal.conf new file mode 100644 index 000000000..2124eb345 --- /dev/null +++ b/deploy/systemd/genarrative-database-backup-files-minimal.conf @@ -0,0 +1,5 @@ +[Service] +Environment=GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=4 +Environment=GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND=20971520 +ExecStart= +ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode full --minimal --retain-snapshots 2 --data-dir /stdb --work-dir /var/lib/genarrative/database-backups/files-minimal diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index a82811675..5912a73b2 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -4,6 +4,8 @@ `Genarrative-Stdb-Module-Publish` 先进入维护模式、停掉 API/controller/worker，再执行发布前冷备份；archive 口径要求 `data × 1.1`（40.6GiB 数据 → 44.7GiB，加上生产根盘只剩 13.5GiB），于是 2026-09-21 的 release 发布在停服后失败并保持维护态，站点 503 直到人工恢复。规则：空间预检必须先于 `maintenance-on` 与停服；archive 不足且未显式禁用降级时改用 files（`max(data × 0.05, 2GiB)`，不落地本地归档，但 files 不支持 `--defer-upload`，会从 async 收敛为 sync）；只有真正开始 `spacetime publish` 之后的失败才允许保持维护态。另：定时备份的失效锁在当前仓库版本会自动清理，但生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若是旧版会拒绝抢锁并要求人工删锁，需随 provision 更新。 +按上游 [#5555](https://github.com/clockworklabs/SpacetimeDB/pull/5555) 的 retention 语义，只有最近 `retain-snapshots`（默认 2）份 snapshot 与覆盖它之后的 commitlog 段是重启所需，其余历史可丢；因此备份改为 `files + full + --minimal --retain-snapshots 2`（release 实测 40G → 3.6G，热备不停服），不再做增量差异计算，也不需要 44.7G 冷备空间。 + ## copyArtifacts 报「Unable to find project for artifact copy」的用户触发构建差异 Copy Artifact 插件在**非 SYSTEM 认证**下按「认证用户」判权：只有当被复制 Job 的 `CopyArtifactPermissionProperty`（仓库里由 Declarative 的 `copyArtifactPermission(...)` 维护）显式列出当前消费者，或者该 Job 对认证用户开放 Item.Read 时才放行；`ACL.SYSTEM2` 的定时构建会短路通过。因此会出现「定时调度一路成功、手动发布必挂」的现象（2026-09-21 手动发布 #6/#7 与同期的用户触发探测全部命中，定时调度 #104+ 正常）。`Genarrative-Agc-Global-Version-Issue` 生产权限模式的授权名单必须同时包含 `Genarrative-Scheduled-Revision-Trigger` 与 `Genarrative-Manual-Build-And-Deploy`；改完 `copyArtifactPermission` 后要先跑一次发号 Job 把 Job property 写回 Jenkins，只改仓库文件不生效。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 0dca6f394..801261b85 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -449,7 +449,7 @@ UI 相关修改要重点验证： npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service ``` -脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 `2026-09-21` 又命中一次同类事故：release 冷备按 `archive` 口径要求 `data × 1.1`（40.6GiB → 44.7GiB），生产根盘只剩 13.5GiB，备份在停服后才失败，生产被留在维护态。现在 `production-stdb-publish.sh` 在进入维护模式与停服务之前先执行 `database-backup-to-oss.mjs --check-space-only`：archive 口径为 `max(data × 1.1, data + 512MiB)`，files 口径为 `max(data × 0.05, 2GiB)`，空间不足以退出码 `3` 返回。archive 空间不足且未显式设置 `GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=0` 时，脚本自动降级为 `--storage-format files`（不落地本地归档；files 不支持 `--defer-upload`，因此 `DATABASE_BACKUP_MODE=async` 会收敛为 `sync`，窗口内直传 OSS）；也可用 `GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=files` 直接指定。任何「尚未开始 `spacetime publish`」的失败（空间预检、备份执行等）都会自动把本次停掉的 API/controller/worker 拉回并退出维护，只有真正开始发布之后的失败才保持维护态等待人工；该自动恢复可用 `GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=0` 关闭。定时备份的失效锁：当前仓库版本在持有者进程退出后会自动清理并重试，而生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若仍是旧版（2026-07-16 前）会拒绝抢锁并需要人工删锁，必须随 provision 一起更新。 +脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 `2026-09-21` 又命中一次同类事故：release 冷备按 `archive` 口径要求 `data × 1.1`（40.6GiB → 44.7GiB），生产根盘只剩 13.5GiB，备份在停服后才失败，生产被留在维护态。现在 `production-stdb-publish.sh` 在进入维护模式与停服务之前先执行 `database-backup-to-oss.mjs --check-space-only`：archive 口径为 `max(data × 1.1, data + 512MiB)`，files 口径为 `max(data × 0.05, 2GiB)`，空间不足以退出码 `3` 返回。archive 空间不足且未显式设置 `GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=0` 时，脚本自动降级为 `--storage-format files`（不落地本地归档；files 不支持 `--defer-upload`，因此 `DATABASE_BACKUP_MODE=async` 会收敛为 `sync`，窗口内直传 OSS）；也可用 `GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=files` 直接指定。任何「尚未开始 `spacetime publish`」的失败（空间预检、备份执行等）都会自动把本次停掉的 API/controller/worker 拉回并退出维护，只有真正开始发布之后的失败才保持维护态等待人工；该自动恢复可用 `GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=0` 关闭。定时备份的失效锁：当前仓库版本在持有者进程退出后会自动清理并重试，而生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若仍是旧版（2026-07-16 前）会拒绝抢锁并需要人工删锁，必须随 provision 一起更新。 上游确认（[#5542](https://github.com/clockworklabs/SpacetimeDB/issues/5542) / [#5555](https://github.com/clockworklabs/SpacetimeDB/pull/5555)）只保留最近 `retain-snapshots`（默认 2）份 snapshot 与覆盖它之后的 commitlog 段就足以重启数据库，更早的 snapshot/commitlog 属于可丢弃历史；因此备份不再需要"保留每一版 + 计算增量"，改为 `database-backup-to-oss.mjs --storage-format files --mode full --minimal --retain-snapshots 2`：每个 replica 取最近 2 份 snapshot、覆盖最老保留 snapshot 的 commitlog 边界段及其后全部段，外加身份/`data/config.toml`/`data/metadata.toml`/`data/control-db`/`data/program-bytes`，其余历史不备份。实测 release `replicas/2` 为 55 个 snapshot(18G)+108 个 clog 文件(22G)，minimal 口径只需约 3.6GiB（16 个候选），因此定时任务与发布前备份都可以在**不停服**的前提下完成，也不再需要 44.7GiB 的冷备空间；Server-Provision 新增 `DATABASE_BACKUP_PROFILE=files-minimal`（release 推荐，`files-history` 继续禁止），发布前备份默认走 minimal（`GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=0` 可回到冷备）。本地磁盘上的历史数据仍由 SpacetimeDB 自己保留（PR #5555 尚未合并），升级到含该特性的版本后可配置 `[retention] policy = "delete"` 回收。 生产环境变量模板在 `deploy/env/api-server.env.example`： diff --git a/jenkins/Jenkinsfile.production-server-provision b/jenkins/Jenkinsfile.production-server-provision index d2104e68c..28a940332 100644 --- a/jenkins/Jenkinsfile.production-server-provision +++ b/jenkins/Jenkinsfile.production-server-provision @@ -34,7 +34,7 @@ pipeline { string(name: 'WEB_LINK', defaultValue: '/srv/genarrative/web', description: 'Nginx 静态站点目录或软链接') string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'api-server 环境文件') string(name: 'API_PORT', defaultValue: '8082', description: 'api-server 本机监听端口') - choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profile；release 仅允许 archive-full，files-history 仅供 development 在指定 work-dir 已有完整 full baseline 后启用') + choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history'], description: '数据库定时备份 profile；release 用 files-minimal（最近 snapshot + 其后 commitlog 的热备）或 archive-full，files-history 因内存峰值仅 development 使用') string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'development files-history 的本地 state/catalog 目录；必须使用已建立 full baseline 的独立目录') choice(name: 'NGINX_CONFIG_MODE', choices: ['none', 'production-https', 'development-http'], description: 'Nginx 配置模式；开发服无域名时选 development-http，release 正式入口选 production-https') booleanParam(name: 'ENABLE_SERVICES', defaultValue: true, description: '启用并启动 spacetimedb 与 api-server systemd 服务') @@ -110,11 +110,11 @@ pipeline { error('release 目标禁止安装 development-http Nginx 配置；无证书初始化请使用 NGINX_CONFIG_MODE=none。') } def databaseBackupProfile = params.DATABASE_BACKUP_PROFILE?.trim() - if (!(databaseBackupProfile in ['archive-full', 'files-history'])) { - error("DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${params.DATABASE_BACKUP_PROFILE}") + if (!(databaseBackupProfile in ['archive-full', 'files-minimal', 'files-history'])) { + error("DATABASE_BACKUP_PROFILE 只能是 archive-full、files-minimal 或 files-history，当前值: ${params.DATABASE_BACKUP_PROFILE}") } if (params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history') { - error('release 仅允许 archive-full；files-history 会把整棵历史目录加载到 Node 内存，需先完成流式 catalog 改造后才能重新启用。') + error('release 不允许 files-history；files-history 会把整棵历史目录加载到 Node 内存。release 请选择 files-minimal（最近 snapshot + 其后 commitlog）或 archive-full。') } def databaseBackupFilesHistoryWorkDir = params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR?.trim() if (!(databaseBackupFilesHistoryWorkDir ==~ /^\/var\/lib\/genarrative\/database-backups\/[A-Za-z0-9._\/-]+$/) || databaseBackupFilesHistoryWorkDir.contains('..')) { diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index f097b8e37..049d7d851 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -29,6 +29,7 @@ import { describeBackupSpaceRequirement, discoverDeferredArchiveUploads, discoverHistoryPlan, + discoverMinimalPlan, restoreDirectFilesBackup, restoreDirectFilesLatest, resumeUploadedHistoryBatch, @@ -66,6 +67,7 @@ async function main() { assertCanonicalQueryAndAuthorizationIncludeMultipartParameters(); assertInsufficientSpaceStopsBeforeServiceChanges(); assertCheckSpaceOnlyUsesFormatSpecificRequirement(); + assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog(); assertStopFailureRetainsRecoveryMarker(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); @@ -2562,6 +2564,106 @@ async function assertHistoryResumeReverifiesArchiveAndManifest() { } } +function assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog() { + const fixture = createHistoryFixture('minimal-plan', { nestedData: true }); + const replicaDir = path.join(fixture.replicasDir, '2'); + const snapshotsDir = path.join(replicaDir, 'snapshots'); + const clogDir = path.join(replicaDir, 'clog'); + mkdirSync(snapshotsDir, { recursive: true }); + mkdirSync(clogDir, { recursive: true }); + for (const transaction of ['100', '200', '300']) { + const padded = transaction.padStart(20, '0'); + const snapshotDir = path.join(snapshotsDir, `${padded}.snapshot_dir`); + mkdirSync(snapshotDir, { recursive: true }); + writeFileSync( + path.join(snapshotDir, `${padded}.snapshot_bsatn`), + 'snapshot', + ); + } + for (const transaction of ['50', '150', '250', '350']) { + const padded = transaction.padStart(20, '0'); + writeFileSync(path.join(clogDir, `${padded}.stdb.log`), 'log'); + writeFileSync(path.join(clogDir, `${padded}.stdb.ofs`), 'ofs'); + } + for (const relativeDir of ['config', 'data/control-db', 'data/program-bytes']) { + const directory = path.join(fixture.dataDir, relativeDir); + mkdirSync(directory, { recursive: true }); + writeFileSync(path.join(directory, 'state.bin'), 'state'); + } + writeFileSync(path.join(fixture.dataDir, 'data/config.toml'), 'config'); + writeFileSync(path.join(fixture.dataDir, 'data/metadata.toml'), 'metadata'); + + const plan = discoverMinimalPlan({ dataDir: fixture.dataDir }); + const paths = plan.candidates.map((item) => item.path); + const expect = (condition, reason) => { + if (!condition) { + failures.push(reason); + } + }; + + expect( + plan.retainSnapshots === 2, + `minimal 默认应保留 2 份 snapshot，实际 ${plan.retainSnapshots}`, + ); + expect( + paths.includes( + 'data/replicas/2/snapshots/00000000000000000200.snapshot_dir', + ), + 'minimal 必须保留次新 snapshot。', + ); + expect( + paths.includes( + 'data/replicas/2/snapshots/00000000000000000300.snapshot_dir', + ), + 'minimal 必须保留最新 snapshot。', + ); + expect( + !paths.includes( + 'data/replicas/2/snapshots/00000000000000000100.snapshot_dir', + ), + 'minimal 不得备份更早的 snapshot。', + ); + expect( + paths.includes('data/replicas/2/clog/00000000000000000150.stdb.log'), + 'minimal 必须保留覆盖最老保留 snapshot 的边界 commitlog 段。', + ); + expect( + paths.includes('data/replicas/2/clog/00000000000000000350.stdb.log'), + 'minimal 必须保留最新 commitlog 段。', + ); + expect( + !paths.includes('data/replicas/2/clog/00000000000000000050.stdb.log'), + 'minimal 不得备份更早的 commitlog 段。', + ); + for (const staticPath of [ + 'config', + 'data/config.toml', + 'data/metadata.toml', + 'data/control-db', + 'data/program-bytes', + ]) { + expect( + paths.includes(staticPath), + `minimal 必须保留状态路径 ${staticPath}。`, + ); + } + const replica = plan.replicas.find((item) => item.replicaId === '2'); + expect( + replica?.retainedSnapshots === 2 && replica?.droppedSnapshots === 1, + 'minimal 必须报告保留/丢弃的 snapshot 数量。', + ); + expect( + replica?.droppedSegments === 1, + `minimal 必须报告丢弃的 commitlog 段数量，实际 ${replica?.droppedSegments}`, + ); + + assertThrows( + () => discoverMinimalPlan({ dataDir: fixture.dataDir, retainSnapshots: 0 }), + '--retain-snapshots 必须是 >= 1 的整数', + 'minimal 必须校验保留 snapshot 数量。', + ); +} + function createHistoryFixture(name, { nestedData }) { const root = path.join(tmpRoot, name); const dataDir = path.join(root, 'stdb'); diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index bfb4bdbe9..4c6e65d85 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -938,7 +938,7 @@ const checks = [ }, { file: 'scripts/jenkins-server-provision.sh', - includes: 'archive-full|files-history)', + includes: 'archive-full|files-history|files-minimal)', reason: 'Server-Provision 必须拒绝未知数据库备份 profile。', }, { @@ -963,7 +963,7 @@ const checks = [ { file: 'jenkins/Jenkinsfile.production-server-provision', includes: - "choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history']", + "choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history']", reason: 'Server-Provision Job 必须显式暴露 archive-first 的数据库备份 profile。', }, @@ -1016,10 +1016,28 @@ const checks = [ }, { file: 'jenkins/Jenkinsfile.production-server-provision', - includes: 'release 仅允许 archive-full；files-history', + includes: 'release 不允许 files-history', reason: 'release 必须拒绝 files-history，避免逐文件 catalog 扫描再次触发生产内存峰值。', }, + { + file: 'jenkins/Jenkinsfile.production-server-provision', + includes: 'files-minimal', + reason: + 'release 定时备份必须提供 files-minimal profile：只备最近 snapshot 与其后 commitlog，热备不停服。', + }, + { + file: 'deploy/systemd/genarrative-database-backup-files-minimal.conf', + includes: '--mode full --minimal --retain-snapshots 2', + reason: + 'files-minimal 定时备份必须使用 minimal 口径并保留上游默认的最近 2 份 snapshot。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: '--minimal --retain-snapshots', + reason: + '发布前备份默认使用 minimal 热备，避免 40G 级冷备空间门槛与停服。', + }, { file: 'scripts/database-backup-to-oss.mjs', includes: diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index 2202017df..ba6e236fd 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -56,6 +56,16 @@ const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join( const DEFAULT_SPACE_SAFETY_RATIO = 1.1; const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024; // files 模式不落地本地归档（只写 catalog/state），所需空间远小于数据目录本身。 +// minimal 备份保留的最近 snapshot 数（与上游 retention 默认值一致）。 +const DEFAULT_RETAIN_SNAPSHOTS = 2; +// minimal 备份额外保留的小体积状态：身份、standalone 配置、控制库与模块字节。 +const MINIMAL_STATIC_PATHS = [ + 'config', + 'data/config.toml', + 'data/metadata.toml', + 'data/control-db', + 'data/program-bytes', +]; const DEFAULT_FILES_SPACE_SAFETY_RATIO = 0.05; const DEFAULT_FILES_EXTRA_FREE_BYTES = 2 * 1024 * 1024 * 1024; // 空间不足用独立退出码，调用方可据此决定是否降级存储格式。 @@ -84,7 +94,7 @@ const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1; function usage() { console.log(`用法: - npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] + npm run database:backup:oss -- [--mode full|history] [--minimal] [--retain-snapshots 2] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] npm run database:backup:oss -- --check-space-only [--storage-format archive|files] [--data-dir ] [--work-dir ] node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] node -- scripts/database-backup-to-oss.mjs --upload-archive @@ -97,6 +107,7 @@ function usage() { 将 SpacetimeDB 数据目录以 .tar.gz 或逐文件 catalog 形式上传到阿里云 OSS 指定 bucket。 默认 full 模式保持原有全量冷备行为；history 模式只归档已被最新 snapshot 覆盖的历史 commitlog 与旧 snapshot。 --storage-format files 不打包：按原相对路径建立 catalog，文件内容以 SHA-256 不可变对象上传；重复运行只上传新增或变化内容。 + --minimal 只备份上游 retention 语义下仍需要的部分（最近 --retain-snapshots 份 snapshot + 其后 commitlog 段 + 身份/配置/控制库/模块字节）；只支持 files 格式，且不停止任何服务。 archive history 必须有已验真的 full baseline state；files history 必须复用同一 work-dir 中已发布的 full catalog state。 --defer-upload 只生成本地冷备份和 manifest，不上传；后续用 --upload-archive 异步上传。 --upload-deferred-dir 串行收敛目录内 deferred/pending 及已上传未清理归档；只有 OSS 上传与验真完成后才按 keep-local 规则删除。 @@ -192,6 +203,8 @@ function parseArgs(argv) { resultFile: '', minFreeBytes: '', mode: 'full', + minimal: false, + retainSnapshots: DEFAULT_RETAIN_SNAPSHOTS, baselineState: '', baselineManifest: '', publishManifest: '', @@ -282,6 +295,12 @@ function parseArgs(argv) { case '--mode': options.mode = readValue(); break; + case '--minimal': + options.minimal = true; + break; + case '--retain-snapshots': + options.retainSnapshots = readValue(); + break; case '--baseline-state': options.baselineState = readValue(); break; @@ -1383,6 +1402,217 @@ function candidateKey(candidate) { return `${candidate.kind}\0${candidate.path}`; } +/** + * minimal 备份只保留上游 retention 语义下仍然需要的部分： + * 每个 replica 最近 retainSnapshots 份 snapshot、覆盖最老保留 snapshot 的 + * commitlog 边界段及其后的全部 commitlog（段边界与上游 pruner 一致）， + * 外加身份、standalone 配置、控制库与模块字节等小体积状态。 + * 历史 snapshot 与更早的 commitlog 段不再备份，因此也不需要计算增量差异。 + */ +export function discoverMinimalPlan({ + dataDir, + retainSnapshots = DEFAULT_RETAIN_SNAPSHOTS, +}) { + const retainCount = Number(retainSnapshots); + if (!Number.isInteger(retainCount) || retainCount < 1) { + throw new Error( + `--retain-snapshots 必须是 >= 1 的整数，实际: ${retainSnapshots}`, + ); + } + const resolvedDataDir = resolvePath(dataDir); + const replicasDir = findReplicasDir(resolvedDataDir); + const replicas = []; + const candidates = []; + + const replicaIds = readdirSync(replicasDir, { withFileTypes: true }) + .sort((left, right) => left.name.localeCompare(right.name)) + .flatMap((entry) => { + if (!entry.isDirectory()) { + return []; + } + if (!/^\d+$/u.test(entry.name)) { + throw new Error(`replica 目录名不符合预期: ${entry.name}`); + } + return [entry.name]; + }); + + for (const replicaId of replicaIds) { + const replicaDir = join(replicasDir, replicaId); + const snapshotsDir = join(replicaDir, 'snapshots'); + const clogDir = join(replicaDir, 'clog'); + if (!existsSync(snapshotsDir) || !lstatSync(snapshotsDir).isDirectory()) { + replicas.push({ + replicaId, + status: 'skipped', + reason: 'no-snapshots-directory', + }); + continue; + } + const snapshots = readdirSync(snapshotsDir, { withFileTypes: true }) + .flatMap((entry) => { + const match = /^(\d{20})[.]snapshot_dir$/u.exec(entry.name); + if (!match) { + return []; + } + const transaction = BigInt(match[1]); + if (transaction > 0xffff_ffff_ffff_ffffn) { + throw new Error(`snapshot transaction 超出 u64: ${entry.name}`); + } + if (!entry.isDirectory()) { + throw new Error( + `snapshot 候选必须是目录: ${join(snapshotsDir, entry.name)}`, + ); + } + const snapshotDir = join(snapshotsDir, entry.name); + const lockPath = join(snapshotsDir, `${match[1]}.lock`); + const snapshotFile = join(snapshotDir, `${match[1]}.snapshot_bsatn`); + if ( + existsSync(lockPath) || + !existsSync(snapshotFile) || + !lstatSync(snapshotFile).isFile() + ) { + return []; + } + return [{ name: entry.name, transaction }]; + }) + .sort((left, right) => + left.transaction < right.transaction + ? -1 + : left.transaction > right.transaction + ? 1 + : 0, + ); + if (snapshots.length === 0) { + replicas.push({ replicaId, status: 'skipped', reason: 'no-snapshot' }); + continue; + } + if (!existsSync(clogDir) || !lstatSync(clogDir).isDirectory()) { + throw new Error(`replica ${replicaId} 缺少 clog 目录。`); + } + const segmentFiles = new Map(); + for (const entry of readdirSync(clogDir, { withFileTypes: true })) { + const match = /^(\d{20})[.]stdb[.](log|ofs)$/u.exec(entry.name); + if (!match) { + throw new Error(`commitlog 文件名不符合预期: ${entry.name}`); + } + if (!entry.isFile()) { + throw new Error( + `commitlog 候选必须是普通文件: ${join(clogDir, entry.name)}`, + ); + } + const transaction = BigInt(match[1]); + if (transaction > 0xffff_ffff_ffff_ffffn) { + throw new Error(`commitlog transaction 超出 u64: ${entry.name}`); + } + const key = transaction.toString(); + const group = segmentFiles.get(key) ?? { transaction }; + group[match[2]] = entry.name; + segmentFiles.set(key, group); + } + for (const group of segmentFiles.values()) { + if (group.ofs && !group.log) { + throw new Error( + `commitlog offset 缺少对应 log: replica=${replicaId}, transaction=${group.transaction}`, + ); + } + } + const segments = [...segmentFiles.values()] + .filter((group) => group.log) + .sort((left, right) => + left.transaction < right.transaction + ? -1 + : left.transaction > right.transaction + ? 1 + : 0, + ); + + const retainedSnapshots = snapshots.slice(-retainCount); + const oldestRetainedSnapshot = retainedSnapshots[0].transaction; + const boundarySegment = segments + .filter((segment) => segment.transaction <= oldestRetainedSnapshot) + .at(-1); + if (!boundarySegment) { + throw new Error( + `replica ${replicaId} 无法找到覆盖最老保留 snapshot ${oldestRetainedSnapshot} 的 commitlog 边界。`, + ); + } + const retainedSegments = segments.filter( + (segment) => segment.transaction >= boundarySegment.transaction, + ); + + for (const snapshot of retainedSnapshots) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(snapshotsDir, snapshot.name), + kind: 'snapshot', + replicaId, + transaction: snapshot.transaction, + }), + ); + } + for (const segment of retainedSegments) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.log), + kind: 'commitlog', + replicaId, + transaction: segment.transaction, + }), + ); + if (segment.ofs) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.ofs), + kind: 'commitlog-offset', + replicaId, + transaction: segment.transaction, + }), + ); + } + } + replicas.push({ + replicaId, + status: 'ready', + retainedSnapshots: retainedSnapshots.length, + droppedSnapshots: snapshots.length - retainedSnapshots.length, + oldestRetainedSnapshot: oldestRetainedSnapshot.toString(), + boundarySegment: boundarySegment.transaction.toString(), + droppedSegments: segments.length - retainedSegments.length, + }); + } + + for (const relativePath of MINIMAL_STATIC_PATHS) { + const absolutePath = resolve(resolvedDataDir, relativePath); + if (!existsSync(absolutePath)) { + continue; + } + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath, + kind: 'state', + replicaId: 'state', + transaction: 0n, + }), + ); + } + + candidates.sort((left, right) => left.path.localeCompare(right.path)); + return { + dataDir: resolvedDataDir, + replicasDir: assertSafeRelativePath(resolvedDataDir, replicasDir), + replicas, + retainSnapshots: retainCount, + candidates, + totalSizeBytes: candidates + .reduce((sum, item) => sum + BigInt(item.sizeBytes), 0n) + .toString(), + }; +} + export function cleanupHistoryCandidates({ dataDir, candidates }) { const currentPlan = discoverHistoryPlan({ dataDir }); const eligible = new Map( @@ -2114,6 +2344,8 @@ async function publishDirectFilesLatest({ export async function runDirectFilesBackup({ mode, + minimal = false, + retainSnapshots = DEFAULT_RETAIN_SNAPSHOTS, dataDir, workDir, database, @@ -2139,7 +2371,17 @@ export async function runDirectFilesBackup({ `files history 模式缺少已发布 full baseline catalog: ${statePath}`, ); } - const plan = mode === 'history' ? discoverHistoryPlan({ dataDir }) : null; + const plan = + mode === 'history' + ? discoverHistoryPlan({ dataDir }) + : minimal + ? discoverMinimalPlan({ dataDir, retainSnapshots }) + : null; + if (minimal) { + console.log( + `[database-backup] files minimal: retainSnapshots=${plan.retainSnapshots}, candidates=${plan.candidates.length}, size=${plan.totalSizeBytes}`, + ); + } const collected = await collectDirectFileEntries({ dataDir, candidates: plan?.candidates ?? null, @@ -4217,6 +4459,12 @@ async function main() { `--storage-format 只能是 archive 或 files，实际: ${storageFormat}`, ); } + if (args.minimal && storageFormat !== 'files') { + throw new Error('--minimal 只支持 --storage-format files。'); + } + if (args.minimal && args.mode === 'history') { + throw new Error('--minimal 只支持 --mode full：minimal 自身就是自包含快照。'); + } if (args.checkSpaceOnly) { assertSufficientWorkDirSpace({ @@ -4351,11 +4599,13 @@ async function main() { let backupError = null; let restoreError = null; try { - if (args.mode === 'full' && !args.dryRun) { + if (args.mode === 'full' && !args.minimal && !args.dryRun) { serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); } await runDirectFilesBackup({ mode: args.mode, + minimal: args.minimal, + retainSnapshots: args.retainSnapshots, dataDir, workDir, database, diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index b44eccf9c..3b16188f8 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -24,6 +24,8 @@ usage() { 环境变量： GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files（默认 archive） GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0（默认 1：archive 空间不足自动降级 files） + GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=1|0（默认 1：发布前备份只保留最近 N 份 snapshot + 其后 commitlog，热备不停服） + GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS=N（默认 2，仅在 minimal 模式下生效） GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0（默认 1：尚未开始 publish 的失败自动恢复服务并退出维护） migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。 如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。 @@ -65,6 +67,8 @@ BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}" BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}" AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}" AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}" +BACKUP_MINIMAL="${GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL:-1}" +BACKUP_RETAIN_SNAPSHOTS="${GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS:-2}" DEPLOY_COMPLETED=0 PUBLISH_STARTED=0 MAINTENANCE_ENTERED=0 @@ -334,6 +338,14 @@ precheck_backup_space_before_maintenance() { echo "[production-stdb-publish] 已跳过发布前备份空间预检（--backup-mode skip）" return 0 fi + if [[ "${BACKUP_MINIMAL}" == "1" ]]; then + # minimal 备份是热备：只保留最近 N 份 snapshot + 其后 commitlog，不落地归档也不停服务。 + BACKUP_STORAGE_FORMAT="files" + if [[ "${BACKUP_MODE}" == "async" ]]; then + echo "[production-stdb-publish] minimal 备份为同步热备（无本地归档），备份模式由 async 调整为 sync。" >&2 + BACKUP_MODE="sync" + fi + fi local status=0 run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$? @@ -836,14 +848,21 @@ case "${BACKUP_MODE}" in SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service) fi - echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" - node -- "${BACKUP_SCRIPT}" \ - --env-file /etc/genarrative/api-server.env \ - --data-dir "${SPACETIME_ROOT_DIR}" \ - --database "${DATABASE}" \ - --storage-format "${BACKUP_STORAGE_FORMAT}" \ - --stop-service spacetimedb.service \ - "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}" + SYNC_BACKUP_ARGS=( + --env-file /etc/genarrative/api-server.env + --data-dir "${SPACETIME_ROOT_DIR}" + --database "${DATABASE}" + --storage-format "${BACKUP_STORAGE_FORMAT}" + ) + if [[ "${BACKUP_MINIMAL}" == "1" ]]; then + echo "[production-stdb-publish] publish 前执行 minimal 热备（最近 ${BACKUP_RETAIN_SNAPSHOTS} 份 snapshot + 其后 commitlog），不停服务" + SYNC_BACKUP_ARGS+=(--mode full --minimal --retain-snapshots "${BACKUP_RETAIN_SNAPSHOTS}") + else + echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" + SYNC_BACKUP_ARGS+=(--stop-service spacetimedb.service) + SYNC_BACKUP_ARGS+=("${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}") + fi + node -- "${BACKUP_SCRIPT}" "${SYNC_BACKUP_ARGS[@]}" ;; skip) echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份" diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 259b6f6c8..9d2bda38a 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -15,6 +15,8 @@ DATABASE_BACKUP_PROFILE="${DATABASE_BACKUP_PROFILE:-archive-full}" DATABASE_BACKUP_FILES_HISTORY_WORK_DIR="${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR:-/var/lib/genarrative/database-backups/files-history}" DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR="/etc/systemd/system/genarrative-database-backup.service.d" DATABASE_BACKUP_FILES_HISTORY_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-history.conf" +DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR="${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR:-/var/lib/genarrative/database-backups/files-minimal}" +DATABASE_BACKUP_FILES_MINIMAL_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-minimal.conf" DATABASE_BACKUP_LEGACY_DEV_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-dev-files.conf" require_non_root_relative_path() { @@ -71,10 +73,10 @@ validate_server_names() { validate_database_backup_profile() { case "${DATABASE_BACKUP_PROFILE}" in - archive-full|files-history) + archive-full|files-history|files-minimal) ;; *) - echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${DATABASE_BACKUP_PROFILE}" >&2 + echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full、files-history 或 files-minimal，当前值: ${DATABASE_BACKUP_PROFILE}" >&2 exit 1 ;; esac @@ -1323,12 +1325,29 @@ render_database_backup_files_history_drop_in() { deploy/systemd/genarrative-database-backup-files-history.conf } +render_database_backup_files_minimal_drop_in() { + local current_escaped env_escaped work_dir_escaped + current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" + env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")" + work_dir_escaped="$(escape_sed_replacement "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}")" + sed \ + -e "s|/opt/genarrative/current|${current_escaped}|g" \ + -e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \ + -e "s|/var/lib/genarrative/database-backups/files-minimal|${work_dir_escaped}|g" \ + deploy/systemd/genarrative-database-backup-files-minimal.conf +} + configure_database_backup_profile() { local rendered_drop_in if [[ "${DATABASE_BACKUP_PROFILE}" == "archive-full" ]]; then echo "[server-provision] 数据库备份 profile=archive-full，保留主 service 的全量冷备行为。" - run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + return + fi + + if [[ "${DATABASE_BACKUP_PROFILE}" == "files-minimal" ]]; then + configure_database_backup_files_minimal_profile return fi @@ -1350,13 +1369,26 @@ configure_database_backup_profile() { run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}" - run_cmd rm -f "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + run_cmd rm -f "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" rendered_drop_in="$(mktemp)" render_database_backup_files_history_drop_in >"${rendered_drop_in}" install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" 0644 rm -f "${rendered_drop_in}" } +configure_database_backup_files_minimal_profile() { + local rendered_drop_in + + echo "[server-provision] 数据库备份 profile=files-minimal，只保留最近 snapshot 与其后 commitlog（热备、不停服）。" + run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}" + run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}" + run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + rendered_drop_in="$(mktemp)" + render_database_backup_files_minimal_drop_in >"${rendered_drop_in}" + install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" 0644 + rm -f "${rendered_drop_in}" +} + render_health_patrol_service() { local current_escaped current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" @@ -1372,6 +1404,7 @@ require_path deploy/systemd/genarrative-external-generation-controller.service require_path deploy/systemd/genarrative-bgfilter-worker.service require_path deploy/systemd/genarrative-database-backup.service require_path deploy/systemd/genarrative-database-backup-files-history.conf +require_path deploy/systemd/genarrative-database-backup-files-minimal.conf require_path deploy/systemd/genarrative-database-backup.timer require_path deploy/systemd/genarrative-health-patrol.service require_path deploy/systemd/genarrative-health-patrol.timer From 90207bc1d550e12d72972716aae80aa97c914c40 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 11:37:14 +0800 Subject: [PATCH 04/14] =?UTF-8?q?minimal=20=E5=A4=87=E4=BB=BD=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=20freeze=20=E5=89=AF=E6=9C=AC=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - database-backup-to-oss.mjs 新增 --freeze-dir：minimal 模式先把选中子集 rsync 成冻结副本（保持原始相对路径）再从副本上传，避免热备时最新 commitlog 持续追加触发 stat 漂移失败 - files-minimal systemd 模板与发布前 minimal 备份默认传入 freeze 目录 - 生产运维门禁同步 freeze 口径断言 --- ...rrative-database-backup-files-minimal.conf | 2 +- scripts/check-production-ops-guardrails.mjs | 4 +- scripts/database-backup-to-oss.mjs | 52 ++++++++++++++++++- scripts/deploy/production-stdb-publish.sh | 7 ++- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/deploy/systemd/genarrative-database-backup-files-minimal.conf b/deploy/systemd/genarrative-database-backup-files-minimal.conf index 2124eb345..c8d2e43fe 100644 --- a/deploy/systemd/genarrative-database-backup-files-minimal.conf +++ b/deploy/systemd/genarrative-database-backup-files-minimal.conf @@ -2,4 +2,4 @@ Environment=GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=4 Environment=GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND=20971520 ExecStart= -ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode full --minimal --retain-snapshots 2 --data-dir /stdb --work-dir /var/lib/genarrative/database-backups/files-minimal +ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode full --minimal --retain-snapshots 2 --freeze-dir /var/lib/genarrative/database-backups/files-minimal-freeze --data-dir /stdb --work-dir /var/lib/genarrative/database-backups/files-minimal diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 4c6e65d85..93d81f106 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -1028,13 +1028,13 @@ const checks = [ }, { file: 'deploy/systemd/genarrative-database-backup-files-minimal.conf', - includes: '--mode full --minimal --retain-snapshots 2', + includes: '--mode full --minimal --retain-snapshots 2 --freeze-dir', reason: 'files-minimal 定时备份必须使用 minimal 口径并保留上游默认的最近 2 份 snapshot。', }, { file: 'scripts/deploy/production-stdb-publish.sh', - includes: '--minimal --retain-snapshots', + includes: '--freeze-dir', reason: '发布前备份默认使用 minimal 热备，避免 40G 级冷备空间门槛与停服。', }, diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index ba6e236fd..26d8cf232 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -107,6 +107,7 @@ function usage() { 将 SpacetimeDB 数据目录以 .tar.gz 或逐文件 catalog 形式上传到阿里云 OSS 指定 bucket。 默认 full 模式保持原有全量冷备行为；history 模式只归档已被最新 snapshot 覆盖的历史 commitlog 与旧 snapshot。 --storage-format files 不打包：按原相对路径建立 catalog，文件内容以 SHA-256 不可变对象上传；重复运行只上传新增或变化内容。 + --freeze-dir 仅用于 --minimal：先把选中子集 rsync 成冻结副本再上传，避免热备时最新 commitlog 追加导致 stat 漂移；副本内保持原始相对路径。 --minimal 只备份上游 retention 语义下仍需要的部分（最近 --retain-snapshots 份 snapshot + 其后 commitlog 段 + 身份/配置/控制库/模块字节）；只支持 files 格式，且不停止任何服务。 archive history 必须有已验真的 full baseline state；files history 必须复用同一 work-dir 中已发布的 full catalog state。 --defer-upload 只生成本地冷备份和 manifest，不上传；后续用 --upload-archive 异步上传。 @@ -204,6 +205,7 @@ function parseArgs(argv) { minFreeBytes: '', mode: 'full', minimal: false, + freezeDir: '', retainSnapshots: DEFAULT_RETAIN_SNAPSHOTS, baselineState: '', baselineManifest: '', @@ -301,6 +303,9 @@ function parseArgs(argv) { case '--retain-snapshots': options.retainSnapshots = readValue(); break; + case '--freeze-dir': + options.freezeDir = readValue(); + break; case '--baseline-state': options.baselineState = readValue(); break; @@ -4387,7 +4392,7 @@ async function main() { const env = loadEffectiveEnv(args.envFiles); const isProductionLike = existsSync(DEFAULT_PRODUCTION_DATA_DIR) && process.platform !== 'win32'; - const dataDir = resolvePath( + let dataDir = resolvePath( firstNonEmpty( args.dataDir, env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, @@ -4466,6 +4471,51 @@ async function main() { throw new Error('--minimal 只支持 --mode full：minimal 自身就是自包含快照。'); } + if (args.minimal && args.freezeDir) { + if (storageFormat !== 'files') { + throw new Error('--freeze-dir 只支持 --minimal --storage-format files。'); + } + const freezeDir = resolvePath(args.freezeDir); + const plan = discoverMinimalPlan({ + dataDir, + retainSnapshots: args.retainSnapshots, + }); + console.log( + `[database-backup] minimal freeze: candidates=${plan.candidates.length}, size=${plan.totalSizeBytes}, freezeDir=${freezeDir}`, + ); + if (!args.dryRun) { + mkdirSync(freezeDir, { recursive: true }); + const listPath = join(workDir, 'minimal-freeze-files.txt'); + mkdirSync(workDir, { recursive: true }); + writeFileSync( + listPath, + `${plan.candidates.map((item) => item.path).join('\n')}\n`, + 'utf8', + ); + const result = spawnSync( + 'rsync', + [ + '-a', + '-r', + '--delete', + `--files-from=${listPath}`, + `${dataDir}/`, + `${freezeDir}/`, + ], + { stdio: 'inherit' }, + ); + if (result.status !== 0) { + throw new Error( + `minimal freeze 拷贝失败: rsync exit=${result.status ?? ''}`, + ); + } + console.log( + `[database-backup] minimal freeze 完成，后续从冻结副本上传: ${freezeDir}`, + ); + } + dataDir = freezeDir; + } + if (args.checkSpaceOnly) { assertSufficientWorkDirSpace({ dataDir, diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index 3b16188f8..115cf43cf 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -856,7 +856,12 @@ case "${BACKUP_MODE}" in ) if [[ "${BACKUP_MINIMAL}" == "1" ]]; then echo "[production-stdb-publish] publish 前执行 minimal 热备（最近 ${BACKUP_RETAIN_SNAPSHOTS} 份 snapshot + 其后 commitlog），不停服务" - SYNC_BACKUP_ARGS+=(--mode full --minimal --retain-snapshots "${BACKUP_RETAIN_SNAPSHOTS}") + SYNC_BACKUP_ARGS+=( + --mode full + --minimal + --retain-snapshots "${BACKUP_RETAIN_SNAPSHOTS}" + --freeze-dir "${GENARRATIVE_STDB_PUBLISH_BACKUP_FREEZE_DIR:-/var/lib/genarrative/database-backups/publish-minimal-freeze}" + ) else echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" SYNC_BACKUP_ARGS+=(--stop-service spacetimedb.service) From 9bf35e87457c338352706382ec3821fab179b622 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 11:53:32 +0800 Subject: [PATCH 05/14] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20macOS=20=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E4=B8=8A=E7=9A=84=E6=B8=A0=E9=81=93=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E7=94=A8=E4=BE=8B=E5=AE=BF=E4=B8=BB=E4=BE=9D?= =?UTF-8?q?=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build-release.test.mjs 的「packaged renderer receives the same channel as the updater manifest」用 windows target 却未注入 stageRuntime，真实 staging 要求宿主 Node 平台/架构与目标一致，导致 macOS(arm64)/Linux 上必然失败 - 该用例只断言传给 spawn 的 VITE_AGC_PLATFORM_CHANNEL，改为注入空 stageRuntime，与同文件其它用例一致 --- apps/ai-game-creator-shell/scripts/build-release.test.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/ai-game-creator-shell/scripts/build-release.test.mjs b/apps/ai-game-creator-shell/scripts/build-release.test.mjs index 732302eb1..b5f8a44da 100644 --- a/apps/ai-game-creator-shell/scripts/build-release.test.mjs +++ b/apps/ai-game-creator-shell/scripts/build-release.test.mjs @@ -215,6 +215,9 @@ test('packaged renderer receives the same channel as the updater manifest', () = spawnOptions = options; return { status: 0 }; }, + // 该用例只关心渠道环境变量；Node 运行时 staging 与宿主平台绑定， + // 真实 staging 会让同一用例在 macOS/Linux 上对 windows target 失败。 + stageRuntime: () => {}, }); assert.equal(spawnOptions?.env?.VITE_AGC_PLATFORM_CHANNEL, 'release'); }); From beebcca4dec56d94ca719a0b6be9628a1285bd58 Mon Sep 17 00:00:00 2001 From: Suzumiya Date: Mon, 21 Sep 2026 11:58:13 +0800 Subject: [PATCH 06/14] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20AGC=20=E7=94=BB?= =?UTF-8?q?=E5=B8=83=E5=9B=9B=E9=A1=B9=E9=AA=8C=E6=94=B6=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=9A=E9=94=9A=E7=82=B9=E8=AE=A9=E4=BD=8D=E6=A0=87=E9=A2=98?= =?UTF-8?q?=E6=A0=8F=E3=80=81=E6=B5=AE=E5=B1=82=E4=B8=8D=E5=86=8D=E6=92=9E?= =?UTF-8?q?=E7=94=BB=E5=B8=83=E8=BE=B9=E7=BC=98=E3=80=81=E6=8B=96=E5=8A=A8?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E8=A7=A6=E5=8F=91=E7=82=B9=E5=87=BB=E3=80=81?= =?UTF-8?q?=E5=9B=BE=E6=A0=87=E8=A7=84=E8=8C=83=E5=8F=82=E8=80=83=E5=8F=A3?= =?UTF-8?q?=E5=BE=84=E5=AF=B9=E9=BD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 生成任务锚点新增「资源总览」档：资源总览页仍贴画布顶边内缩 0.85rem，资源栏目画布页让开顶部钉住的整宽栏目标题栏（42px + 0.5rem），两档切换走 margin-top 过渡（减动效下关掉）。 - 生成浮层的顶边与可用高度改为按占位卡的真实屏幕位置算（卡下沿到安全底边），不再假设卡片贴在安全带上沿——占位落到下半屏时浮层底边不再越过画布下沿被切掉。 - 生成浮层新增水平收口：按「浮层左右各留 12px」把锚点收进画布可视范围，卡片靠边时浮层不再有一半落到画布之外；宽度口径与面板 CSS 同源（min(560px, 100% - 24px)）。 - 新增画布卡片手势的共享判据 resourceCanvasGestureExceededDragThreshold（阈值仍取 RESOURCE_CANVAS_DRAG_THRESHOLD）：资源卡与生成占位卡共用同一份「拖动还是点击」判断。 - 生成占位卡修掉拖动收尾误触点击：越过阈值才算拖动、登记一次性点击抑制（consumeDragClick）由宿主消费，「拖动中」样式也从越过阈值那一刻起才亮。 - 生成占位卡的删除按钮起手手势不再被当成「点卡片」：按起手点判定，松手落回卡片时不触发开合生成浮层。 - 图标规范面板的参考选择注释按现行判据改写（只有图集 icon-spritesheet 不接受用户参考），入口矩阵文档同步补「参考图（@ 引用）」口径并修掉「本地 IPC 没有参考图入参」的旧说法。 - 补齐回归用例：手势模型判据、占位卡拖动/点击与阈值、宿主级「拖完不弹浮层且普通点击仍开合」、浮层顶边与水平收口、锚点分档与跨文件标题栏高度守卫、逐条生成入口的 @ 与判据一致、图标规范面板的 @ 与提交带引用。 - 共享记忆 pitfalls.md 记录两条排障口径：卡类手势必须只有一份阈值与点击抑制判据；画布浮层几何只能按真实屏幕位置算、分页锚点必须让开钉死的标题栏。 --- ...ResourceCanvasAssetGenerationPanelView.tsx | 10 +- ...rceCanvasAssetGenerationTasksPanelView.tsx | 14 +- ...rceCanvasGenerationPlaceholderCardView.tsx | 26 +++- ...ourceCanvasAssetGenerationTasksSidebar.css | 33 +++++ .../resourceCanvasCardGestureModel.ts | 46 ++++++ .../resourceCanvasGenerationPanel.css | 10 +- ...resourceCanvasGenerationVisibilityModel.ts | 118 +++++++++++++-- ...useResourceCanvasGenerationPlaceholders.ts | 75 +++++++++- .../src/view/project-development/index.tsx | 99 +++++++++---- ...ceCanvasAssetGenerationReferences.test.tsx | 121 ++++++++++++++- ...ceCanvasAssetGenerationTasksPanel.test.tsx | 14 +- ...asAssetGenerationTasksSidebarStyle.test.ts | 82 +++++++++++ .../resourceCanvasCardGestureModel.test.ts | 49 ++++++ ...nvasGenerationFloatingPanelChrome.test.tsx | 16 ++ ...urceCanvasGenerationHostLifecycle.test.tsx | 53 +++++++ ...sourceCanvasGenerationPlaceholder.test.tsx | 133 ++++++++++++++++- ...nvasGenerationTasksSidebarDismiss.test.tsx | 8 +- ...resourceCanvasGenerationVisibility.test.ts | 139 ++++++++++++++++-- docs/project-memory/shared-memory/pitfalls.md | 22 +++ ...C】栏目画布底部工具栏入口矩阵-2026-09-13.md | 3 +- 20 files changed, 1001 insertions(+), 70 deletions(-) create mode 100644 apps/ai-game-creator-shell/src/features/resource-canvas/resourceCanvasCardGestureModel.ts create mode 100644 apps/ai-game-creator-shell/tests/resourceCanvasCardGestureModel.test.ts diff --git a/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasAssetGenerationPanelView.tsx b/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasAssetGenerationPanelView.tsx index 68886de45..f5b099e27 100644 --- a/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasAssetGenerationPanelView.tsx +++ b/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasAssetGenerationPanelView.tsx @@ -235,8 +235,14 @@ export function ResourceCanvasAssetGenerationPanelView({ /** * 参考选择只对有真实参考能力的入口呈现。 * - * 图集只接受单张规范引用、图标规范本身就是权威规范图产出方：这两类入口不给选择器， - * 原生侧同样拒绝额外参考（不是静默丢弃）。 + * `@` 引用输入区是**共享组件**（`ResourceReferenceInput`，聊天 / 快速编辑 / 这里同一份）， + * 这一处只是「要不要挂它」的唯一分支：判据在 `resourceCanvasAssetGenerationAcceptsReferences`， + * 不在这里另写一套 kind 清单。 + * + * 拒绝用户参考的只有**图集**（`icon-spritesheet`）：按合同它只接受单张权威规范引用， + * 原生侧对多余参考是**显式拒绝**（不是静默丢弃）。图标规范（`icon-spec`）虽然产出权威规范图， + * 但生成时同样可以带参考图，上限与普通生成一致——它**有** `@` 选择器。 + * 两端必须同一个词：Rust 侧是 `platform_art_asset_kind_accepts_user_reference_assets`。 */ const referenceEnabled = resourceCanvasAssetGenerationAcceptsReferences(action); diff --git a/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasAssetGenerationTasksPanelView.tsx b/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasAssetGenerationTasksPanelView.tsx index 539131016..216f8800d 100644 --- a/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasAssetGenerationTasksPanelView.tsx +++ b/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasAssetGenerationTasksPanelView.tsx @@ -44,12 +44,18 @@ export type ResourceCanvasAssetGenerationTasksPanelViewProps = { /** 定位到该任务产出的素材卡（宿主复用既有 `pendingResourceFocusRef` 聚焦链）。 */ onFocusTask: (task: ResourceCanvasAssetGenerationTask) => void; /** - * 锚点落在哪个工作面：资源栏目画布 / 运行表现层 / UI 编辑器。 + * 锚点落在哪个工作面：资源总览页 / 资源栏目画布 / 运行表现层 / UI 编辑器。 * - * 三处右上角的既有 chrome 不同（运行表现层右上角是版本入口、UI 编辑器是整页编辑器壳）， - * 所以坐标由这一档在样式里分档给出；缺省按资源栏目画布算。 + * 四处的既有 chrome 不同，所以坐标由这一档在样式里分档给出；缺省按资源栏目画布算： + * - `canvas-overview`（资源总览 / 画本）：顶部没有钉死的栏目标题栏，锚点贴画布顶边内缩； + * - `canvas`（资源栏目画布）：顶部钉着**整宽**的栏目标题栏（含「返回资源总览」入口）， + * 锚点必须让开它，否则会压在那条标题栏上（验收现场那条「生成任务和标题栏重叠」）； + * - `run`：右上角被版本入口占着； + * - `editor`：UI 编辑器是整页编辑器壳。 + * + * 两页之间切换时锚点高度走样式里的 transition，不瞬移。 */ - placement?: 'canvas' | 'run' | 'editor'; + placement?: 'canvas-overview' | 'canvas' | 'run' | 'editor'; }; /** diff --git a/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasGenerationPlaceholderCardView.tsx b/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasGenerationPlaceholderCardView.tsx index 2c0305d3b..503a6489a 100644 --- a/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasGenerationPlaceholderCardView.tsx +++ b/apps/ai-game-creator-shell/src/features/resource-canvas/ResourceCanvasGenerationPlaceholderCardView.tsx @@ -2,6 +2,7 @@ import './resourceCanvasGenerationPanel.css'; import { Sparkles, X } from 'lucide-react'; import type { PointerEvent as ReactPointerEvent } from 'react'; +import { useRef } from 'react'; import { RESOURCE_CANVAS_GENERATION_PLACEHOLDER_STATUS_LABELS, @@ -44,6 +45,14 @@ export function ResourceCanvasGenerationPlaceholderCardView({ onTogglePanel, onRemove, }: ResourceCanvasGenerationPlaceholderCardViewProps) { + /** + * 这一次点击手势是从哪儿起手的。 + * + * 删除按钮自己会 `stopPropagation`，所以「按在删除按钮上、松手落回卡片」这种手势里， + * 浏览器把 `click` 派给两者的共同祖先（也就是这张卡）——删除按钮的删除没发生，卡片的 + * 「开合生成浮层」却执行了。判据不看移动距离：**从删除按钮起手的手势永远不是「点卡片」**。 + */ + const gestureOriginRef = useRef<'card' | 'remove'>('card'); return (
{ + gestureOriginRef.current = 'card'; + onPointerDown(event); + }} onPointerMove={onPointerMove} onPointerUp={onPointerUp} onPointerCancel={onPointerCancel} onLostPointerCapture={onPointerCancel} onClick={(event) => { event.stopPropagation(); + const origin = gestureOriginRef.current; + gestureOriginRef.current = 'card'; + if (origin === 'remove') { + return; + } onTogglePanel(); }} onKeyDown={(event) => { @@ -87,7 +104,12 @@ export function ResourceCanvasGenerationPlaceholderCardView({
{task.error ? (

@@ -166,14 +211,7 @@ function assetGenerationTaskRow( * 给的待办记录里没有产物 id，编一个指向不明的跳转不如不给；② 提示词单独一行，用户要能认出手上 * 这行是哪一次修改。 */ -function resourceEditTaskRow( - task: ResourceCanvasResourceEditTask, - nowMillis: number, -) { - const elapsedMillis = resourceCanvasResourceEditTaskElapsedMillis( - task, - nowMillis, - ); +function resourceEditTaskRow(task: ResourceCanvasResourceEditTask) { return (

  • {task.phaseDetail} - {elapsedMillis === null ? null : ( - - {`已耗时 ${resourceCanvasResourceEditElapsedLabel(elapsedMillis)}`} - - )} + {task.error ? (

    @@ -224,12 +258,11 @@ function resourceEditTaskRow( function resourceCanvasGenerationTaskRowNode( row: ResourceCanvasGenerationTaskRow, - nowMillis: number, onFocusTask: (task: ResourceCanvasAssetGenerationTask) => void, ) { return row.source === 'asset-generation' - ? assetGenerationTaskRow(row.task, nowMillis, onFocusTask) - : resourceEditTaskRow(row.task, nowMillis); + ? assetGenerationTaskRow(row.task, onFocusTask) + : resourceEditTaskRow(row.task); } /** @@ -257,7 +290,6 @@ export function ResourceCanvasAssetGenerationTasksPanelView({ onFocusTask, placement = 'canvas', }: ResourceCanvasAssetGenerationTasksPanelViewProps) { - const [nowMillis, setNowMillis] = useState(() => Date.now()); /** * 收起动画期：`open` 已经变 false，但侧栏还要留在 DOM 里把 `…-leave` 播完 * （`entering` 是刚打开时给根节点挂进场动画的那一档）。 @@ -291,7 +323,6 @@ export function ResourceCanvasAssetGenerationTasksPanelView({ ); const done = ordered.filter(resourceCanvasGenerationTaskRowIsTerminal); const inFlightCount = active.length; - const hasLiveTask = inFlightCount > 0; const visibleDone = done.slice( 0, RESOURCE_CANVAS_ASSET_GENERATION_DONE_SECTION_LIMIT, @@ -320,14 +351,12 @@ export function ResourceCanvasAssetGenerationTasksPanelView({ ? { inFlightCount, ordered, active, visibleDone, done } : lastRenderedRef.current; - // 已耗时是前端计时（后端只给时间戳）：只在还有未终态任务时走秒表，全部收口后停掉。 - useEffect(() => { - if (!hasLiveTask) { - return undefined; - } - const timer = setInterval(() => setNowMillis(Date.now()), 1_000); - return () => clearInterval(timer); - }, [hasLiveTask]); + /* + * 已耗时是前端计时（后端只给时间戳）：秒表**不在这一层**——面板上通常同时挂着多条任务， + * 在这里订阅 100ms 的 tick 等于每 0.1 秒重建整个侧栏。计时下沉到每一行的 + * `AssetGenerationElapsedLabel` / `ResourceEditElapsedLabel`，只在**还有未终态任务**时 + * 订阅（终态行定格在账本的结束时间上）。 + */ /** * open → 进场，close → 先留一帧播 `…-leave` 再卸载。 @@ -436,11 +465,7 @@ export function ResourceCanvasAssetGenerationTasksPanelView({ ) : (

      {rendered.active.map((row) => - resourceCanvasGenerationTaskRowNode( - row, - nowMillis, - onFocusTask, - ), + resourceCanvasGenerationTaskRowNode(row, onFocusTask), )}
    )} @@ -463,11 +488,7 @@ export function ResourceCanvasAssetGenerationTasksPanelView({ <>
      {rendered.visibleDone.map((row) => - resourceCanvasGenerationTaskRowNode( - row, - nowMillis, - onFocusTask, - ), + resourceCanvasGenerationTaskRowNode(row, onFocusTask), )}
    {rendered.done.length > rendered.visibleDone.length ? ( diff --git a/apps/ai-game-creator-shell/tests/resourceCanvasAssetGenerationTasksPanel.test.tsx b/apps/ai-game-creator-shell/tests/resourceCanvasAssetGenerationTasksPanel.test.tsx index 3e911ea8f..5d01d2ac8 100644 --- a/apps/ai-game-creator-shell/tests/resourceCanvasAssetGenerationTasksPanel.test.tsx +++ b/apps/ai-game-creator-shell/tests/resourceCanvasAssetGenerationTasksPanel.test.tsx @@ -133,6 +133,100 @@ describe('「生成任务」侧栏', () => { ).toBeGreaterThan(0); }); + /** + * 读秒粒度：与对话侧同一份 `useLiveNow`（`LIVE_TIMER_TICK_MS = 100`）。 + * + * 不足一分钟的耗时显示到 0.1 秒，刷新就必须是 100ms——按 1 秒一跳时，用户看到的是一块 + * 「带小数却一格一格跳」的表，像卡住不动（验收现场「生成任务里的读秒更新频率应为 0.1s」）。 + */ + test('读秒按 100ms 刷新：不足一分钟的耗时以 0.1 秒递增', () => { + vi.useFakeTimers(); + try { + // 假时钟与任务的创建时间对齐（夹具的 `nowMillis` 是 1000）。 + vi.setSystemTime(1_000); + renderSidebar([ + task({ + taskId: 't1', + dispatched: true, + status: 'running', + phaseDetail: '正在生成。', + }), + ]); + const elapsed = () => screen.getByText(/^已耗时 /u).textContent; + expect(elapsed()).toBe('已耗时 0.0秒'); + + act(() => { + vi.advanceTimersByTime(100); + }); + expect(elapsed()).toBe('已耗时 0.1秒'); + + // 1 秒一跳的实现会停在上面的 0.1 秒：再走 400ms 必须继续往上加。 + act(() => { + vi.advanceTimersByTime(400); + }); + expect(elapsed()).toBe('已耗时 0.5秒'); + } finally { + vi.useRealTimers(); + } + }); + + test('任务收口后不再走表：已耗时定格在账本给的结束时间上', () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(1_000); + renderSidebar([ + task({ + taskId: 't1', + dispatched: true, + status: 'completed', + phaseDetail: '生成已完成。', + assetId: 'asset-1', + finishedAtMillis: 2_500, + }), + ]); + const elapsed = () => screen.getByText(/^已耗时 /u).textContent; + expect(elapsed()).toBe('已耗时 1.5秒'); + + act(() => { + vi.advanceTimersByTime(5_000); + }); + expect(elapsed()).toBe('已耗时 1.5秒'); + } finally { + vi.useRealTimers(); + } + }); + + /** + * 秒表只订在**运行中的那一行**上：终态行不订阅，面板本身也不订阅。 + * + * 这条是性能约束的可执行版本——100ms 的 tick 若落在整块面板或终态行上，每 0.1 秒都会重建 + * 整个侧栏（`useLiveNow` 的注释写明「调用方必须是叶子节点」）。 + */ + test('秒表只订在运行中的那一行上，终态行与面板本身都不订阅时钟', () => { + const spy = vi.spyOn(globalThis, 'setInterval'); + try { + renderSidebar([ + task({ + taskId: 'live', + dispatched: true, + status: 'running', + phaseDetail: '正在生成。', + }), + task({ + taskId: 'done', + dispatched: true, + status: 'completed', + phaseDetail: '生成已完成。', + assetId: 'asset-done', + finishedAtMillis: 2_000, + }), + ]); + expect(spy).toHaveBeenCalledTimes(1); + } finally { + spy.mockRestore(); + } + }); + test('收起只有一个入口：头部那枚关闭按钮，收起即整块让出画布', async () => { const user = userEvent.setup(); const { onToggleOpen } = renderSidebar([task({ taskId: 't1' })]); From 886bdc06ed5d36a1869ed19d19bbf422df74d0e1 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 12:49:34 +0800 Subject: [PATCH 13/14] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=B8=A0=E9=81=93?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=E7=94=A8=E4=BE=8B=E9=87=8C?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E7=9A=84=20stageRuntime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 两次独立修复（#444 与本仓库 55cb04765）在同一处各加了一个 stageRuntime，合并后成为重复对象键，Web-Build 的 ESLint(no-dupe-keys) 失败 - 保留一条带说明的 stageRuntime stub，用例语义不变 --- apps/ai-game-creator-shell/scripts/build-release.test.mjs | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/ai-game-creator-shell/scripts/build-release.test.mjs b/apps/ai-game-creator-shell/scripts/build-release.test.mjs index 967380d0c..59e1a878b 100644 --- a/apps/ai-game-creator-shell/scripts/build-release.test.mjs +++ b/apps/ai-game-creator-shell/scripts/build-release.test.mjs @@ -218,9 +218,6 @@ test('packaged renderer receives the same channel as the updater manifest', () = spawnOptions = options; return { status: 0 }; }, - // 该用例只关心渠道环境变量；Node 运行时 staging 与宿主平台绑定， - // 真实 staging 会让同一用例在 macOS/Linux 上对 windows target 失败。 - stageRuntime: () => {}, }); assert.equal(spawnOptions?.env?.VITE_AGC_PLATFORM_CHANNEL, 'release'); }); From 2e31a485d871b8a8bb4b8a1dbcf8af8666b14bc6 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 13:27:37 +0800 Subject: [PATCH 14/14] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20Stdb=20publish=20?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=E7=A1=AE=E8=AE=A4=E8=B6=85=E6=97=B6=E8=A2=AB?= =?UTF-8?q?=E8=AF=AF=E5=88=A4=E4=B8=BA=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - production-stdb-publish 新增 run_spacetime_publish：仅在出现 timeout waiting for transaction confirmation 时重试一次同版本 publish（SpacetimeDB 对已生效的同版本为幂等），重试成功即视为模块已在位，不再把生产留在维护态 - 其它失败原因保持原有失败关闭语义；生产运维门禁断言补上该重试口径 --- scripts/check-production-ops-guardrails.mjs | 6 ++++ scripts/deploy/production-stdb-publish.sh | 32 +++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 93d81f106..f6f2cf056 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -277,6 +277,12 @@ const checks = [ reason: 'archive 冷备份空间不足时必须能自动降级为 files 存储格式（不落地本地归档）。', }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: 'timeout waiting for transaction confirmation', + reason: + 'publish 客户端等确认超时不能直接判失败：必须重试同版本 publish 以确认模块是否已生效，避免把生产留在维护态。', + }, { file: 'scripts/deploy/production-stdb-publish.sh', includes: 'restore_runtime_services_before_publish', diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index 115cf43cf..37933968d 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -904,6 +904,34 @@ else echo "[production-stdb-publish] 发布 SpacetimeDB module: ${DATABASE} -> ${SERVER_ALIAS}, root=${SPACETIME_ROOT_DIR}" fi +# 迁移已提交但客户端等确认超时（HTTP 504 / timeout waiting for transaction confirmation） +# 时不能直接判失败：重试一次同版本 publish，SpacetimeDB 对已生效的同版本是幂等的 no-op， +# 重试成功即说明目标模块已在位，避免把生产留在维护态。 +run_spacetime_publish() { + local attempt=1 + local output="" + local status=0 + while :; do + output="" + status=0 + if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then + output="$(runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}" 2>&1)" || status=$? + else + output="$(spacetime "${PUBLISH_ARGS[@]}" 2>&1)" || status=$? + fi + printf '%s\n' "${output}" + if [[ "${status}" -eq 0 ]]; then + return 0 + fi + if [[ "${attempt}" -ge 2 || "${output}" != *"timeout waiting for transaction confirmation"* ]]; then + return "${status}" + fi + echo "[production-stdb-publish] publish 客户端等事务确认超时，可能迁移已提交；重试一次同版本 publish 以确认模块状态。" >&2 + attempt=$((attempt + 1)) + sleep 5 + done +} + if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then if ! id "${RUN_AS_USER}" >/dev/null 2>&1; then echo "[production-stdb-publish] 发布用户不存在: ${RUN_AS_USER}" >&2 @@ -931,10 +959,10 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then PUBLISH_ARGS+=(--server "${SERVER_ALIAS}") fi PUBLISH_STARTED=1 - runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}" + run_spacetime_publish else PUBLISH_STARTED=1 - spacetime "${PUBLISH_ARGS[@]}" + run_spacetime_publish fi RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR="$(dirname "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}")"