修复 release 发布与冷备份恢复

冷备份增加工作目录空间预检并确保失败后恢复依赖服务

API deploy 增加 Pingora 产物硬校验并区分 current 切换前后维护模式处理

Jenkins release 默认构建和部署 Pingora 影子网关并在全量流水线透传参数

补充备份与 API deploy 回归检查和生产运维护栏

更新生产运维文档与项目记忆的事故处理口径

deploy/env/api-server.env.example

@@ -139,6 +139,8 @@ GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET=
 GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT=oss-cn-shanghai.aliyuncs.com
 GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX=database-backups
 GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL=false
+# 可选：显式要求备份工作目录所在文件系统至少保留的可用空间；为空时按数据目录大小 + 安全余量估算。
+GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES=
 # 可选：定时 / publish 前备份使用独立最小权限 AccessKey；为空时回退 ALIYUN_OSS_ACCESS_KEY_*。
 GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID=
 GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET=

jenkins/Jenkinsfile.production-api-build

@@ -25,7 +25,7 @@ pipeline {
     string(name: 'COMMIT_HASH', defaultValue: '', description: '可选，指定属于 SOURCE_BRANCH 的 Git commit')
     string(name: 'BUILD_VERSION', defaultValue: '', description: '发布版本号，留空则使用 Jenkins BUILD_NUMBER')
     string(name: 'NOTIFICATION_EMAILS', defaultValue: '', description: '本次运行追加通知邮箱；会与 Jenkins Secret Text 凭据 genarrative-notification-emails 合并发送')
-    booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: false, description: '是否额外构建并归档 Pingora 影子网关二进制')
+    booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: true, description: '是否构建并归档 Pingora 影子网关二进制；release 发布默认必须包含')
     booleanParam(name: 'PUBLISH_AFTER_BUILD', defaultValue: false, description: '构建成功后是否触发 API 发布')
     string(name: 'DEPLOY_JOB_NAME', defaultValue: 'Genarrative-Api-Deploy', description: 'API 发布流水线作业名')
     choice(name: 'DEPLOY_TARGET', choices: ['development', 'release'], description: 'PUBLISH_AFTER_BUILD=true 时的逻辑部署目标；development 使用当前 Linux 开发/构建/开发部署 agent')

jenkins/Jenkinsfile.production-api-deploy

@@ -16,7 +16,7 @@ pipeline {
     string(name: 'BUILD_VERSION', defaultValue: '', description: '待发布版本号')
     string(name: 'BUILD_JOB_NAME', defaultValue: 'Genarrative-Api-Build', description: 'API 构建流水线作业名')
     string(name: 'BUILD_NUMBER_TO_DEPLOY', defaultValue: '', description: '要复制归档产物的上游构建号')
-    booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: false, description: '上游构建是否包含 Pingora 影子网关产物')
+    booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: true, description: '上游构建是否包含 Pingora 影子网关产物；release 发布默认必须包含')
     string(name: 'RELEASE_ROOT', defaultValue: '/opt/genarrative/releases', description: '生产 release 根目录')
     string(name: 'CURRENT_LINK', defaultValue: '/opt/genarrative/current', description: '当前版本软链接')
     string(name: 'SERVICE_NAME', defaultValue: 'genarrative-api.service', description: 'systemd 服务名')
@@ -93,12 +93,17 @@ pipeline {
           bash -lc '
             set -euo pipefail
             chmod +x "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-on.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-off.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs"
+            pingora_deploy_args=()
+            if [[ "${INCLUDE_PINGORA_GATEWAY:-false}" == "true" ]]; then
+              pingora_deploy_args+=(--require-pingora-gateway)
+            fi
             "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" \
               --source-dir "build/${BUILD_VERSION}" \
               --version "${BUILD_VERSION}" \
               --release-root "${RELEASE_ROOT}" \
               --current-link "${CURRENT_LINK}" \
               --service "${SERVICE_NAME}" \
+              "${pingora_deploy_args[@]}" \
               --health-url "${HEALTH_URL}" \
               --api-env-file "${API_ENV_FILE:-/etc/genarrative/api-server.env}" \
               --database "${DATABASE}" \

jenkins/Jenkinsfile.production-full-build-and-deploy

@@ -25,6 +25,7 @@ pipeline {
     booleanParam(name: 'RUN_NPM_CI', defaultValue: true, description: 'Web 构建前是否执行 npm ci')
     string(name: 'NOTIFICATION_EMAILS', defaultValue: '', description: '本次运行追加通知邮箱；会与 Jenkins Secret Text 凭据 genarrative-notification-emails 合并发送')
     string(name: 'MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID', defaultValue: '', description: '可选，透传给 Stdb module 构建的迁移 bootstrap secret 凭据 ID；留空则由 Stdb 构建自动生成')
+    booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: true, description: 'API release 是否构建、归档并部署 Pingora 影子网关；release 默认必须包含')
     string(name: 'WEB_BUILD_JOB_NAME', defaultValue: 'Genarrative-Web-Build', description: 'Web 构建流水线作业名')
     string(name: 'API_BUILD_JOB_NAME', defaultValue: 'Genarrative-Api-Build', description: 'API 构建流水线作业名')
     string(name: 'STDB_BUILD_JOB_NAME', defaultValue: 'Genarrative-Stdb-Module-Build', description: 'Stdb 构建流水线作业名')
@@ -134,6 +135,7 @@ pipeline {
                   string(name: 'COMMIT_HASH', value: env.SOURCE_COMMIT),
                   string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION),
                   string(name: 'NOTIFICATION_EMAILS', value: params.NOTIFICATION_EMAILS ?: ''),
+                  booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', value: params.INCLUDE_PINGORA_GATEWAY),
                 ]
               env.API_BUILD_NUMBER = apiRun.number.toString()
             }
@@ -196,6 +198,7 @@ pipeline {
             booleanParam(name: 'CONFIRM_RELEASE_DEPLOY_AGENT', value: params.CONFIRM_RELEASE_DEPLOY_AGENT),
             string(name: 'BUILD_JOB_NAME', value: params.API_BUILD_JOB_NAME),
             string(name: 'BUILD_NUMBER_TO_DEPLOY', value: env.API_BUILD_NUMBER),
+            booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', value: params.INCLUDE_PINGORA_GATEWAY),
             string(name: 'DATABASE', value: params.DATABASE),
             string(name: 'SPACETIME_SERVER_URL', value: params.SPACETIME_SERVER_URL ?: ''),
           ]
@@ -234,6 +237,7 @@ pipeline {
           string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION ?: (params.BUILD_VERSION ?: '')),
           string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET ?: ''),
           string(name: 'DATABASE', value: params.DATABASE ?: ''),
+          string(name: 'INCLUDE_PINGORA_GATEWAY', value: String.valueOf(params.INCLUDE_PINGORA_GATEWAY)),
           string(name: 'SUMMARY', value: '全量构建发布编排结束'),
         ]
         def notificationRecipients = params.NOTIFICATION_EMAILS?.trim()

package.json

@@ -29,6 +29,7 @@
     "check:encoding": "node scripts/check-encoding.mjs",
     "check:spacetime-schema": "node scripts/check-spacetime-schema-guard.mjs",
     "check:production-ops": "node scripts/check-production-ops-guardrails.mjs",
+    "check:database-backup": "node scripts/check-database-backup-to-oss.mjs",
     "check:production-health-patrol": "node scripts/check-production-health-patrol.mjs",
     "check:production-health-patrol-env": "node scripts/check-production-health-patrol-env-check.mjs",
     "check:production-api-release": "node scripts/check-production-api-release.mjs",

scripts/check-database-backup-to-oss.mjs

@@ -0,0 +1,167 @@
+#!/usr/bin/env node
+
+import {spawnSync} from 'node:child_process';
+import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import path from 'node:path';
+
+const BACKUP_SCRIPT = path.resolve('scripts/database-backup-to-oss.mjs');
+const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-database-backup-check-'));
+const failures = [];
+
+try {
+  main();
+} finally {
+  rmSync(tmpRoot, {recursive: true, force: true});
+}
+
+if (failures.length > 0) {
+  console.error('[check:database-backup] FAILED');
+  for (const failure of failures) {
+    console.error(`- ${failure}`);
+  }
+  process.exit(1);
+}
+
+console.log('[check:database-backup] OK');
+
+function main() {
+  assertInsufficientSpaceStopsBeforeServiceChanges();
+  assertArchiveFailureStillRestoresDependentServices();
+}
+
+function assertInsufficientSpaceStopsBeforeServiceChanges() {
+  const fixture = createFixture('insufficient-space');
+  const result = runBackup(fixture, [
+    '--stop-service',
+    'spacetimedb.service',
+    '--restart-service-after',
+    'genarrative-api.service',
+    '--min-free-bytes',
+    '999999999999999999',
+  ]);
+
+  assertStatus(result, 1, '空间不足时必须失败。');
+  assertIncludes(result.stdout, '备份空间预检', '空间不足失败前应打印空间预检。');
+  assertIncludes(result.stderr, '剩余空间不足', '空间不足失败应说明剩余空间不足。');
+  assertFileMissing(fixture.systemctlLog, '空间不足时不能调用 systemctl。');
+  assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
+}
+
+function assertArchiveFailureStillRestoresDependentServices() {
+  const fixture = createFixture('tar-failure');
+  const result = runBackup(fixture, [
+    '--stop-service',
+    'spacetimedb.service',
+    '--restart-service-after',
+    'genarrative-api.service',
+    '--restart-service-after',
+    'genarrative-external-generation-worker@1.service',
+    '--restart-service-after',
+    'genarrative-external-generation-controller.service',
+    '--min-free-bytes',
+    '1',
+  ]);
+
+  assertStatus(result, 1, 'tar 失败时备份脚本必须失败。');
+  assertIncludes(result.stderr, 'fake tar failure', 'tar 失败原因应保留在错误输出中。');
+  const systemctlLog = readFile(fixture.systemctlLog);
+  const expectedCommands = [
+    'systemctl stop spacetimedb.service',
+    'systemctl start spacetimedb.service',
+    'systemctl restart genarrative-api.service',
+    'systemctl restart genarrative-external-generation-worker@1.service',
+    'systemctl restart genarrative-external-generation-controller.service',
+  ];
+  for (const command of expectedCommands) {
+    assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`);
+  }
+}
+
+function createFixture(name) {
+  const root = path.join(tmpRoot, name);
+  const binDir = path.join(root, 'bin');
+  const dataDir = path.join(root, 'data');
+  const workDir = path.join(root, 'work');
+  const systemctlLog = path.join(root, 'systemctl.log');
+  const tarLog = path.join(root, 'tar.log');
+  mkdirSync(binDir, {recursive: true});
+  mkdirSync(dataDir, {recursive: true});
+  writeFileSync(path.join(dataDir, 'sample.bin'), 'sample backup payload\n', 'utf8');
+  writeExecutable(
+    path.join(binDir, 'systemctl'),
+    `#!/usr/bin/env bash
+printf 'systemctl %s\\n' "$*" >> "${systemctlLog}"
+exit 0
+`,
+  );
+  writeExecutable(
+    path.join(binDir, 'tar'),
+    `#!/usr/bin/env bash
+printf 'tar %s\\n' "$*" >> "${tarLog}"
+echo 'fake tar failure' >&2
+exit 2
+`,
+  );
+  return {root, binDir, dataDir, workDir, systemctlLog, tarLog};
+}
+
+function runBackup(fixture, extraArgs = []) {
+  return spawnSync(
+    process.execPath,
+    [
+      BACKUP_SCRIPT,
+      '--data-dir',
+      fixture.dataDir,
+      '--work-dir',
+      fixture.workDir,
+      '--bucket',
+      'genarrative-test',
+      '--endpoint',
+      'oss-cn-shanghai.aliyuncs.com',
+      '--access-key-id',
+      'test',
+      '--access-key-secret',
+      'test',
+      ...extraArgs,
+    ],
+    {
+      cwd: process.cwd(),
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`,
+      },
+    },
+  );
+}
+
+function writeExecutable(filePath, content) {
+  writeFileSync(filePath, content, 'utf8');
+  spawnSync('chmod', ['0755', filePath], {encoding: 'utf8'});
+}
+
+function readFile(filePath) {
+  return existsSync(filePath) ? readFileSync(filePath, 'utf8') : '';
+}
+
+function assertStatus(result, expected, reason) {
+  const actual = result.status ?? 0;
+  if (actual !== expected) {
+    failures.push(
+      `${reason} 预期退出码 ${expected}，实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
+    );
+  }
+}
+
+function assertIncludes(content, expected, reason) {
+  if (!String(content).includes(expected)) {
+    failures.push(`${reason} 缺少: ${expected}`);
+  }
+}
+
+function assertFileMissing(filePath, reason) {
+  if (existsSync(filePath)) {
+    failures.push(`${reason} 实际存在: ${filePath}\n${readFile(filePath)}`);
+  }
+}

scripts/check-production-api-deploy.mjs

@@ -41,6 +41,9 @@ function main() {
   assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded();
   assertDeployRejectsPingoraPublicListenWhenArtifactIncluded();
   assertDeployRejectsPingoraArtifactMissingManifestEntry();
+  assertDeployRejectsPingoraManifestEntryMissingArtifact();
+  assertDeployRequiresPingoraWhenRequested();
+  assertReadinessFailureKeepsMaintenanceAfterCurrentSwitch();
   assertMissingReleaseManifestFails();
   assertReleaseManifestMissingApiArtifactFails();
   assertDeployRejectsDotVersion();
@@ -82,6 +85,18 @@ function readOptionalCommandsLog(fixture) {
   return readFileSync(fixture.commandsLog, 'utf8');
 }
 
+function assertMaintenanceCleared(fixture, reason) {
+  if (existsSync(fixture.maintenanceFile)) {
+    failures.push(`${reason} 时应退出本次打开的维护模式。`);
+  }
+}
+
+function assertMaintenanceKept(fixture, reason) {
+  if (!existsSync(fixture.maintenanceFile)) {
+    failures.push(`${reason} 时必须保持维护模式。`);
+  }
+}
+
 function assertDeployCopiesPingoraDirectReleaseDependencies() {
   const fixture = prepareFixture('with-direct-checks');
   const result = runDeploy(fixture);
@@ -409,9 +424,7 @@ function assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded() {
   ) {
     failures.push('direct-entry capability 存在时不能自动 restart Pingora。');
   }
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('direct-entry capability 导致部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, 'direct-entry capability 导致部署失败');
   assertNoReleasePromoted(
     fixture,
     'direct-entry capability 导致部署失败时不能提升正式 release。',
@@ -442,9 +455,7 @@ function assertDeployRejectsPingoraPublicListenWhenArtifactIncluded() {
   ) {
     failures.push('公网监听 env 存在时不能自动 restart Pingora。');
   }
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('公网监听 env 导致部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '公网监听 env 导致部署失败');
   assertNoReleasePromoted(
     fixture,
     '公网监听 env 导致部署失败时不能提升正式 release。',
@@ -464,8 +475,67 @@ function assertDeployRejectsPingoraArtifactMissingManifestEntry() {
     'release-manifest.json 缺少 pingora-gateway artifact',
     'manifest 未登记 Pingora 时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('manifest 未登记 Pingora 导致部署失败时必须保持维护模式。');
+  assertMaintenanceCleared(fixture, 'manifest 未登记 Pingora 导致部署失败');
+}
+
+function assertDeployRejectsPingoraManifestEntryMissingArtifact() {
+  const fixture = prepareFixture('pingora-manifest-entry-missing-artifact');
+  const manifestPath = path.join(fixture.sourceDir, 'release-manifest.json');
+  const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
+  manifest.artifacts.push({
+    component: 'pingora-gateway',
+    path: 'pingora-gateway',
+    checksum_path: 'pingora-gateway.sha256',
+  });
+  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
+  const result = runDeploy(fixture);
+
+  if (result.status === 0) {
+    failures.push('manifest 登记 Pingora 但发布目录缺少二进制时部署必须失败。');
+  }
+  assertIncludes(
+    result.stderr,
+    'release-manifest.json 登记了 pingora-gateway artifact',
+    'manifest 登记 Pingora 但文件缺失时必须给出明确错误。',
+  );
+  assertMaintenanceCleared(fixture, 'manifest 登记 Pingora 但文件缺失导致部署失败');
+}
+
+function assertDeployRequiresPingoraWhenRequested() {
+  const fixture = prepareFixture('require-pingora-missing-artifact');
+  const result = runDeploy(fixture, { requirePingoraGateway: true });
+
+  if (result.status === 0) {
+    failures.push('--require-pingora-gateway 但发布目录缺少 Pingora 时部署必须失败。');
+  }
+  assertIncludes(
+    result.stderr,
+    '本次部署要求 Pingora',
+    '--require-pingora-gateway 缺少 Pingora 文件时必须给出明确错误。',
+  );
+  assertMaintenanceCleared(fixture, '--require-pingora-gateway 缺少 Pingora 文件导致部署失败');
+}
+
+function assertReadinessFailureKeepsMaintenanceAfterCurrentSwitch() {
+  const fixture = prepareFixture('readiness-failure');
+  const result = runDeploy(fixture, { curlFails: true });
+
+  if (result.status === 0) {
+    failures.push('current 切换后的 readiness 失败必须让部署失败。');
+  }
+  assertIncludes(
+    result.stderr,
+    'readiness 检查超时',
+    'readiness 失败时必须给出明确错误。',
+  );
+  assertMaintenanceKept(fixture, 'current 切换后的 readiness 失败');
+
+  const releaseDir = path.join(fixture.releaseRoot, fixture.version);
+  const currentTarget = readlinkSync(fixture.currentLink);
+  if (currentTarget !== releaseDir) {
+    failures.push(
+      `readiness 失败发生在 current 切换后，current link 应指向新 release。实际 ${currentTarget}，预期 ${releaseDir}`,
+    );
   }
 }
 
@@ -482,9 +552,7 @@ function assertMissingReleaseManifestFails() {
     '发布产物缺少 release-manifest.json',
     '缺少 release-manifest.json 时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('缺少 release-manifest.json 导致部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 release-manifest.json 导致部署失败');
 }
 
 function assertReleaseManifestMissingApiArtifactFails() {
@@ -504,9 +572,7 @@ function assertReleaseManifestMissingApiArtifactFails() {
     'release-manifest.json 缺少 api-server artifact',
     'manifest 缺少 api-server artifact 时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('manifest 缺少 api-server artifact 导致部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, 'manifest 缺少 api-server artifact 导致部署失败');
 }
 
 function assertDeployRejectsDotVersion() {
@@ -679,9 +745,7 @@ function assertDeployCleansStagingReleaseOnFailure() {
   if (stagingEntries.length > 0) {
     failures.push(`部署失败时不应留下 staging release: ${stagingEntries.join(', ')}`);
   }
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, 'staging 构建中失败');
 }
 
 function assertDeployRejectsFinalReleaseRaceAndCleansStaging() {
@@ -714,9 +778,7 @@ function assertDeployRejectsFinalReleaseRaceAndCleansStaging() {
   if (stagingEntries.length > 0) {
     failures.push(`目标 release 竞态失败后不应留下 staging release: ${stagingEntries.join(', ')}`);
   }
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('目标 release 竞态失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '目标 release 竞态失败');
 }
 
 function assertMissingPingoraDirectCheckFails() {
@@ -732,9 +794,7 @@ function assertMissingPingoraDirectCheckFails() {
     '发布产物缺少 Pingora 直连 live smoke 脚本',
     '缺少 direct live smoke 脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 direct live smoke 脚本导致部署失败');
 }
 
 function assertMissingPingoraCanaryLiveFails() {
@@ -750,9 +810,7 @@ function assertMissingPingoraCanaryLiveFails() {
     '发布产物缺少 Pingora canary live smoke 脚本',
     '缺少 canary live smoke 脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 canary live smoke 脚本导致部署失败');
 }
 
 function assertMissingPingoraCanaryAccessLogParityFails() {
@@ -773,9 +831,7 @@ function assertMissingPingoraCanaryAccessLogParityFails() {
     '发布产物缺少 Pingora canary access log 对账脚本',
     '缺少 canary access log 对账脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 canary access log 对账脚本导致部署失败');
 }
 
 function assertMissingBackupScriptFails() {
@@ -791,9 +847,7 @@ function assertMissingBackupScriptFails() {
     '发布产物缺少数据库备份脚本',
     '缺少数据库备份脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少数据库备份脚本导致部署失败');
 }
 
 function assertMissingHealthPatrolScriptFails() {
@@ -811,9 +865,7 @@ function assertMissingHealthPatrolScriptFails() {
     '发布产物缺少生产健康巡检脚本',
     '缺少生产健康巡检脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少生产健康巡检脚本导致部署失败');
 }
 
 function assertMissingPingoraCurrentReleaseAuditFails() {
@@ -834,9 +886,7 @@ function assertMissingPingoraCurrentReleaseAuditFails() {
     '发布产物缺少 Pingora current release 自审脚本',
     '缺少 Pingora current release 自审脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora current release 自审脚本导致部署失败');
 }
 
 function assertMissingPingoraDirectRehearsalStatusFails() {
@@ -857,9 +907,7 @@ function assertMissingPingoraDirectRehearsalStatusFails() {
     '发布产物缺少 Pingora 直连彩排状态脚本',
     '缺少 Pingora 直连彩排状态脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora 直连彩排状态脚本导致部署失败');
 }
 
 function assertMissingPingoraCutoverStatusSnapshotFails() {
@@ -880,9 +928,7 @@ function assertMissingPingoraCutoverStatusSnapshotFails() {
     '发布产物缺少 Pingora 直连切换状态快照脚本',
     '缺少 Pingora 直连切换状态快照脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换状态快照脚本导致部署失败');
 }
 
 function assertMissingPingoraCutoverEvidenceBundleFails() {
@@ -903,9 +949,7 @@ function assertMissingPingoraCutoverEvidenceBundleFails() {
     '发布产物缺少 Pingora 直连切换证据包脚本',
     '缺少 Pingora 直连切换证据包脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换证据包脚本导致部署失败');
 }
 
 function assertMissingPingoraCutoverCommandEvidenceFails() {
@@ -926,9 +970,7 @@ function assertMissingPingoraCutoverCommandEvidenceFails() {
     '发布产物缺少 Pingora 直连切换命令证据脚本',
     '缺少 Pingora 直连切换命令证据脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换命令证据脚本导致部署失败');
 }
 
 function assertMissingPingoraCutoverEvidenceVerifyFails() {
@@ -949,9 +991,7 @@ function assertMissingPingoraCutoverEvidenceVerifyFails() {
     '发布产物缺少 Pingora 直连切换证据验真脚本',
     '缺少 Pingora 直连切换证据验真脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换证据验真脚本导致部署失败');
 }
 
 function assertMissingPingoraCutoverEvidenceAuditFails() {
@@ -974,9 +1014,7 @@ function assertMissingPingoraCutoverEvidenceAuditFails() {
     '发布产物缺少 Pingora 直连切换证据根目录审计脚本',
     '缺少 Pingora 直连切换证据根目录审计脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换证据根目录审计脚本导致部署失败');
 }
 
 function assertMissingHealthPatrolEnvCheckFails() {
@@ -994,9 +1032,7 @@ function assertMissingHealthPatrolEnvCheckFails() {
     '发布产物缺少生产健康巡检 env 复核脚本',
     '缺少生产健康巡检 env 复核脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少生产健康巡检 env 复核脚本导致部署失败');
 }
 
 function assertMissingPingoraReleaseReadinessFails() {
@@ -1016,9 +1052,7 @@ function assertMissingPingoraReleaseReadinessFails() {
     '发布产物缺少 Pingora release readiness 聚合门禁脚本',
     '缺少 Pingora release readiness 聚合门禁脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora release readiness 聚合门禁脚本导致部署失败');
 }
 
 function assertMissingPingoraHealthPatrolEnvSwitchFails() {
@@ -1039,9 +1073,7 @@ function assertMissingPingoraHealthPatrolEnvSwitchFails() {
     '发布产物缺少 Pingora health patrol env 切换脚本',
     '缺少 Pingora health patrol env 切换脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora health patrol env 切换脚本导致部署失败');
 }
 
 function assertMissingPingoraGatewayEnvShadowSwitchFails() {
@@ -1064,9 +1096,7 @@ function assertMissingPingoraGatewayEnvShadowSwitchFails() {
     '发布产物缺少 Pingora gateway env shadow 切换脚本',
     '缺少 Pingora gateway env shadow 切换脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora gateway env shadow 切换脚本导致部署失败');
 }
 
 function assertMissingPingoraRealpathCanaryEnableFails() {
@@ -1087,9 +1117,7 @@ function assertMissingPingoraRealpathCanaryEnableFails() {
     '发布产物缺少 Pingora realpath canary 启用脚本',
     '缺少 Pingora realpath canary 启用脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora realpath canary 启用脚本导致部署失败');
 }
 
 function assertMissingPingoraRealpathCanaryDisableFails() {
@@ -1110,9 +1138,7 @@ function assertMissingPingoraRealpathCanaryDisableFails() {
     '发布产物缺少 Pingora realpath canary 关闭脚本',
     '缺少 Pingora realpath canary 关闭脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora realpath canary 关闭脚本导致部署失败');
 }
 
 function assertMissingPingoraTlsCertSyncFails() {
@@ -1133,9 +1159,7 @@ function assertMissingPingoraTlsCertSyncFails() {
     '发布产物缺少 Pingora TLS 证书同步脚本',
     '缺少 Pingora TLS 证书同步脚本时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少 Pingora TLS 证书同步脚本导致部署失败');
 }
 
 function assertMissingEnvExamplesFails() {
@@ -1154,9 +1178,7 @@ function assertMissingEnvExamplesFails() {
     '发布产物缺少环境变量示例目录',
     '缺少环境变量示例目录时必须给出明确错误。',
   );
-  if (!existsSync(fixture.maintenanceFile)) {
-    failures.push('部署失败时必须保持维护模式。');
-  }
+  assertMaintenanceCleared(fixture, '缺少环境变量示例目录导致部署失败');
 }
 
 function prepareFixture(name) {
@@ -1432,6 +1454,19 @@ function prepareFixture(name) {
     [
       '#!/usr/bin/env bash',
       `printf 'curl %s\\n' "$*" >> ${shellQuote(commandsLog)}`,
+      'if [[ "${FAKE_CURL_FAIL:-false}" == "true" ]]; then',
+      '  exit 22',
+      'fi',
+      'exit 0',
+      '',
+    ].join('\n'),
+    'utf8',
+  );
+  writeFileSync(
+    path.join(fakeBin, 'sleep'),
+    [
+      '#!/usr/bin/env bash',
+      `printf 'sleep %s\\n' "$*" >> ${shellQuote(commandsLog)}`,
       'exit 0',
       '',
     ].join('\n'),
@@ -1478,6 +1513,7 @@ function prepareFixture(name) {
   );
   chmodExecutable(path.join(fakeBin, 'systemctl'));
   chmodExecutable(path.join(fakeBin, 'curl'));
+  chmodExecutable(path.join(fakeBin, 'sleep'));
   chmodExecutable(path.join(fakeBin, 'cp'));
   chmodExecutable(path.join(fakeBin, 'sudo'));
 
@@ -1543,29 +1579,33 @@ function runDeploy(fixture, options = {}) {
     fixture.sourceDir,
     'scripts/deploy/production-api-deploy.sh',
   );
+  const args = [
+    deployScript,
+    '--source-dir',
+    fixture.sourceDir,
+    '--version',
+    options.version ?? fixture.version,
+    '--release-root',
+    options.releaseRoot ?? fixture.releaseRoot,
+    '--current-link',
+    options.currentLink ?? fixture.currentLink,
+    '--service',
+    'genarrative-api.service',
+    '--health-url',
+    'http://127.0.0.1:18082/readyz',
+    '--api-env-file',
+    options.apiEnvFile ?? fixture.apiEnvFile,
+    '--database',
+    'genarrative-prod',
+    '--spacetime-server-url',
+    'http://127.0.0.1:3101',
+  ];
+  if (options.requirePingoraGateway) {
+    args.push('--require-pingora-gateway');
+  }
   return spawnSync(
     'bash',
-    [
-      deployScript,
-      '--source-dir',
-      fixture.sourceDir,
-      '--version',
-      options.version ?? fixture.version,
-      '--release-root',
-      options.releaseRoot ?? fixture.releaseRoot,
-      '--current-link',
-      options.currentLink ?? fixture.currentLink,
-      '--service',
-      'genarrative-api.service',
-      '--health-url',
-      'http://127.0.0.1:18082/readyz',
-      '--api-env-file',
-      options.apiEnvFile ?? fixture.apiEnvFile,
-      '--database',
-      'genarrative-prod',
-      '--spacetime-server-url',
-      'http://127.0.0.1:3101',
-    ],
+    args,
     {
       cwd: process.cwd(),
       encoding: 'utf8',
@@ -1578,6 +1618,7 @@ function runDeploy(fixture, options = {}) {
           options.pingoraDirectEntry === true ? 'true' : 'false',
         FAKE_PINGORA_ENV_FILE: fixture.pingoraEnvFile,
         FAKE_PINGORA_STATE_FILE: fixture.pingoraStateFile,
+        FAKE_CURL_FAIL: options.curlFails === true ? 'true' : 'false',
         FAKE_CREATE_RELEASE_DURING_COPY:
           options.createReleaseDuringCopy === true ? 'true' : 'false',
         FAKE_RELEASE_ROOT: fixture.releaseRoot,

scripts/check-production-ops-guardrails.mjs

@@ -30,6 +30,31 @@ const checks = [
     reason:
       '生产冷备份 service 必须用 node -- 分隔脚本参数，避免 Node 22 抢占业务 --env-file。',
   },
+  {
+    file: 'scripts/database-backup-to-oss.mjs',
+    includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})',
+    reason: '生产冷备份必须先做工作目录剩余空间预检，避免停库后写满磁盘。',
+  },
+  {
+    file: 'scripts/database-backup-to-oss.mjs',
+    includes: 'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter})',
+    reason: '生产冷备份打包失败时也必须恢复 SpacetimeDB 及依赖服务。',
+  },
+  {
+    file: 'package.json',
+    includes: '"check:database-backup": "node scripts/check-database-backup-to-oss.mjs"',
+    reason: '数据库冷备份事故路径必须有本地 smoke 检查入口。',
+  },
+  {
+    file: 'scripts/check-database-backup-to-oss.mjs',
+    includes: 'assertArchiveFailureStillRestoresDependentServices',
+    reason: '数据库冷备份 smoke 必须覆盖 tar 失败后恢复依赖服务。',
+  },
+  {
+    file: 'scripts/check-database-backup-to-oss.mjs',
+    includes: 'assertInsufficientSpaceStopsBeforeServiceChanges',
+    reason: '数据库冷备份 smoke 必须覆盖空间不足时不触碰 systemctl。',
+  },
   {
     file: 'deploy/systemd/genarrative-health-patrol.service',
     includes: 'scripts/ops/production-health-patrol.mjs',
@@ -4756,11 +4781,29 @@ const checks = [
     reason:
       'API deploy 动态烟测必须覆盖 Pingora 二进制与 release manifest 漂移负例。',
   },
+  {
+    file: 'scripts/check-production-api-deploy.mjs',
+    includes: 'assertDeployRequiresPingoraWhenRequested',
+    reason:
+      'API deploy 动态烟测必须覆盖 --require-pingora-gateway 缺失产物负例。',
+  },
+  {
+    file: 'scripts/check-production-api-deploy.mjs',
+    includes: 'assertReadinessFailureKeepsMaintenanceAfterCurrentSwitch',
+    reason:
+      'API deploy 动态烟测必须覆盖 current 切换后的 readiness 失败仍保留维护模式。',
+  },
+  {
+    file: 'scripts/deploy/production-api-deploy.sh',
+    includes: '--require-pingora-gateway',
+    reason:
+      'API deploy 必须支持 Jenkins release 对 Pingora 产物做硬要求。',
+  },
   {
     file: 'scripts/check-production-api-deploy.mjs',
     includes: 'assertMissingReleaseManifestFails',
     reason:
-      'API deploy 动态烟测必须覆盖缺少 release manifest 的失败维护模式。',
+      'API deploy 动态烟测必须覆盖缺少 release manifest 的失败维护回滚。',
   },
   {
     file: 'scripts/check-production-api-deploy.mjs',
@@ -4831,18 +4874,18 @@ const checks = [
   {
     file: 'scripts/check-production-api-deploy.mjs',
     includes: 'assertMissingBackupScriptFails',
-    reason: 'API deploy 动态烟测必须覆盖缺少数据库备份脚本的失败维护模式。',
+    reason: 'API deploy 动态烟测必须覆盖缺少数据库备份脚本的失败维护回滚。',
   },
   {
     file: 'scripts/check-production-api-deploy.mjs',
     includes: 'assertMissingHealthPatrolScriptFails',
-    reason: 'API deploy 动态烟测必须覆盖缺少健康巡检脚本的失败维护模式。',
+    reason: 'API deploy 动态烟测必须覆盖缺少健康巡检脚本的失败维护回滚。',
   },
   {
     file: 'scripts/check-production-api-deploy.mjs',
     includes: 'assertMissingHealthPatrolEnvCheckFails',
     reason:
-      'API deploy 动态烟测必须覆盖缺少健康巡检 env 复核脚本的失败维护模式。',
+      'API deploy 动态烟测必须覆盖缺少健康巡检 env 复核脚本的失败维护回滚。',
   },
   {
     file: 'scripts/check-production-api-deploy.mjs',

scripts/database-backup-to-oss.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {spawnSync} from 'node:child_process';
 import {createHash, createHmac} from 'node:crypto';
-import {createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs';
+import {createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, statfsSync, writeFileSync} from 'node:fs';
 import {basename, dirname, isAbsolute, resolve} from 'node:path';
 import {fileURLToPath} from 'node:url';
 
@@ -12,6 +12,8 @@ const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/
 const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups');
 const DEFAULT_PRODUCTION_DATA_DIR = '/stdb';
 const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups';
+const DEFAULT_SPACE_SAFETY_RATIO = 1.1;
+const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024;
 const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
 const OSS_SERVICE = 'oss';
 const OSS_REQUEST = 'aliyun_v4_request';
@@ -36,6 +38,7 @@ function usage() {
   GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX      对象前缀，默认 database-backups
   GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT    OSS endpoint；未设置时回退 ALIYUN_OSS_ENDPOINT
   GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL      true 时保留本地 tar.gz
+  GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES  备份前要求 work dir 所在文件系统至少有这些可用字节；未设置时按数据目录大小估算
   ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET
 `);
 }
@@ -108,6 +111,7 @@ function parseArgs(argv) {
     manifestFile: '',
     objectKey: '',
     resultFile: '',
+    minFreeBytes: '',
   };
 
   for (let index = 0; index < argv.length; index += 1) {
@@ -182,6 +186,9 @@ function parseArgs(argv) {
       case '--result-file':
         options.resultFile = readValue();
         break;
+      case '--min-free-bytes':
+        options.minFreeBytes = readValue();
+        break;
       default:
         throw new Error(`未知参数: ${arg}`);
     }
@@ -253,6 +260,119 @@ function runCommand(command, args, options = {}) {
   return result;
 }
 
+function parseByteSize(rawValue, label) {
+  const value = String(rawValue ?? '').trim();
+  if (!value) {
+    return null;
+  }
+  const match = /^(\d+)(?:\s*([KMGTPE]?)(?:I?B?)?)?$/iu.exec(value);
+  if (!match) {
+    throw new Error(`${label} 必须是字节数或 K/M/G/T/P/E 后缀大小，实际: ${rawValue}`);
+  }
+  const [, amountText, unitText = ''] = match;
+  const multipliers = {
+    '': 1n,
+    K: 1024n,
+    M: 1024n ** 2n,
+    G: 1024n ** 3n,
+    T: 1024n ** 4n,
+    P: 1024n ** 5n,
+    E: 1024n ** 6n,
+  };
+  return BigInt(amountText) * multipliers[unitText.toUpperCase()];
+}
+
+function formatBytes(bytes) {
+  const value = BigInt(bytes);
+  const gib = Number(value) / (1024 ** 3);
+  if (gib >= 1) {
+    return `${gib.toFixed(1)}GiB`;
+  }
+  const mib = Number(value) / (1024 ** 2);
+  if (mib >= 1) {
+    return `${mib.toFixed(1)}MiB`;
+  }
+  return `${value}B`;
+}
+
+function getDirectorySizeBytes(dataDir) {
+  const result = runCommand('du', ['-sk', dataDir]);
+  const [sizeKbText] = String(result.stdout ?? '').trim().split(/\s+/u);
+  if (!sizeKbText || !/^\d+$/u.test(sizeKbText)) {
+    throw new Error(`无法解析数据目录大小: ${result.stdout}`);
+  }
+  return BigInt(sizeKbText) * 1024n;
+}
+
+function getAvailableBytes(fileSystemPath) {
+  const stat = statfsSync(fileSystemPath, {bigint: true});
+  return stat.bavail * stat.bsize;
+}
+
+function parseSafetyRatio(rawValue) {
+  const value = String(rawValue ?? '').trim();
+  if (!value) {
+    return DEFAULT_SPACE_SAFETY_RATIO;
+  }
+  const ratio = Number(value);
+  if (!Number.isFinite(ratio) || ratio < 1) {
+    throw new Error(`GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO 必须是 >= 1 的数字，实际: ${rawValue}`);
+  }
+  return ratio;
+}
+
+function calculateRequiredFreeBytes({dataSizeBytes, args, env}) {
+  const explicitMinFreeBytes = parseByteSize(
+    firstNonEmpty(args.minFreeBytes, env.GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES),
+    'GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES',
+  );
+  if (explicitMinFreeBytes !== null) {
+    return explicitMinFreeBytes;
+  }
+  const ratio = parseSafetyRatio(env.GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO);
+  const ratioBasisPoints = BigInt(Math.ceil(ratio * 10000));
+  const ratioRequirement = (dataSizeBytes * ratioBasisPoints + 9999n) / 10000n;
+  const extraFreeBytes = parseByteSize(
+    firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_EXTRA_FREE_BYTES, String(DEFAULT_EXTRA_FREE_BYTES)),
+    'GENARRATIVE_DATABASE_BACKUP_EXTRA_FREE_BYTES',
+  );
+  const extraRequirement = dataSizeBytes + extraFreeBytes;
+  return ratioRequirement > extraRequirement ? ratioRequirement : extraRequirement;
+}
+
+function assertSufficientWorkDirSpace({dataDir, workDir, args, env}) {
+  mkdirSync(workDir, {recursive: true});
+  const dataSizeBytes = getDirectorySizeBytes(dataDir);
+  const availableBytes = getAvailableBytes(workDir);
+  const requiredFreeBytes = calculateRequiredFreeBytes({dataSizeBytes, args, env});
+  console.log(
+    `[database-backup] 备份空间预检: data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`,
+  );
+  if (availableBytes < requiredFreeBytes) {
+    throw new Error(
+      [
+        `备份工作目录所在文件系统剩余空间不足: ${workDir}`,
+        `available=${formatBytes(availableBytes)}`,
+        `required=${formatBytes(requiredFreeBytes)}`,
+        `dataDir=${dataDir}`,
+        `dataSize=${formatBytes(dataSizeBytes)}`,
+        '为避免停库后写满磁盘，本次备份已在停止服务前中止。',
+      ].join('；'),
+    );
+  }
+}
+
+function collectRestartServicesAfterBackup({args, env}) {
+  const serviceNames = [
+    ...String(env.GENARRATIVE_DATABASE_BACKUP_RESTART_SERVICE_AFTER ?? '')
+      .split(',')
+      .map((value) => value.trim())
+      .filter(Boolean),
+    ...args.restartServicesAfter,
+  ];
+  return [...new Set(serviceNames.filter(Boolean))];
+}
+
 function stopServiceIfNeeded(serviceName) {
   if (!serviceName) {
     return false;
@@ -271,12 +391,37 @@ function startServiceIfNeeded(serviceName, wasStopped) {
 }
 
 function restartServicesAfterBackup(serviceNames) {
+  const errors = [];
   for (const serviceName of serviceNames) {
     if (!serviceName) {
       continue;
     }
     console.log(`[database-backup] 冷备份后重启依赖服务: ${serviceName}`);
-    runCommand('systemctl', ['restart', serviceName], {stdio: 'inherit'});
+    try {
+      runCommand('systemctl', ['restart', serviceName], {stdio: 'inherit'});
+    } catch (error) {
+      errors.push(error);
+    }
+  }
+  if (errors.length > 0) {
+    throw new AggregateError(errors, `冷备份后重启依赖服务失败: ${errors.map((error) => error.message).join('; ')}`);
+  }
+}
+
+function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}) {
+  const errors = [];
+  try {
+    startServiceIfNeeded(stopService, serviceStopped);
+  } catch (error) {
+    errors.push(error);
+  }
+  try {
+    restartServicesAfterBackup(restartServicesAfter);
+  } catch (error) {
+    errors.push(error);
+  }
+  if (errors.length > 0) {
+    throw new AggregateError(errors, `恢复冷备份相关服务失败: ${errors.map((error) => error.message).join('; ')}`);
   }
 }
 
@@ -523,19 +668,36 @@ async function main() {
 
   let archivePath = '';
   let serviceStopped = false;
+  let backupError = null;
+  let restoreError = null;
+  const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE);
+  const restartServicesAfter = collectRestartServicesAfterBackup({args, env});
   try {
-    serviceStopped = stopServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE));
+    assertSufficientWorkDirSpace({dataDir, workDir, args, env});
+    serviceStopped = stopServiceIfNeeded(stopService);
     archivePath = createArchive({dataDir, workDir, fileName});
+  } catch (error) {
+    backupError = error;
   } finally {
-    startServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE), serviceStopped);
+    try {
+      if (serviceStopped) {
+        restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter});
+      } else if (!backupError) {
+        restartServicesAfterBackup(restartServicesAfter);
+      }
+    } catch (error) {
+      restoreError = error;
+    }
+  }
+  if (backupError) {
+    if (restoreError) {
+      throw new AggregateError([backupError, restoreError], `数据库备份失败，且恢复依赖服务时也失败: ${backupError.message}; ${restoreError.message}`);
+    }
+    throw backupError;
+  }
+  if (restoreError) {
+    throw restoreError;
   }
-  restartServicesAfterBackup([
-    ...String(env.GENARRATIVE_DATABASE_BACKUP_RESTART_SERVICE_AFTER ?? '')
-      .split(',')
-      .map((value) => value.trim())
-      .filter(Boolean),
-    ...args.restartServicesAfter,
-  ]);
 
   const manifestPath = `${archivePath}.manifest.json`;
   writeManifest({

scripts/deploy/production-api-deploy.sh

@@ -5,14 +5,14 @@ set -euo pipefail
 usage() {
   cat <<'EOF'
 用法:
-  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
+  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
 
 说明:
   进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
   默认同时重启外部生成 worker controller 和已加载的 worker 实例；未启用 worker 单元时会自动跳过。
   若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。
-  若发布包包含 pingora-gateway，部署脚本会在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，再启动或重启 Pingora 影子服务并复核 active。
-  失败时保留维护模式。
+  若发布包包含 pingora-gateway，或传入 --require-pingora-gateway，部署脚本会要求 release manifest、二进制与 checksum 一致，再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，启动或重启 Pingora 影子服务并复核 active。
+  current 链接切换前失败时会退出本次打开的维护模式；current 链接切换后失败时保留维护模式，避免暴露半发布版本。
 EOF
 }
 
@@ -347,17 +347,20 @@ ensure_pingora_shadow_service() {
 validate_release_manifest() {
   local manifest_path="$1"
   local require_pingora="$2"
+  local source_dir="$3"
 
   if [[ ! -f "${manifest_path}" ]]; then
     echo "[production-api-deploy] 发布产物缺少 release-manifest.json: ${manifest_path}" >&2
     exit 1
   fi
 
-  node - "${manifest_path}" "${require_pingora}" <<'NODE'
+  node - "${manifest_path}" "${require_pingora}" "${source_dir}" <<'NODE'
 const fs = require('fs');
+const path = require('path');
 
 const manifestPath = process.argv[2];
 const requirePingora = process.argv[3] === '1';
+const sourceDir = process.argv[4];
 let manifest;
 try {
   manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
@@ -380,6 +383,16 @@ if (requirePingora) {
     process.exit(1);
   }
 }
+
+const hasPingora = artifacts.some((artifact) => artifact?.path === 'pingora-gateway');
+if (hasPingora) {
+  const binaryPath = path.join(sourceDir, 'pingora-gateway');
+  const checksumPath = path.join(sourceDir, 'pingora-gateway.sha256');
+  if (!fs.existsSync(binaryPath) || !fs.existsSync(checksumPath)) {
+    console.error('[production-api-deploy] release-manifest.json 登记了 pingora-gateway artifact，但发布目录缺少 pingora-gateway 或 pingora-gateway.sha256。');
+    process.exit(1);
+  }
+}
 NODE
 }
 
@@ -576,6 +589,9 @@ DATABASE=""
 SPACETIME_SERVER_URL=""
 DEPLOY_COMPLETED=0
 PINGORA_INCLUDED=0
+REQUIRE_PINGORA_GATEWAY=0
+MAINTENANCE_ENABLED_BY_DEPLOY=0
+CURRENT_LINK_SWITCHED=0
 RELEASE_DIR=""
 STAGING_RELEASE_DIR=""
 
@@ -609,6 +625,10 @@ while [[ $# -gt 0 ]]; do
       PINGORA_SERVICE_NAME="${2:?缺少 --pingora-service 的值}"
       shift 2
       ;;
+    --require-pingora-gateway)
+      REQUIRE_PINGORA_GATEWAY=1
+      shift
+      ;;
     --worker-service-pattern)
       WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}"
       shift 2
@@ -709,7 +729,14 @@ on_exit() {
   local exit_code=$?
   if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
     cleanup_staging_release
-    echo "[production-api-deploy] 部署失败，保持维护模式。" >&2
+    if [[ "${MAINTENANCE_ENABLED_BY_DEPLOY}" -eq 1 && "${CURRENT_LINK_SWITCHED}" -ne 1 ]]; then
+      echo "[production-api-deploy] 部署失败且尚未切换 current，退出本次打开的维护模式。" >&2
+      if ! bash "${SCRIPT_DIR}/maintenance-off.sh"; then
+        echo "[production-api-deploy] 退出维护模式失败，请人工检查维护文件。" >&2
+      fi
+    else
+      echo "[production-api-deploy] 部署失败，current 可能已切换或维护模式不是本次打开，保持维护模式。" >&2
+    fi
   fi
   exit "${exit_code}"
 }
@@ -717,6 +744,7 @@ on_exit() {
 trap on_exit EXIT
 
 bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}"
+MAINTENANCE_ENABLED_BY_DEPLOY=1
 
 echo "[production-api-deploy] 校验 api-server"
 (
@@ -730,10 +758,14 @@ echo "[production-api-deploy] 校验 api-server"
     sha256sum -c pingora-gateway.sha256
   fi
 )
+if [[ "${REQUIRE_PINGORA_GATEWAY}" -eq 1 && ( ! -f "${SOURCE_DIR}/pingora-gateway" || ! -f "${SOURCE_DIR}/pingora-gateway.sha256" ) ]]; then
+  echo "[production-api-deploy] 本次部署要求 Pingora，但发布目录缺少 pingora-gateway 或 pingora-gateway.sha256。" >&2
+  exit 1
+fi
 if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then
   PINGORA_INCLUDED=1
 fi
-validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "${PINGORA_INCLUDED}"
+validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "$(( PINGORA_INCLUDED || REQUIRE_PINGORA_GATEWAY ))" "${SOURCE_DIR}"
 
 mkdir -p "${RELEASE_ROOT}"
 mkdir "${STAGING_RELEASE_DIR}"
@@ -960,6 +992,7 @@ fi
 mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}"
 STAGING_RELEASE_DIR=""
 ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}"
+CURRENT_LINK_SWITCHED=1
 
 if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
   ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}"