发布前备份空间预检前置并支持自动降级

- database-backup-to-oss.mjs 新增 --check-space-only：archive 按 data×1.1、files 按 max(data×0.05, 2GiB) 计算门槛，空间不足以退出码 3 返回
- production-stdb-publish.sh 在进入维护与停服务之前先做空间预检，archive 不足时自动降级 files（files 不支持 defer，async 收敛为 sync），可用环境变量关闭
- 尚未开始 publish 的失败自动恢复本次停掉的 API/controller/worker 并退出维护，只有真正开始发布之后的失败才保持维护态
- 备份检查脚本与生产运维门禁补上新口径与执行顺序断言，运维文档与共享记忆记录事故与规则

scripts/check-database-backup-to-oss.mjs

@@ -26,6 +26,7 @@ import {
   cleanupHistoryCandidates,
   collectDirectFileEntries,
   createUploadBandwidthLimiter,
+  describeBackupSpaceRequirement,
   discoverDeferredArchiveUploads,
   discoverHistoryPlan,
   restoreDirectFilesBackup,
@@ -64,6 +65,7 @@ async function main() {
   assertDeferredArchiveDiscoveryIsBoundedAndDeterministic();
   assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
   assertInsufficientSpaceStopsBeforeServiceChanges();
+  assertCheckSpaceOnlyUsesFormatSpecificRequirement();
   assertStopFailureRetainsRecoveryMarker();
   assertArchiveFailureStillRestoresDependentServices();
   await assertMultipartUploadRetriesAndVerifiesRemoteLength();
@@ -1198,7 +1200,7 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() {
     '999999999999999999',
   ]);
 
-  assertStatus(result, 1, '空间不足时必须失败。');
+  assertStatus(result, 3, '空间不足必须用独立退出码 3 失败。');
   assertIncludes(
     result.stdout,
     '备份空间预检',
@@ -1213,6 +1215,66 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() {
   assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
 }
 
+function assertCheckSpaceOnlyUsesFormatSpecificRequirement() {
+  const fixture = createFixture('check-space-only');
+
+  const filesOk = runBackup(
+    fixture,
+    ['--check-space-only', '--storage-format', 'files'],
+    { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1M' },
+  );
+  assertStatus(filesOk, 0, 'files 模式空间预检应在阈值满足时通过。');
+  assertIncludes(
+    filesOk.stdout,
+    '备份空间预检(files)',
+    'files 预检必须打印 files 口径的预检结果。',
+  );
+  assertIncludes(
+    filesOk.stdout,
+    '空间预检通过(check-space-only)',
+    'check-space-only 通过时必须给出显式成功标记。',
+  );
+  assertFileMissing(fixture.systemctlLog, '空间预检不得调用 systemctl。');
+  assertFileMissing(fixture.tarLog, '空间预检不得调用 tar。');
+
+  const filesInsufficient = runBackup(
+    fixture,
+    ['--check-space-only', '--storage-format', 'files'],
+    { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1000T' },
+  );
+  assertStatus(filesInsufficient, 3, 'files 空间不足同样使用退出码 3。');
+
+  const archiveInsufficient = runBackup(
+    fixture,
+    ['--check-space-only', '--storage-format', 'archive'],
+    { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1000T' },
+  );
+  assertStatus(archiveInsufficient, 3, 'archive 空间不足必须使用退出码 3。');
+
+  const archive = describeBackupSpaceRequirement({
+    dataDir: fixture.dataDir,
+    workDir: fixture.workDir,
+    storageFormat: 'archive',
+    args: {},
+    env: { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1G' },
+  });
+  const files = describeBackupSpaceRequirement({
+    dataDir: fixture.dataDir,
+    workDir: fixture.workDir,
+    storageFormat: 'files',
+    args: {},
+    env: { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1G' },
+  });
+  if (archive.requiredFreeBytes !== 1024n ** 3n || files.requiredFreeBytes !== 1024n ** 3n) {
+    failures.push(
+      `空间口径覆盖参数应生效：archive=${archive.requiredFreeBytes} files=${files.requiredFreeBytes}`,
+    );
+  }
+  if (archive.storageFormat !== 'archive' || files.storageFormat !== 'files') {
+    failures.push('空间预检结果必须回显实际 storage-format。');
+  }
+}
+
 function assertStopFailureRetainsRecoveryMarker() {
   const fixture = createFixture('stop-failure-marker');
   writeExecutable(
@@ -2695,7 +2757,7 @@ exit 2
   return { root, binDir, dataDir, workDir, systemctlLog, tarLog };
 }
 
-function runBackup(fixture, extraArgs = []) {
+function runBackup(fixture, extraArgs = [], envOverrides = {}) {
   return spawnSync(
     process.execPath,
     [
@@ -2720,6 +2782,7 @@ function runBackup(fixture, extraArgs = []) {
       env: {
         ...process.env,
         PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`,
+        ...envOverrides,
       },
     },
   );

scripts/check-production-ops-guardrails.mjs

@@ -258,6 +258,44 @@ const checks = [
     reason:
       'Stdb 先于 API 发布时必须先补齐 api-server env 的 FILE 路径，保证首次 rollout 重启即可读取 secret。',
   },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes:
+      'precheck_backup_space_before_maintenance\n\n"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"',
+    reason:
+      '备份空间预检必须先于进入维护模式执行：磁盘不足时不得停服务或把生产留在维护态。',
+  },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes: '--check-space-only',
+    reason:
+      '生产 Stdb publish 必须复用备份脚本的 --check-space-only 预检，保证空间口径与真实备份一致。',
+  },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes: 'GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK',
+    reason:
+      'archive 冷备份空间不足时必须能自动降级为 files 存储格式（不落地本地归档）。',
+  },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes: 'restore_runtime_services_before_publish',
+    reason:
+      '尚未开始 publish 的失败必须自动恢复运行时服务并退出维护，避免生产停在维护态等人工。',
+  },
+  {
+    file: 'scripts/deploy/production-stdb-publish.sh',
+    includes:
+      'if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then',
+    reason:
+      '只有尚未开始 publish 的失败才允许自动恢复并退出维护，半发布状态必须保持维护。',
+  },
+  {
+    file: 'scripts/database-backup-to-oss.mjs',
+    includes: 'SPACE_INSUFFICIENT_EXIT_CODE',
+    reason:
+      '备份空间不足必须使用独立退出码，调用方据此决定降级存储格式而不是一律失败。',
+  },
   {
     file: 'scripts/deploy/production-stdb-publish.sh',
     includes: 'stop_runtime_services_for_rollout_gate',
@@ -984,7 +1022,8 @@ const checks = [
   },
   {
     file: 'scripts/database-backup-to-oss.mjs',
-    includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})',
+    includes:
+      "assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' })",
     normalizeWhitespace: true,
     reason: '生产冷备份必须先做工作目录剩余空间预检，避免停库后写满磁盘。',
   },

scripts/database-backup-to-oss.mjs

@@ -55,6 +55,11 @@ const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join(
 );
 const DEFAULT_SPACE_SAFETY_RATIO = 1.1;
 const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024;
+// files 模式不落地本地归档（只写 catalog/state），所需空间远小于数据目录本身。
+const DEFAULT_FILES_SPACE_SAFETY_RATIO = 0.05;
+const DEFAULT_FILES_EXTRA_FREE_BYTES = 2 * 1024 * 1024 * 1024;
+// 空间不足用独立退出码，调用方可据此决定是否降级存储格式。
+const SPACE_INSUFFICIENT_EXIT_CODE = 3;
 const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
 const OSS_SERVICE = 'oss';
 const OSS_REQUEST = 'aliyun_v4_request';
@@ -80,6 +85,7 @@ const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1;
 function usage() {
   console.log(`用法:
   npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir <path>] [--work-dir <path>] [--bucket <bucket>] [--object-prefix <prefix>] [--keep-local]
+  npm run database:backup:oss -- --check-space-only [--storage-format archive|files] [--data-dir <path>] [--work-dir <path>]
   node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload]
   node -- scripts/database-backup-to-oss.mjs --upload-archive <path>
   node -- scripts/database-backup-to-oss.mjs --upload-deferred-dir <path>
@@ -106,6 +112,8 @@ function usage() {
   GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT  archive（默认）或 files
   GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL      true 时保留本地 tar.gz
   GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES  备份前要求 work dir 所在文件系统至少有这些可用字节；未设置时按数据目录大小估算
+  GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES  files 模式的最小可用字节（默认 2GiB）
+  GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO  files 模式按数据目录比例要求的可用空间（默认 0.05）
   GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE  history 使用的 full baseline 与追加批次状态文件
   GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST 首次初始化 history state 的 uploaded full manifest
   ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET
@@ -175,6 +183,7 @@ function parseArgs(argv) {
     restartServicesAfter: [],
     database: '',
     dryRun: false,
+    checkSpaceOnly: false,
     deferUpload: false,
     uploadArchive: '',
     uploadDeferredDir: '',
@@ -282,6 +291,9 @@ function parseArgs(argv) {
       case '--publish-manifest':
         options.publishManifest = readValue();
         break;
+      case '--check-space-only':
+        options.checkSpaceOnly = true;
+        break;
       case '--storage-format':
         options.storageFormat = readValue();
         break;
@@ -1013,6 +1025,20 @@ function parseSafetyRatio(rawValue) {
   return ratio;
 }
 
+function parseFilesSpaceSafetyRatio(rawValue) {
+  const value = String(rawValue ?? '').trim();
+  if (!value) {
+    return DEFAULT_FILES_SPACE_SAFETY_RATIO;
+  }
+  const ratio = Number(value);
+  if (!Number.isFinite(ratio) || ratio <= 0) {
+    throw new Error(
+      `GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO 必须是 > 0 的数字，实际: ${rawValue}`,
+    );
+  }
+  return ratio;
+}
+
 function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) {
   const explicitMinFreeBytes = parseByteSize(
     firstNonEmpty(
@@ -1042,22 +1068,57 @@ function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) {
     : extraRequirement;
 }
 
-function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) {
+function calculateFilesRequiredFreeBytes({ dataSizeBytes, env }) {
+  const explicitMinFreeBytes = parseByteSize(
+    env.GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES,
+    'GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES',
+  );
+  if (explicitMinFreeBytes !== null) {
+    return explicitMinFreeBytes;
+  }
+  const ratio = parseFilesSpaceSafetyRatio(
+    env.GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO,
+  );
+  const ratioBasisPoints = BigInt(Math.ceil(ratio * 10000));
+  const ratioRequirement = (dataSizeBytes * ratioBasisPoints + 9999n) / 10000n;
+  const floorRequirement = BigInt(DEFAULT_FILES_EXTRA_FREE_BYTES);
+  return ratioRequirement > floorRequirement ? ratioRequirement : floorRequirement;
+}
+
+export function describeBackupSpaceRequirement({
+  dataDir,
+  workDir,
+  storageFormat = 'archive',
+  args = {},
+  env = {},
+}) {
   mkdirSync(workDir, { recursive: true });
   const dataSizeBytes = getDirectorySizeBytes(dataDir);
   const availableBytes = getAvailableBytes(workDir);
-  const requiredFreeBytes = calculateRequiredFreeBytes({
-    dataSizeBytes,
-    args,
-    env,
-  });
+  const requiredFreeBytes =
+    storageFormat === 'files'
+      ? calculateFilesRequiredFreeBytes({ dataSizeBytes, env })
+      : calculateRequiredFreeBytes({ dataSizeBytes, args, env });
+  return { storageFormat, dataSizeBytes, availableBytes, requiredFreeBytes };
+}
+
+function assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat = 'archive' }) {
+  const { dataSizeBytes, availableBytes, requiredFreeBytes } =
+    describeBackupSpaceRequirement({
+      dataDir,
+      workDir,
+      storageFormat,
+      args,
+      env,
+    });
   console.log(
-    `[database-backup] 备份空间预检: data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`,
+    `[database-backup] 备份空间预检(${storageFormat}): data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`,
   );
   if (availableBytes < requiredFreeBytes) {
-    throw new Error(
+    const error = new Error(
       [
         `备份工作目录所在文件系统剩余空间不足: ${workDir}`,
+        `storage-format=${storageFormat}`,
         `available=${formatBytes(availableBytes)}`,
         `required=${formatBytes(requiredFreeBytes)}`,
         `dataDir=${dataDir}`,
@@ -1065,7 +1126,10 @@ function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) {
         '为避免停库后写满磁盘，本次备份已在停止服务前中止。',
       ].join('；'),
     );
+    error.spaceInsufficient = true;
+    throw error;
   }
+  return { dataSizeBytes, availableBytes, requiredFreeBytes };
 }
 
 function assertSufficientHistoryWorkDirSpace({
@@ -4154,6 +4218,20 @@ async function main() {
     );
   }
 
+  if (args.checkSpaceOnly) {
+    assertSufficientWorkDirSpace({
+      dataDir,
+      workDir,
+      args,
+      env,
+      storageFormat,
+    });
+    console.log(
+      `[database-backup] 空间预检通过(check-space-only): storage-format=${storageFormat}, workDir=${workDir}`,
+    );
+    return;
+  }
+
   for (const [label, value] of Object.entries({
     bucket,
     endpoint,
@@ -4369,7 +4447,7 @@ async function main() {
   const restartServicesAfter = collectRestartServicesAfterBackup({ args, env });
   const stopMarkerPath = databaseBackupStopMarkerPath(workDir);
   try {
-    assertSufficientWorkDirSpace({ dataDir, workDir, args, env });
+    assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' });
     serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath);
     archivePath = createArchive({ dataDir, workDir, fileName });
   } catch (error) {
@@ -4558,6 +4636,6 @@ if (
     } catch {
       // 参数解析或状态文件写入失败时，不能掩盖原始备份错误。
     }
-    process.exit(1);
+    process.exit(error?.spaceInsufficient ? SPACE_INSUFFICIENT_EXIT_CODE : 1);
   });
 }

scripts/deploy/production-stdb-publish.sh

@@ -18,9 +18,16 @@ usage() {
   如需强制等待备份完成并在失败时阻断 publish，传入 --backup-mode sync。
   发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径，再重启并验活重启前 active 的服务。
   --keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker，并在成功后保持维护态，交由后续 API deploy 恢复服务。
+  发布前先做备份空间预检（不进入维护、不停服务）；archive 空间不足且未显式关闭自动降级时，
+  自动改用 files 存储格式（不落地本地归档，改为同步直传 OSS），避免磁盘不足把生产留在维护态。
+
+环境变量：
+  GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files（默认 archive）
+  GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0（默认 1：archive 空间不足自动降级 files）
+  GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0（默认 1：尚未开始 publish 的失败自动恢复服务并退出维护）
   migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。
   如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。
-  失败时保留维护模式。
+  失败时：尚未开始 publish 的失败会自动恢复运行时服务并退出维护；真正开始 publish 之后的失败保留维护模式。
 EOF
 }
 
@@ -55,7 +62,13 @@ API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-serv
 WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
 KEEP_MAINTENANCE_MODE=0
 BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}"
+BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}"
+AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}"
+AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}"
 DEPLOY_COMPLETED=0
+PUBLISH_STARTED=0
+MAINTENANCE_ENTERED=0
+STOPPED_RUNTIME_SERVICES=()
 PUBLISH_TMP_DIR=""
 ASYNC_BACKUP_STATUS_FILE=""
 ASYNC_BACKUP_SCRIPT=""
@@ -287,6 +300,106 @@ restart_runtime_services_after_bootstrap_secret_install() {
   fi
 }
 
+backup_script_path() {
+  local candidate=""
+  for candidate in \
+    "${SCRIPT_DIR}/../database-backup-to-oss.mjs" \
+    "${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"; do
+    if [[ -f "${candidate}" ]]; then
+      printf '%s\n' "${candidate}"
+      return 0
+    fi
+  done
+  return 1
+}
+
+run_backup_space_precheck() {
+  local storage_format="$1"
+  local backup_script=""
+  if ! backup_script="$(backup_script_path)"; then
+    echo "[production-stdb-publish] 缺少数据库备份脚本，无法做备份空间预检" >&2
+    return 1
+  fi
+  node -- "${backup_script}" \
+    --env-file /etc/genarrative/api-server.env \
+    --data-dir "${SPACETIME_ROOT_DIR}" \
+    --database "${DATABASE}" \
+    --storage-format "${storage_format}" \
+    --check-space-only
+}
+
+# 空间预检必须发生在进入维护模式与停服务之前：磁盘不够时不允许再动生产。
+precheck_backup_space_before_maintenance() {
+  if [[ "${BACKUP_MODE}" == "skip" ]]; then
+    echo "[production-stdb-publish] 已跳过发布前备份空间预检（--backup-mode skip）"
+    return 0
+  fi
+
+  local status=0
+  run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$?
+  if [[ "${status}" -eq 0 ]]; then
+    echo "[production-stdb-publish] 发布前备份空间预检通过: storage-format=${BACKUP_STORAGE_FORMAT}（尚未进入维护模式、未停服务）"
+    return 0
+  fi
+  if [[ "${status}" -ne 3 ]]; then
+    echo "[production-stdb-publish] 发布前备份空间预检失败（非空间原因），中止发布；未进入维护模式、未停服务。" >&2
+    exit 1
+  fi
+  if [[ "${BACKUP_STORAGE_FORMAT}" == "files" ]]; then
+    echo "[production-stdb-publish] files 模式备份空间仍不足，中止发布；未进入维护模式、未停服务。" >&2
+    exit 1
+  fi
+  if [[ "${AUTO_FILES_FALLBACK}" != "1" ]]; then
+    echo "[production-stdb-publish] archive 备份空间不足且已禁用自动降级（GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=${AUTO_FILES_FALLBACK}），中止发布；未进入维护模式、未停服务。" >&2
+    exit 1
+  fi
+
+  echo "[production-stdb-publish] archive 冷备份空间不足：自动降级为 files 存储格式（不落地本地归档，改为文件级 catalog 直传 OSS）。" >&2
+  BACKUP_STORAGE_FORMAT="files"
+  if [[ "${BACKUP_MODE}" == "async" ]]; then
+    echo "[production-stdb-publish] files 模式不支持 --defer-upload，本次备份改为同步执行。" >&2
+    BACKUP_MODE="sync"
+  fi
+  status=0
+  run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$?
+  if [[ "${status}" -ne 0 ]]; then
+    echo "[production-stdb-publish] 降级为 files 后空间预检仍失败，中止发布；未进入维护模式、未停服务。" >&2
+    exit 1
+  fi
+  echo "[production-stdb-publish] 已降级为 files 存储格式且空间预检通过。"
+}
+
+# 仅在「尚未开始 publish」的失败路径调用：把本次停掉的运行时服务拉回来。
+restore_runtime_services_before_publish() {
+  if [[ "${#STOPPED_RUNTIME_SERVICES[@]}" -eq 0 ]]; then
+    return 0
+  fi
+  local service=""
+  local state=""
+  local attempt=0
+  echo "[production-stdb-publish] 发布尚未开始，恢复本次停掉的运行时服务: ${STOPPED_RUNTIME_SERVICES[*]}"
+  if ! run_privileged systemctl start "${STOPPED_RUNTIME_SERVICES[@]}"; then
+    echo "[production-stdb-publish] 启动运行时服务失败: ${STOPPED_RUNTIME_SERVICES[*]}" >&2
+    return 1
+  fi
+  for service in "${STOPPED_RUNTIME_SERVICES[@]}"; do
+    state=""
+    for attempt in $(seq 1 15); do
+      state="$(get_runtime_service_active_state "${service}" 2>/dev/null || true)"
+      if [[ "${state}" == "active" ]]; then
+        break
+      fi
+      sleep 1
+    done
+    if [[ "${state}" != "active" ]]; then
+      echo "[production-stdb-publish] 运行时服务未恢复 active: ${service}, state=${state}" >&2
+      return 1
+    fi
+    echo "[production-stdb-publish] 运行时服务已恢复 active: ${service}"
+  done
+  return 0
+}
+
 stop_runtime_services_for_rollout_gate() {
   local api_state=""
   local controller_state=""
@@ -326,6 +439,7 @@ stop_runtime_services_for_rollout_gate() {
   fi
 
   echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}"
+  STOPPED_RUNTIME_SERVICES=("${services_to_stop[@]}")
   run_privileged systemctl stop "${services_to_stop[@]}"
   for worker_service in "${services_to_stop[@]}"; do
     if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then
@@ -402,6 +516,10 @@ while [[ $# -gt 0 ]]; do
       BACKUP_MODE="${2:?缺少 --backup-mode 的值}"
       shift 2
       ;;
+    --backup-storage-format)
+      BACKUP_STORAGE_FORMAT="${2:?缺少 --backup-storage-format 的值}"
+      shift 2
+      ;;
     *)
       echo "[production-stdb-publish] 未知参数: $1" >&2
       usage >&2
@@ -431,6 +549,14 @@ for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do
   fi
 done
 
+if [[ ! "${BACKUP_STORAGE_FORMAT}" =~ ^(archive|files)$ ]]; then
+  echo "[production-stdb-publish] --backup-storage-format 只能是 archive 或 files: ${BACKUP_STORAGE_FORMAT}" >&2
+  exit 1
+fi
+if [[ "${BACKUP_STORAGE_FORMAT}" == "files" && "${BACKUP_MODE}" == "async" ]]; then
+  echo "[production-stdb-publish] files 存储格式不支持 --defer-upload，备份模式由 async 调整为 sync" >&2
+  BACKUP_MODE="sync"
+fi
 if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then
   echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2
   exit 1
@@ -497,7 +623,22 @@ on_exit() {
     rm -rf "${PUBLISH_TMP_DIR}"
   fi
   if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
-    echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2
+    if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then
+      # 尚未开始 publish 就失败（例如备份空间/备份执行失败）：本次没有任何发布变更，
+      # 必须把停掉的运行时服务拉回来并退出维护，避免生产停在维护态等人工救。
+      if restore_runtime_services_before_publish; then
+        if [[ "${MAINTENANCE_ENTERED}" -eq 1 ]]; then
+          if ! "${SCRIPT_DIR}/maintenance-off.sh"; then
+            echo "[production-stdb-publish] 自动退出维护模式失败，请手工执行 maintenance-off.sh。" >&2
+          fi
+        fi
+        echo "[production-stdb-publish] 发布尚未开始即失败，已自动恢复运行时服务并退出维护模式。"
+      else
+        echo "[production-stdb-publish] 自动恢复运行时服务失败，保持维护模式，请手工处理。" >&2
+      fi
+    else
+      echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2
+    fi
   fi
   exit "${exit_code}"
 }
@@ -506,12 +647,8 @@ trap on_exit EXIT
 
 prepare_async_backup() {
   local -a restart_service_args=()
-  ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
-  if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
-    ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
-  fi
-  if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
-    echo "[production-stdb-publish] 缺少数据库备份脚本: ${ASYNC_BACKUP_SCRIPT}" >&2
+  if ! ASYNC_BACKUP_SCRIPT="$(backup_script_path)"; then
+    echo "[production-stdb-publish] 缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
     exit 1
   fi
 
@@ -527,6 +664,7 @@ prepare_async_backup() {
     --env-file /etc/genarrative/api-server.env \
     --data-dir "${SPACETIME_ROOT_DIR}" \
     --database "${DATABASE}" \
+    --storage-format "${BACKUP_STORAGE_FORMAT}" \
     --stop-service spacetimedb.service \
     "${restart_service_args[@]}" \
     --defer-upload \
@@ -676,7 +814,10 @@ wait_for_api_healthz_ready() {
   return 1
 }
 
+precheck_backup_space_before_maintenance
+
 "${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"
+MAINTENANCE_ENTERED=1
 if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
   stop_runtime_services_for_rollout_gate
 fi
@@ -687,23 +828,20 @@ case "${BACKUP_MODE}" in
     ;;
   sync)
     SYNC_BACKUP_RESTART_SERVICE_ARGS=()
-    BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
-    if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
-      BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
-    fi
-    if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
-      echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2
+    if ! BACKUP_SCRIPT="$(backup_script_path)"; then
+      echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
       exit 1
     fi
     if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
       SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service)
     fi
 
-    echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布"
+    echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布"
     node -- "${BACKUP_SCRIPT}" \
       --env-file /etc/genarrative/api-server.env \
       --data-dir "${SPACETIME_ROOT_DIR}" \
       --database "${DATABASE}" \
+      --storage-format "${BACKUP_STORAGE_FORMAT}" \
       --stop-service spacetimedb.service \
       "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}"
     ;;
@@ -768,8 +906,10 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then
   else
     PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
   fi
+  PUBLISH_STARTED=1
   runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}"
 else
+  PUBLISH_STARTED=1
   spacetime "${PUBLISH_ARGS[@]}"
 fi