发布前备份空间预检前置并支持自动降级

- database-backup-to-oss.mjs 新增 --check-space-only：archive 按 data×1.1、files 按 max(data×0.05, 2GiB) 计算门槛，空间不足以退出码 3 返回
- production-stdb-publish.sh 在进入维护与停服务之前先做空间预检，archive 不足时自动降级 files（files 不支持 defer，async 收敛为 sync），可用环境变量关闭
- 尚未开始 publish 的失败自动恢复本次停掉的 API/controller/worker 并退出维护，只有真正开始发布之后的失败才保持维护态
- 备份检查脚本与生产运维门禁补上新口径与执行顺序断言，运维文档与共享记忆记录事故与规则

scripts/deploy/production-stdb-publish.sh

@@ -18,9 +18,16 @@ usage() {
   如需强制等待备份完成并在失败时阻断 publish，传入 --backup-mode sync。
   发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径，再重启并验活重启前 active 的服务。
   --keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker，并在成功后保持维护态，交由后续 API deploy 恢复服务。
+  发布前先做备份空间预检（不进入维护、不停服务）；archive 空间不足且未显式关闭自动降级时，
+  自动改用 files 存储格式（不落地本地归档，改为同步直传 OSS），避免磁盘不足把生产留在维护态。
+
+环境变量：
+  GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files（默认 archive）
+  GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0（默认 1：archive 空间不足自动降级 files）
+  GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0（默认 1：尚未开始 publish 的失败自动恢复服务并退出维护）
   migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。
   如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。
-  失败时保留维护模式。
+  失败时：尚未开始 publish 的失败会自动恢复运行时服务并退出维护；真正开始 publish 之后的失败保留维护模式。
 EOF
 }
 
@@ -55,7 +62,13 @@ API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-serv
 WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
 KEEP_MAINTENANCE_MODE=0
 BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}"
+BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}"
+AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}"
+AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}"
 DEPLOY_COMPLETED=0
+PUBLISH_STARTED=0
+MAINTENANCE_ENTERED=0
+STOPPED_RUNTIME_SERVICES=()
 PUBLISH_TMP_DIR=""
 ASYNC_BACKUP_STATUS_FILE=""
 ASYNC_BACKUP_SCRIPT=""
@@ -287,6 +300,106 @@ restart_runtime_services_after_bootstrap_secret_install() {
   fi
 }
 
+backup_script_path() {
+  local candidate=""
+  for candidate in \
+    "${SCRIPT_DIR}/../database-backup-to-oss.mjs" \
+    "${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"; do
+    if [[ -f "${candidate}" ]]; then
+      printf '%s\n' "${candidate}"
+      return 0
+    fi
+  done
+  return 1
+}
+
+run_backup_space_precheck() {
+  local storage_format="$1"
+  local backup_script=""
+  if ! backup_script="$(backup_script_path)"; then
+    echo "[production-stdb-publish] 缺少数据库备份脚本，无法做备份空间预检" >&2
+    return 1
+  fi
+  node -- "${backup_script}" \
+    --env-file /etc/genarrative/api-server.env \
+    --data-dir "${SPACETIME_ROOT_DIR}" \
+    --database "${DATABASE}" \
+    --storage-format "${storage_format}" \
+    --check-space-only
+}
+
+# 空间预检必须发生在进入维护模式与停服务之前：磁盘不够时不允许再动生产。
+precheck_backup_space_before_maintenance() {
+  if [[ "${BACKUP_MODE}" == "skip" ]]; then
+    echo "[production-stdb-publish] 已跳过发布前备份空间预检（--backup-mode skip）"
+    return 0
+  fi
+
+  local status=0
+  run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$?
+  if [[ "${status}" -eq 0 ]]; then
+    echo "[production-stdb-publish] 发布前备份空间预检通过: storage-format=${BACKUP_STORAGE_FORMAT}（尚未进入维护模式、未停服务）"
+    return 0
+  fi
+  if [[ "${status}" -ne 3 ]]; then
+    echo "[production-stdb-publish] 发布前备份空间预检失败（非空间原因），中止发布；未进入维护模式、未停服务。" >&2
+    exit 1
+  fi
+  if [[ "${BACKUP_STORAGE_FORMAT}" == "files" ]]; then
+    echo "[production-stdb-publish] files 模式备份空间仍不足，中止发布；未进入维护模式、未停服务。" >&2
+    exit 1
+  fi
+  if [[ "${AUTO_FILES_FALLBACK}" != "1" ]]; then
+    echo "[production-stdb-publish] archive 备份空间不足且已禁用自动降级（GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=${AUTO_FILES_FALLBACK}），中止发布；未进入维护模式、未停服务。" >&2
+    exit 1
+  fi
+
+  echo "[production-stdb-publish] archive 冷备份空间不足：自动降级为 files 存储格式（不落地本地归档，改为文件级 catalog 直传 OSS）。" >&2
+  BACKUP_STORAGE_FORMAT="files"
+  if [[ "${BACKUP_MODE}" == "async" ]]; then
+    echo "[production-stdb-publish] files 模式不支持 --defer-upload，本次备份改为同步执行。" >&2
+    BACKUP_MODE="sync"
+  fi
+  status=0
+  run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$?
+  if [[ "${status}" -ne 0 ]]; then
+    echo "[production-stdb-publish] 降级为 files 后空间预检仍失败，中止发布；未进入维护模式、未停服务。" >&2
+    exit 1
+  fi
+  echo "[production-stdb-publish] 已降级为 files 存储格式且空间预检通过。"
+}
+
+# 仅在「尚未开始 publish」的失败路径调用：把本次停掉的运行时服务拉回来。
+restore_runtime_services_before_publish() {
+  if [[ "${#STOPPED_RUNTIME_SERVICES[@]}" -eq 0 ]]; then
+    return 0
+  fi
+  local service=""
+  local state=""
+  local attempt=0
+  echo "[production-stdb-publish] 发布尚未开始，恢复本次停掉的运行时服务: ${STOPPED_RUNTIME_SERVICES[*]}"
+  if ! run_privileged systemctl start "${STOPPED_RUNTIME_SERVICES[@]}"; then
+    echo "[production-stdb-publish] 启动运行时服务失败: ${STOPPED_RUNTIME_SERVICES[*]}" >&2
+    return 1
+  fi
+  for service in "${STOPPED_RUNTIME_SERVICES[@]}"; do
+    state=""
+    for attempt in $(seq 1 15); do
+      state="$(get_runtime_service_active_state "${service}" 2>/dev/null || true)"
+      if [[ "${state}" == "active" ]]; then
+        break
+      fi
+      sleep 1
+    done
+    if [[ "${state}" != "active" ]]; then
+      echo "[production-stdb-publish] 运行时服务未恢复 active: ${service}, state=${state}" >&2
+      return 1
+    fi
+    echo "[production-stdb-publish] 运行时服务已恢复 active: ${service}"
+  done
+  return 0
+}
+
 stop_runtime_services_for_rollout_gate() {
   local api_state=""
   local controller_state=""
@@ -326,6 +439,7 @@ stop_runtime_services_for_rollout_gate() {
   fi
 
   echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}"
+  STOPPED_RUNTIME_SERVICES=("${services_to_stop[@]}")
   run_privileged systemctl stop "${services_to_stop[@]}"
   for worker_service in "${services_to_stop[@]}"; do
     if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then
@@ -402,6 +516,10 @@ while [[ $# -gt 0 ]]; do
       BACKUP_MODE="${2:?缺少 --backup-mode 的值}"
       shift 2
       ;;
+    --backup-storage-format)
+      BACKUP_STORAGE_FORMAT="${2:?缺少 --backup-storage-format 的值}"
+      shift 2
+      ;;
     *)
       echo "[production-stdb-publish] 未知参数: $1" >&2
       usage >&2
@@ -431,6 +549,14 @@ for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do
   fi
 done
 
+if [[ ! "${BACKUP_STORAGE_FORMAT}" =~ ^(archive|files)$ ]]; then
+  echo "[production-stdb-publish] --backup-storage-format 只能是 archive 或 files: ${BACKUP_STORAGE_FORMAT}" >&2
+  exit 1
+fi
+if [[ "${BACKUP_STORAGE_FORMAT}" == "files" && "${BACKUP_MODE}" == "async" ]]; then
+  echo "[production-stdb-publish] files 存储格式不支持 --defer-upload，备份模式由 async 调整为 sync" >&2
+  BACKUP_MODE="sync"
+fi
 if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then
   echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2
   exit 1
@@ -497,7 +623,22 @@ on_exit() {
     rm -rf "${PUBLISH_TMP_DIR}"
   fi
   if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
-    echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2
+    if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then
+      # 尚未开始 publish 就失败（例如备份空间/备份执行失败）：本次没有任何发布变更，
+      # 必须把停掉的运行时服务拉回来并退出维护，避免生产停在维护态等人工救。
+      if restore_runtime_services_before_publish; then
+        if [[ "${MAINTENANCE_ENTERED}" -eq 1 ]]; then
+          if ! "${SCRIPT_DIR}/maintenance-off.sh"; then
+            echo "[production-stdb-publish] 自动退出维护模式失败，请手工执行 maintenance-off.sh。" >&2
+          fi
+        fi
+        echo "[production-stdb-publish] 发布尚未开始即失败，已自动恢复运行时服务并退出维护模式。"
+      else
+        echo "[production-stdb-publish] 自动恢复运行时服务失败，保持维护模式，请手工处理。" >&2
+      fi
+    else
+      echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2
+    fi
   fi
   exit "${exit_code}"
 }
@@ -506,12 +647,8 @@ trap on_exit EXIT
 
 prepare_async_backup() {
   local -a restart_service_args=()
-  ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
-  if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
-    ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
-  fi
-  if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
-    echo "[production-stdb-publish] 缺少数据库备份脚本: ${ASYNC_BACKUP_SCRIPT}" >&2
+  if ! ASYNC_BACKUP_SCRIPT="$(backup_script_path)"; then
+    echo "[production-stdb-publish] 缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
     exit 1
   fi
 
@@ -527,6 +664,7 @@ prepare_async_backup() {
     --env-file /etc/genarrative/api-server.env \
     --data-dir "${SPACETIME_ROOT_DIR}" \
     --database "${DATABASE}" \
+    --storage-format "${BACKUP_STORAGE_FORMAT}" \
     --stop-service spacetimedb.service \
     "${restart_service_args[@]}" \
     --defer-upload \
@@ -676,7 +814,10 @@ wait_for_api_healthz_ready() {
   return 1
 }
 
+precheck_backup_space_before_maintenance
+
 "${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"
+MAINTENANCE_ENTERED=1
 if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
   stop_runtime_services_for_rollout_gate
 fi
@@ -687,23 +828,20 @@ case "${BACKUP_MODE}" in
     ;;
   sync)
     SYNC_BACKUP_RESTART_SERVICE_ARGS=()
-    BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
-    if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
-      BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
-    fi
-    if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
-      echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2
+    if ! BACKUP_SCRIPT="$(backup_script_path)"; then
+      echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
       exit 1
     fi
     if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
       SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service)
     fi
 
-    echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布"
+    echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布"
     node -- "${BACKUP_SCRIPT}" \
       --env-file /etc/genarrative/api-server.env \
       --data-dir "${SPACETIME_ROOT_DIR}" \
       --database "${DATABASE}" \
+      --storage-format "${BACKUP_STORAGE_FORMAT}" \
       --stop-service spacetimedb.service \
       "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}"
     ;;
@@ -768,8 +906,10 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then
   else
     PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
   fi
+  PUBLISH_STARTED=1
   runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}"
 else
+  PUBLISH_STARTED=1
   spacetime "${PUBLISH_ARGS[@]}"
 fi