修复 release 发布与冷备份恢复

冷备份增加工作目录空间预检并确保失败后恢复依赖服务

API deploy 增加 Pingora 产物硬校验并区分 current 切换前后维护模式处理

Jenkins release 默认构建和部署 Pingora 影子网关并在全量流水线透传参数

补充备份与 API deploy 回归检查和生产运维护栏

更新生产运维文档与项目记忆的事故处理口径

scripts/deploy/production-api-deploy.sh

@@ -5,14 +5,14 @@ set -euo pipefail
 usage() {
   cat <<'EOF'
 用法:
-  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
+  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
 
 说明:
   进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
   默认同时重启外部生成 worker controller 和已加载的 worker 实例；未启用 worker 单元时会自动跳过。
   若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。
-  若发布包包含 pingora-gateway，部署脚本会在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，再启动或重启 Pingora 影子服务并复核 active。
-  失败时保留维护模式。
+  若发布包包含 pingora-gateway，或传入 --require-pingora-gateway，部署脚本会要求 release manifest、二进制与 checksum 一致，再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，启动或重启 Pingora 影子服务并复核 active。
+  current 链接切换前失败时会退出本次打开的维护模式；current 链接切换后失败时保留维护模式，避免暴露半发布版本。
 EOF
 }
 
@@ -347,17 +347,20 @@ ensure_pingora_shadow_service() {
 validate_release_manifest() {
   local manifest_path="$1"
   local require_pingora="$2"
+  local source_dir="$3"
 
   if [[ ! -f "${manifest_path}" ]]; then
     echo "[production-api-deploy] 发布产物缺少 release-manifest.json: ${manifest_path}" >&2
     exit 1
   fi
 
-  node - "${manifest_path}" "${require_pingora}" <<'NODE'
+  node - "${manifest_path}" "${require_pingora}" "${source_dir}" <<'NODE'
 const fs = require('fs');
+const path = require('path');
 
 const manifestPath = process.argv[2];
 const requirePingora = process.argv[3] === '1';
+const sourceDir = process.argv[4];
 let manifest;
 try {
   manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
@@ -380,6 +383,16 @@ if (requirePingora) {
     process.exit(1);
   }
 }
+
+const hasPingora = artifacts.some((artifact) => artifact?.path === 'pingora-gateway');
+if (hasPingora) {
+  const binaryPath = path.join(sourceDir, 'pingora-gateway');
+  const checksumPath = path.join(sourceDir, 'pingora-gateway.sha256');
+  if (!fs.existsSync(binaryPath) || !fs.existsSync(checksumPath)) {
+    console.error('[production-api-deploy] release-manifest.json 登记了 pingora-gateway artifact，但发布目录缺少 pingora-gateway 或 pingora-gateway.sha256。');
+    process.exit(1);
+  }
+}
 NODE
 }
 
@@ -576,6 +589,9 @@ DATABASE=""
 SPACETIME_SERVER_URL=""
 DEPLOY_COMPLETED=0
 PINGORA_INCLUDED=0
+REQUIRE_PINGORA_GATEWAY=0
+MAINTENANCE_ENABLED_BY_DEPLOY=0
+CURRENT_LINK_SWITCHED=0
 RELEASE_DIR=""
 STAGING_RELEASE_DIR=""
 
@@ -609,6 +625,10 @@ while [[ $# -gt 0 ]]; do
       PINGORA_SERVICE_NAME="${2:?缺少 --pingora-service 的值}"
       shift 2
       ;;
+    --require-pingora-gateway)
+      REQUIRE_PINGORA_GATEWAY=1
+      shift
+      ;;
     --worker-service-pattern)
       WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}"
       shift 2
@@ -709,7 +729,14 @@ on_exit() {
   local exit_code=$?
   if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
     cleanup_staging_release
-    echo "[production-api-deploy] 部署失败，保持维护模式。" >&2
+    if [[ "${MAINTENANCE_ENABLED_BY_DEPLOY}" -eq 1 && "${CURRENT_LINK_SWITCHED}" -ne 1 ]]; then
+      echo "[production-api-deploy] 部署失败且尚未切换 current，退出本次打开的维护模式。" >&2
+      if ! bash "${SCRIPT_DIR}/maintenance-off.sh"; then
+        echo "[production-api-deploy] 退出维护模式失败，请人工检查维护文件。" >&2
+      fi
+    else
+      echo "[production-api-deploy] 部署失败，current 可能已切换或维护模式不是本次打开，保持维护模式。" >&2
+    fi
   fi
   exit "${exit_code}"
 }
@@ -717,6 +744,7 @@ on_exit() {
 trap on_exit EXIT
 
 bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}"
+MAINTENANCE_ENABLED_BY_DEPLOY=1
 
 echo "[production-api-deploy] 校验 api-server"
 (
@@ -730,10 +758,14 @@ echo "[production-api-deploy] 校验 api-server"
     sha256sum -c pingora-gateway.sha256
   fi
 )
+if [[ "${REQUIRE_PINGORA_GATEWAY}" -eq 1 && ( ! -f "${SOURCE_DIR}/pingora-gateway" || ! -f "${SOURCE_DIR}/pingora-gateway.sha256" ) ]]; then
+  echo "[production-api-deploy] 本次部署要求 Pingora，但发布目录缺少 pingora-gateway 或 pingora-gateway.sha256。" >&2
+  exit 1
+fi
 if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then
   PINGORA_INCLUDED=1
 fi
-validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "${PINGORA_INCLUDED}"
+validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "$(( PINGORA_INCLUDED || REQUIRE_PINGORA_GATEWAY ))" "${SOURCE_DIR}"
 
 mkdir -p "${RELEASE_ROOT}"
 mkdir "${STAGING_RELEASE_DIR}"
@@ -960,6 +992,7 @@ fi
 mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}"
 STAGING_RELEASE_DIR=""
 ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}"
+CURRENT_LINK_SWITCHED=1
 
 if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
   ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}"