Genarrative/scripts/deploy/production-api-deploy.sh

#!/usr/bin/env bash

set -euo pipefail

usage() {
  cat <<'EOF'
用法:
  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url <url>] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--controller-env-file /etc/genarrative/external-generation-controller.env] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode]

说明:
  进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
  默认先停止、启动并验活唯一 BgFilter worker，再重启 API、外部生成 worker controller 和已加载的 worker 实例。
  --bgfilter-worker-health-url 缺省时从已校验的 worker env HOST/PORT 派生；显式传入时必须与父子 env 三方一致，否则预检失败。
  随包 worker systemd unit 按 --current-link 与各 env 参数渲染后安装（与 provision 同语义），自定义路径不会被打回模板默认值；默认参数下渲染输出与模板一致。
  若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。
  若发布包包含 pingora-gateway，或传入 --require-pingora-gateway，部署脚本会要求 release manifest、二进制与 checksum 一致，再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，启动或重启 Pingora 影子服务并复核 active。
  默认在 readiness 通过后退出维护模式；传入 --keep-maintenance-mode 时保留维护文件，供人工验收后再恢复公网。
  current 链接切换前失败时会退出本次打开的维护模式；current 链接切换后失败时保留维护模式，避免暴露半发布版本。
EOF
}

require_argument() {
  local value="$1"
  local label="$2"

  if [[ -z "${value}" ]]; then
    echo "[production-api-deploy] 缺少参数: ${label}" >&2
    exit 1
  fi
}

require_absolute_path() {
  local value="$1"
  local label="$2"

  if [[ ! "${value}" = /* ]]; then
    echo "[production-api-deploy] ${label} 必须使用绝对路径: ${value}" >&2
    exit 1
  fi
  if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then
    echo "[production-api-deploy] ${label} 不能包含换行符。" >&2
    exit 1
  fi
}

validate_spacetime_database_name() {
  local database="$1"

  if [[ ! "${database}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then
    echo "[production-api-deploy] --database 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$，只能使用小写字母、数字，并用单个短横线分隔: ${database}" >&2
    exit 1
  fi
}

write_env_value() {
  local file_path="$1"
  local key="$2"
  local value="$3"

  local can_write_direct=0

  if [[ -e "${file_path}" ]]; then
    [[ -w "${file_path}" ]] && can_write_direct=1
  else
    mkdir -p "$(dirname "${file_path}")"
    [[ -w "$(dirname "${file_path}")" ]] && can_write_direct=1
  fi

  # api-server 环境文件通常由 server-provision 以 root:root 0600 创建。
  # 发布流水线可能以非 root Jenkins 用户运行，因此仅在不能直接写入时使用 sudo -n，避免因为 env 文件权限导致 API 发布中断。
  if [[ "$(id -u)" -eq 0 || "${can_write_direct}" -eq 1 ]]; then
    python3 - "${file_path}" "${key}" "${value}" <<'PY'
import os
import sys
from pathlib import Path

file_path = Path(sys.argv[1])
key = sys.argv[2]
value = sys.argv[3]
was_missing = not file_path.exists()
file_path.parent.mkdir(parents=True, exist_ok=True)

lines = []
if file_path.exists():
    lines = file_path.read_text(encoding="utf-8").splitlines()

updated = False
next_lines = []
for line in lines:
    if line.startswith(f"{key}="):
        next_lines.append(f"{key}={value}")
        updated = True
    else:
        next_lines.append(line)

if not updated:
    next_lines.append(f"{key}={value}")

file_path.write_text("\n".join(next_lines) + "\n", encoding="utf-8")
if was_missing:
    os.chmod(file_path, 0o600)
PY
  else
    if ! sudo -n true >/dev/null 2>&1; then
      echo "[production-api-deploy] 当前用户无权写入 ${file_path}，且 sudo -n 不可用；请给部署用户配置免密写入该环境文件或以 root 执行发布。" >&2
      exit 1
    fi
    sudo -n python3 - "${file_path}" "${key}" "${value}" <<'PY'
import os
import sys
from pathlib import Path

file_path = Path(sys.argv[1])
key = sys.argv[2]
value = sys.argv[3]
was_missing = not file_path.exists()
file_path.parent.mkdir(parents=True, exist_ok=True)

lines = []
if file_path.exists():
    lines = file_path.read_text(encoding="utf-8").splitlines()

updated = False
next_lines = []
for line in lines:
    if line.startswith(f"{key}="):
        next_lines.append(f"{key}={value}")
        updated = True
    else:
        next_lines.append(line)

if not updated:
    next_lines.append(f"{key}={value}")

file_path.write_text("\n".join(next_lines) + "\n", encoding="utf-8")
if was_missing:
    os.chmod(file_path, 0o600)
PY
  fi
}

read_env_value() {
  local file_path="$1"
  local key="$2"

  if [[ ! -f "${file_path}" ]]; then
    return 0
  fi

  local python_script='
import sys
from pathlib import Path

path = Path(sys.argv[1])
key = sys.argv[2]
if not path.exists():
    raise SystemExit(0)
matched_value = None
for raw_line in path.read_text(encoding="utf-8").splitlines():
    line = raw_line.strip()
    if not line or line.startswith("#") or "=" not in line:
        continue
    current_key, value = line.split("=", 1)
    if current_key == key:
        value = value.strip()
        if len(value) >= 2 and value[0] == value[-1] and value[0] in ("\"", "'\''"):
            value = value[1:-1]
        matched_value = value
if matched_value is not None:
    print(matched_value)
'

  if [[ -r "${file_path}" ]]; then
    python3 -c "${python_script}" "${file_path}" "${key}"
  else
    if ! sudo -n true >/dev/null 2>&1; then
      echo "[production-api-deploy] 当前用户无权读取 ${file_path}，且 sudo -n 不可用；无法检查运行态环境变量。" >&2
      exit 1
    fi
    sudo -n python3 -c "${python_script}" "${file_path}" "${key}"
  fi
}

env_contains_nonempty_assignment() {
  local file_path="$1"
  local key="$2"

  if [[ ! -f "${file_path}" ]]; then
    return 1
  fi

  local python_script='
import sys
from pathlib import Path

path = Path(sys.argv[1])
key = sys.argv[2]
for raw_line in path.read_text(encoding="utf-8").splitlines():
    line = raw_line.strip()
    if not line or line.startswith("#") or "=" not in line:
        continue
    current_key, value = line.split("=", 1)
    if current_key.strip() != key:
        continue
    value = value.strip()
    if len(value) >= 2 and value[0] == value[-1] and value[0] in ("\"", chr(39)):
        value = value[1:-1]
    if value.strip():
        raise SystemExit(0)
raise SystemExit(1)
'

  if [[ -r "${file_path}" ]]; then
    python3 -c "${python_script}" "${file_path}" "${key}"
  else
    if ! sudo -n true >/dev/null 2>&1; then
      echo "[production-api-deploy] 当前用户无权读取 ${file_path}，且 sudo -n 不可用；无法检查运行态环境变量。" >&2
      exit 1
    fi
    sudo -n python3 -c "${python_script}" "${file_path}" "${key}"
  fi
}

env_has_assignment() {
  local file_path="$1"
  local key="$2"

  if [[ ! -f "${file_path}" ]]; then
    return 1
  fi

  local python_script='
import sys
from pathlib import Path

path = Path(sys.argv[1])
key = sys.argv[2]
for raw_line in path.read_text(encoding="utf-8").splitlines():
    line = raw_line.strip()
    if not line or line.startswith("#") or "=" not in line:
        continue
    current_key, _ = line.split("=", 1)
    if current_key.strip() == key:
        raise SystemExit(0)
raise SystemExit(1)
'

  if [[ -r "${file_path}" ]]; then
    python3 -c "${python_script}" "${file_path}" "${key}"
  else
    if ! sudo -n true >/dev/null 2>&1; then
      echo "[production-api-deploy] 当前用户无权读取 ${file_path}，且 sudo -n 不可用；无法检查运行态环境变量。" >&2
      exit 1
    fi
    sudo -n python3 -c "${python_script}" "${file_path}" "${key}"
  fi
}

ensure_env_value() {
  local file_path="$1"
  local key="$2"
  local default_value="$3"
  local current_value

  current_value="$(read_env_value "${file_path}" "${key}")"
  if [[ -n "${current_value}" ]]; then
    return
  fi

  echo "[production-api-deploy] 补齐运行态环境变量: ${key} -> ${file_path}"
  write_env_value "${file_path}" "${key}" "${default_value}"
}

remove_env_key_if_present() {
  local file_path="$1"
  local key="$2"
  local can_update_direct=0
  local removal_result
  local python_script='
import sys
import re
from pathlib import Path

file_path = Path(sys.argv[1])
key = sys.argv[2]
assignment_pattern = re.compile(rf"^(?:export\s+)?{re.escape(key)}\s*=")
lines = file_path.read_text(encoding="utf-8").splitlines()
next_lines = []
removed = False
for raw_line in lines:
    if assignment_pattern.match(raw_line.strip()):
        removed = True
        continue
    next_lines.append(raw_line)

if removed:
    content = "\n".join(next_lines)
    file_path.write_text(f"{content}\n" if content else "", encoding="utf-8")
    print("removed")
'

  if [[ ! -f "${file_path}" ]]; then
    return
  fi

  if [[ -r "${file_path}" && -w "${file_path}" ]]; then
    can_update_direct=1
  fi
  if [[ "$(id -u)" -eq 0 || "${can_update_direct}" -eq 1 ]]; then
    removal_result="$(python3 -c "${python_script}" "${file_path}" "${key}")"
  else
    if ! sudo -n true >/dev/null 2>&1; then
      echo "[production-api-deploy] 当前用户无权更新 ${file_path}，且 sudo -n 不可用；无法移除已退役环境变量 ${key}。" >&2
      exit 1
    fi
    removal_result="$(sudo -n python3 -c "${python_script}" "${file_path}" "${key}")"
  fi
  if [[ "${removal_result}" == "removed" ]]; then
    echo "[production-api-deploy] 移除已退役的环境变量: ${key} <- ${file_path}"
  fi
}

ensure_env_value_migrates_old_default() {
  local file_path="$1"
  local key="$2"
  local old_default="$3"
  local new_default="$4"
  local current_value

  current_value="$(read_env_value "${file_path}" "${key}")"
  if [[ -z "${current_value}" ]]; then
    ensure_env_value "${file_path}" "${key}" "${new_default}"
    return
  fi
  if [[ "${current_value}" != "${old_default}" ]]; then
    return
  fi

  echo "[production-api-deploy] 迁移运行态环境变量旧默认值: ${key} ${old_default} -> ${new_default} (${file_path})"
  write_env_value "${file_path}" "${key}" "${new_default}"
}

ensure_runtime_bootstrap_secret_file_env() {
  local file_path="$1"
  local key="GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE"
  local direct_key="GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET"
  local canonical_path="/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt"
  local current_value direct_value

  direct_value="$(read_env_value "${file_path}" "${direct_key}")"
  if [[ -n "${direct_value}" ]]; then
    unset direct_value
    echo "[production-api-deploy] ${file_path} 不得保存 ${direct_key} 明文；生产环境只允许使用 ${key}。" >&2
    exit 1
  fi
  unset direct_value

  current_value="$(read_env_value "${file_path}" "${key}")"
  if [[ -z "${current_value}" ]]; then
    ensure_env_value "${file_path}" "${key}" "${canonical_path}"
    return
  fi
  if [[ "${current_value}" != "${canonical_path}" ]]; then
    echo "[production-api-deploy] ${key} 必须使用 Stdb publish 同步写入的固定路径 ${canonical_path}: ${file_path}" >&2
    exit 1
  fi
}

run_privileged() {
  if [[ "$(id -u)" -eq 0 ]]; then
    "$@"
    return
  fi
  if ! sudo -n true >/dev/null 2>&1; then
    echo "[production-api-deploy] 当前用户不是 root，且 sudo -n 不可用；无法执行: $*" >&2
    exit 1
  fi
  sudo -n "$@"
}

ensure_runtime_dir() {
  local path="$1"
  local mode="$2"

  if [[ -z "${path}" ]]; then
    return
  fi
  if [[ "${path}" != /* ]]; then
    echo "[production-api-deploy] 运行态目录必须使用绝对路径，避免写入只读发布目录: ${path}" >&2
    exit 1
  fi

  echo "[production-api-deploy] 确保运行态目录可写: ${path}"
  run_privileged install -d -o genarrative -g genarrative -m "${mode}" "${path}"
}

migrate_legacy_editor_generation_pricing_override() {
  local current_link="$1"
  local target_dir="/var/lib/genarrative/editor-generation-pricing"
  local target_file="${target_dir}/editor-generation-pricing.override.json"
  local legacy_file="${current_link}/.app/editor-generation-pricing.override.json"

  if [[ -f "${target_file}" ]]; then
    return
  fi
  if [[ ! -f "${legacy_file}" ]]; then
    return
  fi

  echo "[production-api-deploy] 迁移旧模型定价 override 到运行态目录: ${legacy_file} -> ${target_file}"
  ensure_runtime_dir "${target_dir}" "0750"
  run_privileged install -o genarrative -g genarrative -m 0640 "${legacy_file}" "${target_file}"
}

ensure_runtime_env_and_dirs() {
  local api_env_file="$1"
  local tracking_enabled tracking_outbox_dir wallet_refund_enabled wallet_refund_outbox_dir

  # 旧生产环境文件会被 server-provision 保留，不一定包含新增的运行态写入路径。
  # 发布前只补缺省值，不覆盖线上已经定制过的目录或开关。
  ensure_env_value "${api_env_file}" "GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS" "5000"
  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED" "true"
  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR" "/var/lib/genarrative/tracking-outbox"
  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE" "500"
  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS" "1000"
  ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES" "268435456"
  ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_ENABLED" "true"
  ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_DIR" "/var/lib/genarrative/wallet-refund-outbox"
  ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE" "100"
  ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS" "1000"
  ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES" "67108864"
  ensure_env_value "${api_env_file}" "WECHAT_PAY_REFUND_RECONCILIATION_ENABLED" "true"
  ensure_env_value_migrates_old_default "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
  ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
  ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
  ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL" "http://127.0.0.1:8083"
  ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE" "/etc/genarrative/secrets/bgfilter-worker.token"
  ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS" "2000"
  ensure_runtime_bootstrap_secret_file_env "${api_env_file}"
  # N 与单图估时是父子共同派生 attempt/callBudget 公式的输入，必须放共享 API env 单一来源；
  # 旧固定 attempt timeout 已由公式取代。
  ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY" "16"
  ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS" "5000"
  remove_env_key_if_present "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS"

  tracking_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED")"
  tracking_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR")"
  if [[ "$(printf "%s" "${tracking_enabled}" | tr '[:upper:]' '[:lower:]')" != "false" ]]; then
    ensure_runtime_dir "${tracking_outbox_dir}" "0750"
  fi
  wallet_refund_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_ENABLED")"
  wallet_refund_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_DIR")"
  if [[ "$(printf "%s" "${wallet_refund_enabled}" | tr '[:upper:]' '[:lower:]')" != "false" ]]; then
    ensure_runtime_dir "${wallet_refund_outbox_dir}" "0750"
  fi
}

validate_real_wechat_pay_refund_reconciliation() {
  local api_env_file="$1"
  local pay_enabled pay_provider reconciliation_enabled

  pay_enabled="$(read_env_value "${api_env_file}" "WECHAT_PAY_ENABLED" | tr '[:upper:]' '[:lower:]')"
  pay_provider="$(read_env_value "${api_env_file}" "WECHAT_PAY_PROVIDER" | tr '[:upper:]' '[:lower:]')"
  reconciliation_enabled="$(read_env_value "${api_env_file}" "WECHAT_PAY_REFUND_RECONCILIATION_ENABLED" | tr '[:upper:]' '[:lower:]')"
  if [[ "${pay_enabled}" == "true" && "${pay_provider}" == "real" && "${reconciliation_enabled}" != "true" ]]; then
    echo "[production-api-deploy] 真实微信支付必须设置 WECHAT_PAY_REFUND_RECONCILIATION_ENABLED=true: ${api_env_file}" >&2
    exit 1
  fi
  echo "[production-api-deploy] 微信退款 reconciliation 配置已复核: enabled=${reconciliation_enabled:-unset}, real_pay=$([[ "${pay_enabled}" == "true" && "${pay_provider}" == "real" ]] && printf true || printf false)"
}

ensure_worker_runtime_env_defaults() {
  local worker_env_file="$1"

  if [[ -z "${worker_env_file}" ]]; then
    return
  fi
  if [[ ! -f "${worker_env_file}" ]]; then
    echo "[production-api-deploy] worker 环境文件不存在，跳过运行态默认值补齐: ${worker_env_file}"
    return
  fi

  ensure_env_value_migrates_old_default "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
  ensure_env_value "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
  ensure_env_value "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
  ensure_runtime_bootstrap_secret_file_env "${worker_env_file}"
}

ensure_bgfilter_worker_runtime_env_defaults() {
  local bgfilter_env_file="$1"

  if [[ -z "${bgfilter_env_file}" ]]; then
    return
  fi
  if [[ ! -f "${bgfilter_env_file}" ]]; then
    echo "[production-api-deploy] BgFilter worker 环境文件不存在: ${bgfilter_env_file}" >&2
    return 1
  fi

  ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_HOST" "127.0.0.1"
  ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_PORT" "8083"
  # Q 仅是 admission 保险丝；把历史模板默认 128 定向迁到新默认 2048，
  # 其它显式定制值继续保留。
  ensure_env_value_migrates_old_default "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS" "128" "2048"
  ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
  ensure_env_value_migrates_old_default "${bgfilter_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300" "120"
  # N 已迁入共享 API env；worker 专属文件中的旧值会与共享值形成双写风险，直接移除。
  remove_env_key_if_present "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY"
}

validate_bgfilter_shared_runtime_env() {
  local api_env_file="$1"
  local shared_concurrency shared_estimate_ms connect_timeout_ms

  shared_concurrency="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY")"
  if [[ ! "${shared_concurrency}" =~ ^[1-9][0-9]*$ ]]; then
    echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_CONCURRENCY 必须在共享 API env 中配置为正整数: ${api_env_file}" >&2
    return 1
  fi

  shared_estimate_ms="$(read_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS")"
  if [[ ! "${shared_estimate_ms}" =~ ^[1-9][0-9]*$ ]]; then
    echo "[production-api-deploy] GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS 必须在共享 API env 中配置为正整数毫秒: ${api_env_file}" >&2
    return 1
  fi

  connect_timeout_ms="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS")"
  if [[ ! "${connect_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
    echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${api_env_file}" >&2
    return 1
  fi
}

validate_bgfilter_worker_capacity() {
  local api_env_file="$1"
  local bgfilter_env_file="$2"
  local concurrency max_requests

  # N 已迁入共享 API env（worker unit 先加载它）；Q 是可选保险丝（代码默认 2048），
  # 显式配置时必须为正且不小于 N。
  concurrency="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY")"
  max_requests="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS")"
  if [[ ! "${concurrency}" =~ ^[1-9][0-9]{0,8}$ ]]; then
    echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_CONCURRENCY 必须在共享 API env 中配置为正整数: ${api_env_file}" >&2
    return 1
  fi
  if [[ -z "${max_requests}" ]]; then
    return 0
  fi
  if [[ ! "${max_requests}" =~ ^[1-9][0-9]{0,8}$ ]]; then
    echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS 必须是正整数: ${bgfilter_env_file}" >&2
    return 1
  fi
  if (( 10#${max_requests} < 10#${concurrency} )); then
    echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS 必须大于或等于 CONCURRENCY: ${bgfilter_env_file}" >&2
    return 1
  fi
}

validate_bgfilter_worker_shared_env_alignment() {
  local api_env_file="$1"
  local bgfilter_env_file="$2"
  local key shared_value dedicated_value

  for key in \
    GENARRATIVE_BGFILTER_WORKER_CONCURRENCY \
    GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS \
    GENARRATIVE_EDITOR_BGFILTER_BASE_URL \
    GENARRATIVE_EDITOR_BGFILTER_TOKEN \
    GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE \
    ALIYUN_OSS_BUCKET \
    ALIYUN_OSS_ENDPOINT \
    ALIYUN_OSS_ACCESS_KEY_ID \
    ALIYUN_OSS_ACCESS_KEY_SECRET \
    ALIYUN_OSS_READ_EXPIRE_SECONDS; do
    if ! env_has_assignment "${bgfilter_env_file}" "${key}"; then
      continue
    fi
    dedicated_value="$(read_env_value "${bgfilter_env_file}" "${key}")"
    shared_value="$(read_env_value "${api_env_file}" "${key}")"
    if [[ "${dedicated_value}" != "${shared_value}" ]]; then
      echo "[production-api-deploy] BgFilter 专属 env 中的共享配置与 API env 不一致: ${key}；请迁移到 ${api_env_file} 并从 ${bgfilter_env_file} 删除重复项。" >&2
      return 1
    fi
  done
}

validate_external_generation_worker_bgfilter_env_alignment() {
  local api_env_file="$1"
  local worker_env_file="$2"
  local key shared_value worker_value

  if [[ -z "${worker_env_file}" || ! -f "${worker_env_file}" ]]; then
    return 0
  fi

  for key in \
    GENARRATIVE_BGFILTER_WORKER_CONCURRENCY \
    GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS \
    GENARRATIVE_BGFILTER_WORKER_BASE_URL \
    GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE \
    GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS \
    ALIYUN_OSS_BUCKET \
    ALIYUN_OSS_ENDPOINT; do
    if ! env_has_assignment "${worker_env_file}" "${key}"; then
      continue
    fi
    worker_value="$(read_env_value "${worker_env_file}" "${key}")"
    shared_value="$(read_env_value "${api_env_file}" "${key}")"
    if [[ "${worker_value}" != "${shared_value}" ]]; then
      echo "[production-api-deploy] 外部生成 worker env 中的 BgFilter 共享配置与 API env 不一致: ${key}；${worker_env_file} 会在 systemd 中后加载并覆盖父侧有效值。" >&2
      return 1
    fi
  done
}

validate_bgfilter_loopback_endpoint_alignment() {
  local api_env_file="$1"
  local bgfilter_env_file="$2"
  local health_url="$3"
  local base_url host port expected_base_url expected_health_url

  base_url="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL")"
  host="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_HOST")"
  port="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_PORT")"

  if [[ "${host}" != "127.0.0.1" ]]; then
    echo "[production-api-deploy] BgFilter worker 首版必须监听 127.0.0.1，当前 GENARRATIVE_BGFILTER_WORKER_HOST=${host:-<empty>}: ${bgfilter_env_file}" >&2
    return 1
  fi
  if [[ ! "${port}" =~ ^[1-9][0-9]{0,4}$ ]] || (( 10#${port} > 65535 )); then
    echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_PORT 必须是 1-65535 的有效端口: ${bgfilter_env_file}" >&2
    return 1
  fi

  expected_base_url="http://${host}:${port}"
  if [[ "${base_url%/}" != "${expected_base_url}" ]]; then
    echo "[production-api-deploy] 父进程 GENARRATIVE_BGFILTER_WORKER_BASE_URL 必须与 BgFilter worker 有效监听地址一致: expected=${expected_base_url}, actual=${base_url:-<empty>}" >&2
    return 1
  fi

  expected_health_url="${expected_base_url}/readyz"
  if [[ -z "${health_url}" ]]; then
    # 合法自定义端口不应仅因调用方未重复传入可派生值而阻断发布（Jenkins job 不传该 flag）。
    BGFILTER_WORKER_HEALTH_URL="${expected_health_url}"
  elif [[ "${health_url}" != "${expected_health_url}" ]]; then
    echo "[production-api-deploy] --bgfilter-worker-health-url 必须与父进程 base URL 和子 worker listener 指向同一 loopback endpoint: expected=${expected_health_url}, actual=${health_url}" >&2
    return 1
  fi
}

validate_no_bgfilter_internal_token_plaintext() {
  local env_file

  for env_file in "$@"; do
    if [[ -z "${env_file}" ]]; then
      continue
    fi
    if env_contains_nonempty_assignment "${env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN"; then
      echo "[production-api-deploy] ${env_file} 不得保存 GENARRATIVE_BGFILTER_INTERNAL_TOKEN 明文；生产环境只允许使用 GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE。" >&2
      return 1
    fi
  done
}

bgfilter_internal_token_file_is_single_segment() {
  local token_file="$1"

  run_privileged awk '
    /[^[:space:]]/ {
      non_empty_lines += 1
      if ($0 !~ /^[[:space:]]*[^[:space:]]+[[:space:]]*$/) {
        invalid = 1
      }
    }
    END {
      exit !(non_empty_lines == 1 && invalid == 0)
    }
  ' "${token_file}"
}

validate_bgfilter_internal_token_file() {
  local api_env_file="$1"
  local token_file token_metadata

  token_file="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE")"
  if [[ -z "${token_file}" || "${token_file}" != /* ]]; then
    echo "[production-api-deploy] GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE 必须指向绝对路径: ${api_env_file}" >&2
    return 1
  fi
  if [[ -L "${token_file}" || ! -f "${token_file}" || ! -s "${token_file}" ]]; then
    echo "[production-api-deploy] BgFilter 内部 Token 必须是非空普通文件且不能是符号链接: ${token_file}" >&2
    return 1
  fi
  if ! bgfilter_internal_token_file_is_single_segment "${token_file}"; then
    echo "[production-api-deploy] BgFilter 内部 Token 文件必须为不含空白字符的单段值: ${token_file}" >&2
    return 1
  fi

  token_metadata="$(run_privileged stat -c '%U:%G:%a' -- "${token_file}")"
  if [[ "${token_metadata}" != "root:genarrative:440" ]]; then
    echo "[production-api-deploy] BgFilter 内部 Token 权限必须为 root:genarrative 0440，且必须是普通文件: ${token_file} (${token_metadata})" >&2
    return 1
  fi
}

extract_pingora_env_files_from_unit() {
  local service_name="$1"
  local unit_content

  if ! unit_content="$(systemctl cat "${service_name}")"; then
    echo "[production-api-deploy] 无法读取 Pingora systemd 最终配置: ${service_name}" >&2
    exit 1
  fi

  if printf "%s\n" "${unit_content}" | grep -Eq '^[[:space:]]*(AmbientCapabilities|CapabilityBoundingSet)=.*CAP_NET_BIND_SERVICE'; then
    echo "[production-api-deploy] Pingora systemd 已包含 CAP_NET_BIND_SERVICE，疑似直连入口配置；API deploy 不会自动启动或重启直连服务: ${service_name}" >&2
    exit 1
  fi

  printf "%s\n" "${unit_content}" | while IFS= read -r raw_line; do
    local line value token env_file
    line="${raw_line#"${raw_line%%[![:space:]]*}"}"
    [[ "${line}" == EnvironmentFile=* ]] || continue
    value="${line#EnvironmentFile=}"
    for token in ${value}; do
      env_file="${token#-}"
      env_file="${env_file%\"}"
      env_file="${env_file#\"}"
      env_file="${env_file%\'}"
      env_file="${env_file#\'}"
      [[ -n "${env_file}" ]] && printf "%s\n" "${env_file}"
    done
  done
}

find_pingora_gateway_env_file() {
  local service_name="$1"
  local env_file listen unit_env_files

  if ! unit_env_files="$(extract_pingora_env_files_from_unit "${service_name}")"; then
    return 1
  fi

  if [[ -n "${unit_env_files}" ]]; then
    while IFS= read -r env_file; do
      if [[ "${env_file}" != /* ]]; then
        echo "[production-api-deploy] Pingora EnvironmentFile 必须使用绝对路径: ${env_file}" >&2
        exit 1
      fi
      listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")"
      if [[ -n "${listen}" ]]; then
        printf "%s\n" "${env_file}"
        return
      fi
    done <<< "${unit_env_files}"
  fi

  echo "[production-api-deploy] Pingora systemd 配置缺少包含 GENARRATIVE_PINGORA_GATEWAY_LISTEN 的 EnvironmentFile: ${service_name}" >&2
  exit 1
}

require_pingora_shadow_env() {
  local env_file="$1"
  local listen tls_listen redirect_listen

  listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")"
  tls_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN")"
  redirect_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN")"

  if [[ "${listen}" != "127.0.0.1:18081" ]]; then
    echo "[production-api-deploy] Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081，当前 GENARRATIVE_PINGORA_GATEWAY_LISTEN=${listen:-<empty>}" >&2
    exit 1
  fi
  if [[ -n "${tls_listen}" ]]; then
    echo "[production-api-deploy] Pingora 自动启动不允许启用 TLS_LISTEN，当前 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=${tls_listen}" >&2
    exit 1
  fi
  if [[ -n "${redirect_listen}" ]]; then
    echo "[production-api-deploy] Pingora 自动启动不允许启用 HTTP_REDIRECT_LISTEN，当前 GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=${redirect_listen}" >&2
    exit 1
  fi
}

check_pingora_shadow_service_config() {
  local service_name="$1"
  local env_file

  if ! env_file="$(find_pingora_gateway_env_file "${service_name}")"; then
    return 1
  fi
  require_pingora_shadow_env "${env_file}"
  printf "%s\n" "${env_file}"
}

ensure_pingora_shadow_service() {
  local service_name="$1"
  local env_file="${2:-}"

  if [[ -z "${env_file}" ]]; then
    env_file="$(check_pingora_shadow_service_config "${service_name}")"
  else
    require_pingora_shadow_env "${env_file}"
  fi

  echo "[production-api-deploy] 发布包包含 Pingora，启动或重启 shadow 影子服务: ${service_name} (${env_file})"
  systemctl restart "${service_name}"
  if ! systemctl is-active --quiet "${service_name}"; then
    echo "[production-api-deploy] Pingora shadow 影子服务启动或重启后不是 active: ${service_name}" >&2
    exit 1
  fi
}

validate_release_manifest() {
  local manifest_path="$1"
  local require_pingora="$2"
  local source_dir="$3"

  if [[ ! -f "${manifest_path}" ]]; then
    echo "[production-api-deploy] 发布产物缺少 release-manifest.json: ${manifest_path}" >&2
    exit 1
  fi

  node - "${manifest_path}" "${require_pingora}" "${source_dir}" <<'NODE'
const fs = require('fs');
const path = require('path');

const manifestPath = process.argv[2];
const requirePingora = process.argv[3] === '1';
const sourceDir = process.argv[4];
let manifest;
try {
  manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
} catch (error) {
  console.error(`[production-api-deploy] release-manifest.json 不是合法 JSON: ${error.message}`);
  process.exit(1);
}

const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : [];
const hasApiServer = artifacts.some((artifact) => artifact?.path === 'api-server');
if (!hasApiServer) {
  console.error('[production-api-deploy] release-manifest.json 缺少 api-server artifact。');
  process.exit(1);
}

if (requirePingora) {
  const hasPingora = artifacts.some((artifact) => artifact?.path === 'pingora-gateway');
  if (!hasPingora) {
    console.error('[production-api-deploy] release-manifest.json 缺少 pingora-gateway artifact。');
    process.exit(1);
  }
}

const hasPingora = artifacts.some((artifact) => artifact?.path === 'pingora-gateway');
if (hasPingora) {
  const binaryPath = path.join(sourceDir, 'pingora-gateway');
  const checksumPath = path.join(sourceDir, 'pingora-gateway.sha256');
  if (!fs.existsSync(binaryPath) || !fs.existsSync(checksumPath)) {
    console.error('[production-api-deploy] release-manifest.json 登记了 pingora-gateway artifact，但发布目录缺少 pingora-gateway 或 pingora-gateway.sha256。');
    process.exit(1);
  }
}
NODE
}

list_worker_services() {
  local pattern="$1"

  if [[ -z "${pattern}" ]]; then
    return 0
  fi

  systemctl list-units --all --plain --no-legend "${pattern}" 2>/dev/null | awk '{print $1}' | sort -u
}

ensure_default_worker_service() {
  local pattern="$1"
  local default_service="genarrative-external-generation-worker@1.service"
  local template_service="genarrative-external-generation-worker@.service"
  local services=()

  if [[ -z "${pattern}" ]]; then
    return 0
  fi

  if [[ "${pattern}" != "genarrative-external-generation-worker@*.service" ]]; then
    return 0
  fi

  if ! systemctl cat "${template_service}" >/dev/null 2>&1; then
    echo "[production-api-deploy] 缺少外部生成 worker systemd 模板: ${template_service}" >&2
    return 1
  fi

  while IFS= read -r service; do
    services+=("${service}")
  done < <(list_worker_services "${pattern}")
  if [[ "${#services[@]}" -gt 0 ]]; then
    return 0
  fi

  echo "[production-api-deploy] 未发现外部生成 worker 实例，启用并启动默认实例: ${default_service}"
  systemctl enable --now "${default_service}"
}

install_release_systemd_unit() {
  local source_path="$1"
  local unit_name="$2"
  local label="$3"
  local unit_dir="${GENARRATIVE_SYSTEMD_UNIT_DIR:-/etc/systemd/system}"

  if [[ ! -f "${source_path}" ]]; then
    echo "[production-api-deploy] 发布产物缺少${label}: ${source_path}" >&2
    return 1
  fi
  if [[ "${unit_dir}" != /* ]]; then
    echo "[production-api-deploy] systemd unit 目录必须使用绝对路径: ${unit_dir}" >&2
    return 1
  fi

  echo "[production-api-deploy] 安装${label}: ${unit_name}"
  if ! run_privileged install -d -m 0755 "${unit_dir}"; then
    return 1
  fi
  run_privileged install -m 0644 "${source_path}" "${unit_dir}/${unit_name}"
}

render_and_install_release_systemd_unit() {
  local source_path="$1"
  local unit_name="$2"
  local label="$3"
  local rendered_path
  local install_status=0
  shift 3

  if [[ ! -f "${source_path}" ]]; then
    echo "[production-api-deploy] 发布产物缺少${label}: ${source_path}" >&2
    return 1
  fi
  if [[ "$#" -eq 0 || $(( $# % 2 )) -ne 0 ]]; then
    echo "[production-api-deploy] ${label} 缺少成对的模板路径和实际路径。" >&2
    return 1
  fi

  RENDERED_SYSTEMD_UNIT_FILE="$(mktemp)"
  rendered_path="${RENDERED_SYSTEMD_UNIT_FILE}"
  if ! python3 - "${source_path}" "${rendered_path}" "$@" <<'PY'
import re
import sys
from pathlib import Path

source_path = Path(sys.argv[1])
rendered_path = Path(sys.argv[2])
replacements = sys.argv[3:]
content = source_path.read_text(encoding="utf-8")
replacement_map = dict(zip(replacements[::2], replacements[1::2]))

for template_path in replacement_map:
    if template_path not in content:
        raise SystemExit(f"systemd unit 模板缺少占位路径: {template_path}")

pattern = re.compile("|".join(re.escape(path) for path in replacement_map))
content = pattern.sub(lambda match: replacement_map[match.group(0)], content)

rendered_path.write_text(content, encoding="utf-8")
PY
  then
    cleanup_rendered_systemd_unit
    return 1
  fi

  install_release_systemd_unit "${rendered_path}" "${unit_name}" "${label}" || install_status=$?
  cleanup_rendered_systemd_unit
  return "${install_status}"
}

install_worker_systemd_units() {
  local release_dir="$1"
  local pattern="$2"
  local controller_service="$3"
  local bgfilter_service="$4"
  local current_link="$5"
  local api_env_file="$6"
  local worker_env_file="$7"
  local controller_env_file="$8"
  local bgfilter_env_file="$9"
  local installed_any=0

  if [[ "${bgfilter_service}" == "genarrative-bgfilter-worker.service" ]]; then
    render_and_install_release_systemd_unit \
      "${release_dir}/deploy/systemd/genarrative-bgfilter-worker.service" \
      "genarrative-bgfilter-worker.service" \
      "BgFilter worker systemd 单元" \
      "/opt/genarrative/current" "${current_link}" \
      "/etc/genarrative/api-server.env" "${api_env_file}" \
      "/etc/genarrative/bgfilter-worker.env" "${bgfilter_env_file}"
    installed_any=1
  fi

  if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" ]]; then
    render_and_install_release_systemd_unit \
      "${release_dir}/deploy/systemd/genarrative-external-generation-worker@.service" \
      "genarrative-external-generation-worker@.service" \
      "外部生成 worker systemd 模板" \
      "/opt/genarrative/current" "${current_link}" \
      "/etc/genarrative/api-server.env" "${api_env_file}" \
      "/etc/genarrative/external-generation-worker.env" "${worker_env_file}"
    installed_any=1
  fi

  if [[ "${controller_service}" == "genarrative-external-generation-controller.service" ]]; then
    render_and_install_release_systemd_unit \
      "${release_dir}/deploy/systemd/genarrative-external-generation-controller.service" \
      "genarrative-external-generation-controller.service" \
      "外部生成 worker controller systemd 单元" \
      "/opt/genarrative/current" "${current_link}" \
      "/etc/genarrative/api-server.env" "${api_env_file}" \
      "/etc/genarrative/external-generation-controller.env" "${controller_env_file}"
    installed_any=1
  fi

  if [[ "${installed_any}" -eq 1 ]]; then
    echo "[production-api-deploy] 重新加载 systemd unit。"
    systemctl daemon-reload
  fi
}

restart_worker_services() {
  local pattern="$1"
  local services=()

  if [[ -z "${pattern}" ]]; then
    echo "[production-api-deploy] 跳过外部生成 worker 重启。"
    return 0
  fi

  ensure_default_worker_service "${pattern}"
  while IFS= read -r service; do
    services+=("${service}")
  done < <(list_worker_services "${pattern}")
  if [[ "${#services[@]}" -eq 0 ]]; then
    echo "[production-api-deploy] 未发现已加载的外部生成 worker 单元: ${pattern}" >&2
    return 1
  fi

  echo "[production-api-deploy] 重启外部生成 worker: ${services[*]}"
  systemctl restart "${services[@]}"
}

wait_for_worker_services() {
  local pattern="$1"
  local services=()
  local all_active

  if [[ -z "${pattern}" ]]; then
    return 0
  fi

  while IFS= read -r service; do
    services+=("${service}")
  done < <(list_worker_services "${pattern}")
  if [[ "${#services[@]}" -eq 0 ]]; then
    echo "[production-api-deploy] 外部生成 worker 单元不存在，发布失败: ${pattern}" >&2
    return 1
  fi

  echo "[production-api-deploy] 等待外部生成 worker active: ${services[*]}"
  for _ in {1..30}; do
    all_active=1
    for service in "${services[@]}"; do
      if ! systemctl is-active --quiet "${service}"; then
        all_active=0
        break
      fi
    done
    if [[ "${all_active}" -eq 1 ]]; then
      return 0
    fi
    sleep 2
  done

  systemctl --no-pager --full status "${services[@]}" || true
  echo "[production-api-deploy] 外部生成 worker 未在超时时间内进入 active，发布失败。" >&2
  return 1
}

ensure_worker_controller_service() {
  local service="$1"

  if [[ -z "${service}" ]]; then
    return 0
  fi

  if ! systemctl cat "${service}" >/dev/null 2>&1; then
    echo "[production-api-deploy] 缺少外部生成 worker controller systemd 单元: ${service}" >&2
    return 1
  fi

  echo "[production-api-deploy] 启用并重启外部生成 worker controller: ${service}"
  systemctl enable "${service}"
  systemctl restart "${service}"
}

wait_for_worker_controller_service() {
  local service="$1"

  if [[ -z "${service}" ]]; then
    return 0
  fi

  echo "[production-api-deploy] 等待外部生成 worker controller active: ${service}"
  for _ in {1..30}; do
    if systemctl is-active --quiet "${service}"; then
      return 0
    fi
    sleep 2
  done

  systemctl --no-pager --full status "${service}" || true
  echo "[production-api-deploy] 外部生成 worker controller 未在超时时间内进入 active，发布失败。" >&2
  return 1
}

restart_and_wait_for_bgfilter_worker() {
  local service="$1"
  local health_url="$2"

  if [[ -z "${service}" ]]; then
    echo "[production-api-deploy] 跳过 BgFilter worker 启动。"
    return 0
  fi
  if ! systemctl cat "${service}" >/dev/null 2>&1; then
    echo "[production-api-deploy] 缺少 BgFilter worker systemd 单元: ${service}" >&2
    return 1
  fi

  echo "[production-api-deploy] 停止旧 BgFilter worker 并等待在途请求排空: ${service}"
  systemctl stop "${service}"
  systemctl enable "${service}"
  echo "[production-api-deploy] 启动唯一 BgFilter worker: ${service}"
  systemctl start "${service}"

  for _ in {1..30}; do
    if systemctl is-active --quiet "${service}" && curl -fsS --max-time 2 "${health_url}" >/dev/null; then
      echo "[production-api-deploy] BgFilter worker readiness 通过: ${health_url}"
      return 0
    fi
    sleep 2
  done

  systemctl --no-pager --full status "${service}" || true
  echo "[production-api-deploy] BgFilter worker readiness 检查超时: ${health_url}" >&2
  return 1
}

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
SOURCE_DIR=""
VERSION=""
RELEASE_ROOT="/opt/genarrative/releases"
CURRENT_LINK="/opt/genarrative/current"
SERVICE_NAME="genarrative-api.service"
PINGORA_SERVICE_NAME="genarrative-pingora-gateway.service"
WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service"
BGFILTER_WORKER_SERVICE="genarrative-bgfilter-worker.service"
# 缺省时在预检阶段从已校验的 worker HOST/PORT 派生；显式传入时仍强制三方一致。
BGFILTER_WORKER_HEALTH_URL=""
BGFILTER_WORKER_ENV_FILE="/etc/genarrative/bgfilter-worker.env"
HEALTH_URL="http://127.0.0.1:8082/readyz"
API_ENV_FILE="/etc/genarrative/api-server.env"
WORKER_ENV_FILE="/etc/genarrative/external-generation-worker.env"
# controller unit 渲染专用：deploy 不读写该 env 内容，只保证安装的 unit 指向正确路径。
CONTROLLER_ENV_FILE="/etc/genarrative/external-generation-controller.env"
DATABASE=""
SPACETIME_SERVER_URL=""
DEPLOY_COMPLETED=0
PINGORA_INCLUDED=0
REQUIRE_PINGORA_GATEWAY=0
KEEP_MAINTENANCE_MODE=0
MAINTENANCE_ENABLED_BY_DEPLOY=0
MAINTENANCE_FILE="${GENARRATIVE_MAINTENANCE_FILE:-/var/lib/genarrative/maintenance/enabled}"
CURRENT_LINK_SWITCHED=0
RELEASE_DIR=""
STAGING_RELEASE_DIR=""
RENDERED_SYSTEMD_UNIT_FILE=""

while [[ $# -gt 0 ]]; do
  case "$1" in
    -h|--help)
      usage
      exit 0
      ;;
    --source-dir)
      SOURCE_DIR="${2:?缺少 --source-dir 的值}"
      shift 2
      ;;
    --version)
      VERSION="${2:?缺少 --version 的值}"
      shift 2
      ;;
    --release-root)
      RELEASE_ROOT="${2:?缺少 --release-root 的值}"
      shift 2
      ;;
    --current-link)
      CURRENT_LINK="${2:?缺少 --current-link 的值}"
      shift 2
      ;;
    --service)
      SERVICE_NAME="${2:?缺少 --service 的值}"
      shift 2
      ;;
    --pingora-service)
      PINGORA_SERVICE_NAME="${2:?缺少 --pingora-service 的值}"
      shift 2
      ;;
    --require-pingora-gateway)
      REQUIRE_PINGORA_GATEWAY=1
      shift
      ;;
    --keep-maintenance-mode)
      KEEP_MAINTENANCE_MODE=1
      shift
      ;;
    --worker-service-pattern)
      WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}"
      shift 2
      ;;
    --no-worker-services)
      WORKER_SERVICE_PATTERN=""
      shift
      ;;
    --worker-controller-service)
      WORKER_CONTROLLER_SERVICE="${2:?缺少 --worker-controller-service 的值}"
      shift 2
      ;;
    --controller-env-file)
      CONTROLLER_ENV_FILE="${2:?缺少 --controller-env-file 的值}"
      shift 2
      ;;
    --no-worker-controller)
      WORKER_CONTROLLER_SERVICE=""
      shift
      ;;
    --bgfilter-worker-service)
      BGFILTER_WORKER_SERVICE="${2:?缺少 --bgfilter-worker-service 的值}"
      shift 2
      ;;
    --bgfilter-worker-health-url)
      BGFILTER_WORKER_HEALTH_URL="${2:?缺少 --bgfilter-worker-health-url 的值}"
      shift 2
      ;;
    --bgfilter-worker-env-file)
      BGFILTER_WORKER_ENV_FILE="${2:?缺少 --bgfilter-worker-env-file 的值}"
      shift 2
      ;;
    --no-bgfilter-worker)
      BGFILTER_WORKER_SERVICE=""
      BGFILTER_WORKER_ENV_FILE=""
      shift
      ;;
    --health-url)
      HEALTH_URL="${2:?缺少 --health-url 的值}"
      shift 2
      ;;
    --api-env-file)
      API_ENV_FILE="${2:?缺少 --api-env-file 的值}"
      shift 2
      ;;
    --worker-env-file)
      WORKER_ENV_FILE="${2:?缺少 --worker-env-file 的值}"
      shift 2
      ;;
    --database)
      DATABASE="${2:?缺少 --database 的值}"
      shift 2
      ;;
    --spacetime-server-url)
      SPACETIME_SERVER_URL="${2:?缺少 --spacetime-server-url 的值}"
      shift 2
      ;;
    *)
      echo "[production-api-deploy] 未知参数: $1" >&2
      usage >&2
      exit 1
      ;;
  esac
done

require_argument "${SOURCE_DIR}" "--source-dir"
require_absolute_path "${RELEASE_ROOT}" "--release-root"
require_absolute_path "${CURRENT_LINK}" "--current-link"
require_absolute_path "${API_ENV_FILE}" "--api-env-file"
if [[ -n "${WORKER_ENV_FILE}" ]]; then
  require_absolute_path "${WORKER_ENV_FILE}" "--worker-env-file"
fi
if [[ -n "${CONTROLLER_ENV_FILE}" ]]; then
  require_absolute_path "${CONTROLLER_ENV_FILE}" "--controller-env-file"
fi
if [[ -n "${BGFILTER_WORKER_ENV_FILE}" ]]; then
  require_absolute_path "${BGFILTER_WORKER_ENV_FILE}" "--bgfilter-worker-env-file"
fi

if [[ -n "${DATABASE}" ]]; then
  validate_spacetime_database_name "${DATABASE}"
fi

if [[ ! -d "${SOURCE_DIR}" ]]; then
  echo "[production-api-deploy] 发布目录不存在: ${SOURCE_DIR}" >&2
  exit 1
fi

SOURCE_DIR="$(cd "${SOURCE_DIR}" && pwd)"
VERSION="${VERSION:-$(basename "${SOURCE_DIR}")}"

if [[ ! "${VERSION}" =~ ^[0-9A-Za-z][0-9A-Za-z._-]*$ ]]; then
  echo "[production-api-deploy] --version 必须以数字或字母开头，且只能包含数字、字母、点、下划线和短横线: ${VERSION}" >&2
  exit 1
fi

if [[ "${VERSION}" == "." || "${VERSION}" == ".." ]]; then
  echo "[production-api-deploy] --version 不能是点目录: ${VERSION}" >&2
  exit 1
fi

if [[ ! -f "${SOURCE_DIR}/api-server" || ! -f "${SOURCE_DIR}/api-server.sha256" ]]; then
  echo "[production-api-deploy] 缺少 api-server 或 api-server.sha256: ${SOURCE_DIR}" >&2
  exit 1
fi

RELEASE_DIR="${RELEASE_ROOT}/${VERSION}"
STAGING_RELEASE_DIR="${RELEASE_ROOT}/.${VERSION}.staging.$$"

if [[ -e "${RELEASE_DIR}" ]]; then
  echo "[production-api-deploy] 目标 release 已存在，拒绝覆盖或合并旧文件: ${RELEASE_DIR}" >&2
  exit 1
fi

if [[ -e "${STAGING_RELEASE_DIR}" ]]; then
  echo "[production-api-deploy] 临时 staging release 已存在: ${STAGING_RELEASE_DIR}" >&2
  exit 1
fi

if [[ -e "${CURRENT_LINK}" && ! -L "${CURRENT_LINK}" ]]; then
  echo "[production-api-deploy] current 链接路径已存在但不是符号链接，拒绝覆盖: ${CURRENT_LINK}" >&2
  exit 1
fi

cleanup_staging_release() {
  if [[ -n "${STAGING_RELEASE_DIR:-}" && -d "${STAGING_RELEASE_DIR}" ]]; then
    rm -rf "${STAGING_RELEASE_DIR}"
  fi
}

cleanup_rendered_systemd_unit() {
  if [[ -n "${RENDERED_SYSTEMD_UNIT_FILE:-}" && -f "${RENDERED_SYSTEMD_UNIT_FILE}" ]]; then
    rm -f "${RENDERED_SYSTEMD_UNIT_FILE}"
  fi
  RENDERED_SYSTEMD_UNIT_FILE=""
}

on_exit() {
  local exit_code=$?
  cleanup_rendered_systemd_unit
  if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
    cleanup_staging_release
    if [[ "${MAINTENANCE_ENABLED_BY_DEPLOY}" -eq 1 && "${CURRENT_LINK_SWITCHED}" -ne 1 ]]; then
      echo "[production-api-deploy] 部署失败且尚未切换 current，退出本次打开的维护模式。" >&2
      if ! bash "${SCRIPT_DIR}/maintenance-off.sh"; then
        echo "[production-api-deploy] 退出维护模式失败，请人工检查维护文件。" >&2
      fi
    else
      echo "[production-api-deploy] 部署失败，current 可能已切换或维护模式不是本次打开，保持维护模式。" >&2
    fi
  fi
  exit "${exit_code}"
}

trap on_exit EXIT

if [[ ! -f "${MAINTENANCE_FILE}" ]]; then
  MAINTENANCE_ENABLED_BY_DEPLOY=1
else
  echo "[production-api-deploy] 继承已有维护模式；部署失败时不得误退出上游维护窗口: ${MAINTENANCE_FILE}"
fi
bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}"

echo "[production-api-deploy] 校验 api-server"
(
  cd "${SOURCE_DIR}"
  sha256sum -c api-server.sha256
  if [[ -f pingora-gateway || -f pingora-gateway.sha256 ]]; then
    if [[ ! -f pingora-gateway || ! -f pingora-gateway.sha256 ]]; then
      echo "[production-api-deploy] pingora-gateway 与 pingora-gateway.sha256 必须同时存在。" >&2
      exit 1
    fi
    sha256sum -c pingora-gateway.sha256
  fi
)
if [[ "${REQUIRE_PINGORA_GATEWAY}" -eq 1 && ( ! -f "${SOURCE_DIR}/pingora-gateway" || ! -f "${SOURCE_DIR}/pingora-gateway.sha256" ) ]]; then
  echo "[production-api-deploy] 本次部署要求 Pingora，但发布目录缺少 pingora-gateway 或 pingora-gateway.sha256。" >&2
  exit 1
fi
if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then
  PINGORA_INCLUDED=1
fi
validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "$(( PINGORA_INCLUDED || REQUIRE_PINGORA_GATEWAY ))" "${SOURCE_DIR}"

mkdir -p "${RELEASE_ROOT}"
mkdir "${STAGING_RELEASE_DIR}"
RELEASE_CONTENT_DIR="${STAGING_RELEASE_DIR}"
cp "${SOURCE_DIR}/api-server" "${RELEASE_CONTENT_DIR}/api-server"
cp "${SOURCE_DIR}/api-server.sha256" "${RELEASE_CONTENT_DIR}/api-server.sha256"
chmod +x "${RELEASE_CONTENT_DIR}/api-server"
if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then
  cp "${SOURCE_DIR}/pingora-gateway" "${RELEASE_CONTENT_DIR}/pingora-gateway"
  cp "${SOURCE_DIR}/pingora-gateway.sha256" "${RELEASE_CONTENT_DIR}/pingora-gateway.sha256"
  chmod +x "${RELEASE_CONTENT_DIR}/pingora-gateway"
  echo "[production-api-deploy] 已复制 Pingora 影子网关；current 链接切换后将复核 shadow 配置并启动或重启 ${PINGORA_SERVICE_NAME}"
fi

BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
API_DEPLOY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/production-api-deploy.sh"
MAINTENANCE_ON_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/maintenance-on.sh"
MAINTENANCE_OFF_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/maintenance-off.sh"
HEALTH_PATROL_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs"
PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs"
PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs"
PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs"
PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs"
PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs"
PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs"
HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs"
PINGORA_RELEASE_READINESS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs"
PINGORA_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh"
PINGORA_ROLLBACK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh"
PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh"
PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh"
PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs"
PINGORA_GATEWAY_ENV_SHADOW_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs"
PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs"
PINGORA_PREFLIGHT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs"
PINGORA_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs"
PINGORA_CANARY_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs"
PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs"
PINGORA_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/pingora"
SYSTEMD_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/systemd"
NGINX_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/nginx"
ENV_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/env"
mkdir -p "${RELEASE_CONTENT_DIR}/scripts" "${RELEASE_CONTENT_DIR}/scripts/deploy" "${RELEASE_CONTENT_DIR}/scripts/ops" "${RELEASE_CONTENT_DIR}/deploy"
if [[ ! -f "${API_DEPLOY_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 API 部署脚本: ${SOURCE_DIR}/scripts/deploy/production-api-deploy.sh" >&2
  exit 1
fi
cp "${API_DEPLOY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/production-api-deploy.sh"
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/production-api-deploy.sh"
if [[ ! -f "${MAINTENANCE_ON_SCRIPT_SOURCE}" || ! -f "${MAINTENANCE_OFF_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少维护模式脚本: ${SOURCE_DIR}/scripts/deploy" >&2
  exit 1
fi
cp "${MAINTENANCE_ON_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/maintenance-on.sh"
cp "${MAINTENANCE_OFF_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/maintenance-off.sh"
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/maintenance-on.sh" "${RELEASE_CONTENT_DIR}/scripts/deploy/maintenance-off.sh"
if [[ ! -f "${BACKUP_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
  exit 1
fi
cp "${BACKUP_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs"
if [[ ! -f "${HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少生产健康巡检脚本: ${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs" >&2
  exit 1
fi
cp "${HEALTH_PATROL_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs"
if [[ ! -f "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora current release 自审脚本: ${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs" >&2
  exit 1
fi
cp "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs"
if [[ ! -f "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连彩排状态脚本: ${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" >&2
  exit 1
fi
cp "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
if [[ ! -f "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换状态快照脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" >&2
  exit 1
fi
cp "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs"
if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据包脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" >&2
  exit 1
fi
cp "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs"
if [[ ! -f "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换命令证据脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" >&2
  exit 1
fi
cp "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs"
if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据验真脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" >&2
  exit 1
fi
cp "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs"
if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据根目录审计脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" >&2
  exit 1
fi
cp "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs"
if [[ ! -f "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少生产健康巡检 env 复核脚本: ${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs" >&2
  exit 1
fi
cp "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs"
if [[ ! -f "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora release readiness 聚合门禁脚本: ${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs" >&2
  exit 1
fi
cp "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs"
if [[ ! -f "${PINGORA_ENABLE_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连启用脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" >&2
  exit 1
fi
cp "${PINGORA_ENABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh"
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh"
if [[ ! -f "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连回退脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" >&2
  exit 1
fi
cp "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh"
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh"
if [[ ! -f "${PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora realpath canary 启用脚本: ${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" >&2
  exit 1
fi
cp "${PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh"
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh"
if [[ ! -f "${PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora realpath canary 关闭脚本: ${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" >&2
  exit 1
fi
cp "${PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh"
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh"
if [[ ! -f "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora health patrol env 切换脚本: ${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" >&2
  exit 1
fi
cp "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs"
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs"
if [[ ! -f "${PINGORA_GATEWAY_ENV_SHADOW_SWITCH_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora gateway env shadow 切换脚本: ${SOURCE_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" >&2
  exit 1
fi
cp "${PINGORA_GATEWAY_ENV_SHADOW_SWITCH_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs"
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs"
if [[ ! -f "${PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora TLS 证书同步脚本: ${SOURCE_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" >&2
  exit 1
fi
cp "${PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs"
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs"
if [[ ! -f "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连预检脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" >&2
  exit 1
fi
cp "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs"
if [[ ! -f "${PINGORA_LIVE_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 直连 live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs" >&2
  exit 1
fi
cp "${PINGORA_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs"
if [[ ! -f "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora canary live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs" >&2
  exit 1
fi
cp "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs"
if [[ ! -f "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora canary access log 对账脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" >&2
  exit 1
fi
cp "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs"
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs"
if [[ ! -d "${PINGORA_DEPLOY_DIR_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Pingora 配置目录: ${SOURCE_DIR}/deploy/pingora" >&2
  exit 1
fi
rm -rf "${RELEASE_CONTENT_DIR}/deploy/pingora"
cp -R "${PINGORA_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/pingora"
if [[ ! -d "${SYSTEMD_DEPLOY_DIR_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 systemd 配置目录: ${SOURCE_DIR}/deploy/systemd" >&2
  exit 1
fi
rm -rf "${RELEASE_CONTENT_DIR}/deploy/systemd"
cp -R "${SYSTEMD_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/systemd"
if [[ ! -d "${NGINX_DEPLOY_DIR_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少 Nginx 配置目录: ${SOURCE_DIR}/deploy/nginx" >&2
  exit 1
fi
rm -rf "${RELEASE_CONTENT_DIR}/deploy/nginx"
cp -R "${NGINX_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/nginx"
if [[ ! -d "${ENV_DEPLOY_DIR_SOURCE}" ]]; then
  echo "[production-api-deploy] 发布产物缺少环境变量示例目录: ${SOURCE_DIR}/deploy/env" >&2
  exit 1
fi
rm -rf "${RELEASE_CONTENT_DIR}/deploy/env"
cp -R "${ENV_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/env"

cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_CONTENT_DIR}/release-manifest.api-server.json"

PINGORA_SHADOW_ENV_FILE=""

if [[ -n "${DATABASE}" ]]; then
  echo "[production-api-deploy] 写入 api-server SpacetimeDB database: ${DATABASE} -> ${API_ENV_FILE}"
  write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_DATABASE" "${DATABASE}"
fi

if [[ -n "${SPACETIME_SERVER_URL}" ]]; then
  echo "[production-api-deploy] 写入 api-server SpacetimeDB server: ${SPACETIME_SERVER_URL} -> ${API_ENV_FILE}"
  write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_SERVER_URL" "${SPACETIME_SERVER_URL}"
fi

ensure_runtime_env_and_dirs "${API_ENV_FILE}"
validate_bgfilter_shared_runtime_env "${API_ENV_FILE}"
validate_real_wechat_pay_refund_reconciliation "${API_ENV_FILE}"
ensure_worker_runtime_env_defaults "${WORKER_ENV_FILE}"
ensure_bgfilter_worker_runtime_env_defaults "${BGFILTER_WORKER_ENV_FILE}"
validate_external_generation_worker_bgfilter_env_alignment "${API_ENV_FILE}" "${WORKER_ENV_FILE}"
validate_no_bgfilter_internal_token_plaintext "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"
if [[ -n "${BGFILTER_WORKER_SERVICE}" ]]; then
  validate_bgfilter_internal_token_file "${API_ENV_FILE}"
  validate_bgfilter_loopback_endpoint_alignment "${API_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}" "${BGFILTER_WORKER_HEALTH_URL}"
fi
if [[ -n "${BGFILTER_WORKER_ENV_FILE}" ]]; then
  validate_bgfilter_worker_capacity "${API_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"
  validate_bgfilter_worker_shared_env_alignment "${API_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"
fi
migrate_legacy_editor_generation_pricing_override "${CURRENT_LINK}"

if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
  PINGORA_SHADOW_ENV_FILE="$(check_pingora_shadow_service_config "${PINGORA_SERVICE_NAME}")"
fi

mkdir -p "$(dirname "${CURRENT_LINK}")"
if [[ -e "${RELEASE_DIR}" ]]; then
  echo "[production-api-deploy] 目标 release 在发布过程中出现，拒绝合并 staging: ${RELEASE_DIR}" >&2
  exit 1
fi
mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}"
STAGING_RELEASE_DIR=""
ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}"
CURRENT_LINK_SWITCHED=1

if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
  ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}"
fi

install_worker_systemd_units \
  "${RELEASE_DIR}" \
  "${WORKER_SERVICE_PATTERN}" \
  "${WORKER_CONTROLLER_SERVICE}" \
  "${BGFILTER_WORKER_SERVICE}" \
  "${CURRENT_LINK}" \
  "${API_ENV_FILE}" \
  "${WORKER_ENV_FILE}" \
  "${CONTROLLER_ENV_FILE}" \
  "${BGFILTER_WORKER_ENV_FILE}"

restart_and_wait_for_bgfilter_worker "${BGFILTER_WORKER_SERVICE}" "${BGFILTER_WORKER_HEALTH_URL}"

echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}"
systemctl restart "${SERVICE_NAME}"
restart_worker_services "${WORKER_SERVICE_PATTERN}"
wait_for_worker_services "${WORKER_SERVICE_PATTERN}"
ensure_worker_controller_service "${WORKER_CONTROLLER_SERVICE}"
wait_for_worker_controller_service "${WORKER_CONTROLLER_SERVICE}"

echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}"
for _ in {1..30}; do
  if curl -fsS --max-time 2 "${HEALTH_URL}" >/dev/null; then
    if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
      echo "[production-api-deploy] readiness 通过，按参数保持维护模式: ${MAINTENANCE_FILE}"
    else
      bash "${SCRIPT_DIR}/maintenance-off.sh"
    fi
    DEPLOY_COMPLETED=1
    echo "[production-api-deploy] 完成: ${RELEASE_DIR}/api-server"
    exit 0
  fi
  sleep 2
done

echo "[production-api-deploy] readiness 检查超时: ${HEALTH_URL}" >&2
exit 1