#!/usr/bin/env bash

set -euo pipefail
umask 077

usage() {
  cat <<'EOF'
用法:
  ./scripts/deploy/production-stdb-publish.sh --source-dir build/<version> --database <database> --migration-bootstrap-secret-file <protected-file> [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--api-health-url http://127.0.0.1:8082/healthz] [--api-readiness-timeout-seconds 60] [--keep-maintenance-mode] [--backup-mode async|sync|skip]

说明:
  进入维护模式，校验 spacetime_module.wasm.sha256，并在生产实例本机执行 spacetime publish。
  publish 固定使用 --delete-data=never 与 scoped --yes=migrate,break-clients；任何需要删除数据的 schema 冲突都会阻断发布。
  默认使用 http://127.0.0.1:3101，避免与部署机本机 Git/Web 服务的 3000 端口冲突。
  默认使用 /stdb 作为 spacetime CLI root-dir，并以 spacetimedb 用户发布，避免 root CLI 身份污染自托管实例。
  发布时固定追加 --no-config，只使用显式参数，避免工作区或用户目录里的 spacetime 配置干扰目标。
  async 模式会在 publish 前先做本地冷备份，再在 publish 完成后后台上传 OSS，避免低带宽上传阻塞部署。
  如需强制等待备份完成并在失败时阻断 publish，传入 --backup-mode sync。
  发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径，再重启并验活重启前 active 的服务。
  --keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker，并在成功后保持维护态，交由后续 API deploy 恢复服务。
  migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。
  如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。
  失败时保留维护模式。
EOF
}

require_argument() {
  local value="$1"
  local label="$2"

  if [[ -z "${value}" ]]; then
    echo "[production-stdb-publish] 缺少参数: ${label}" >&2
    exit 1
  fi
}

validate_spacetime_database_name() {
  local database="$1"

  if [[ ! "${database}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then
    echo "[production-stdb-publish] --database 必须匹配 ^[a-z0-9]+(-[a-z0-9]+)*$: ${database}" >&2
    exit 1
  fi
}

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
SOURCE_DIR=""
DATABASE=""
SERVER_ALIAS="local"
SERVER_URL="http://127.0.0.1:3101"
SPACETIME_ROOT_DIR="/stdb"
RUN_AS_USER="spacetimedb"
MIGRATION_BOOTSTRAP_SECRET_FILE=""
API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-server.env}"
WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
KEEP_MAINTENANCE_MODE=0
BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}"
DEPLOY_COMPLETED=0
PUBLISH_TMP_DIR=""
ASYNC_BACKUP_STATUS_FILE=""
ASYNC_BACKUP_SCRIPT=""
ASYNC_BACKUP_ARCHIVE=""
ASYNC_BACKUP_MANIFEST=""
ASYNC_BACKUP_LOG=""
SPACETIME_READY_TIMEOUT_SECONDS="${GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS:-60}"
API_HEALTH_URL="${GENARRATIVE_STDB_PUBLISH_API_HEALTH_URL:-http://127.0.0.1:8082/healthz}"
API_READINESS_TIMEOUT_SECONDS="${GENARRATIVE_STDB_PUBLISH_API_READINESS_TIMEOUT_SECONDS:-60}"
API_WAS_ACTIVE=0
RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE="${GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE:-/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt}"
RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE="/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt"

run_privileged() {
  if [[ "$(id -u)" -eq 0 ]]; then
    "$@"
  elif command -v sudo >/dev/null 2>&1; then
    sudo -n "$@"
  else
    echo "[production-stdb-publish] 当前用户不是 root，且 sudo 不可用；无法完成受保护运行态文件或服务操作。" >&2
    return 1
  fi
}

ensure_runtime_bootstrap_secret_env_file() {
  local env_file="$1"
  local required="$2"

  if [[ ! -e "${env_file}" ]]; then
    if [[ "${required}" == "true" ]]; then
      echo "[production-stdb-publish] 运行时环境文件不存在，无法补齐 bootstrap secret FILE: ${env_file}" >&2
      exit 1
    fi
    echo "[production-stdb-publish] worker 环境文件不存在，跳过 bootstrap secret FILE 补齐: ${env_file}"
    return
  fi
  if [[ ! -f "${env_file}" || -L "${env_file}" ]]; then
    echo "[production-stdb-publish] 运行时环境文件必须是普通文件且不能是符号链接: ${env_file}" >&2
    exit 1
  fi

  run_privileged python3 - "${env_file}" "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE}" <<'PY'
import os
import stat
import sys
import tempfile
from pathlib import Path

env_path = Path(sys.argv[1])
canonical_path = sys.argv[2]
file_key = "GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE"
direct_key = "GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET"


def normalize_value(value: str) -> str:
    value = value.strip()
    if len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'"):
        value = value[1:-1]
    return value.strip()


lines = env_path.read_text(encoding="utf-8").splitlines()
next_lines: list[str] = []
file_key_written = False
changed = False

for raw_line in lines:
    stripped = raw_line.strip()
    if not stripped or stripped.startswith("#") or "=" not in stripped:
        next_lines.append(raw_line)
        continue

    key, value = stripped.split("=", 1)
    if key == direct_key and normalize_value(value):
        print(
            f"[production-stdb-publish] {env_path} 不得保存 {direct_key} 明文；生产环境只允许使用 {file_key}。",
            file=sys.stderr,
        )
        raise SystemExit(1)
    if key != file_key:
        next_lines.append(raw_line)
        continue

    current_value = normalize_value(value)
    if current_value and current_value != canonical_path:
        print(
            f"[production-stdb-publish] {file_key} 必须使用固定路径 {canonical_path}: {env_path}",
            file=sys.stderr,
        )
        raise SystemExit(1)
    if file_key_written:
        changed = True
        continue

    next_lines.append(f"{file_key}={canonical_path}")
    file_key_written = True
    changed = changed or raw_line != next_lines[-1]

if not file_key_written:
    next_lines.append(f"{file_key}={canonical_path}")
    changed = True

if changed:
    metadata = env_path.stat()
    fd, temp_name = tempfile.mkstemp(prefix=f".{env_path.name}.", dir=env_path.parent)
    try:
        os.fchmod(fd, stat.S_IMODE(metadata.st_mode))
        os.fchown(fd, metadata.st_uid, metadata.st_gid)
        with os.fdopen(fd, "w", encoding="utf-8") as temp_file:
            temp_file.write("\n".join(next_lines) + "\n")
        os.replace(temp_name, env_path)
    finally:
        if os.path.exists(temp_name):
            os.unlink(temp_name)
PY
  echo "[production-stdb-publish] 已确认 bootstrap secret FILE 配置: ${env_file}"
}

get_runtime_service_active_state() {
  local service_name="$1"
  local state=""
  local exit_code=0

  if state="$(run_privileged systemctl is-active "${service_name}")"; then
    if [[ "${state}" != "active" ]]; then
      echo "[production-stdb-publish] systemctl is-active 返回成功但状态异常: ${service_name}, state=${state}" >&2
      return 1
    fi
    printf '%s\n' "${state}"
    return 0
  else
    exit_code=$?
  fi

  if [[ "${exit_code}" -eq 3 && "${state}" =~ ^(inactive|failed|activating|deactivating|reloading|maintenance|refreshing)$ ]]; then
    printf '%s\n' "${state}"
    return 0
  fi

  echo "[production-stdb-publish] 查询运行时服务状态失败: ${service_name}, exit=${exit_code}, state=${state:-<empty>}" >&2
  return 1
}

restart_runtime_service_and_require_active() {
  local service_name="$1"
  local state=""

  echo "[production-stdb-publish] 重启运行时服务以加载新引导密钥: ${service_name}"
  if ! run_privileged systemctl restart "${service_name}"; then
    echo "[production-stdb-publish] 运行时服务重启失败: ${service_name}" >&2
    return 1
  fi

  if ! state="$(get_runtime_service_active_state "${service_name}")"; then
    return 1
  fi
  if [[ "${state}" != "active" ]]; then
    echo "[production-stdb-publish] 运行时服务重启后未恢复 active: ${service_name}, state=${state}" >&2
    return 1
  fi
  echo "[production-stdb-publish] 运行时服务重启后已恢复 active: ${service_name}"
}

restart_runtime_services_after_bootstrap_secret_install() {
  local api_state=""
  local controller_state=""
  local worker_service=""
  local worker_state=""
  local worker_units_output=""
  local list_units_exit_code=0
  local -a active_worker_services=()

  if ! api_state="$(get_runtime_service_active_state genarrative-api.service)"; then
    return 1
  fi
  if ! controller_state="$(get_runtime_service_active_state genarrative-external-generation-controller.service)"; then
    return 1
  fi

  if worker_units_output="$(
    run_privileged systemctl list-units \
      --type=service \
      --state=active \
      --no-legend \
      --plain \
      'genarrative-external-generation-worker@*.service'
  )"; then
    :
  else
    list_units_exit_code=$?
    echo "[production-stdb-publish] 查询 active worker 服务失败，阻断退出维护模式: exit=${list_units_exit_code}" >&2
    return 1
  fi

  while read -r worker_service _; do
    if [[ "${worker_service}" =~ ^genarrative-external-generation-worker@[A-Za-z0-9_.@:-]+\.service$ ]]; then
      active_worker_services+=("${worker_service}")
    fi
  done <<<"${worker_units_output}"

  if [[ "${api_state}" == "active" ]]; then
    API_WAS_ACTIVE=1
    if ! restart_runtime_service_and_require_active genarrative-api.service; then
      return 1
    fi
  fi
  if [[ "${controller_state}" == "active" ]]; then
    if ! restart_runtime_service_and_require_active genarrative-external-generation-controller.service; then
      return 1
    fi
  fi

  if [[ "${#active_worker_services[@]}" -gt 0 ]]; then
    echo "[production-stdb-publish] 重启运行时 worker 以加载新引导密钥: ${active_worker_services[*]}"
    if ! run_privileged systemctl restart "${active_worker_services[@]}"; then
      echo "[production-stdb-publish] 运行时 worker 重启失败: ${active_worker_services[*]}" >&2
      return 1
    fi
    for worker_service in "${active_worker_services[@]}"; do
      if ! worker_state="$(get_runtime_service_active_state "${worker_service}")"; then
        return 1
      fi
      if [[ "${worker_state}" != "active" ]]; then
        echo "[production-stdb-publish] 运行时 worker 重启后未恢复 active: ${worker_service}, state=${worker_state}" >&2
        return 1
      fi
      echo "[production-stdb-publish] 运行时 worker 重启后已恢复 active: ${worker_service}"
    done
  fi
}

stop_runtime_services_for_rollout_gate() {
  local api_state=""
  local controller_state=""
  local worker_service=""
  local worker_units_output=""
  local -a services_to_stop=()

  api_state="$(get_runtime_service_active_state genarrative-api.service)"
  controller_state="$(get_runtime_service_active_state genarrative-external-generation-controller.service)"
  if ! worker_units_output="$(
    run_privileged systemctl list-units \
      --type=service \
      --state=active \
      --no-legend \
      --plain \
      'genarrative-external-generation-worker@*.service'
  )"; then
    echo "[production-stdb-publish] 查询 active worker 服务失败，无法建立受控维护窗口。" >&2
    return 1
  fi

  if [[ "${controller_state}" == "active" ]]; then
    services_to_stop+=(genarrative-external-generation-controller.service)
  fi
  while read -r worker_service _; do
    if [[ "${worker_service}" =~ ^genarrative-external-generation-worker@[A-Za-z0-9_.@:-]+\.service$ ]]; then
      services_to_stop+=("${worker_service}")
    fi
  done <<<"${worker_units_output}"
  if [[ "${api_state}" == "active" ]]; then
    services_to_stop+=(genarrative-api.service)
  fi

  if [[ "${#services_to_stop[@]}" -eq 0 ]]; then
    echo "[production-stdb-publish] 受控维护窗口开始前没有 active API/controller/worker。"
    return 0
  fi

  echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}"
  run_privileged systemctl stop "${services_to_stop[@]}"
  for worker_service in "${services_to_stop[@]}"; do
    if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then
      echo "[production-stdb-publish] 运行时服务停止后仍为 active: ${worker_service}" >&2
      return 1
    fi
  done
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    -h|--help)
      usage
      exit 0
      ;;
    --source-dir)
      SOURCE_DIR="${2:?缺少 --source-dir 的值}"
      shift 2
      ;;
    --database)
      DATABASE="${2:?缺少 --database 的值}"
      shift 2
      ;;
    --server)
      SERVER_ALIAS="${2:?缺少 --server 的值}"
      SERVER_URL=""
      shift 2
      ;;
    --server-url)
      SERVER_URL="${2:?缺少 --server-url 的值}"
      shift 2
      ;;
    --root-dir)
      SPACETIME_ROOT_DIR="${2:?缺少 --root-dir 的值}"
      shift 2
      ;;
    --run-as-user)
      RUN_AS_USER="${2:?缺少 --run-as-user 的值}"
      shift 2
      ;;
    --migration-bootstrap-secret-file)
      MIGRATION_BOOTSTRAP_SECRET_FILE="${2:?缺少 --migration-bootstrap-secret-file 的值}"
      shift 2
      ;;
    --api-env-file)
      API_ENV_FILE="${2:?缺少 --api-env-file 的值}"
      shift 2
      ;;
    --worker-env-file)
      WORKER_ENV_FILE="${2:?缺少 --worker-env-file 的值}"
      shift 2
      ;;
    --api-health-url)
      API_HEALTH_URL="${2:?缺少 --api-health-url 的值}"
      shift 2
      ;;
    --api-readiness-timeout-seconds)
      API_READINESS_TIMEOUT_SECONDS="${2:?缺少 --api-readiness-timeout-seconds 的值}"
      shift 2
      ;;
    --keep-maintenance-mode)
      KEEP_MAINTENANCE_MODE=1
      shift
      ;;
    --skip-backup)
      BACKUP_MODE="skip"
      shift
      ;;
    --sync-backup)
      BACKUP_MODE="sync"
      shift
      ;;
    --backup-mode)
      BACKUP_MODE="${2:?缺少 --backup-mode 的值}"
      shift 2
      ;;
    *)
      echo "[production-stdb-publish] 未知参数: $1" >&2
      usage >&2
      exit 1
      ;;
  esac
done

require_argument "${SOURCE_DIR}" "--source-dir"
require_argument "${DATABASE}" "--database"
validate_spacetime_database_name "${DATABASE}"

if [[ ! "${SPACETIME_ROOT_DIR}" == /* || "${SPACETIME_ROOT_DIR}" == *".."* ]]; then
  echo "[production-stdb-publish] --root-dir 必须是 Linux 绝对路径且不能包含 ..: ${SPACETIME_ROOT_DIR}" >&2
  exit 1
fi

if [[ "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" != "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE}" ]]; then
  echo "[production-stdb-publish] GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE 必须使用固定路径 ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE}: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" >&2
  exit 1
fi

for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do
  if [[ "${runtime_env_file}" != /* || "${runtime_env_file}" == *".."* || "${runtime_env_file}" == "/" ]]; then
    echo "[production-stdb-publish] 运行时环境文件必须是非根绝对路径且不能包含 ..: ${runtime_env_file}" >&2
    exit 1
  fi
done

if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then
  echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2
  exit 1
fi

if [[ -n "${RUN_AS_USER}" && ! "${RUN_AS_USER}" =~ ^[A-Za-z_][A-Za-z0-9_-]*$ ]]; then
  echo "[production-stdb-publish] --run-as-user 只能是本机用户名: ${RUN_AS_USER}" >&2
  exit 1
fi

if [[ ! "${SPACETIME_READY_TIMEOUT_SECONDS}" =~ ^[0-9]+$ || "${SPACETIME_READY_TIMEOUT_SECONDS}" -le 0 ]]; then
  echo "[production-stdb-publish] GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS 必须是正整数: ${SPACETIME_READY_TIMEOUT_SECONDS}" >&2
  exit 1
fi

if [[ ! "${API_READINESS_TIMEOUT_SECONDS}" =~ ^[0-9]+$ || "${API_READINESS_TIMEOUT_SECONDS}" -le 0 ]]; then
  echo "[production-stdb-publish] API readiness timeout 必须是正整数: ${API_READINESS_TIMEOUT_SECONDS}" >&2
  exit 1
fi

if [[ ! "${API_HEALTH_URL}" =~ ^http://(127\.0\.0\.1|localhost)(:[0-9]{1,5})?/healthz$ ]]; then
  echo "[production-stdb-publish] API readiness 必须使用本机 HTTP /healthz: ${API_HEALTH_URL}" >&2
  exit 1
fi

if [[ ! -d "${SOURCE_DIR}" ]]; then
  echo "[production-stdb-publish] 发布目录不存在: ${SOURCE_DIR}" >&2
  exit 1
fi

SOURCE_DIR="$(cd "${SOURCE_DIR}" && pwd)"

if [[ ! -f "${SOURCE_DIR}/spacetime_module.wasm" || ! -f "${SOURCE_DIR}/spacetime_module.wasm.sha256" || ! -f "${SOURCE_DIR}/release-manifest.json" ]]; then
  echo "[production-stdb-publish] 缺少 spacetime_module.wasm、checksum 或 release-manifest.json: ${SOURCE_DIR}" >&2
  exit 1
fi

if [[ -z "${MIGRATION_BOOTSTRAP_SECRET_FILE}" || ! -f "${MIGRATION_BOOTSTRAP_SECRET_FILE}" || -L "${MIGRATION_BOOTSTRAP_SECRET_FILE}" || ! -r "${MIGRATION_BOOTSTRAP_SECRET_FILE}" ]]; then
  echo "[production-stdb-publish] --migration-bootstrap-secret-file 必须是可读、非符号链接的受保护普通文件。" >&2
  exit 1
fi

MIGRATION_BOOTSTRAP_SECRET="$(cat "${MIGRATION_BOOTSTRAP_SECRET_FILE}")"
MIGRATION_BOOTSTRAP_SECRET="${MIGRATION_BOOTSTRAP_SECRET%$'\r'}"
if [[ ! "${MIGRATION_BOOTSTRAP_SECRET}" =~ ^[0-9a-fA-F]{64}$ ]]; then
  echo "[production-stdb-publish] 运行时服务身份引导密钥必须是 64 位十六进制高熵值。" >&2
  exit 1
fi
MIGRATION_BOOTSTRAP_SECRET_DIGEST_LINE="$(printf '%s' "${MIGRATION_BOOTSTRAP_SECRET}" | sha256sum)"
MIGRATION_BOOTSTRAP_SECRET_SHA256="${MIGRATION_BOOTSTRAP_SECRET_DIGEST_LINE%% *}"
EXPECTED_MIGRATION_BOOTSTRAP_SECRET_SHA256="$(node -e 'const fs=require("node:fs"); const manifest=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.stdout.write(manifest.migration_bootstrap_secret_sha256 || "");' "${SOURCE_DIR}/release-manifest.json")"
if [[ ! "${EXPECTED_MIGRATION_BOOTSTRAP_SECRET_SHA256}" =~ ^[0-9a-f]{64}$ || "${MIGRATION_BOOTSTRAP_SECRET_SHA256}" != "${EXPECTED_MIGRATION_BOOTSTRAP_SECRET_SHA256}" ]]; then
  echo "[production-stdb-publish] Secret File 与构建 WASM 的 bootstrap secret 摘要不一致。" >&2
  exit 1
fi
unset MIGRATION_BOOTSTRAP_SECRET MIGRATION_BOOTSTRAP_SECRET_DIGEST_LINE

on_exit() {
  local exit_code=$?
  if [[ "${BACKUP_MODE}" == "async" && -n "${ASYNC_BACKUP_STATUS_FILE}" && -f "${ASYNC_BACKUP_STATUS_FILE}" ]]; then
    start_async_backup_upload || true
  fi
  if [[ -n "${PUBLISH_TMP_DIR}" && -d "${PUBLISH_TMP_DIR}" ]]; then
    rm -rf "${PUBLISH_TMP_DIR}"
  fi
  if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
    echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2
  fi
  exit "${exit_code}"
}

trap on_exit EXIT

prepare_async_backup() {
  local -a restart_service_args=()
  ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
  if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
    ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
  fi
  if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
    echo "[production-stdb-publish] 缺少数据库备份脚本: ${ASYNC_BACKUP_SCRIPT}" >&2
    exit 1
  fi

  if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
    restart_service_args+=(--restart-service-after genarrative-api.service)
  fi

  ASYNC_BACKUP_STATUS_FILE="$(mktemp /tmp/genarrative-stdb-backup-status.XXXXXX.json)"
  echo "[production-stdb-publish] publish 前生成本地冷备份，随后会异步上传 OSS"
  node -- "${ASYNC_BACKUP_SCRIPT}" \
    --env-file /etc/genarrative/api-server.env \
    --data-dir "${SPACETIME_ROOT_DIR}" \
    --database "${DATABASE}" \
    --stop-service spacetimedb.service \
    "${restart_service_args[@]}" \
    --defer-upload \
    --result-file "${ASYNC_BACKUP_STATUS_FILE}"
}

start_async_backup_upload() {
  if [[ -z "${ASYNC_BACKUP_STATUS_FILE}" || ! -f "${ASYNC_BACKUP_STATUS_FILE}" ]]; then
    echo "[production-stdb-publish] 警告：未找到可上传的本地备份状态文件，跳过异步上传" >&2
    return 0
  fi

  ASYNC_BACKUP_ARCHIVE="$(node -e 'const fs=require("node:fs"); const p=process.argv[1]; const o=JSON.parse(fs.readFileSync(p,"utf8")); process.stdout.write(o.archivePath || "");' "${ASYNC_BACKUP_STATUS_FILE}")"
  ASYNC_BACKUP_MANIFEST="$(node -e 'const fs=require("node:fs"); const p=process.argv[1]; const o=JSON.parse(fs.readFileSync(p,"utf8")); process.stdout.write(o.manifestPath || "");' "${ASYNC_BACKUP_STATUS_FILE}")"
  if [[ -z "${ASYNC_BACKUP_ARCHIVE}" || -z "${ASYNC_BACKUP_MANIFEST}" ]]; then
    echo "[production-stdb-publish] 警告：备份状态文件缺少 archivePath 或 manifestPath，跳过异步上传" >&2
    return 0
  fi

  mkdir -p "$(dirname "${ASYNC_BACKUP_ARCHIVE}")"
  ASYNC_BACKUP_LOG="$(dirname "${ASYNC_BACKUP_ARCHIVE}")/${DATABASE}-upload.log"
  echo "[production-stdb-publish] 后台上传本地备份到 OSS: ${ASYNC_BACKUP_ARCHIVE}"
  nohup node -- "${ASYNC_BACKUP_SCRIPT}" \
    --env-file /etc/genarrative/api-server.env \
    --upload-archive "${ASYNC_BACKUP_ARCHIVE}" \
    --manifest-file "${ASYNC_BACKUP_MANIFEST}" \
    >"${ASYNC_BACKUP_LOG}" 2>&1 &
  echo "[production-stdb-publish] OSS 后台上传日志: ${ASYNC_BACKUP_LOG}"
  rm -f "${ASYNC_BACKUP_STATUS_FILE}"
  ASYNC_BACKUP_STATUS_FILE=""
}

wait_for_spacetime_ready() {
  if [[ -z "${SERVER_URL}" ]]; then
    echo "[production-stdb-publish] 使用 server alias=${SERVER_ALIAS}，跳过 URL 健康检查等待"
    return 0
  fi

  local ping_url="${SERVER_URL%/}/v1/ping"
  local deadline=$((SECONDS + SPACETIME_READY_TIMEOUT_SECONDS))
  local last_status=""

  echo "[production-stdb-publish] 等待 SpacetimeDB 就绪: ${ping_url}，timeout=${SPACETIME_READY_TIMEOUT_SECONDS}s"
  while (( SECONDS < deadline )); do
    # curl 失败时通常表示服务尚未监听；不立即失败，等待冷备份恢复后的 systemd 启动完成。
    if last_status="$(curl -fsS --max-time 2 "${ping_url}" 2>&1)"; then
      echo "[production-stdb-publish] SpacetimeDB 已就绪: ${ping_url}"
      return 0
    fi
    sleep 2
  done

  echo "[production-stdb-publish] SpacetimeDB 未在超时内就绪: ${ping_url}" >&2
  if [[ -n "${last_status}" ]]; then
    echo "[production-stdb-publish] 最后一次健康检查输出: ${last_status}" >&2
  fi
  return 1
}

wait_for_api_healthz_ready() {
  if [[ "${API_WAS_ACTIVE}" -ne 1 ]]; then
    echo "[production-stdb-publish] API 重启前不是 active，跳过 /healthz readiness"
    return 0
  fi

  local deadline=$((SECONDS + API_READINESS_TIMEOUT_SECONDS))
  local last_status=""

  echo "[production-stdb-publish] 等待 API /healthz readiness: ${API_HEALTH_URL}，timeout=${API_READINESS_TIMEOUT_SECONDS}s"
  while (( SECONDS < deadline )); do
    if last_status="$(curl -fsS --max-time 2 "${API_HEALTH_URL}" 2>&1)"; then
      echo "[production-stdb-publish] API /healthz readiness 已通过: ${API_HEALTH_URL}"
      return 0
    fi
    sleep 2
  done

  echo "[production-stdb-publish] API /healthz readiness 未在超时内通过: ${API_HEALTH_URL}" >&2
  if [[ -n "${last_status}" ]]; then
    echo "[production-stdb-publish] 最后一次 /healthz 检查输出: ${last_status}" >&2
  fi
  return 1
}

"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"
if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
  stop_runtime_services_for_rollout_gate
fi

case "${BACKUP_MODE}" in
  async)
    prepare_async_backup
    ;;
  sync)
    SYNC_BACKUP_RESTART_SERVICE_ARGS=()
    BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
    if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
      BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
    fi
    if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
      echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2
      exit 1
    fi
    if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
      SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service)
    fi

    echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布"
    node -- "${BACKUP_SCRIPT}" \
      --env-file /etc/genarrative/api-server.env \
      --data-dir "${SPACETIME_ROOT_DIR}" \
      --database "${DATABASE}" \
      --stop-service spacetimedb.service \
      "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}"
    ;;
  skip)
    echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份"
    ;;
esac

echo "[production-stdb-publish] 校验 wasm"
(
  cd "${SOURCE_DIR}"
  sha256sum -c spacetime_module.wasm.sha256
)

wait_for_spacetime_ready

PUBLISH_ARGS=(
  --root-dir="${SPACETIME_ROOT_DIR}"
  publish
  "${DATABASE}"
  --bin-path "${SOURCE_DIR}/spacetime_module.wasm"
  --delete-data=never
  --yes=migrate,break-clients
  --no-config
)

if [[ -n "${SERVER_URL}" ]]; then
  PUBLISH_ARGS+=(--server "${SERVER_URL}")
else
  PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
fi

if [[ -n "${SERVER_URL}" ]]; then
  echo "[production-stdb-publish] 发布 SpacetimeDB module: ${DATABASE} -> ${SERVER_URL}, root=${SPACETIME_ROOT_DIR}"
else
  echo "[production-stdb-publish] 发布 SpacetimeDB module: ${DATABASE} -> ${SERVER_ALIAS}, root=${SPACETIME_ROOT_DIR}"
fi

if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then
  if ! id "${RUN_AS_USER}" >/dev/null 2>&1; then
    echo "[production-stdb-publish] 发布用户不存在: ${RUN_AS_USER}" >&2
    exit 1
  fi
  PUBLISH_TMP_DIR="$(mktemp -d /tmp/genarrative-stdb-publish.XXXXXX)"
  install -m 0644 "${SOURCE_DIR}/spacetime_module.wasm" "${PUBLISH_TMP_DIR}/spacetime_module.wasm"
  chown -R "${RUN_AS_USER}:${RUN_AS_USER}" "${PUBLISH_TMP_DIR}"
  PUBLISH_ARGS=(
    --root-dir="${SPACETIME_ROOT_DIR}"
    publish
    "${DATABASE}"
    --bin-path "${PUBLISH_TMP_DIR}/spacetime_module.wasm"
    --delete-data=never
    --yes=migrate,break-clients
    --no-config
  )
  if [[ -n "${SERVER_URL}" ]]; then
    PUBLISH_ARGS+=(--server "${SERVER_URL}")
  else
    PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
  fi
  runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}"
else
  spacetime "${PUBLISH_ARGS[@]}"
fi

RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR="$(dirname "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}")"
if [[ -L "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR}" || -L "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" ]]; then
  echo "[production-stdb-publish] 运行时服务身份引导密钥路径不能是符号链接: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" >&2
  exit 1
fi
if ! id genarrative >/dev/null 2>&1; then
  echo "[production-stdb-publish] 缺少运行时服务用户 genarrative，无法安全安装引导密钥。" >&2
  exit 1
fi

run_privileged install -d -o root -g genarrative -m 0750 "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR}"
run_privileged install -o root -g genarrative -m 0440 \
  "${MIGRATION_BOOTSTRAP_SECRET_FILE}" \
  "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
RUNTIME_SERVICE_BOOTSTRAP_SECRET_METADATA="$(run_privileged stat -c '%U:%G:%a' "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}")"
if [[ "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_METADATA}" != "root:genarrative:440" ]]; then
  echo "[production-stdb-publish] 运行时服务身份引导密钥权限不符合 root:genarrative:0440: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_METADATA}" >&2
  exit 1
fi
run_privileged runuser -u genarrative -- test -r "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
echo "[production-stdb-publish] 已安装运行时服务身份引导密钥: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
ensure_runtime_bootstrap_secret_env_file "${API_ENV_FILE}" true
ensure_runtime_bootstrap_secret_env_file "${WORKER_ENV_FILE}" false
if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
  echo "[production-stdb-publish] module 发布完成；按参数保持维护模式和旧运行时服务停止状态，等待人工维护与 API deploy。"
  DEPLOY_COMPLETED=1
  exit 0
fi
restart_runtime_services_after_bootstrap_secret_install
wait_for_api_healthz_ready

"${SCRIPT_DIR}/maintenance-off.sh"
DEPLOY_COMPLETED=1

echo "[production-stdb-publish] 完成"