Genarrative/scripts/jenkins-server-provision.sh

#!/usr/bin/env bash
set -euo pipefail

PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}"
SPACETIME_BIN_SOURCE="${SPACETIME_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/spacetime/spacetime}"
OTELCOL_BIN_SOURCE="${OTELCOL_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/otelcol-contrib}"
WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
CONTROLLER_ENV_FILE="${CONTROLLER_ENV_FILE:-/etc/genarrative/external-generation-controller.env}"
BGFILTER_WORKER_ENV_FILE="${BGFILTER_WORKER_ENV_FILE:-/etc/genarrative/bgfilter-worker.env}"
GENARRATIVE_OPENSSL_VERSION="${GENARRATIVE_OPENSSL_VERSION:-3.2.0}"
GENARRATIVE_OPENSSL_PREFIX="${GENARRATIVE_OPENSSL_PREFIX:-/opt/genarrative/openssl-3.2.0}"
GENARRATIVE_OPENSSL_SOURCE_URL="${GENARRATIVE_OPENSSL_SOURCE_URL:-https://github.com/openssl/openssl/releases/download/openssl-${GENARRATIVE_OPENSSL_VERSION}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz}"
GENARRATIVE_OPENSSL_SOURCE_SHA256="${GENARRATIVE_OPENSSL_SOURCE_SHA256:-14c826f07c7e433706fb5c69fa9e25dab95684844b4c962a2cf1bf183eb4690e}"
DATABASE_BACKUP_PROFILE="${DATABASE_BACKUP_PROFILE:-archive-full}"
DATABASE_BACKUP_FILES_HISTORY_WORK_DIR="${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR:-/var/lib/genarrative/database-backups/files-history}"
DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR="/etc/systemd/system/genarrative-database-backup.service.d"
DATABASE_BACKUP_FILES_HISTORY_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-history.conf"
DATABASE_BACKUP_LEGACY_DEV_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-dev-files.conf"

require_non_root_relative_path() {
  local label="$1"
  local path="$2"

  if [[ -z "${path}" ]]; then
    echo "[server-provision] ${label} 不能为空。" >&2
    exit 1
  fi
  if [[ "${path}" == /* || "${path}" == *..* ]]; then
    echo "[server-provision] ${label} 只能是工作区内的相对路径: ${path}" >&2
    exit 1
  fi
}

require_path() {
  local path="$1"
  if [[ ! -e "${path}" ]]; then
    echo "[server-provision] 缺少必要文件: ${path}" >&2
    exit 1
  fi
}

require_cmd() {
  local name="$1"
  if ! command -v "${name}" >/dev/null 2>&1; then
    echo "[server-provision] 缺少命令: ${name}" >&2
    exit 1
  fi
}

normalize_server_aliases() {
  printf "%s" "${SERVER_ALIASES:-}" | tr ',' ' ' | xargs
}

validate_server_names() {
  local alias_name
  if [[ -z "${SERVER_NAME:-}" ]]; then
    echo "[server-provision] SERVER_NAME 不能为空。" >&2
    exit 1
  fi
  if [[ ! "${SERVER_NAME}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]]; then
    echo "[server-provision] SERVER_NAME 只能填写单个域名或 IP，不能包含空格、路径或协议: ${SERVER_NAME}" >&2
    exit 1
  fi
  for alias_name in $(normalize_server_aliases); do
    if [[ ! "${alias_name}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]]; then
      echo "[server-provision] SERVER_ALIASES 只能填写域名或 IP，多个用空格或逗号分隔: ${alias_name}" >&2
      exit 1
    fi
  done
}

validate_database_backup_profile() {
  case "${DATABASE_BACKUP_PROFILE}" in
    archive-full|files-history)
      ;;
    *)
      echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${DATABASE_BACKUP_PROFILE}" >&2
      exit 1
      ;;
  esac
  if [[ "${DEPLOY_TARGET}" == "release" && "${DATABASE_BACKUP_PROFILE}" == "files-history" ]]; then
    echo "[server-provision] release 仅允许 archive-full；files-history 会把整棵历史目录加载到 Node 内存，需先完成流式 catalog 改造后才能重新启用。" >&2
    exit 1
  fi
  if [[ ! "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" =~ ^/var/lib/genarrative/database-backups/[A-Za-z0-9._/-]+$ || "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" == *..* ]]; then
    echo "[server-provision] DATABASE_BACKUP_FILES_HISTORY_WORK_DIR 必须是 /var/lib/genarrative/database-backups/ 下不含连续点号的绝对路径，当前值: ${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" >&2
    exit 1
  fi
}

run_cmd() {
  echo "+ $*"
  if [[ "${DRY_RUN}" != "true" ]]; then
    "$@"
  fi
}

require_root_for_real_provision() {
  if [[ "${DRY_RUN}" == "true" ]]; then
    return
  fi

  if [[ "$(id -u)" != "0" ]]; then
    echo "[server-provision] 非 dry-run 会安装系统包、写入 systemd/Nginx 和创建系统用户，必须在 root agent 上执行。" >&2
    echo "[server-provision] 当前用户: $(id -un) uid=$(id -u)。请确认 DEPLOY_TARGET=${DEPLOY_TARGET:-} 对应的目标服务器 agent 以 root 运行，或保持 DRY_RUN=true。" >&2
    exit 1
  fi
}

install_file() {
  local source="$1"
  local target="$2"
  local mode="$3"
  echo "+ install -m ${mode} ${source} ${target}"
  if [[ "${DRY_RUN}" != "true" ]]; then
    install -m "${mode}" "${source}" "${target}"
  fi
}

install_nginx_brotli_modules() {
  echo "[server-provision] 安装 Nginx Brotli 动态模块依赖"
  if command -v apt-get >/dev/null 2>&1; then
    run_cmd apt-get install -y libnginx-mod-http-brotli-filter libnginx-mod-http-brotli-static
  else
    echo "[server-provision] 当前系统未使用 apt，无法自动安装 Nginx Brotli 动态模块；将继续通过 nginx -t 能力探测决定是否启用 Brotli。"
  fi
}

download_file() {
  local url="$1"
  local output="$2"

  if command -v curl >/dev/null 2>&1; then
    curl -fsSL --retry 3 --retry-delay 2 "${url}" -o "${output}"
  elif command -v wget >/dev/null 2>&1; then
    wget -O "${output}" "${url}"
  else
    echo "[server-provision] 需要 curl 或 wget 下载: ${url}" >&2
    exit 1
  fi
}

openssl_lib_dir_candidates() {
  printf "%s\n" \
    "${GENARRATIVE_OPENSSL_PREFIX}/lib64" \
    "${GENARRATIVE_OPENSSL_PREFIX}/lib"
}

find_genarrative_openssl_lib_dir() {
  local lib_dir
  while IFS= read -r lib_dir; do
    if [[ -f "${lib_dir}/libssl.so.3" && -f "${lib_dir}/libcrypto.so.3" ]]; then
      printf "%s" "${lib_dir}"
      return 0
    fi
  done < <(openssl_lib_dir_candidates)
  return 1
}

genarrative_openssl_has_required_symbol() {
  local lib_dir
  lib_dir="$(find_genarrative_openssl_lib_dir 2>/dev/null || true)"
  if [[ -z "${lib_dir}" ]]; then
    return 1
  fi
  grep -a -q "OPENSSL_${GENARRATIVE_OPENSSL_VERSION}" "${lib_dir}/libssl.so.3"
}

verify_genarrative_openssl_install() {
  local lib_dir
  lib_dir="$(find_genarrative_openssl_lib_dir 2>/dev/null || true)"
  if [[ -z "${lib_dir}" ]]; then
    echo "[server-provision] OpenSSL ${GENARRATIVE_OPENSSL_VERSION} 安装后缺少 libssl.so.3/libcrypto.so.3: ${GENARRATIVE_OPENSSL_PREFIX}" >&2
    exit 1
  fi
  if ! grep -a -q "OPENSSL_${GENARRATIVE_OPENSSL_VERSION}" "${lib_dir}/libssl.so.3"; then
    echo "[server-provision] OpenSSL 动态库缺少 OPENSSL_${GENARRATIVE_OPENSSL_VERSION} 符号: ${lib_dir}/libssl.so.3" >&2
    exit 1
  fi
  if ! env "LD_LIBRARY_PATH=${lib_dir}" "${GENARRATIVE_OPENSSL_PREFIX}/bin/openssl" version | grep -q "OpenSSL ${GENARRATIVE_OPENSSL_VERSION}"; then
    echo "[server-provision] OpenSSL ${GENARRATIVE_OPENSSL_VERSION} 安装后命令验证失败: ${GENARRATIVE_OPENSSL_PREFIX}/bin/openssl" >&2
    exit 1
  fi
  echo "[server-provision] OpenSSL ${GENARRATIVE_OPENSSL_VERSION} 已就绪: ${lib_dir}"
}

install_genarrative_openssl_runtime() {
  local tmp_dir archive source_dir jobs lib_dir

  echo "[server-provision] 检查 api-server/libcurl 运行时 OpenSSL ${GENARRATIVE_OPENSSL_VERSION}"
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ install OpenSSL ${GENARRATIVE_OPENSSL_VERSION} into ${GENARRATIVE_OPENSSL_PREFIX}"
    echo "+ verify OPENSSL_${GENARRATIVE_OPENSSL_VERSION} symbol for api-server/libcurl"
    return
  fi

  if genarrative_openssl_has_required_symbol; then
    verify_genarrative_openssl_install
    return
  fi

  if command -v apt-get >/dev/null 2>&1; then
    run_cmd apt-get install -y build-essential ca-certificates curl perl tar
  else
    echo "[server-provision] 当前系统未使用 apt，无法自动构建 OpenSSL ${GENARRATIVE_OPENSSL_VERSION}；请手动安装到 ${GENARRATIVE_OPENSSL_PREFIX}。" >&2
    exit 1
  fi
  require_cmd sha256sum
  require_cmd tar

  tmp_dir="$(mktemp -d)"
  archive="${tmp_dir}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz"
  echo "[server-provision] 下载 OpenSSL ${GENARRATIVE_OPENSSL_VERSION}: ${GENARRATIVE_OPENSSL_SOURCE_URL}"
  download_file "${GENARRATIVE_OPENSSL_SOURCE_URL}" "${archive}"
  printf "%s  %s\n" "${GENARRATIVE_OPENSSL_SOURCE_SHA256}" "${archive}" | sha256sum -c -

  tar -xzf "${archive}" -C "${tmp_dir}"
  source_dir="${tmp_dir}/openssl-${GENARRATIVE_OPENSSL_VERSION}"
  jobs="$(nproc 2>/dev/null || echo 2)"
  (
    cd "${source_dir}"
    ./config --prefix="${GENARRATIVE_OPENSSL_PREFIX}" --openssldir="${GENARRATIVE_OPENSSL_PREFIX}/ssl" shared
    make -j "${jobs}"
    make install_sw
  )
  rm -rf "${tmp_dir}"

  lib_dir="$(find_genarrative_openssl_lib_dir 2>/dev/null || true)"
  if [[ -n "${lib_dir}" ]]; then
    chmod 0755 "${GENARRATIVE_OPENSSL_PREFIX}" "${lib_dir}" || true
    chmod 0644 "${lib_dir}/libssl.so.3" "${lib_dir}/libcrypto.so.3" || true
  fi
  verify_genarrative_openssl_install
}

sync_otelcol_install() {
  local target_bin="/usr/local/bin/otelcol-contrib"
  local source_bin="${OTELCOL_BIN_SOURCE}"
  local version="${OTELCOL_VERSION:-0.151.0}"
  local resolved_source="${source_bin}"

  if [[ "${ENABLE_OTELCOL:-true}" != "true" ]]; then
    echo "[server-provision] ENABLE_OTELCOL=${ENABLE_OTELCOL:-}，跳过 otelcol-contrib 配置。"
    return
  fi

  if command -v readlink >/dev/null 2>&1; then
    resolved_source="$(readlink -f "${source_bin}" 2>/dev/null || echo "${source_bin}")"
  fi

  if [[ ! -x "${resolved_source}" ]]; then
    echo "[server-provision] otelcol-contrib 不存在或不可执行: ${source_bin}" >&2
    echo "[server-provision] 请确认 Prepare Provision Tools 已在目标 agent 生成 otelcol-contrib ${version}: ${source_bin}" >&2
    exit 1
  fi

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ install -m 0755 ${resolved_source} ${target_bin}"
    return
  fi

  install -m 0755 "${resolved_source}" "${target_bin}"
  if ! "${target_bin}" --version >/dev/null 2>&1; then
    echo "[server-provision] otelcol-contrib 安装后无法执行: ${target_bin}" >&2
    exit 1
  fi
  if ! "${target_bin}" --version 2>/dev/null | grep -q "${version}"; then
    echo "[server-provision] 警告: otelcol-contrib 版本不是期望的 ${version}: $("${target_bin}" --version 2>/dev/null || true)" >&2
  fi
}

ensure_otelcol_runtime() {
  if [[ "${ENABLE_OTELCOL:-true}" != "true" ]]; then
    return
  fi

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ ensure system user/group otelcol"
    echo "+ install -d -m 0755 -o otelcol -g otelcol /var/lib/otelcol"
    echo "+ install -d -m 0755 -o root -g root /etc/otelcol"
    echo "+ install -d -m 0755 -o genarrative -g genarrative /var/log/genarrative"
    echo "+ install -m 0644 deploy/otelcol/genarrative-debug.yaml /etc/otelcol/genarrative-debug.yaml"
    return
  fi

  if ! getent group otelcol >/dev/null 2>&1; then
    groupadd --system otelcol
  fi
  if ! id otelcol >/dev/null 2>&1; then
    useradd --system --gid otelcol --home-dir /var/lib/otelcol --shell /usr/sbin/nologin otelcol
  fi

  install -d -m 0755 -o otelcol -g otelcol /var/lib/otelcol
  install -d -m 0755 -o root -g root /etc/otelcol
  install -d -m 0755 -o genarrative -g genarrative /var/log/genarrative
  install -m 0644 deploy/otelcol/genarrative-debug.yaml /etc/otelcol/genarrative-debug.yaml
  chown root:root /etc/otelcol/genarrative-debug.yaml
}

stamp_database_backup_timer_now() {
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ install -d -m 0755 /var/lib/systemd/timers"
    echo "+ touch /var/lib/systemd/timers/stamp-genarrative-database-backup.timer"
    return
  fi

  install -d -m 0755 /var/lib/systemd/timers
  # 避免 provision 在当天 03:20 之后启动 timer 时因 Persistent=true 立刻补跑冷备份、
  # 进而在初始化/发布窗口中意外停止 spacetimedb.service。
  touch /var/lib/systemd/timers/stamp-genarrative-database-backup.timer
}

sync_spacetime_install() {
  local root_dir="$1"
  local target_bin_dir="${root_dir}/bin/current"
  local target_cli="${target_bin_dir}/spacetimedb-cli"
  local target_standalone="${target_bin_dir}/spacetimedb-standalone"
  local resolved_command="${SPACETIME_BIN_SOURCE}"
  local install_dir=""
  local root_bin="${root_dir}/bin"

  echo "[server-provision] 同步 SpacetimeDB current 目录到 ${target_bin_dir}"
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ mkdir -p ${target_bin_dir}"
    echo "+ copy spacetimedb-cli and spacetimedb-standalone into ${target_bin_dir}"
    return
  fi

  if command -v readlink >/dev/null 2>&1; then
    resolved_command="$(readlink -f "${SPACETIME_BIN_SOURCE}" 2>/dev/null || echo "${SPACETIME_BIN_SOURCE}")"
  fi
  install_dir="$(cd -- "$(dirname -- "${resolved_command}")" && pwd)"
  mkdir -p "${root_bin}"

  if [[ -d "${install_dir}/bin" ]]; then
    echo "[server-provision] 同步 SpacetimeDB 安装: ${install_dir}/bin -> ${root_bin}"
    rm -rf "${root_bin}"
    mkdir -p "${root_bin}"
    cp -a "${install_dir}/bin/." "${root_bin}/"
  elif [[ -x "${install_dir}/spacetimedb-cli" && -x "${install_dir}/spacetimedb-standalone" ]]; then
    echo "[server-provision] 同步 SpacetimeDB 安装: ${install_dir} -> ${target_bin_dir}"
    rm -rf "${target_bin_dir}"
    mkdir -p "${target_bin_dir}"
    cp -f "${install_dir}/spacetimedb-cli" "${target_cli}"
    cp -f "${install_dir}/spacetimedb-standalone" "${target_standalone}"
    chmod +x "${target_cli}" "${target_standalone}"
  else
    echo "[server-provision] 未能从 SpacetimeDB 交付包推断完整安装目录: ${resolved_command}" >&2
  fi

  if [[ ! -x "${target_cli}" || ! -x "${target_standalone}" ]]; then
    echo "[server-provision] 同步 SpacetimeDB 安装后仍缺少 current 目录。" >&2
    echo "[server-provision] 需要同时存在: ${target_cli} 与 ${target_standalone}" >&2
    exit 1
  fi

  chown -R spacetimedb:spacetimedb "${root_bin}"
}

is_spacetimedb_ready() {
  local server_url="http://127.0.0.1:3101"

  if command -v curl >/dev/null 2>&1 && curl -fsS "${server_url}/v1/ping" >/dev/null 2>&1; then
    return 0
  fi

  return 1
}

wait_for_spacetimedb_service() {
  local deadline=$((SECONDS + 60))

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ wait for spacetimedb.service on http://127.0.0.1:3101"
    return
  fi

  while ((SECONDS < deadline)); do
    if is_spacetimedb_ready; then
      echo "[server-provision] spacetimedb.service 已就绪: http://127.0.0.1:3101"
      return
    fi
    sleep 1
  done

  echo "[server-provision] 等待 spacetimedb.service 就绪超时。" >&2
  systemctl status spacetimedb.service --no-pager -l >&2 || true
  journalctl -u spacetimedb.service --no-pager -n 80 >&2 || true
  ss -ltnp >&2 || true
  exit 1
}

read_env_value() {
  local file="$1"
  local key="$2"
  local line value quote_char
  quote_char='"'

  if [[ ! -f "${file}" ]]; then
    return
  fi

  while IFS= read -r line || [[ -n "${line}" ]]; do
    if [[ "${line}" == "${key}="* ]]; then
      value="${line#*=}"
      value="$(printf "%s" "${value}" | tr -d "\r")"
      if [[ ${#value} -ge 2 && "${value:0:1}" == "${quote_char}" && "${value: -1}" == "${quote_char}" ]]; then
        value="${value:1:${#value}-2}"
      fi
      printf "%s" "${value}"
      return
    fi
  done <"${file}"
}

env_contains_nonempty_assignment() {
  local file="$1"
  local key="$2"
  local line value first_char last_char

  if [[ ! -f "${file}" ]]; then
    return 1
  fi

  while IFS= read -r line || [[ -n "${line}" ]]; do
    line="${line#"${line%%[![:space:]]*}"}"
    if [[ -z "${line}" || "${line}" == \#* || "${line}" != "${key}="* ]]; then
      continue
    fi
    value="${line#*=}"
    value="${value%$'\r'}"
    value="${value#"${value%%[![:space:]]*}"}"
    value="${value%"${value##*[![:space:]]}"}"
    if [[ ${#value} -ge 2 ]]; then
      first_char="${value:0:1}"
      last_char="${value: -1}"
      if [[ "${first_char}" == "${last_char}" && ( "${first_char}" == '"' || "${first_char}" == "'" ) ]]; then
        value="${value:1:${#value}-2}"
      fi
    fi
    if [[ "${value}" =~ [^[:space:]] ]]; then
      return 0
    fi
  done <"${file}"
  return 1
}

env_has_assignment() {
  local file="$1"
  local key="$2"
  local line current_key

  if [[ ! -f "${file}" ]]; then
    return 1
  fi

  while IFS= read -r line || [[ -n "${line}" ]]; do
    line="${line%$'\r'}"
    line="${line#"${line%%[![:space:]]*}"}"
    if [[ -z "${line}" || "${line}" == \#* || "${line}" != *"="* ]]; then
      continue
    fi
    current_key="${line%%=*}"
    current_key="${current_key%"${current_key##*[![:space:]]}"}"
    if [[ "${current_key}" == "${key}" ]]; then
      return 0
    fi
  done <"${file}"
  return 1
}

read_effective_env_value() {
  local file="$1"
  local key="$2"
  local line current_key value first_char last_char matched_value="" found="false"

  if [[ ! -f "${file}" ]]; then
    return
  fi

  while IFS= read -r line || [[ -n "${line}" ]]; do
    line="${line%$'\r'}"
    line="${line#"${line%%[![:space:]]*}"}"
    if [[ -z "${line}" || "${line}" == \#* || "${line}" != *"="* ]]; then
      continue
    fi
    current_key="${line%%=*}"
    current_key="${current_key%"${current_key##*[![:space:]]}"}"
    if [[ "${current_key}" != "${key}" ]]; then
      continue
    fi
    value="${line#*=}"
    value="${value#"${value%%[![:space:]]*}"}"
    value="${value%"${value##*[![:space:]]}"}"
    if [[ ${#value} -ge 2 ]]; then
      first_char="${value:0:1}"
      last_char="${value: -1}"
      if [[ "${first_char}" == "${last_char}" && ( "${first_char}" == '"' || "${first_char}" == "'" ) ]]; then
        value="${value:1:${#value}-2}"
      fi
    fi
    matched_value="${value}"
    found="true"
  done <"${file}"

  if [[ "${found}" == "true" ]]; then
    printf "%s" "${matched_value}"
  fi
}

write_env_value() {
  local file="$1"
  local key="$2"
  local value="$3"
  local tmp updated line

  tmp="$(mktemp)"
  updated="false"
  while IFS= read -r line || [[ -n "${line}" ]]; do
    if [[ "${line}" == "${key}="* ]]; then
      if [[ "${updated}" != "true" ]]; then
        printf "%s=%s\\n" "${key}" "${value}" >>"${tmp}"
        updated="true"
      fi
    else
      printf "%s\\n" "${line}" >>"${tmp}"
    fi
  done <"${file}"
  if [[ "${updated}" != "true" ]]; then
    printf "%s=%s\\n" "${key}" "${value}" >>"${tmp}"
  fi

  cat "${tmp}" >"${file}"
  rm -f "${tmp}"
  chmod 0600 "${file}"
  chown root:root "${file}"
}

ensure_env_value() {
  local file="$1"
  local key="$2"
  local default_value="$3"
  local current_value

  current_value="$(read_env_value "${file}" "${key}")"
  if [[ -n "${current_value}" ]]; then
    return
  fi

  echo "[server-provision] 补齐运行态环境变量: ${key} -> ${file}"
  if [[ "${DRY_RUN}" != "true" ]]; then
    write_env_value "${file}" "${key}" "${default_value}"
  fi
}

remove_env_key_if_present() {
  local file="$1"
  local key="$2"

  if [[ ! -f "${file}" ]]; then
    return
  fi
  if ! grep -Eq "^[[:space:]]*(export[[:space:]]+)?${key}=" "${file}"; then
    return
  fi

  echo "[server-provision] 移除已退役的环境变量: ${key} <- ${file}"
  if [[ "${DRY_RUN}" != "true" ]]; then
    local tmp_file
    tmp_file="$(mktemp)"
    grep -Ev "^[[:space:]]*(export[[:space:]]+)?${key}=" "${file}" >"${tmp_file}"
    cat "${tmp_file}" >"${file}"
    rm -f "${tmp_file}"
  fi
}

ensure_env_value_migrates_old_default() {
  local file="$1"
  local key="$2"
  local old_default="$3"
  local new_default="$4"
  local current_value

  # 迁移判断必须读取最后一次有效赋值；生产 env 若暂时存在重复键，后写值才是
  # systemd EnvironmentFile 的实际语义，不能因前面的历史默认覆盖后面的自定义值。
  current_value="$(read_effective_env_value "${file}" "${key}")"
  if [[ -z "${current_value}" ]]; then
    ensure_env_value "${file}" "${key}" "${new_default}"
    return
  fi
  if [[ "${current_value}" != "${old_default}" ]]; then
    return
  fi

  echo "[server-provision] 迁移运行态环境变量旧默认值: ${key} ${old_default} -> ${new_default} (${file})"
  if [[ "${DRY_RUN}" != "true" ]]; then
    write_env_value "${file}" "${key}" "${new_default}"
  fi
}

ensure_runtime_bootstrap_secret_file_env() {
  local file="$1"
  local key="GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE"
  local direct_key="GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET"
  local canonical_path="/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt"
  local current_value direct_value

  direct_value="$(read_env_value "${file}" "${direct_key}")"
  if [[ -n "${direct_value}" ]]; then
    unset direct_value
    echo "[server-provision] ${file} 不得保存 ${direct_key} 明文；生产环境只允许使用 ${key}。" >&2
    exit 1
  fi
  unset direct_value

  current_value="$(read_env_value "${file}" "${key}")"
  if [[ -z "${current_value}" ]]; then
    ensure_env_value "${file}" "${key}" "${canonical_path}"
    return
  fi
  if [[ "${current_value}" != "${canonical_path}" ]]; then
    echo "[server-provision] ${key} 必须使用 Stdb publish 同步写入的固定路径 ${canonical_path}: ${file}" >&2
    exit 1
  fi
}

ensure_api_runtime_env_defaults() {
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ ensure api-server runtime env defaults in ${API_ENV_FILE}"
    return
  fi
  if [[ ! -f "${API_ENV_FILE}" ]]; then
    echo "[server-provision] 环境文件不存在，无法补齐 api-server 运行态目录变量: ${API_ENV_FILE}" >&2
    exit 1
  fi

  # 已存在的生产 env 会被保留，不会整文件覆盖；这里仅补后续版本新增的运行态写入路径。
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED" "true"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_DIR" "/var/lib/genarrative/tracking-outbox"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE" "500"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS" "1000"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES" "268435456"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_ENABLED" "true"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_DIR" "/var/lib/genarrative/wallet-refund-outbox"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE" "100"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS" "1000"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES" "67108864"
  ensure_env_value_migrates_old_default "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL" "http://127.0.0.1:8083"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE" "/etc/genarrative/secrets/bgfilter-worker.token"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS" "2000"
  ensure_runtime_bootstrap_secret_file_env "${API_ENV_FILE}"
  # N 与单图估时是父子共同派生 attempt/callBudget 公式的输入，必须放共享 API env 单一来源。
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY" "16"
  ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS" "5000"
  remove_env_key_if_present "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS"
}

validate_bgfilter_shared_runtime_env() {
  local shared_concurrency shared_estimate_ms connect_timeout_ms

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ validate shared BgFilter concurrency/estimate inputs in ${API_ENV_FILE}"
    return
  fi

  shared_concurrency="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY")"
  if [[ ! "${shared_concurrency}" =~ ^[1-9][0-9]*$ ]]; then
    echo "[server-provision] GENARRATIVE_BGFILTER_WORKER_CONCURRENCY 必须在共享 API env 中配置为正整数: ${API_ENV_FILE}" >&2
    exit 1
  fi

  shared_estimate_ms="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS")"
  if [[ ! "${shared_estimate_ms}" =~ ^[1-9][0-9]*$ ]]; then
    echo "[server-provision] GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS 必须在共享 API env 中配置为正整数毫秒: ${API_ENV_FILE}" >&2
    exit 1
  fi

  connect_timeout_ms="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS")"
  if [[ ! "${connect_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
    echo "[server-provision] GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${API_ENV_FILE}" >&2
    exit 1
  fi
}

validate_no_bgfilter_internal_token_plaintext() {
  local env_file

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ reject non-empty GENARRATIVE_BGFILTER_INTERNAL_TOKEN in ${API_ENV_FILE}, ${WORKER_ENV_FILE}, and ${BGFILTER_WORKER_ENV_FILE}"
    return
  fi

  for env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"; do
    if env_contains_nonempty_assignment "${env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN"; then
      echo "[server-provision] ${env_file} 不得保存 GENARRATIVE_BGFILTER_INTERNAL_TOKEN 明文；生产环境只允许使用 GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE。" >&2
      exit 1
    fi
  done
}

validate_bgfilter_env_file_alignment() {
  local env_file="$1"
  local label="$2"
  local include_provider_credentials="$3"
  local key shared_value dedicated_value
  local -a shared_keys=(
    GENARRATIVE_BGFILTER_WORKER_CONCURRENCY
    GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS
    GENARRATIVE_BGFILTER_WORKER_BASE_URL
    GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE
    GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS
    ALIYUN_OSS_BUCKET
    ALIYUN_OSS_ENDPOINT
  )

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ validate ${label} BgFilter shared configuration alignment with ${API_ENV_FILE}"
    return
  fi
  if [[ ! -f "${env_file}" ]]; then
    echo "[server-provision] ${label} 不存在，无法检查 BgFilter 共享配置: ${env_file}" >&2
    exit 1
  fi

  if [[ "${include_provider_credentials}" == "true" ]]; then
    shared_keys+=(
      GENARRATIVE_EDITOR_BGFILTER_BASE_URL
      GENARRATIVE_EDITOR_BGFILTER_TOKEN
      ALIYUN_OSS_ACCESS_KEY_ID
      ALIYUN_OSS_ACCESS_KEY_SECRET
      ALIYUN_OSS_READ_EXPIRE_SECONDS
    )
  fi

  for key in "${shared_keys[@]}"; do
    if ! env_has_assignment "${env_file}" "${key}"; then
      continue
    fi
    dedicated_value="$(read_effective_env_value "${env_file}" "${key}")"
    shared_value="$(read_effective_env_value "${API_ENV_FILE}" "${key}")"
    if [[ "${dedicated_value}" != "${shared_value}" ]]; then
      echo "[server-provision] ${label} 中的 BgFilter 共享配置与 API env 不一致: ${key}；后加载 env 会覆盖进程有效值。" >&2
      exit 1
    fi
  done
}

validate_bgfilter_loopback_endpoint_alignment() {
  local base_url host port expected_base_url

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ validate BgFilter parent base URL and child listener alignment"
    return
  fi

  base_url="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL")"
  host="$(read_effective_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_HOST")"
  port="$(read_effective_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_PORT")"
  if [[ "${host}" != "127.0.0.1" ]]; then
    echo "[server-provision] BgFilter worker 首版必须监听 127.0.0.1，当前 GENARRATIVE_BGFILTER_WORKER_HOST=${host:-<empty>}" >&2
    exit 1
  fi
  if [[ ! "${port}" =~ ^[1-9][0-9]{0,4}$ ]] || (( 10#${port} > 65535 )); then
    echo "[server-provision] GENARRATIVE_BGFILTER_WORKER_PORT 必须是 1-65535 的有效端口: ${BGFILTER_WORKER_ENV_FILE}" >&2
    exit 1
  fi
  expected_base_url="http://${host}:${port}"
  if [[ "${base_url%/}" != "${expected_base_url}" ]]; then
    echo "[server-provision] 父进程 GENARRATIVE_BGFILTER_WORKER_BASE_URL 必须与 BgFilter worker 有效监听地址一致: expected=${expected_base_url}, actual=${base_url:-<empty>}" >&2
    exit 1
  fi
  # 验活 URL 从这里已验证的 host/port 派生：校验允许 8083 以外的合法自定义端口，
  # 后续 readiness 必须访问同一 endpoint，不能硬编码默认端口造成假阴性。
  BGFILTER_WORKER_READYZ_URL="${expected_base_url}/readyz"
}

ensure_worker_runtime_env_defaults() {
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ ensure external-generation worker runtime env defaults in ${WORKER_ENV_FILE}"
    return
  fi
  if [[ ! -f "${WORKER_ENV_FILE}" ]]; then
    echo "[server-provision] worker 环境文件不存在，无法补齐运行态变量: ${WORKER_ENV_FILE}" >&2
    exit 1
  fi

  ensure_env_value_migrates_old_default "${WORKER_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
  ensure_env_value "${WORKER_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
  ensure_env_value "${WORKER_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
  ensure_runtime_bootstrap_secret_file_env "${WORKER_ENV_FILE}"
}

ensure_bgfilter_worker_runtime_env_defaults() {
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ ensure BgFilter worker runtime env defaults in ${BGFILTER_WORKER_ENV_FILE}"
    return
  fi
  if [[ ! -f "${BGFILTER_WORKER_ENV_FILE}" ]]; then
    echo "[server-provision] BgFilter worker 环境文件不存在，无法补齐运行态变量: ${BGFILTER_WORKER_ENV_FILE}" >&2
    exit 1
  fi

  ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_HOST" "127.0.0.1"
  ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_PORT" "8083"
  remove_env_key_if_present "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY"
  # Q 已降级为可选 admission 保险丝（代码默认 2048）；只迁移历史模板默认 128，
  # 其它显式定制值继续保留。
  ensure_env_value_migrates_old_default "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS" "128" "2048"
  ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
  ensure_env_value_migrates_old_default "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300" "120"
}

bgfilter_internal_token_file_is_single_segment() {
  local token_file="$1"

  awk '
    /[^[:space:]]/ {
      non_empty_lines += 1
      if ($0 !~ /^[[:space:]]*[^[:space:]]+[[:space:]]*$/) {
        invalid = 1
      }
    }
    END {
      exit !(non_empty_lines == 1 && invalid == 0)
    }
  ' "${token_file}"
}

ensure_bgfilter_internal_token_file() {
  local token_file="/etc/genarrative/secrets/bgfilter-worker.token"
  local token_dir="/etc/genarrative/secrets"
  local openssl_bin="${GENARRATIVE_OPENSSL_PREFIX}/bin/openssl"
  local configured_token_file temporary_file token_metadata

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ ensure shared BgFilter internal token file ${token_file} (root:genarrative 0440)"
    return
  fi
  configured_token_file="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE")"
  if [[ "${configured_token_file}" != "${token_file}" ]]; then
    echo "[server-provision] GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE 必须与 Provision 管理路径一致: ${token_file}" >&2
    exit 1
  fi
  if [[ -L "${token_dir}" || -L "${token_file}" ]]; then
    echo "[server-provision] BgFilter 内部 Token 目录和文件不能是符号链接: ${token_file}" >&2
    exit 1
  fi
  if [[ -e "${token_file}" && ! -f "${token_file}" ]]; then
    echo "[server-provision] BgFilter 内部 Token 必须是普通文件: ${token_file}" >&2
    exit 1
  fi
  install -d -o root -g genarrative -m 0750 "${token_dir}"
  if [[ ! -f "${token_file}" ]]; then
    temporary_file="$(mktemp "${token_dir}/.bgfilter-worker.token.XXXXXX")"
    if ! "${openssl_bin}" rand -hex 32 >"${temporary_file}"; then
      rm -f "${temporary_file}"
      echo "[server-provision] 生成 BgFilter 内部 Token 失败。" >&2
      exit 1
    fi
    if ! bgfilter_internal_token_file_is_single_segment "${temporary_file}"; then
      rm -f "${temporary_file}"
      echo "[server-provision] 生成的 BgFilter 内部 Token 必须为不含空白字符的单段值。" >&2
      exit 1
    fi
    chown root:genarrative "${temporary_file}"
    chmod 0440 "${temporary_file}"
    mv -T "${temporary_file}" "${token_file}"
    echo "[server-provision] 已生成 BgFilter 内部 Token 文件: ${token_file}"
  else
    if ! bgfilter_internal_token_file_is_single_segment "${token_file}"; then
      echo "[server-provision] BgFilter 内部 Token 文件必须为不含空白字符的单段值: ${token_file}" >&2
      exit 1
    fi
    chown root:genarrative "${token_file}"
    chmod 0440 "${token_file}"
    echo "[server-provision] BgFilter 内部 Token 文件已存在，保留内容并收紧权限。"
  fi
  token_metadata="$(stat -c '%U:%G:%a' -- "${token_file}")"
  if [[ "${token_metadata}" != "root:genarrative:440" ]]; then
    echo "[server-provision] BgFilter 内部 Token 权限必须为 root:genarrative 0440: ${token_file} (${token_metadata})" >&2
    exit 1
  fi
}

parse_json_string_field() {
  local json="$1"
  local key="$2"

  printf "%s" "${json}" | sed -n "s/.*\\\"${key}\\\"[[:space:]]*:[[:space:]]*\\\"\\([^\\\"]*\\)\\\".*/\\1/p" | head -n 1
}

ensure_spacetime_owner_client_token() {
  local server_url="http://127.0.0.1:3101"
  local cli_path="${SPACETIME_ROOT}/bin/current/spacetimedb-cli"
  local token identity response login_output existing_token identity_preview

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ ensure GENARRATIVE_SPACETIME_TOKEN in ${API_ENV_FILE}"
    echo "+ generate SpacetimeDB client identity when token is missing"
    echo "+ runuser -u spacetimedb -- ${cli_path} --root-dir ${SPACETIME_ROOT} login --token [REDACTED]"
    return
  fi

  if [[ ! -f "${API_ENV_FILE}" ]]; then
    echo "[server-provision] 环境文件不存在，无法写入 GENARRATIVE_SPACETIME_TOKEN: ${API_ENV_FILE}" >&2
    exit 1
  fi
  if [[ ! -x "${cli_path}" ]]; then
    echo "[server-provision] SpacetimeDB CLI 不存在或不可执行: ${cli_path}" >&2
    exit 1
  fi

  existing_token="$(read_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_TOKEN")"
  if [[ -n "${existing_token}" ]]; then
    token="${existing_token}"
    echo "[server-provision] GENARRATIVE_SPACETIME_TOKEN 已存在，保留并同步 SpacetimeDB CLI 登录态。"
  else
    response="$(curl -fsS -X POST "${server_url}/v1/identity")"
    identity="$(parse_json_string_field "${response}" "identity")"
    identity="${identity:-$(parse_json_string_field "${response}" "Identity")}"
    identity="${identity:-$(parse_json_string_field "${response}" "identity_hex")}"
    identity="${identity:-$(parse_json_string_field "${response}" "identityHex")}"
    token="$(parse_json_string_field "${response}" "token")"
    token="${token:-$(parse_json_string_field "${response}" "Token")}"
    if [[ -z "${identity}" || -z "${token}" ]]; then
      echo "[server-provision] 生成 SpacetimeDB client identity 失败，响应缺少 identity/token。" >&2
      exit 1
    fi

    write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_TOKEN" "${token}"
    identity_preview="${identity:0:12}"
    echo "[server-provision] 已生成 SpacetimeDB client identity 并写入 GENARRATIVE_SPACETIME_TOKEN: ${identity_preview}..."
  fi

  # 中文注释：这里是 provision 内部为 spacetimedb 运行用户隔离 CLI 登录态的受控用法，不作为人工 spacetime 命令示例。
  if ! login_output="$(runuser -u spacetimedb -- "${cli_path}" --root-dir "${SPACETIME_ROOT}" login --token "${token}" 2>&1)"; then
    echo "[server-provision] 使用 GENARRATIVE_SPACETIME_TOKEN 登录 SpacetimeDB CLI 失败。" >&2
    printf "%s\\n" "${login_output}" | sed -E "s/[A-Za-z0-9_.=-]{24,}/[REDACTED]/g" >&2
    exit 1
  fi
  echo "[server-provision] 已同步 SpacetimeDB CLI 登录态；后续首次 publish 将使用同一 client identity。"
}

render_nginx_brotli_directives() {
  if ! command -v nginx >/dev/null 2>&1; then
    echo "    # Brotli 未启用：目标服务器未找到 nginx 命令。"
    return
  fi

  local brotli_snippet
  brotli_snippet="$(mktemp)"
  cat >"${brotli_snippet}" <<'EOF'
include /etc/nginx/modules-enabled/*.conf;
events {}
http {
    brotli on;
    brotli_comp_level 4;
    brotli_min_length 1024;
    brotli_types application/json;
}
EOF
  if nginx -t -c "${brotli_snippet}" >/dev/null 2>&1; then
    cat <<'EOF'
    brotli on;
    brotli_comp_level 4;
    brotli_min_length 1024;
    brotli_types
        text/plain
        text/css
        text/javascript
        application/javascript
        application/json
        application/xml
        application/xml+rss
        image/svg+xml;
EOF
  else
    echo "    # Brotli 未启用：nginx -t 不接受 brotli 指令。"
  fi
  rm -f "${brotli_snippet}"
}

render_nginx_template() {
  local template="$1"
  local rendered_brotli server_names
  rendered_brotli="$(render_nginx_brotli_directives)"
  server_names="${SERVER_NAME}"
  if [[ -n "${SERVER_ALIASES:-}" ]]; then
    server_names="${server_names} $(normalize_server_aliases)"
  fi
  sed \
    -e "s/server_name genarrative.example.com;/server_name ${server_names};/g" \
    -e "s|/etc/letsencrypt/live/genarrative.example.com/|/etc/letsencrypt/live/${SERVER_NAME}/|g" \
    -e "/# __GENARRATIVE_BROTLI_DIRECTIVES__/r /dev/stdin" \
    -e "/# __GENARRATIVE_BROTLI_DIRECTIVES__/d" \
    "${template}" <<<"${rendered_brotli}"
}

render_nginx_https_config() {
  render_nginx_template deploy/nginx/genarrative.conf
}

render_nginx_development_http_config() {
  render_nginx_template deploy/nginx/genarrative-dev-http.conf
}

render_api_env_example() {
  sed \
    -e "s|^GENARRATIVE_API_PORT=.*|GENARRATIVE_API_PORT=${API_PORT}|" \
    -e "s|^GENARRATIVE_SPACETIME_SERVER_URL=.*|GENARRATIVE_SPACETIME_SERVER_URL=http://127.0.0.1:3101|" \
    deploy/env/api-server.env.example
}

render_external_generation_worker_env_example() {
  cat deploy/env/external-generation-worker.env.example
}

render_external_generation_controller_env_example() {
  cat deploy/env/external-generation-controller.env.example
}

render_bgfilter_worker_env_example() {
  cat deploy/env/bgfilter-worker.env.example
}

render_otelcol_service() {
  cat deploy/systemd/otelcol-contrib.service
}

validate_nginx_tls() {
  local cert_dir="/etc/letsencrypt/live/${SERVER_NAME}"
  if [[ "${SERVER_NAME}" == "genarrative.example.com" ]]; then
    echo "[server-provision] SERVER_NAME 仍是占位域名，拒绝写入 Nginx HTTPS 配置。请填写真实域名，或先设置 NGINX_CONFIG_MODE=none。" >&2
    exit 1
  fi
  if [[ ! -f "${cert_dir}/fullchain.pem" || ! -f "${cert_dir}/privkey.pem" ]]; then
    echo "[server-provision] 未找到 Nginx HTTPS 证书: ${cert_dir}/fullchain.pem 或 ${cert_dir}/privkey.pem" >&2
    echo "[server-provision] 请先完成证书申请，或首次初始化时设置 NGINX_CONFIG_MODE=none，避免写入无法通过 nginx -t 的配置。" >&2
    exit 1
  fi
}

disable_nginx_default_sites_enabled() {
  local moves_file="$1"
  local sites_enabled="/etc/nginx/sites-enabled"
  local sites_disabled="/etc/nginx/sites-disabled"
  local stamp source target base
  local candidates=("${sites_enabled}/default" "${sites_enabled}/default."*)

  stamp="$(date +%Y%m%d%H%M%S)"
  for source in "${candidates[@]}"; do
    if [[ ! -e "${source}" && ! -L "${source}" ]]; then
      continue
    fi

    base="$(basename "${source}")"
    target="${sites_disabled}/${base}.disabled-${stamp}"
    echo "[server-provision] 禁用 Debian 默认 Nginx 站点，避免与 Genarrative server_name 冲突: ${source} -> ${target}"
    mkdir -p "${sites_disabled}"
    mv "${source}" "${target}"
    printf "%s\t%s\n" "${target}" "${source}" >>"${moves_file}"
  done
}

restore_nginx_default_sites_enabled() {
  local moves_file="$1"
  local target source

  if [[ ! -f "${moves_file}" ]]; then
    return
  fi

  while IFS=$'\t' read -r target source || [[ -n "${target:-}" ]]; do
    if [[ -z "${target:-}" || -z "${source:-}" ]]; then
      continue
    fi
    if [[ -e "${target}" || -L "${target}" ]]; then
      mkdir -p "$(dirname "${source}")"
      if [[ ! -e "${source}" && ! -L "${source}" ]]; then
        echo "[server-provision] 恢复 Debian 默认 Nginx 站点: ${target} -> ${source}"
        mv "${target}" "${source}"
      fi
    fi
  done <"${moves_file}"
}

install_nginx_config_with_rollback() {
  local config_target="/etc/nginx/conf.d/genarrative.conf"
  local snippet_target="/etc/nginx/snippets/genarrative-maintenance.conf"
  local pingora_canary_snippet_target="/etc/nginx/snippets/genarrative-pingora-canary.conf"
  local pingora_realpath_canary_snippet_target="/etc/nginx/snippets/genarrative-pingora-realpath-canary.conf"
  local config_source
  local rendered_config rendered_snippet rendered_pingora_canary_snippet rendered_pingora_realpath_canary_snippet
  local config_backup snippet_backup pingora_canary_snippet_backup pingora_realpath_canary_snippet_backup disabled_sites
  local had_config="false"
  local had_snippet="false"
  local had_pingora_canary_snippet="false"
  local had_pingora_realpath_canary_snippet="false"

  run_cmd mkdir -p /etc/nginx/snippets /etc/nginx/conf.d
  if [[ "${NGINX_CONFIG_MODE}" == "production-https" ]]; then
    config_source="deploy/nginx/genarrative.conf"
  elif [[ "${NGINX_CONFIG_MODE}" == "development-http" ]]; then
    config_source="deploy/nginx/genarrative-dev-http.conf"
  else
    echo "[server-provision] NGINX_CONFIG_MODE=${NGINX_CONFIG_MODE} 不需要安装 Nginx 配置。"
    return
  fi

  echo "+ render ${config_source} -> ${config_target}"
  echo "+ install -m 0644 deploy/nginx/snippets/genarrative-maintenance.conf ${snippet_target}"
  echo "+ install -m 0644 deploy/nginx/snippets/genarrative-pingora-canary.conf ${pingora_canary_snippet_target}"
  echo "+ install -m 0644 deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf ${pingora_realpath_canary_snippet_target}"

  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ disable /etc/nginx/sites-enabled/default* if present"
    echo "+ nginx -t"
    echo "+ nginx -s reload"
    return
  fi

  rendered_config="$(mktemp)"
  rendered_snippet="$(mktemp)"
  rendered_pingora_canary_snippet="$(mktemp)"
  rendered_pingora_realpath_canary_snippet="$(mktemp)"
  config_backup="$(mktemp)"
  snippet_backup="$(mktemp)"
  pingora_canary_snippet_backup="$(mktemp)"
  pingora_realpath_canary_snippet_backup="$(mktemp)"
  disabled_sites="$(mktemp)"
  if [[ "${NGINX_CONFIG_MODE}" == "production-https" ]]; then
    validate_nginx_tls
    render_nginx_https_config >"${rendered_config}"
  else
    render_nginx_development_http_config >"${rendered_config}"
  fi
  cp deploy/nginx/snippets/genarrative-maintenance.conf "${rendered_snippet}"
  cp deploy/nginx/snippets/genarrative-pingora-canary.conf "${rendered_pingora_canary_snippet}"
  cp deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf "${rendered_pingora_realpath_canary_snippet}"

  if [[ -f "${config_target}" ]]; then
    cp -p "${config_target}" "${config_backup}"
    had_config="true"
  fi
  if [[ -f "${snippet_target}" ]]; then
    cp -p "${snippet_target}" "${snippet_backup}"
    had_snippet="true"
  fi
  if [[ -f "${pingora_canary_snippet_target}" ]]; then
    cp -p "${pingora_canary_snippet_target}" "${pingora_canary_snippet_backup}"
    had_pingora_canary_snippet="true"
  fi
  if [[ -f "${pingora_realpath_canary_snippet_target}" ]]; then
    cp -p "${pingora_realpath_canary_snippet_target}" "${pingora_realpath_canary_snippet_backup}"
    had_pingora_realpath_canary_snippet="true"
  fi

  install -m 0644 "${rendered_config}" "${config_target}"
  install -m 0644 "${rendered_snippet}" "${snippet_target}"
  install -m 0644 "${rendered_pingora_canary_snippet}" "${pingora_canary_snippet_target}"
  install -m 0644 "${rendered_pingora_realpath_canary_snippet}" "${pingora_realpath_canary_snippet_target}"
  disable_nginx_default_sites_enabled "${disabled_sites}"

  if ! nginx -t; then
    echo "[server-provision] nginx -t 失败，恢复写入前的 Nginx 配置。" >&2
    if [[ "${had_config}" == "true" ]]; then
      cp -p "${config_backup}" "${config_target}"
    else
      rm -f "${config_target}"
    fi
    if [[ "${had_snippet}" == "true" ]]; then
      cp -p "${snippet_backup}" "${snippet_target}"
    else
      rm -f "${snippet_target}"
    fi
    if [[ "${had_pingora_canary_snippet}" == "true" ]]; then
      cp -p "${pingora_canary_snippet_backup}" "${pingora_canary_snippet_target}"
    else
      rm -f "${pingora_canary_snippet_target}"
    fi
    if [[ "${had_pingora_realpath_canary_snippet}" == "true" ]]; then
      cp -p "${pingora_realpath_canary_snippet_backup}" "${pingora_realpath_canary_snippet_target}"
    else
      rm -f "${pingora_realpath_canary_snippet_target}"
    fi
    restore_nginx_default_sites_enabled "${disabled_sites}"
    rm -f "${rendered_config}" "${rendered_snippet}" "${rendered_pingora_canary_snippet}" "${rendered_pingora_realpath_canary_snippet}" "${config_backup}" "${snippet_backup}" "${pingora_canary_snippet_backup}" "${pingora_realpath_canary_snippet_backup}" "${disabled_sites}"
    exit 1
  fi
  echo "+ nginx -s reload"
  nginx -s reload

  rm -f "${rendered_config}" "${rendered_snippet}" "${rendered_pingora_canary_snippet}" "${rendered_pingora_realpath_canary_snippet}" "${config_backup}" "${snippet_backup}" "${pingora_canary_snippet_backup}" "${pingora_realpath_canary_snippet_backup}" "${disabled_sites}"
}

cleanup_placeholder_nginx_config() {
  local config_target="/etc/nginx/conf.d/genarrative.conf"
  local disabled_target

  if [[ ! -f "${config_target}" ]]; then
    return
  fi

  if ! grep -q "/etc/letsencrypt/live/genarrative.example.com/" "${config_target}"; then
    return
  fi

  disabled_target="${config_target}.disabled-placeholder-$(date +%Y%m%d%H%M%S)"
  echo "[server-provision] 发现上一轮初始化留下的占位域名 Nginx 配置，禁用: ${config_target} -> ${disabled_target}"
  if [[ "${DRY_RUN}" != "true" ]]; then
    mv "${config_target}" "${disabled_target}"
    if command -v nginx >/dev/null 2>&1; then
      if ! nginx -t; then
        echo "[server-provision] 占位配置已禁用，但 nginx -t 仍失败；请检查其他 Nginx 配置。" >&2
      else
        echo "+ nginx -s reload"
        nginx -s reload
      fi
    fi
  fi
}

escape_sed_replacement() {
  printf "%s" "$1" | sed "s/[&|]/\\\\&/g"
}

render_spacetimedb_service() {
  local root_escaped
  root_escaped="$(escape_sed_replacement "${SPACETIME_ROOT}")"
  sed \
    -e "s|/stdb|${root_escaped}|g" \
    deploy/systemd/spacetimedb.service
}

render_api_service() {
  local current_escaped env_escaped
  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
  env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
  sed \
    -e "s|/opt/genarrative/current|${current_escaped}|g" \
    -e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \
    deploy/systemd/genarrative-api.service
}

render_external_generation_worker_service() {
  local current_escaped api_env_escaped worker_env_escaped
  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
  api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
  worker_env_escaped="$(escape_sed_replacement "${WORKER_ENV_FILE}")"
  sed \
    -e "s|/opt/genarrative/current|${current_escaped}|g" \
    -e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \
    -e "s|/etc/genarrative/external-generation-worker.env|${worker_env_escaped}|g" \
    deploy/systemd/genarrative-external-generation-worker@.service
}

render_external_generation_controller_service() {
  local current_escaped api_env_escaped controller_env_escaped
  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
  api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
  controller_env_escaped="$(escape_sed_replacement "${CONTROLLER_ENV_FILE}")"
  sed \
    -e "s|/opt/genarrative/current|${current_escaped}|g" \
    -e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \
    -e "s|/etc/genarrative/external-generation-controller.env|${controller_env_escaped}|g" \
    deploy/systemd/genarrative-external-generation-controller.service
}

render_bgfilter_worker_service() {
  local current_escaped api_env_escaped bgfilter_env_escaped
  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
  api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
  bgfilter_env_escaped="$(escape_sed_replacement "${BGFILTER_WORKER_ENV_FILE}")"
  sed \
    -e "s|/opt/genarrative/current|${current_escaped}|g" \
    -e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \
    -e "s|/etc/genarrative/bgfilter-worker.env|${bgfilter_env_escaped}|g" \
    deploy/systemd/genarrative-bgfilter-worker.service
}

wait_for_bgfilter_worker_service() {
  # 真实路径的 URL 由 validate_bgfilter_loopback_endpoint_alignment 从已验证 env 派生；
  # dry-run 分支该校验提前返回，此处仅回显默认值。
  local readyz_url="${BGFILTER_WORKER_READYZ_URL:-http://127.0.0.1:8083/readyz}"
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ curl -fsS --max-time 2 ${readyz_url}"
    return
  fi
  echo "[server-provision] 等待 BgFilter worker readiness: ${readyz_url}"
  for _ in {1..30}; do
    if systemctl is-active --quiet genarrative-bgfilter-worker.service && curl -fsS --max-time 2 "${readyz_url}" >/dev/null; then
      return
    fi
    sleep 2
  done
  systemctl --no-pager --full status genarrative-bgfilter-worker.service || true
  echo "[server-provision] BgFilter worker 未在超时时间内通过 readiness: ${readyz_url}" >&2
  exit 1
}

render_database_backup_service() {
  local current_escaped env_escaped
  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
  env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
  sed \
    -e "s|/opt/genarrative/current|${current_escaped}|g" \
    -e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \
    deploy/systemd/genarrative-database-backup.service
}

render_database_backup_files_history_drop_in() {
  local current_escaped env_escaped work_dir_escaped
  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
  env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
  work_dir_escaped="$(escape_sed_replacement "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}")"
  sed \
    -e "s|/opt/genarrative/current|${current_escaped}|g" \
    -e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \
    -e "s|/var/lib/genarrative/database-backups/files-history|${work_dir_escaped}|g" \
    deploy/systemd/genarrative-database-backup-files-history.conf
}

configure_database_backup_profile() {
  local rendered_drop_in

  if [[ "${DATABASE_BACKUP_PROFILE}" == "archive-full" ]]; then
    echo "[server-provision] 数据库备份 profile=archive-full，保留主 service 的全量冷备行为。"
    run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
    return
  fi

  echo "[server-provision] 数据库备份 profile=files-history，先只读验证 full baseline state。"
  if [[ "${DRY_RUN}" == "true" ]]; then
    echo "+ /usr/bin/node -- ${CURRENT_LINK}/scripts/database-backup-to-oss.mjs --env-file ${API_ENV_FILE} --storage-format files --mode history --work-dir ${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR} --dry-run"
  else
    if [[ ! -f "${CURRENT_LINK}/scripts/database-backup-to-oss.mjs" ]]; then
      echo "[server-provision] current release 缺少数据库备份脚本: ${CURRENT_LINK}/scripts/database-backup-to-oss.mjs" >&2
      exit 1
    fi
    /usr/bin/node -- "${CURRENT_LINK}/scripts/database-backup-to-oss.mjs" \
      --env-file "${API_ENV_FILE}" \
      --storage-format files \
      --mode history \
      --work-dir "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" \
      --dry-run
  fi

  run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}"
  run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}"
  run_cmd rm -f "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
  rendered_drop_in="$(mktemp)"
  render_database_backup_files_history_drop_in >"${rendered_drop_in}"
  install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" 0644
  rm -f "${rendered_drop_in}"
}

render_health_patrol_service() {
  local current_escaped
  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
  sed \
    -e "s|/opt/genarrative/current|${current_escaped}|g" \
    deploy/systemd/genarrative-health-patrol.service
}

require_path deploy/systemd/spacetimedb.service
require_path deploy/systemd/genarrative-api.service
require_path deploy/systemd/genarrative-external-generation-worker@.service
require_path deploy/systemd/genarrative-external-generation-controller.service
require_path deploy/systemd/genarrative-bgfilter-worker.service
require_path deploy/systemd/genarrative-database-backup.service
require_path deploy/systemd/genarrative-database-backup-files-history.conf
require_path deploy/systemd/genarrative-database-backup.timer
require_path deploy/systemd/genarrative-health-patrol.service
require_path deploy/systemd/genarrative-health-patrol.timer
require_path deploy/systemd/genarrative-pingora-gateway.service
require_path deploy/systemd/genarrative-pingora-gateway-direct-entry.conf
require_path deploy/systemd/otelcol-contrib.service
require_path deploy/otelcol/genarrative-debug.yaml
require_path deploy/nginx/genarrative.conf
require_path deploy/nginx/genarrative-dev-http.conf
require_path deploy/nginx/snippets/genarrative-maintenance.conf
require_path deploy/nginx/snippets/genarrative-pingora-canary.conf
require_path deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf
require_path deploy/logrotate/genarrative-pingora-gateway
require_path deploy/env/api-server.env.example
require_path deploy/env/external-generation-worker.env.example
require_path deploy/env/external-generation-controller.env.example
require_path deploy/env/bgfilter-worker.env.example
require_path scripts/deploy/maintenance-on.sh
require_path scripts/deploy/maintenance-off.sh
require_path scripts/deploy/maintenance-status.sh

validate_server_names
validate_database_backup_profile
require_non_root_relative_path "PROVISION_TOOLS_DIR" "${PROVISION_TOOLS_DIR}"

echo "[server-provision] target=${DEPLOY_TARGET}, dry_run=${DRY_RUN}, nginx_config_mode=${NGINX_CONFIG_MODE}, database_backup_profile=${DATABASE_BACKUP_PROFILE}, source_commit=$(cat .jenkins-source-commit)"

run_cmd id
require_root_for_real_provision
install_nginx_brotli_modules
run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /etc/genarrative/pingora /etc/genarrative/secrets /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/wallet-refund-outbox /var/lib/genarrative/editor-generation-pricing /var/lib/genarrative/spacetime /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol /var/log/genarrative

if ! id spacetimedb >/dev/null 2>&1; then
  run_cmd useradd --system --home-dir "${SPACETIME_ROOT}" --shell /usr/sbin/nologin spacetimedb
else
  echo "[server-provision] 用户已存在: spacetimedb"
fi

if ! id genarrative >/dev/null 2>&1; then
  run_cmd useradd --system --home-dir /opt/genarrative --shell /usr/sbin/nologin genarrative
else
  echo "[server-provision] 用户已存在: genarrative"
fi

run_cmd chown -R spacetimedb:spacetimedb "${SPACETIME_ROOT}"
run_cmd chown -R genarrative:genarrative /opt/genarrative /var/lib/genarrative /srv/genarrative
run_cmd chown -R genarrative:genarrative /var/log/genarrative
if [[ -L /var/lib/genarrative/spacetime || -L /var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt ]]; then
  echo "[server-provision] 运行时服务身份引导密钥路径不能是符号链接。" >&2
  exit 1
fi
run_cmd install -d -o root -g genarrative -m 0750 /var/lib/genarrative/spacetime
if [[ -f /var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt ]]; then
  run_cmd chown root:genarrative /var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt
  run_cmd chmod 0440 /var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt
fi
install_genarrative_openssl_runtime

if [[ ! -x "${SPACETIME_BIN_SOURCE}" ]]; then
  echo "[server-provision] spacetime CLI 不存在或不可执行: ${SPACETIME_BIN_SOURCE}" >&2
  exit 1
fi
echo "+ install -m 0755 ${SPACETIME_BIN_SOURCE} ${SPACETIME_ROOT}/spacetime"
if [[ "${DRY_RUN}" != "true" ]]; then
  install -m 0755 "${SPACETIME_BIN_SOURCE}" "${SPACETIME_ROOT}/spacetime"
  chown spacetimedb:spacetimedb "${SPACETIME_ROOT}/spacetime"
fi
sync_spacetime_install "${SPACETIME_ROOT}"

spacetimedb_service="$(mktemp)"
api_service="$(mktemp)"
external_generation_worker_service="$(mktemp)"
external_generation_controller_service="$(mktemp)"
bgfilter_worker_service="$(mktemp)"
database_backup_service="$(mktemp)"
health_patrol_service="$(mktemp)"
render_spacetimedb_service >"${spacetimedb_service}"
render_api_service >"${api_service}"
render_external_generation_worker_service >"${external_generation_worker_service}"
render_external_generation_controller_service >"${external_generation_controller_service}"
render_bgfilter_worker_service >"${bgfilter_worker_service}"
render_database_backup_service >"${database_backup_service}"
render_health_patrol_service >"${health_patrol_service}"
install_file "${spacetimedb_service}" /etc/systemd/system/spacetimedb.service 0644
install_file "${api_service}" /etc/systemd/system/genarrative-api.service 0644
install_file "${external_generation_worker_service}" /etc/systemd/system/genarrative-external-generation-worker@.service 0644
install_file "${external_generation_controller_service}" /etc/systemd/system/genarrative-external-generation-controller.service 0644
install_file "${bgfilter_worker_service}" /etc/systemd/system/genarrative-bgfilter-worker.service 0644
install_file "${database_backup_service}" /etc/systemd/system/genarrative-database-backup.service 0644
install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644
install_file "${health_patrol_service}" /etc/systemd/system/genarrative-health-patrol.service 0644
install_file deploy/systemd/genarrative-health-patrol.timer /etc/systemd/system/genarrative-health-patrol.timer 0644
install_file deploy/systemd/genarrative-pingora-gateway.service /etc/systemd/system/genarrative-pingora-gateway.service 0644
install_file deploy/systemd/genarrative-pingora-gateway-direct-entry.conf /etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf 0644
install_file deploy/logrotate/genarrative-pingora-gateway /etc/logrotate.d/genarrative-pingora-gateway 0644
rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${external_generation_controller_service}" "${bgfilter_worker_service}" "${database_backup_service}" "${health_patrol_service}"

if [[ ! -f "${API_ENV_FILE}" ]]; then
  echo "+ create ${API_ENV_FILE} from example"
  if [[ "${DRY_RUN}" != "true" ]]; then
    render_api_env_example >"${API_ENV_FILE}"
    chmod 0600 "${API_ENV_FILE}"
    chown root:root "${API_ENV_FILE}"
  fi
else
  echo "[server-provision] 已存在环境文件，保留不覆盖: ${API_ENV_FILE}"
fi
ensure_api_runtime_env_defaults
validate_bgfilter_shared_runtime_env
configure_database_backup_profile

if [[ ! -f "${WORKER_ENV_FILE}" ]]; then
  echo "+ create ${WORKER_ENV_FILE} from example"
  if [[ "${DRY_RUN}" != "true" ]]; then
    render_external_generation_worker_env_example >"${WORKER_ENV_FILE}"
    chmod 0600 "${WORKER_ENV_FILE}"
    chown root:root "${WORKER_ENV_FILE}"
  fi
else
  echo "[server-provision] 已存在 worker 环境文件，保留不覆盖: ${WORKER_ENV_FILE}"
fi
ensure_worker_runtime_env_defaults

if [[ ! -f "${BGFILTER_WORKER_ENV_FILE}" ]]; then
  echo "+ create ${BGFILTER_WORKER_ENV_FILE} from example"
  if [[ "${DRY_RUN}" != "true" ]]; then
    render_bgfilter_worker_env_example >"${BGFILTER_WORKER_ENV_FILE}"
    chmod 0600 "${BGFILTER_WORKER_ENV_FILE}"
    chown root:root "${BGFILTER_WORKER_ENV_FILE}"
  fi
else
  echo "[server-provision] 已存在 BgFilter worker 环境文件，保留不覆盖: ${BGFILTER_WORKER_ENV_FILE}"
fi
ensure_bgfilter_worker_runtime_env_defaults
validate_bgfilter_env_file_alignment "${WORKER_ENV_FILE}" "外部生成 worker env" "false"
validate_bgfilter_env_file_alignment "${BGFILTER_WORKER_ENV_FILE}" "BgFilter 专属 env" "true"
validate_bgfilter_loopback_endpoint_alignment
validate_no_bgfilter_internal_token_plaintext
ensure_bgfilter_internal_token_file

if [[ ! -f "${CONTROLLER_ENV_FILE}" ]]; then
  echo "+ create ${CONTROLLER_ENV_FILE} from example"
  if [[ "${DRY_RUN}" != "true" ]]; then
    render_external_generation_controller_env_example >"${CONTROLLER_ENV_FILE}"
    chmod 0600 "${CONTROLLER_ENV_FILE}"
    chown root:root "${CONTROLLER_ENV_FILE}"
  fi
else
  echo "[server-provision] 已存在 controller 环境文件，保留不覆盖: ${CONTROLLER_ENV_FILE}"
fi

if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
  sync_otelcol_install
  ensure_otelcol_runtime
  otelcol_service="$(mktemp)"
  render_otelcol_service >"${otelcol_service}"
  install_file "${otelcol_service}" /etc/systemd/system/otelcol-contrib.service 0644
  rm -f "${otelcol_service}"
else
  echo "[server-provision] ENABLE_OTELCOL=${ENABLE_OTELCOL:-}，跳过 otelcol-contrib service 安装。"
fi

if [[ "${NGINX_CONFIG_MODE}" != "none" ]]; then
  install_nginx_config_with_rollback
else
  cleanup_placeholder_nginx_config
fi

run_cmd systemctl daemon-reload
if [[ "${ENABLE_SERVICES}" == "true" ]]; then
  if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
    run_cmd systemctl enable otelcol-contrib.service
  fi
  stamp_database_backup_timer_now
  run_cmd systemctl enable spacetimedb.service genarrative-bgfilter-worker.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service genarrative-health-patrol.timer
  run_cmd systemctl start genarrative-database-backup.timer
  if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
    run_cmd systemctl restart otelcol-contrib.service
  fi
  run_cmd systemctl restart spacetimedb.service
  wait_for_spacetimedb_service
  ensure_spacetime_owner_client_token
  if [[ -x "${CURRENT_LINK}/api-server" ]]; then
    run_cmd systemctl enable genarrative-bgfilter-worker.service
    run_cmd systemctl stop genarrative-bgfilter-worker.service
    run_cmd systemctl start genarrative-bgfilter-worker.service
    wait_for_bgfilter_worker_service
    run_cmd systemctl restart genarrative-api.service
    run_cmd systemctl enable --now genarrative-external-generation-worker@1.service
    run_cmd systemctl restart genarrative-external-generation-worker@1.service
    run_cmd systemctl enable --now genarrative-external-generation-controller.service
    run_cmd systemctl restart genarrative-external-generation-controller.service
  else
    echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server，跳过 BgFilter worker、api-server、外部生成 worker 和 controller 首次启动。后续 API deploy 会按顺序启动。"
  fi
fi

echo "[server-provision] 完成。若是首次初始化，请补齐 ${API_ENV_FILE} 的真实密钥后再启动 api-server。"