Genarrative/scripts/deploy/pingora-direct-rollback.sh

#!/usr/bin/env bash

set -euo pipefail

SERVICE_NAME="${GENARRATIVE_PINGORA_GATEWAY_SERVICE:-genarrative-pingora-gateway.service}"
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"
DEFAULT_SERVICE_UNIT_PATH="${REPO_ROOT}/deploy/systemd/genarrative-pingora-gateway.service"
SERVICE_UNIT_PATH="${GENARRATIVE_PINGORA_ROLLBACK_SERVICE_UNIT_PATH:-${DEFAULT_SERVICE_UNIT_PATH}}"
DROPIN_PATH="${GENARRATIVE_PINGORA_DIRECT_DROPIN_PATH:-/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf}"
NGINX_SERVICE="${GENARRATIVE_NGINX_SERVICE:-nginx.service}"
NGINX_BINARY="${GENARRATIVE_NGINX_BINARY:-nginx}"
NGINX_SMOKE_URL="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_URL:-}"
NGINX_SMOKE_HOST="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_HOST:-}"
NGINX_SMOKE_EXPECT_BODY="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_EXPECT_BODY:-}"
CURL_BINARY="${GENARRATIVE_CURL_BINARY:-curl}"
HEALTH_PATROL_ENV_FILE="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_ENV_FILE:-}"
HEALTH_PATROL_ENV_CHECK_SCRIPT="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_ENV_CHECK_SCRIPT:-}"
HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL:-}"
HEALTH_PATROL_EXPECTED_PUBLIC_HOST="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_EXPECTED_PUBLIC_HOST:-}"
HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST:-auto}"
PINGORA_SHADOW_PROBE_URL="${GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_URL:-}"
PINGORA_SHADOW_PROBE_TOKEN="${GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN:-}"
APPLY="false"
RELOAD_NGINX="false"
STATUS_AFTER="true"
VERIFY_SYSTEMD_AFTER="true"

usage() {
  cat <<'EOF'
用法:
  scripts/deploy/pingora-direct-rollback.sh [--apply] [--reload-nginx] [--nginx-smoke-url <url>] [--nginx-smoke-host <host>] [--nginx-smoke-expect-body <text>] [--curl-binary <path>] [--health-patrol-env-file <path>] [--health-patrol-env-check-script <path>] [--health-patrol-expected-public-base-url <url>] [--health-patrol-expected-public-host <host>|--health-patrol-require-empty-public-host] [--pingora-shadow-probe-url <url>] [--pingora-shadow-probe-token <token>] [--no-postcheck] [--service <name>] [--service-unit-path <path>] [--dropin-path <path>] [--nginx-service <name>] [--nginx-binary <path>]

说明:
  回退 Pingora 直连低端口入口：移除 direct-entry systemd drop-in，执行 daemon-reload，
  重启 genarrative-pingora-gateway.service，让服务回到 shadow/high-port 口径。

默认是 dry-run，只打印将执行的命令；必须显式传 --apply 才会修改系统状态。
--apply 必须同时传 --reload-nginx，让脚本执行 nginx reload，
脚本会先运行 nginx -t，通过后再 reload，并在 reload 后确认 Nginx 仍处于 active 状态。
--apply 还必须提供 --nginx-smoke-url，reload 后用 curl 验证 Nginx 入口已经真实响应。
若提供 --nginx-smoke-expect-body，脚本会要求 smoke 响应体包含该片段，避免 HTTP 200 命中错误入口。
脚本会以 current release 随包主 service 模板的 ExecStart 为真相源，
在重启 Pingora 后核验 systemd 最终 ExecStart 仍指向同一个 pingora-gateway 二进制。
如果提供 --health-patrol-env-file，脚本会在 Nginx smoke 之后复核 health patrol 已切回 nginx 模式，
并可校验回退后的 public base URL / Host 是否恢复为切换前记录值。
如果同时提供 --pingora-shadow-probe-url 与 --pingora-shadow-probe-token，脚本会在回退后
验证 Pingora shadow 高端口探针仍返回 gateway=pingora-shadow。
EOF
}

resolve_default_health_patrol_env_check_script() {
  printf '%s\n' "${REPO_ROOT}/scripts/check-production-health-patrol-env.mjs"
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    -h|--help)
      usage
      exit 0
      ;;
    --apply)
      APPLY="true"
      shift
      ;;
    --reload-nginx)
      RELOAD_NGINX="true"
      shift
      ;;
    --no-status)
      STATUS_AFTER="false"
      shift
      ;;
    --no-postcheck)
      VERIFY_SYSTEMD_AFTER="false"
      shift
      ;;
    --service)
      SERVICE_NAME="${2:-}"
      if [[ -z "${SERVICE_NAME}" ]]; then
        echo "[pingora-direct-rollback] --service 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --service-unit-path)
      SERVICE_UNIT_PATH="${2:-}"
      if [[ -z "${SERVICE_UNIT_PATH}" ]]; then
        echo "[pingora-direct-rollback] --service-unit-path 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --dropin-path)
      DROPIN_PATH="${2:-}"
      if [[ -z "${DROPIN_PATH}" ]]; then
        echo "[pingora-direct-rollback] --dropin-path 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --nginx-service)
      NGINX_SERVICE="${2:-}"
      if [[ -z "${NGINX_SERVICE}" ]]; then
        echo "[pingora-direct-rollback] --nginx-service 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --nginx-binary)
      NGINX_BINARY="${2:-}"
      if [[ -z "${NGINX_BINARY}" ]]; then
        echo "[pingora-direct-rollback] --nginx-binary 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --nginx-smoke-url)
      NGINX_SMOKE_URL="${2:-}"
      if [[ -z "${NGINX_SMOKE_URL}" ]]; then
        echo "[pingora-direct-rollback] --nginx-smoke-url 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --nginx-smoke-host)
      NGINX_SMOKE_HOST="${2:-}"
      if [[ -z "${NGINX_SMOKE_HOST}" ]]; then
        echo "[pingora-direct-rollback] --nginx-smoke-host 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --nginx-smoke-expect-body)
      NGINX_SMOKE_EXPECT_BODY="${2:-}"
      if [[ -z "${NGINX_SMOKE_EXPECT_BODY}" ]]; then
        echo "[pingora-direct-rollback] --nginx-smoke-expect-body 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --curl-binary)
      CURL_BINARY="${2:-}"
      if [[ -z "${CURL_BINARY}" ]]; then
        echo "[pingora-direct-rollback] --curl-binary 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --health-patrol-env-file)
      HEALTH_PATROL_ENV_FILE="${2:-}"
      if [[ -z "${HEALTH_PATROL_ENV_FILE}" ]]; then
        echo "[pingora-direct-rollback] --health-patrol-env-file 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --health-patrol-env-check-script)
      HEALTH_PATROL_ENV_CHECK_SCRIPT="${2:-}"
      if [[ -z "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then
        echo "[pingora-direct-rollback] --health-patrol-env-check-script 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --health-patrol-expected-public-base-url)
      HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL="${2:-}"
      if [[ -z "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then
        echo "[pingora-direct-rollback] --health-patrol-expected-public-base-url 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --health-patrol-expected-public-host)
      HEALTH_PATROL_EXPECTED_PUBLIC_HOST="${2:-}"
      if [[ -z "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then
        echo "[pingora-direct-rollback] --health-patrol-expected-public-host 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --health-patrol-require-empty-public-host)
      HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST="true"
      shift
      ;;
    --pingora-shadow-probe-url)
      PINGORA_SHADOW_PROBE_URL="${2:-}"
      if [[ -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then
        echo "[pingora-direct-rollback] --pingora-shadow-probe-url 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    --pingora-shadow-probe-token)
      PINGORA_SHADOW_PROBE_TOKEN="${2:-}"
      if [[ -z "${PINGORA_SHADOW_PROBE_TOKEN}" ]]; then
        echo "[pingora-direct-rollback] --pingora-shadow-probe-token 缺少参数" >&2
        exit 1
      fi
      shift 2
      ;;
    *)
      echo "[pingora-direct-rollback] 未知参数: $1" >&2
      usage >&2
      exit 1
      ;;
  esac
done

if [[ -n "${HEALTH_PATROL_ENV_FILE}" && -z "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then
  HEALTH_PATROL_ENV_CHECK_SCRIPT="$(resolve_default_health_patrol_env_check_script)"
fi

reject_control_characters() {
  local label="$1"
  local value="$2"
  if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then
    echo "[pingora-direct-rollback] ${label} 不能包含换行或 NUL 字符。" >&2
    exit 1
  fi
}

is_filesystem_root_path() {
  local value="$1"
  local without_slashes="${value//\//}"
  [[ -n "${value}" && -z "${without_slashes}" ]]
}

reject_filesystem_root_path() {
  local label="$1"
  local value="$2"
  if is_filesystem_root_path "${value}"; then
    echo "[pingora-direct-rollback] ${label} 不能是文件系统根目录。" >&2
    exit 1
  fi
}

reject_control_characters "--service" "${SERVICE_NAME}"
reject_control_characters "--service-unit-path" "${SERVICE_UNIT_PATH}"
reject_control_characters "--dropin-path" "${DROPIN_PATH}"
reject_control_characters "--nginx-service" "${NGINX_SERVICE}"
reject_control_characters "--nginx-binary" "${NGINX_BINARY}"
reject_control_characters "--nginx-smoke-url" "${NGINX_SMOKE_URL}"
reject_control_characters "--nginx-smoke-host" "${NGINX_SMOKE_HOST}"
reject_control_characters "--nginx-smoke-expect-body" "${NGINX_SMOKE_EXPECT_BODY}"
reject_control_characters "--curl-binary" "${CURL_BINARY}"
reject_control_characters "--health-patrol-env-file" "${HEALTH_PATROL_ENV_FILE}"
reject_control_characters "--health-patrol-env-check-script" "${HEALTH_PATROL_ENV_CHECK_SCRIPT}"
reject_control_characters "--health-patrol-expected-public-base-url" "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}"
reject_control_characters "--health-patrol-expected-public-host" "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}"
reject_control_characters "--health-patrol-require-empty-public-host" "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}"
reject_control_characters "--pingora-shadow-probe-url" "${PINGORA_SHADOW_PROBE_URL}"
reject_control_characters "--pingora-shadow-probe-token" "${PINGORA_SHADOW_PROBE_TOKEN}"

reject_filesystem_root_path "--service-unit-path" "${SERVICE_UNIT_PATH}"
reject_filesystem_root_path "--dropin-path" "${DROPIN_PATH}"
reject_filesystem_root_path "--nginx-binary" "${NGINX_BINARY}"
reject_filesystem_root_path "--curl-binary" "${CURL_BINARY}"
reject_filesystem_root_path "--health-patrol-env-file" "${HEALTH_PATROL_ENV_FILE}"
reject_filesystem_root_path "--health-patrol-env-check-script" "${HEALTH_PATROL_ENV_CHECK_SCRIPT}"

if [[ "${DROPIN_PATH}" != /* ]]; then
  echo "[pingora-direct-rollback] --dropin-path 必须是绝对路径: ${DROPIN_PATH}" >&2
  exit 1
fi

if [[ "${SERVICE_UNIT_PATH}" != /* ]]; then
  echo "[pingora-direct-rollback] --service-unit-path 必须是绝对路径: ${SERVICE_UNIT_PATH}" >&2
  exit 1
fi

if [[ -n "${HEALTH_PATROL_ENV_FILE}" && "${HEALTH_PATROL_ENV_FILE}" != /* ]]; then
  echo "[pingora-direct-rollback] --health-patrol-env-file 必须是绝对路径: ${HEALTH_PATROL_ENV_FILE}" >&2
  exit 1
fi

if [[ -n "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" && "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" != /* ]]; then
  echo "[pingora-direct-rollback] --health-patrol-env-check-script 必须是绝对路径: ${HEALTH_PATROL_ENV_CHECK_SCRIPT}" >&2
  exit 1
fi

require_absolute_path_if_path_like() {
  local label="$1"
  local value="$2"
  if [[ "${value}" == */* && "${value}" != /* ]]; then
    echo "[pingora-direct-rollback] ${label} 包含路径分隔符时必须是绝对路径: ${value}" >&2
    exit 1
  fi
}

require_absolute_path_if_path_like "--nginx-binary" "${NGINX_BINARY}"
require_absolute_path_if_path_like "--curl-binary" "${CURL_BINARY}"

if [[ -n "${PINGORA_SHADOW_PROBE_URL}" && -z "${PINGORA_SHADOW_PROBE_TOKEN}" ]]; then
  echo "[pingora-direct-rollback] --pingora-shadow-probe-url 必须同时提供 --pingora-shadow-probe-token。" >&2
  exit 1
fi

if [[ -n "${PINGORA_SHADOW_PROBE_TOKEN}" && -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then
  echo "[pingora-direct-rollback] --pingora-shadow-probe-token 必须同时提供 --pingora-shadow-probe-url。" >&2
  exit 1
fi

if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" && "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" == "true" ]]; then
  echo "[pingora-direct-rollback] --health-patrol-expected-public-host 和 --health-patrol-require-empty-public-host 不能同时使用。" >&2
  exit 1
fi

validate_nginx_smoke_host() {
  validate_host_value "--nginx-smoke-host" "${NGINX_SMOKE_HOST}"
}

validate_host_value() {
  local label="$1"
  local value="$2"
  if [[ "${value}" == *"://"* || "${value}" == */* || "${value}" == *\?* || "${value}" == *"#"* || "${value}" =~ [[:space:]] ]]; then
    echo "[pingora-direct-rollback] ${label} 只能是 host 或 host:port，不能包含 URL、路径、查询、片段或空白字符: ${value}" >&2
    exit 1
  fi
}

is_loopback_smoke_url() {
  local url="$1"
  local authority host
  case "${url}" in
    http://*|https://*) ;;
    *) return 1 ;;
  esac
  authority="${url#*://}"
  authority="${authority%%/*}"
  authority="${authority%%\?*}"
  authority="${authority%%#*}"
  authority="${authority##*@}"
  if [[ "${authority}" == \[*\]* ]]; then
    host="${authority#\[}"
    host="${host%%\]*}"
  else
    host="${authority%%:*}"
  fi
  host="${host,,}"
  [[ "${host}" == "localhost" || "${host}" == "127."* || "${host}" == "::1" ]]
}

validate_http_url() {
  local label="$1"
  local url="$2"
  case "${url}" in
    http://*|https://*) ;;
    *)
      echo "[pingora-direct-rollback] ${label} 必须是 http(s) URL: ${url}" >&2
      exit 1
      ;;
  esac
}

if [[ -n "${NGINX_SMOKE_HOST}" ]]; then
  validate_nginx_smoke_host
fi

if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then
  validate_host_value "--health-patrol-expected-public-host" "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}"
fi

if [[ -n "${PINGORA_SHADOW_PROBE_URL}" ]]; then
  validate_http_url "--pingora-shadow-probe-url" "${PINGORA_SHADOW_PROBE_URL}"
fi

if [[ -n "${NGINX_SMOKE_URL}" ]]; then
  validate_http_url "--nginx-smoke-url" "${NGINX_SMOKE_URL}"
fi

if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then
  validate_http_url "--health-patrol-expected-public-base-url" "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}"
fi

if [[ "${APPLY}" == "true" && "${RELOAD_NGINX}" != "true" ]]; then
  echo "[pingora-direct-rollback] --apply 必须同时提供 --reload-nginx，确保回退后 Nginx 公网入口重新加载并保持 active。" >&2
  exit 1
fi

if [[ "${APPLY}" == "true" && -z "${NGINX_SMOKE_URL}" ]]; then
  echo "[pingora-direct-rollback] --apply 必须同时提供 --nginx-smoke-url，确保回退后 Nginx 入口真实可访问。" >&2
  exit 1
fi

if [[ "${APPLY}" == "true" && -z "${NGINX_SMOKE_HOST}" ]] && is_loopback_smoke_url "${NGINX_SMOKE_URL}"; then
  echo "[pingora-direct-rollback] --nginx-smoke-url 指向本机地址时必须同时提供 --nginx-smoke-host，避免回退 smoke 命中默认 vhost。" >&2
  exit 1
fi

run_cmd() {
  echo "+ $*"
  if [[ "${APPLY}" == "true" ]]; then
    "$@"
  fi
}

run_systemctl() {
  if [[ "${APPLY}" != "true" ]]; then
    run_cmd systemctl "$@"
    return
  fi
  if command -v systemctl >/dev/null 2>&1; then
    run_cmd systemctl "$@"
  else
    echo "[pingora-direct-rollback] 未找到 systemctl，无法执行: systemctl $*" >&2
    exit 1
  fi
}

read_expected_exec_start() {
  local line value
  if [[ ! -f "${SERVICE_UNIT_PATH}" ]]; then
    return 1
  fi
  while IFS= read -r line; do
    line="${line#"${line%%[![:space:]]*}"}"
    line="${line%"${line##*[![:space:]]}"}"
    [[ "${line}" == ExecStart=* ]] || continue
    value="${line#ExecStart=}"
    [[ -n "${value}" ]] || continue
    printf '%s\n' "${value}"
    return 0
  done <"${SERVICE_UNIT_PATH}"
  return 1
}

first_exec_word() {
  local value="$1"
  value="${value#"${value%%[![:space:]]*}"}"
  value="${value%"${value##*[![:space:]]}"}"
  if [[ "${value}" == \"* ]]; then
    value="${value#\"}"
    printf '%s\n' "${value%%\"*}"
    return
  fi
  printf '%s\n' "${value%%[[:space:]]*}"
}

systemd_show_exec_start_matches() {
  local expected_exec="$1"
  local expected_binary actual_show actual_binary
  expected_binary="$(first_exec_word "${expected_exec}")"
  [[ -n "${expected_binary}" ]] || return 1
  actual_show="$(systemctl show "${SERVICE_NAME}" --property=ExecStart --value --no-pager)"
  actual_binary="${actual_show}"
  if [[ "${actual_show}" == *"path="* ]]; then
    actual_binary="${actual_show#*path=}"
    actual_binary="${actual_binary%% ;*}"
  elif [[ "${actual_show}" == ExecStart=* ]]; then
    actual_binary="${actual_show#ExecStart=}"
    actual_binary="$(first_exec_word "${actual_binary}")"
  else
    actual_binary="$(first_exec_word "${actual_binary}")"
  fi
  [[ "${actual_binary}" == "${expected_binary}" ]]
}

verify_systemd_exec_start_after_rollback() {
  if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then
    echo "[pingora-direct-rollback] 已跳过 Pingora ExecStart 指向核验。"
    return
  fi

  expected_exec_start="$(read_expected_exec_start || true)"
  if [[ -z "${expected_exec_start}" ]]; then
    echo "[pingora-direct-rollback] 无法从主 service 模板读取 ExecStart: ${SERVICE_UNIT_PATH}" >&2
    exit 1
  fi

  if [[ "${APPLY}" != "true" ]]; then
    echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager"
    echo "[pingora-direct-rollback] dry-run：--apply 后会核验 systemd ExecStart 指向主 service 模板中的 ${expected_exec_start}。"
    return
  fi

  if ! command -v systemctl >/dev/null 2>&1; then
    echo "[pingora-direct-rollback] 未找到 systemctl，无法核验 Pingora ExecStart。" >&2
    exit 1
  fi

  echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager"
  if ! systemd_show_exec_start_matches "${expected_exec_start}"; then
    echo "[pingora-direct-rollback] systemctl show ExecStart 未指向主 service 模板中的 ${expected_exec_start}，请先修正 ${SERVICE_NAME} 指向 current release 网关二进制。" >&2
    exit 1
  fi
}

verify_systemd_dropin_removed() {
  if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then
    echo "[pingora-direct-rollback] 已跳过 systemd drop-in 移除核验。"
    return
  fi

  if [[ "${APPLY}" != "true" ]]; then
    echo "+ systemctl cat ${SERVICE_NAME}"
    echo "[pingora-direct-rollback] dry-run：--apply 后会校验 systemd 最终配置不再包含 CAP_NET_BIND_SERVICE。"
    return
  fi

  if ! command -v systemctl >/dev/null 2>&1; then
    echo "[pingora-direct-rollback] 未找到 systemctl，无法核验 drop-in 是否已移除。" >&2
    exit 1
  fi

  echo "+ systemctl cat ${SERVICE_NAME}"
  unit_content="$(systemctl cat "${SERVICE_NAME}")"
  if [[ "${unit_content}" == *"AmbientCapabilities=CAP_NET_BIND_SERVICE"* ]]; then
    echo "[pingora-direct-rollback] systemctl cat 仍显示 AmbientCapabilities=CAP_NET_BIND_SERVICE，direct-entry drop-in 未移除。" >&2
    exit 1
  fi
  if [[ "${unit_content}" == *"CapabilityBoundingSet=CAP_NET_BIND_SERVICE"* ]]; then
    echo "[pingora-direct-rollback] systemctl cat 仍显示 CapabilityBoundingSet=CAP_NET_BIND_SERVICE，direct-entry drop-in 未移除。" >&2
    exit 1
  fi
}

verify_nginx_active_after_reload() {
  if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then
    echo "[pingora-direct-rollback] 已跳过 Nginx reload 后状态核验。"
    return
  fi

  if [[ "${APPLY}" != "true" ]]; then
    echo "+ systemctl is-active ${NGINX_SERVICE}"
    echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会 reload Nginx 并校验 service 仍为 active。"
    return
  fi

  if ! command -v systemctl >/dev/null 2>&1; then
    echo "[pingora-direct-rollback] 未找到 systemctl，无法核验 Nginx reload 后状态。" >&2
    exit 1
  fi

  echo "+ systemctl is-active ${NGINX_SERVICE}"
  nginx_state="$(systemctl is-active "${NGINX_SERVICE}")"
  if [[ "${nginx_state}" != "active" ]]; then
    echo "[pingora-direct-rollback] Nginx reload 后状态不是 active: ${nginx_state}" >&2
    exit 1
  fi
}

verify_nginx_smoke_after_reload() {
  if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then
    echo "[pingora-direct-rollback] 已跳过 Nginx reload 后 smoke 核验。"
    return
  fi

  if [[ -z "${NGINX_SMOKE_URL}" ]]; then
    echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 <nginx-smoke-url>"
    echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会用 --nginx-smoke-url 验证 Nginx 入口。"
    return
  fi

  smoke_args=(
    --fail
    --silent
    --show-error
    --max-time
    5
  )
  if [[ -n "${NGINX_SMOKE_HOST}" ]]; then
    smoke_args+=(-H "Host: ${NGINX_SMOKE_HOST}")
  fi
  smoke_args+=("${NGINX_SMOKE_URL}")

  if [[ "${APPLY}" != "true" ]]; then
    echo "+ ${CURL_BINARY} ${smoke_args[*]}"
    if [[ -n "${NGINX_SMOKE_EXPECT_BODY}" ]]; then
      echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会验证 Nginx smoke URL 可访问，且响应体包含预期片段。"
    else
      echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会验证 Nginx smoke URL 可访问。"
    fi
    return
  fi

  if ! command -v "${CURL_BINARY}" >/dev/null 2>&1; then
    echo "[pingora-direct-rollback] 未找到 curl 可执行文件，无法验证 Nginx smoke URL: ${CURL_BINARY}" >&2
    exit 1
  fi

  echo "+ ${CURL_BINARY} ${smoke_args[*]}"
  smoke_body="$("${CURL_BINARY}" "${smoke_args[@]}")"
  if [[ -n "${NGINX_SMOKE_EXPECT_BODY}" ]]; then
    if [[ "${smoke_body}" != *"${NGINX_SMOKE_EXPECT_BODY}"* ]]; then
      echo "[pingora-direct-rollback] Nginx smoke 响应缺少预期片段: ${NGINX_SMOKE_EXPECT_BODY}" >&2
      echo "[pingora-direct-rollback] Nginx smoke 响应前 500 字符: ${smoke_body:0:500}" >&2
      exit 1
    fi
    echo "[pingora-direct-rollback] nginx smoke evidence matched expected body fragment."
  else
    echo "[pingora-direct-rollback] nginx smoke evidence curl --fail passed without body fragment check."
  fi
}

test_nginx_config_before_reload() {
  if [[ "${APPLY}" != "true" ]]; then
    echo "+ ${NGINX_BINARY} -t"
    echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会先执行 nginx -t，通过后再 reload。"
    return
  fi

  if ! command -v "${NGINX_BINARY}" >/dev/null 2>&1; then
    echo "[pingora-direct-rollback] 未找到 Nginx 可执行文件，无法执行: ${NGINX_BINARY} -t" >&2
    exit 1
  fi

  echo "+ ${NGINX_BINARY} -t"
  "${NGINX_BINARY}" -t
}

validate_dropin_path_for_apply() {
  if [[ "${APPLY}" != "true" ]]; then
    return
  fi

  DROPIN_DIR="$(dirname "${DROPIN_PATH}")"
  if [[ -L "${DROPIN_DIR}" ]]; then
    echo "[pingora-direct-rollback] drop-in 目录不能是符号链接: ${DROPIN_DIR}" >&2
    exit 1
  fi
  if [[ -e "${DROPIN_DIR}" && ! -d "${DROPIN_DIR}" ]]; then
    echo "[pingora-direct-rollback] drop-in 父路径必须是目录: ${DROPIN_DIR}" >&2
    exit 1
  fi
  if [[ -L "${DROPIN_PATH}" ]]; then
    echo "[pingora-direct-rollback] drop-in 目标不能是符号链接: ${DROPIN_PATH}" >&2
    exit 1
  fi
  if [[ -e "${DROPIN_PATH}" && ! -f "${DROPIN_PATH}" ]]; then
    echo "[pingora-direct-rollback] drop-in 目标已存在但不是普通文件: ${DROPIN_PATH}" >&2
    exit 1
  fi
}

verify_health_patrol_env_after_rollback() {
  if [[ -z "${HEALTH_PATROL_ENV_FILE}" ]]; then
    return
  fi

  if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then
    echo "[pingora-direct-rollback] 已跳过 health patrol env 回退后复核。"
    return
  fi

  health_patrol_args=(
    "${HEALTH_PATROL_ENV_CHECK_SCRIPT}"
    --env-file
    "${HEALTH_PATROL_ENV_FILE}"
    --expected-gateway-mode
    nginx
  )
  if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then
    health_patrol_args+=(
      --expected-public-base-url
      "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}"
    )
  fi
  if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then
    health_patrol_args+=(
      --expected-public-host
      "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}"
    )
  elif [[ "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" != "false" ]]; then
    health_patrol_args+=(--require-empty-public-host)
  fi

  if [[ "${APPLY}" != "true" ]]; then
    echo "+ node -- ${health_patrol_args[*]}"
    echo "[pingora-direct-rollback] dry-run：--apply 后会复核 health patrol env 已切回 nginx，且 public base URL / Host 已恢复为预期值。"
    return
  fi

  if ! command -v node >/dev/null 2>&1; then
    echo "[pingora-direct-rollback] 未找到 node，无法复核 health patrol env。" >&2
    exit 1
  fi
  if [[ ! -f "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then
    echo "[pingora-direct-rollback] health patrol env 复核脚本不存在: ${HEALTH_PATROL_ENV_CHECK_SCRIPT}" >&2
    exit 1
  fi

  echo "+ node -- ${health_patrol_args[*]}"
  node -- "${health_patrol_args[@]}"
}

verify_pingora_shadow_probe_after_rollback() {
  if [[ -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then
    return
  fi

  if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then
    echo "[pingora-direct-rollback] 已跳过 Pingora shadow 探针回退后核验。"
    return
  fi

  shadow_args=(
    --fail
    --silent
    --show-error
    --max-time
    5
    -H
    "X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_TOKEN}"
    "${PINGORA_SHADOW_PROBE_URL}"
  )

  if [[ "${APPLY}" != "true" ]]; then
    echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 -H X-Genarrative-Pingora-Probe: <redacted> ${PINGORA_SHADOW_PROBE_URL}"
    echo "[pingora-direct-rollback] dry-run：--apply 后会验证 Pingora shadow 探针仍为 gateway=pingora-shadow。"
    return
  fi

  if ! command -v "${CURL_BINARY}" >/dev/null 2>&1; then
    echo "[pingora-direct-rollback] 未找到 curl 可执行文件，无法验证 Pingora shadow 探针: ${CURL_BINARY}" >&2
    exit 1
  fi

  echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 -H X-Genarrative-Pingora-Probe: <redacted> ${PINGORA_SHADOW_PROBE_URL}"
  shadow_body="$("${CURL_BINARY}" "${shadow_args[@]}")"
  if [[ "${shadow_body}" != *'"ok":true'* || "${shadow_body}" != *'"gateway":"pingora-shadow"'* ]]; then
    echo "[pingora-direct-rollback] Pingora shadow 探针响应不是预期 JSON: ${shadow_body}" >&2
    exit 1
  fi
}

echo "[pingora-direct-rollback] apply=${APPLY} service=${SERVICE_NAME} dropin=${DROPIN_PATH}"

if [[ "${APPLY}" != "true" ]]; then
  echo "[pingora-direct-rollback] 当前是 dry-run；确认无误后追加 --apply。"
fi

validate_dropin_path_for_apply
test_nginx_config_before_reload

if [[ -e "${DROPIN_PATH}" || -L "${DROPIN_PATH}" ]]; then
  run_cmd rm -f "${DROPIN_PATH}"
else
  echo "[pingora-direct-rollback] direct-entry drop-in 不存在，视为已移除: ${DROPIN_PATH}"
fi

run_systemctl daemon-reload
run_systemctl restart "${SERVICE_NAME}"
verify_systemd_dropin_removed
verify_systemd_exec_start_after_rollback

run_systemctl reload "${NGINX_SERVICE}"
verify_nginx_active_after_reload
verify_nginx_smoke_after_reload
verify_health_patrol_env_after_rollback
verify_pingora_shadow_probe_after_rollback

if [[ "${STATUS_AFTER}" == "true" ]]; then
  run_systemctl status "${SERVICE_NAME}" --no-pager
fi

cat <<EOF
[pingora-direct-rollback] 完成。后续请确认：
- systemctl cat ${SERVICE_NAME} 不再包含 AmbientCapabilities=CAP_NET_BIND_SERVICE
- systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager 指向 current release 的 pingora-gateway
- ${NGINX_BINARY} -t 已通过
- systemctl is-active ${NGINX_SERVICE} 为 active
- ${CURL_BINARY} --fail ${NGINX_SMOKE_URL:-<nginx-smoke-url>} 已通过
- 如传入 --nginx-smoke-expect-body，Nginx smoke 响应体已匹配预期片段
- 如传入 --health-patrol-env-file，health patrol env 已复核为 nginx 模式且 public base URL / Host 符合预期
- 如传入 --pingora-shadow-probe-url，Pingora shadow 高端口探针已复核为 gateway=pingora-shadow
- 公网入口已回到 Nginx 或其它正式前置入口
EOF