#!/usr/bin/env bash set -euo pipefail SERVICE_NAME="${GENARRATIVE_PINGORA_GATEWAY_SERVICE:-genarrative-pingora-gateway.service}" SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)" DEFAULT_SERVICE_UNIT_PATH="${REPO_ROOT}/deploy/systemd/genarrative-pingora-gateway.service" SERVICE_UNIT_PATH="${GENARRATIVE_PINGORA_ROLLBACK_SERVICE_UNIT_PATH:-${DEFAULT_SERVICE_UNIT_PATH}}" DROPIN_PATH="${GENARRATIVE_PINGORA_DIRECT_DROPIN_PATH:-/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf}" NGINX_SERVICE="${GENARRATIVE_NGINX_SERVICE:-nginx.service}" NGINX_BINARY="${GENARRATIVE_NGINX_BINARY:-nginx}" NGINX_SMOKE_URL="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_URL:-}" NGINX_SMOKE_HOST="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_HOST:-}" NGINX_SMOKE_EXPECT_BODY="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_EXPECT_BODY:-}" CURL_BINARY="${GENARRATIVE_CURL_BINARY:-curl}" HEALTH_PATROL_ENV_FILE="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_ENV_FILE:-}" HEALTH_PATROL_ENV_CHECK_SCRIPT="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_ENV_CHECK_SCRIPT:-}" HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL:-}" HEALTH_PATROL_EXPECTED_PUBLIC_HOST="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_EXPECTED_PUBLIC_HOST:-}" HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST:-auto}" PINGORA_SHADOW_PROBE_URL="${GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_URL:-}" PINGORA_SHADOW_PROBE_TOKEN="${GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN:-}" APPLY="false" RELOAD_NGINX="false" STATUS_AFTER="true" VERIFY_SYSTEMD_AFTER="true" usage() { cat <<'EOF' 用法: scripts/deploy/pingora-direct-rollback.sh [--apply] [--reload-nginx] [--nginx-smoke-url ] [--nginx-smoke-host ] [--nginx-smoke-expect-body ] [--curl-binary ] [--health-patrol-env-file ] [--health-patrol-env-check-script ] [--health-patrol-expected-public-base-url ] [--health-patrol-expected-public-host |--health-patrol-require-empty-public-host] [--pingora-shadow-probe-url ] [--pingora-shadow-probe-token ] [--no-postcheck] [--service ] [--service-unit-path ] [--dropin-path ] [--nginx-service ] [--nginx-binary ] 说明: 回退 Pingora 直连低端口入口:移除 direct-entry systemd drop-in,执行 daemon-reload, 重启 genarrative-pingora-gateway.service,让服务回到 shadow/high-port 口径。 默认是 dry-run,只打印将执行的命令;必须显式传 --apply 才会修改系统状态。 --apply 必须同时传 --reload-nginx,让脚本执行 nginx reload, 脚本会先运行 nginx -t,通过后再 reload,并在 reload 后确认 Nginx 仍处于 active 状态。 --apply 还必须提供 --nginx-smoke-url,reload 后用 curl 验证 Nginx 入口已经真实响应。 若提供 --nginx-smoke-expect-body,脚本会要求 smoke 响应体包含该片段,避免 HTTP 200 命中错误入口。 脚本会以 current release 随包主 service 模板的 ExecStart 为真相源, 在重启 Pingora 后核验 systemd 最终 ExecStart 仍指向同一个 pingora-gateway 二进制。 如果提供 --health-patrol-env-file,脚本会在 Nginx smoke 之后复核 health patrol 已切回 nginx 模式, 并可校验回退后的 public base URL / Host 是否恢复为切换前记录值。 如果同时提供 --pingora-shadow-probe-url 与 --pingora-shadow-probe-token,脚本会在回退后 验证 Pingora shadow 高端口探针仍返回 gateway=pingora-shadow。 EOF } resolve_default_health_patrol_env_check_script() { printf '%s\n' "${REPO_ROOT}/scripts/check-production-health-patrol-env.mjs" } while [[ $# -gt 0 ]]; do case "$1" in -h|--help) usage exit 0 ;; --apply) APPLY="true" shift ;; --reload-nginx) RELOAD_NGINX="true" shift ;; --no-status) STATUS_AFTER="false" shift ;; --no-postcheck) VERIFY_SYSTEMD_AFTER="false" shift ;; --service) SERVICE_NAME="${2:-}" if [[ -z "${SERVICE_NAME}" ]]; then echo "[pingora-direct-rollback] --service 缺少参数" >&2 exit 1 fi shift 2 ;; --service-unit-path) SERVICE_UNIT_PATH="${2:-}" if [[ -z "${SERVICE_UNIT_PATH}" ]]; then echo "[pingora-direct-rollback] --service-unit-path 缺少参数" >&2 exit 1 fi shift 2 ;; --dropin-path) DROPIN_PATH="${2:-}" if [[ -z "${DROPIN_PATH}" ]]; then echo "[pingora-direct-rollback] --dropin-path 缺少参数" >&2 exit 1 fi shift 2 ;; --nginx-service) NGINX_SERVICE="${2:-}" if [[ -z "${NGINX_SERVICE}" ]]; then echo "[pingora-direct-rollback] --nginx-service 缺少参数" >&2 exit 1 fi shift 2 ;; --nginx-binary) NGINX_BINARY="${2:-}" if [[ -z "${NGINX_BINARY}" ]]; then echo "[pingora-direct-rollback] --nginx-binary 缺少参数" >&2 exit 1 fi shift 2 ;; --nginx-smoke-url) NGINX_SMOKE_URL="${2:-}" if [[ -z "${NGINX_SMOKE_URL}" ]]; then echo "[pingora-direct-rollback] --nginx-smoke-url 缺少参数" >&2 exit 1 fi shift 2 ;; --nginx-smoke-host) NGINX_SMOKE_HOST="${2:-}" if [[ -z "${NGINX_SMOKE_HOST}" ]]; then echo "[pingora-direct-rollback] --nginx-smoke-host 缺少参数" >&2 exit 1 fi shift 2 ;; --nginx-smoke-expect-body) NGINX_SMOKE_EXPECT_BODY="${2:-}" if [[ -z "${NGINX_SMOKE_EXPECT_BODY}" ]]; then echo "[pingora-direct-rollback] --nginx-smoke-expect-body 缺少参数" >&2 exit 1 fi shift 2 ;; --curl-binary) CURL_BINARY="${2:-}" if [[ -z "${CURL_BINARY}" ]]; then echo "[pingora-direct-rollback] --curl-binary 缺少参数" >&2 exit 1 fi shift 2 ;; --health-patrol-env-file) HEALTH_PATROL_ENV_FILE="${2:-}" if [[ -z "${HEALTH_PATROL_ENV_FILE}" ]]; then echo "[pingora-direct-rollback] --health-patrol-env-file 缺少参数" >&2 exit 1 fi shift 2 ;; --health-patrol-env-check-script) HEALTH_PATROL_ENV_CHECK_SCRIPT="${2:-}" if [[ -z "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then echo "[pingora-direct-rollback] --health-patrol-env-check-script 缺少参数" >&2 exit 1 fi shift 2 ;; --health-patrol-expected-public-base-url) HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL="${2:-}" if [[ -z "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then echo "[pingora-direct-rollback] --health-patrol-expected-public-base-url 缺少参数" >&2 exit 1 fi shift 2 ;; --health-patrol-expected-public-host) HEALTH_PATROL_EXPECTED_PUBLIC_HOST="${2:-}" if [[ -z "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then echo "[pingora-direct-rollback] --health-patrol-expected-public-host 缺少参数" >&2 exit 1 fi shift 2 ;; --health-patrol-require-empty-public-host) HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST="true" shift ;; --pingora-shadow-probe-url) PINGORA_SHADOW_PROBE_URL="${2:-}" if [[ -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then echo "[pingora-direct-rollback] --pingora-shadow-probe-url 缺少参数" >&2 exit 1 fi shift 2 ;; --pingora-shadow-probe-token) PINGORA_SHADOW_PROBE_TOKEN="${2:-}" if [[ -z "${PINGORA_SHADOW_PROBE_TOKEN}" ]]; then echo "[pingora-direct-rollback] --pingora-shadow-probe-token 缺少参数" >&2 exit 1 fi shift 2 ;; *) echo "[pingora-direct-rollback] 未知参数: $1" >&2 usage >&2 exit 1 ;; esac done if [[ -n "${HEALTH_PATROL_ENV_FILE}" && -z "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then HEALTH_PATROL_ENV_CHECK_SCRIPT="$(resolve_default_health_patrol_env_check_script)" fi reject_control_characters() { local label="$1" local value="$2" if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then echo "[pingora-direct-rollback] ${label} 不能包含换行或 NUL 字符。" >&2 exit 1 fi } is_filesystem_root_path() { local value="$1" local without_slashes="${value//\//}" [[ -n "${value}" && -z "${without_slashes}" ]] } reject_filesystem_root_path() { local label="$1" local value="$2" if is_filesystem_root_path "${value}"; then echo "[pingora-direct-rollback] ${label} 不能是文件系统根目录。" >&2 exit 1 fi } reject_control_characters "--service" "${SERVICE_NAME}" reject_control_characters "--service-unit-path" "${SERVICE_UNIT_PATH}" reject_control_characters "--dropin-path" "${DROPIN_PATH}" reject_control_characters "--nginx-service" "${NGINX_SERVICE}" reject_control_characters "--nginx-binary" "${NGINX_BINARY}" reject_control_characters "--nginx-smoke-url" "${NGINX_SMOKE_URL}" reject_control_characters "--nginx-smoke-host" "${NGINX_SMOKE_HOST}" reject_control_characters "--nginx-smoke-expect-body" "${NGINX_SMOKE_EXPECT_BODY}" reject_control_characters "--curl-binary" "${CURL_BINARY}" reject_control_characters "--health-patrol-env-file" "${HEALTH_PATROL_ENV_FILE}" reject_control_characters "--health-patrol-env-check-script" "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" reject_control_characters "--health-patrol-expected-public-base-url" "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" reject_control_characters "--health-patrol-expected-public-host" "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" reject_control_characters "--health-patrol-require-empty-public-host" "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" reject_control_characters "--pingora-shadow-probe-url" "${PINGORA_SHADOW_PROBE_URL}" reject_control_characters "--pingora-shadow-probe-token" "${PINGORA_SHADOW_PROBE_TOKEN}" reject_filesystem_root_path "--service-unit-path" "${SERVICE_UNIT_PATH}" reject_filesystem_root_path "--dropin-path" "${DROPIN_PATH}" reject_filesystem_root_path "--nginx-binary" "${NGINX_BINARY}" reject_filesystem_root_path "--curl-binary" "${CURL_BINARY}" reject_filesystem_root_path "--health-patrol-env-file" "${HEALTH_PATROL_ENV_FILE}" reject_filesystem_root_path "--health-patrol-env-check-script" "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" if [[ "${DROPIN_PATH}" != /* ]]; then echo "[pingora-direct-rollback] --dropin-path 必须是绝对路径: ${DROPIN_PATH}" >&2 exit 1 fi if [[ "${SERVICE_UNIT_PATH}" != /* ]]; then echo "[pingora-direct-rollback] --service-unit-path 必须是绝对路径: ${SERVICE_UNIT_PATH}" >&2 exit 1 fi if [[ -n "${HEALTH_PATROL_ENV_FILE}" && "${HEALTH_PATROL_ENV_FILE}" != /* ]]; then echo "[pingora-direct-rollback] --health-patrol-env-file 必须是绝对路径: ${HEALTH_PATROL_ENV_FILE}" >&2 exit 1 fi if [[ -n "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" && "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" != /* ]]; then echo "[pingora-direct-rollback] --health-patrol-env-check-script 必须是绝对路径: ${HEALTH_PATROL_ENV_CHECK_SCRIPT}" >&2 exit 1 fi require_absolute_path_if_path_like() { local label="$1" local value="$2" if [[ "${value}" == */* && "${value}" != /* ]]; then echo "[pingora-direct-rollback] ${label} 包含路径分隔符时必须是绝对路径: ${value}" >&2 exit 1 fi } require_absolute_path_if_path_like "--nginx-binary" "${NGINX_BINARY}" require_absolute_path_if_path_like "--curl-binary" "${CURL_BINARY}" if [[ -n "${PINGORA_SHADOW_PROBE_URL}" && -z "${PINGORA_SHADOW_PROBE_TOKEN}" ]]; then echo "[pingora-direct-rollback] --pingora-shadow-probe-url 必须同时提供 --pingora-shadow-probe-token。" >&2 exit 1 fi if [[ -n "${PINGORA_SHADOW_PROBE_TOKEN}" && -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then echo "[pingora-direct-rollback] --pingora-shadow-probe-token 必须同时提供 --pingora-shadow-probe-url。" >&2 exit 1 fi if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" && "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" == "true" ]]; then echo "[pingora-direct-rollback] --health-patrol-expected-public-host 和 --health-patrol-require-empty-public-host 不能同时使用。" >&2 exit 1 fi validate_nginx_smoke_host() { validate_host_value "--nginx-smoke-host" "${NGINX_SMOKE_HOST}" } validate_host_value() { local label="$1" local value="$2" if [[ "${value}" == *"://"* || "${value}" == */* || "${value}" == *\?* || "${value}" == *"#"* || "${value}" =~ [[:space:]] ]]; then echo "[pingora-direct-rollback] ${label} 只能是 host 或 host:port,不能包含 URL、路径、查询、片段或空白字符: ${value}" >&2 exit 1 fi } is_loopback_smoke_url() { local url="$1" local authority host case "${url}" in http://*|https://*) ;; *) return 1 ;; esac authority="${url#*://}" authority="${authority%%/*}" authority="${authority%%\?*}" authority="${authority%%#*}" authority="${authority##*@}" if [[ "${authority}" == \[*\]* ]]; then host="${authority#\[}" host="${host%%\]*}" else host="${authority%%:*}" fi host="${host,,}" [[ "${host}" == "localhost" || "${host}" == "127."* || "${host}" == "::1" ]] } validate_http_url() { local label="$1" local url="$2" case "${url}" in http://*|https://*) ;; *) echo "[pingora-direct-rollback] ${label} 必须是 http(s) URL: ${url}" >&2 exit 1 ;; esac } if [[ -n "${NGINX_SMOKE_HOST}" ]]; then validate_nginx_smoke_host fi if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then validate_host_value "--health-patrol-expected-public-host" "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" fi if [[ -n "${PINGORA_SHADOW_PROBE_URL}" ]]; then validate_http_url "--pingora-shadow-probe-url" "${PINGORA_SHADOW_PROBE_URL}" fi if [[ -n "${NGINX_SMOKE_URL}" ]]; then validate_http_url "--nginx-smoke-url" "${NGINX_SMOKE_URL}" fi if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then validate_http_url "--health-patrol-expected-public-base-url" "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" fi if [[ "${APPLY}" == "true" && "${RELOAD_NGINX}" != "true" ]]; then echo "[pingora-direct-rollback] --apply 必须同时提供 --reload-nginx,确保回退后 Nginx 公网入口重新加载并保持 active。" >&2 exit 1 fi if [[ "${APPLY}" == "true" && -z "${NGINX_SMOKE_URL}" ]]; then echo "[pingora-direct-rollback] --apply 必须同时提供 --nginx-smoke-url,确保回退后 Nginx 入口真实可访问。" >&2 exit 1 fi if [[ "${APPLY}" == "true" && -z "${NGINX_SMOKE_HOST}" ]] && is_loopback_smoke_url "${NGINX_SMOKE_URL}"; then echo "[pingora-direct-rollback] --nginx-smoke-url 指向本机地址时必须同时提供 --nginx-smoke-host,避免回退 smoke 命中默认 vhost。" >&2 exit 1 fi run_cmd() { echo "+ $*" if [[ "${APPLY}" == "true" ]]; then "$@" fi } run_systemctl() { if [[ "${APPLY}" != "true" ]]; then run_cmd systemctl "$@" return fi if command -v systemctl >/dev/null 2>&1; then run_cmd systemctl "$@" else echo "[pingora-direct-rollback] 未找到 systemctl,无法执行: systemctl $*" >&2 exit 1 fi } read_expected_exec_start() { local line value if [[ ! -f "${SERVICE_UNIT_PATH}" ]]; then return 1 fi while IFS= read -r line; do line="${line#"${line%%[![:space:]]*}"}" line="${line%"${line##*[![:space:]]}"}" [[ "${line}" == ExecStart=* ]] || continue value="${line#ExecStart=}" [[ -n "${value}" ]] || continue printf '%s\n' "${value}" return 0 done <"${SERVICE_UNIT_PATH}" return 1 } first_exec_word() { local value="$1" value="${value#"${value%%[![:space:]]*}"}" value="${value%"${value##*[![:space:]]}"}" if [[ "${value}" == \"* ]]; then value="${value#\"}" printf '%s\n' "${value%%\"*}" return fi printf '%s\n' "${value%%[[:space:]]*}" } systemd_show_exec_start_matches() { local expected_exec="$1" local expected_binary actual_show actual_binary expected_binary="$(first_exec_word "${expected_exec}")" [[ -n "${expected_binary}" ]] || return 1 actual_show="$(systemctl show "${SERVICE_NAME}" --property=ExecStart --value --no-pager)" actual_binary="${actual_show}" if [[ "${actual_show}" == *"path="* ]]; then actual_binary="${actual_show#*path=}" actual_binary="${actual_binary%% ;*}" elif [[ "${actual_show}" == ExecStart=* ]]; then actual_binary="${actual_show#ExecStart=}" actual_binary="$(first_exec_word "${actual_binary}")" else actual_binary="$(first_exec_word "${actual_binary}")" fi [[ "${actual_binary}" == "${expected_binary}" ]] } verify_systemd_exec_start_after_rollback() { if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then echo "[pingora-direct-rollback] 已跳过 Pingora ExecStart 指向核验。" return fi expected_exec_start="$(read_expected_exec_start || true)" if [[ -z "${expected_exec_start}" ]]; then echo "[pingora-direct-rollback] 无法从主 service 模板读取 ExecStart: ${SERVICE_UNIT_PATH}" >&2 exit 1 fi if [[ "${APPLY}" != "true" ]]; then echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" echo "[pingora-direct-rollback] dry-run:--apply 后会核验 systemd ExecStart 指向主 service 模板中的 ${expected_exec_start}。" return fi if ! command -v systemctl >/dev/null 2>&1; then echo "[pingora-direct-rollback] 未找到 systemctl,无法核验 Pingora ExecStart。" >&2 exit 1 fi echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" if ! systemd_show_exec_start_matches "${expected_exec_start}"; then echo "[pingora-direct-rollback] systemctl show ExecStart 未指向主 service 模板中的 ${expected_exec_start},请先修正 ${SERVICE_NAME} 指向 current release 网关二进制。" >&2 exit 1 fi } verify_systemd_dropin_removed() { if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then echo "[pingora-direct-rollback] 已跳过 systemd drop-in 移除核验。" return fi if [[ "${APPLY}" != "true" ]]; then echo "+ systemctl cat ${SERVICE_NAME}" echo "[pingora-direct-rollback] dry-run:--apply 后会校验 systemd 最终配置不再包含 CAP_NET_BIND_SERVICE。" return fi if ! command -v systemctl >/dev/null 2>&1; then echo "[pingora-direct-rollback] 未找到 systemctl,无法核验 drop-in 是否已移除。" >&2 exit 1 fi echo "+ systemctl cat ${SERVICE_NAME}" unit_content="$(systemctl cat "${SERVICE_NAME}")" if [[ "${unit_content}" == *"AmbientCapabilities=CAP_NET_BIND_SERVICE"* ]]; then echo "[pingora-direct-rollback] systemctl cat 仍显示 AmbientCapabilities=CAP_NET_BIND_SERVICE,direct-entry drop-in 未移除。" >&2 exit 1 fi if [[ "${unit_content}" == *"CapabilityBoundingSet=CAP_NET_BIND_SERVICE"* ]]; then echo "[pingora-direct-rollback] systemctl cat 仍显示 CapabilityBoundingSet=CAP_NET_BIND_SERVICE,direct-entry drop-in 未移除。" >&2 exit 1 fi } verify_nginx_active_after_reload() { if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then echo "[pingora-direct-rollback] 已跳过 Nginx reload 后状态核验。" return fi if [[ "${APPLY}" != "true" ]]; then echo "+ systemctl is-active ${NGINX_SERVICE}" echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会 reload Nginx 并校验 service 仍为 active。" return fi if ! command -v systemctl >/dev/null 2>&1; then echo "[pingora-direct-rollback] 未找到 systemctl,无法核验 Nginx reload 后状态。" >&2 exit 1 fi echo "+ systemctl is-active ${NGINX_SERVICE}" nginx_state="$(systemctl is-active "${NGINX_SERVICE}")" if [[ "${nginx_state}" != "active" ]]; then echo "[pingora-direct-rollback] Nginx reload 后状态不是 active: ${nginx_state}" >&2 exit 1 fi } verify_nginx_smoke_after_reload() { if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then echo "[pingora-direct-rollback] 已跳过 Nginx reload 后 smoke 核验。" return fi if [[ -z "${NGINX_SMOKE_URL}" ]]; then echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 " echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会用 --nginx-smoke-url 验证 Nginx 入口。" return fi smoke_args=( --fail --silent --show-error --max-time 5 ) if [[ -n "${NGINX_SMOKE_HOST}" ]]; then smoke_args+=(-H "Host: ${NGINX_SMOKE_HOST}") fi smoke_args+=("${NGINX_SMOKE_URL}") if [[ "${APPLY}" != "true" ]]; then echo "+ ${CURL_BINARY} ${smoke_args[*]}" if [[ -n "${NGINX_SMOKE_EXPECT_BODY}" ]]; then echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会验证 Nginx smoke URL 可访问,且响应体包含预期片段。" else echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会验证 Nginx smoke URL 可访问。" fi return fi if ! command -v "${CURL_BINARY}" >/dev/null 2>&1; then echo "[pingora-direct-rollback] 未找到 curl 可执行文件,无法验证 Nginx smoke URL: ${CURL_BINARY}" >&2 exit 1 fi echo "+ ${CURL_BINARY} ${smoke_args[*]}" smoke_body="$("${CURL_BINARY}" "${smoke_args[@]}")" if [[ -n "${NGINX_SMOKE_EXPECT_BODY}" ]]; then if [[ "${smoke_body}" != *"${NGINX_SMOKE_EXPECT_BODY}"* ]]; then echo "[pingora-direct-rollback] Nginx smoke 响应缺少预期片段: ${NGINX_SMOKE_EXPECT_BODY}" >&2 echo "[pingora-direct-rollback] Nginx smoke 响应前 500 字符: ${smoke_body:0:500}" >&2 exit 1 fi echo "[pingora-direct-rollback] nginx smoke evidence matched expected body fragment." else echo "[pingora-direct-rollback] nginx smoke evidence curl --fail passed without body fragment check." fi } test_nginx_config_before_reload() { if [[ "${APPLY}" != "true" ]]; then echo "+ ${NGINX_BINARY} -t" echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会先执行 nginx -t,通过后再 reload。" return fi if ! command -v "${NGINX_BINARY}" >/dev/null 2>&1; then echo "[pingora-direct-rollback] 未找到 Nginx 可执行文件,无法执行: ${NGINX_BINARY} -t" >&2 exit 1 fi echo "+ ${NGINX_BINARY} -t" "${NGINX_BINARY}" -t } validate_dropin_path_for_apply() { if [[ "${APPLY}" != "true" ]]; then return fi DROPIN_DIR="$(dirname "${DROPIN_PATH}")" if [[ -L "${DROPIN_DIR}" ]]; then echo "[pingora-direct-rollback] drop-in 目录不能是符号链接: ${DROPIN_DIR}" >&2 exit 1 fi if [[ -e "${DROPIN_DIR}" && ! -d "${DROPIN_DIR}" ]]; then echo "[pingora-direct-rollback] drop-in 父路径必须是目录: ${DROPIN_DIR}" >&2 exit 1 fi if [[ -L "${DROPIN_PATH}" ]]; then echo "[pingora-direct-rollback] drop-in 目标不能是符号链接: ${DROPIN_PATH}" >&2 exit 1 fi if [[ -e "${DROPIN_PATH}" && ! -f "${DROPIN_PATH}" ]]; then echo "[pingora-direct-rollback] drop-in 目标已存在但不是普通文件: ${DROPIN_PATH}" >&2 exit 1 fi } verify_health_patrol_env_after_rollback() { if [[ -z "${HEALTH_PATROL_ENV_FILE}" ]]; then return fi if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then echo "[pingora-direct-rollback] 已跳过 health patrol env 回退后复核。" return fi health_patrol_args=( "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" --env-file "${HEALTH_PATROL_ENV_FILE}" --expected-gateway-mode nginx ) if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then health_patrol_args+=( --expected-public-base-url "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ) fi if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then health_patrol_args+=( --expected-public-host "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ) elif [[ "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" != "false" ]]; then health_patrol_args+=(--require-empty-public-host) fi if [[ "${APPLY}" != "true" ]]; then echo "+ node -- ${health_patrol_args[*]}" echo "[pingora-direct-rollback] dry-run:--apply 后会复核 health patrol env 已切回 nginx,且 public base URL / Host 已恢复为预期值。" return fi if ! command -v node >/dev/null 2>&1; then echo "[pingora-direct-rollback] 未找到 node,无法复核 health patrol env。" >&2 exit 1 fi if [[ ! -f "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then echo "[pingora-direct-rollback] health patrol env 复核脚本不存在: ${HEALTH_PATROL_ENV_CHECK_SCRIPT}" >&2 exit 1 fi echo "+ node -- ${health_patrol_args[*]}" node -- "${health_patrol_args[@]}" } verify_pingora_shadow_probe_after_rollback() { if [[ -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then return fi if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then echo "[pingora-direct-rollback] 已跳过 Pingora shadow 探针回退后核验。" return fi shadow_args=( --fail --silent --show-error --max-time 5 -H "X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_TOKEN}" "${PINGORA_SHADOW_PROBE_URL}" ) if [[ "${APPLY}" != "true" ]]; then echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 -H X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_URL}" echo "[pingora-direct-rollback] dry-run:--apply 后会验证 Pingora shadow 探针仍为 gateway=pingora-shadow。" return fi if ! command -v "${CURL_BINARY}" >/dev/null 2>&1; then echo "[pingora-direct-rollback] 未找到 curl 可执行文件,无法验证 Pingora shadow 探针: ${CURL_BINARY}" >&2 exit 1 fi echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 -H X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_URL}" shadow_body="$("${CURL_BINARY}" "${shadow_args[@]}")" if [[ "${shadow_body}" != *'"ok":true'* || "${shadow_body}" != *'"gateway":"pingora-shadow"'* ]]; then echo "[pingora-direct-rollback] Pingora shadow 探针响应不是预期 JSON: ${shadow_body}" >&2 exit 1 fi } echo "[pingora-direct-rollback] apply=${APPLY} service=${SERVICE_NAME} dropin=${DROPIN_PATH}" if [[ "${APPLY}" != "true" ]]; then echo "[pingora-direct-rollback] 当前是 dry-run;确认无误后追加 --apply。" fi validate_dropin_path_for_apply test_nginx_config_before_reload if [[ -e "${DROPIN_PATH}" || -L "${DROPIN_PATH}" ]]; then run_cmd rm -f "${DROPIN_PATH}" else echo "[pingora-direct-rollback] direct-entry drop-in 不存在,视为已移除: ${DROPIN_PATH}" fi run_systemctl daemon-reload run_systemctl restart "${SERVICE_NAME}" verify_systemd_dropin_removed verify_systemd_exec_start_after_rollback run_systemctl reload "${NGINX_SERVICE}" verify_nginx_active_after_reload verify_nginx_smoke_after_reload verify_health_patrol_env_after_rollback verify_pingora_shadow_probe_after_rollback if [[ "${STATUS_AFTER}" == "true" ]]; then run_systemctl status "${SERVICE_NAME}" --no-pager fi cat <} 已通过 - 如传入 --nginx-smoke-expect-body,Nginx smoke 响应体已匹配预期片段 - 如传入 --health-patrol-env-file,health patrol env 已复核为 nginx 模式且 public base URL / Host 符合预期 - 如传入 --pingora-shadow-probe-url,Pingora shadow 高端口探针已复核为 gateway=pingora-shadow - 公网入口已回到 Nginx 或其它正式前置入口 EOF