#!/usr/bin/env bash set -euo pipefail usage() { cat <<'EOF' 用法: ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] 说明: 进入维护模式,校验并发布 api-server 单文件,更新 current 链接,重启 systemd 服务并执行 readiness 检查。 默认同时重启外部生成 worker controller 和已加载的 worker 实例;未启用 worker 单元时会自动跳过。 若传入 --database,会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件,避免服务继续读取旧库。 若发布包包含 pingora-gateway,部署脚本会在 current 链接切换后仅对已 active 的 Pingora 影子服务执行 try-restart 并复核 active;未运行时不会主动拉起。 失败时保留维护模式。 EOF } require_argument() { local value="$1" local label="$2" if [[ -z "${value}" ]]; then echo "[production-api-deploy] 缺少参数: ${label}" >&2 exit 1 fi } require_absolute_path() { local value="$1" local label="$2" if [[ ! "${value}" = /* ]]; then echo "[production-api-deploy] ${label} 必须使用绝对路径: ${value}" >&2 exit 1 fi } validate_spacetime_database_name() { local database="$1" if [[ ! "${database}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then echo "[production-api-deploy] --database 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$,只能使用小写字母、数字,并用单个短横线分隔: ${database}" >&2 exit 1 fi } write_env_value() { local file_path="$1" local key="$2" local value="$3" local can_write_direct=0 if [[ -e "${file_path}" ]]; then [[ -w "${file_path}" ]] && can_write_direct=1 else mkdir -p "$(dirname "${file_path}")" [[ -w "$(dirname "${file_path}")" ]] && can_write_direct=1 fi # api-server 环境文件通常由 server-provision 以 root:root 0600 创建。 # 发布流水线可能以非 root Jenkins 用户运行,因此仅在不能直接写入时使用 sudo -n,避免因为 env 文件权限导致 API 发布中断。 if [[ "$(id -u)" -eq 0 || "${can_write_direct}" -eq 1 ]]; then python3 - "${file_path}" "${key}" "${value}" <<'PY' import os import sys from pathlib import Path file_path = Path(sys.argv[1]) key = sys.argv[2] value = sys.argv[3] was_missing = not file_path.exists() file_path.parent.mkdir(parents=True, exist_ok=True) lines = [] if file_path.exists(): lines = file_path.read_text(encoding="utf-8").splitlines() updated = False next_lines = [] for line in lines: if line.startswith(f"{key}="): next_lines.append(f"{key}={value}") updated = True else: next_lines.append(line) if not updated: next_lines.append(f"{key}={value}") file_path.write_text("\n".join(next_lines) + "\n", encoding="utf-8") if was_missing: os.chmod(file_path, 0o600) PY else if ! sudo -n true >/dev/null 2>&1; then echo "[production-api-deploy] 当前用户无权写入 ${file_path},且 sudo -n 不可用;请给部署用户配置免密写入该环境文件或以 root 执行发布。" >&2 exit 1 fi sudo -n python3 - "${file_path}" "${key}" "${value}" <<'PY' import os import sys from pathlib import Path file_path = Path(sys.argv[1]) key = sys.argv[2] value = sys.argv[3] was_missing = not file_path.exists() file_path.parent.mkdir(parents=True, exist_ok=True) lines = [] if file_path.exists(): lines = file_path.read_text(encoding="utf-8").splitlines() updated = False next_lines = [] for line in lines: if line.startswith(f"{key}="): next_lines.append(f"{key}={value}") updated = True else: next_lines.append(line) if not updated: next_lines.append(f"{key}={value}") file_path.write_text("\n".join(next_lines) + "\n", encoding="utf-8") if was_missing: os.chmod(file_path, 0o600) PY fi } read_env_value() { local file_path="$1" local key="$2" if [[ ! -f "${file_path}" ]]; then return 0 fi local python_script=' import sys from pathlib import Path path = Path(sys.argv[1]) key = sys.argv[2] if not path.exists(): raise SystemExit(0) for raw_line in path.read_text(encoding="utf-8").splitlines(): line = raw_line.strip() if not line or line.startswith("#") or "=" not in line: continue current_key, value = line.split("=", 1) if current_key == key: value = value.strip() if len(value) >= 2 and value[0] == value[-1] and value[0] in ("\"", "'\''"): value = value[1:-1] print(value) raise SystemExit(0) ' if [[ -r "${file_path}" ]]; then python3 -c "${python_script}" "${file_path}" "${key}" else if ! sudo -n true >/dev/null 2>&1; then echo "[production-api-deploy] 当前用户无权读取 ${file_path},且 sudo -n 不可用;无法检查运行态环境变量。" >&2 exit 1 fi sudo -n python3 -c "${python_script}" "${file_path}" "${key}" fi } ensure_env_value() { local file_path="$1" local key="$2" local default_value="$3" local current_value current_value="$(read_env_value "${file_path}" "${key}")" if [[ -n "${current_value}" ]]; then return fi echo "[production-api-deploy] 补齐 api-server 环境变量: ${key} -> ${file_path}" write_env_value "${file_path}" "${key}" "${default_value}" } run_privileged() { if [[ "$(id -u)" -eq 0 ]]; then "$@" return fi if ! sudo -n true >/dev/null 2>&1; then echo "[production-api-deploy] 当前用户不是 root,且 sudo -n 不可用;无法执行: $*" >&2 exit 1 fi sudo -n "$@" } ensure_runtime_dir() { local path="$1" local mode="$2" if [[ -z "${path}" ]]; then return fi if [[ "${path}" != /* ]]; then echo "[production-api-deploy] 运行态目录必须使用绝对路径,避免写入只读发布目录: ${path}" >&2 exit 1 fi echo "[production-api-deploy] 确保运行态目录可写: ${path}" run_privileged install -d -o genarrative -g genarrative -m "${mode}" "${path}" } ensure_runtime_env_and_dirs() { local api_env_file="$1" local tracking_enabled tracking_outbox_dir # 旧生产环境文件会被 server-provision 保留,不一定包含新增的运行态写入路径。 # 发布前只补缺省值,不覆盖线上已经定制过的目录或开关。 ensure_env_value "${api_env_file}" "GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS" "5000" ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED" "true" ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR" "/var/lib/genarrative/tracking-outbox" ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE" "500" ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS" "1000" ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES" "268435456" tracking_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED")" tracking_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR")" if [[ "$(printf "%s" "${tracking_enabled}" | tr '[:upper:]' '[:lower:]')" != "false" ]]; then ensure_runtime_dir "${tracking_outbox_dir}" "0750" fi } restart_pingora_if_active() { local service_name="$1" if ! systemctl is-active --quiet "${service_name}"; then echo "[production-api-deploy] Pingora 影子服务未处于 active,跳过自动重启: ${service_name}" return fi echo "[production-api-deploy] 发布包包含 Pingora,重启已运行的影子服务: ${service_name}" systemctl try-restart "${service_name}" if ! systemctl is-active --quiet "${service_name}"; then echo "[production-api-deploy] Pingora 影子服务重启后不是 active: ${service_name}" >&2 exit 1 fi } validate_release_manifest() { local manifest_path="$1" local require_pingora="$2" if [[ ! -f "${manifest_path}" ]]; then echo "[production-api-deploy] 发布产物缺少 release-manifest.json: ${manifest_path}" >&2 exit 1 fi node - "${manifest_path}" "${require_pingora}" <<'NODE' const fs = require('fs'); const manifestPath = process.argv[2]; const requirePingora = process.argv[3] === '1'; let manifest; try { manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); } catch (error) { console.error(`[production-api-deploy] release-manifest.json 不是合法 JSON: ${error.message}`); process.exit(1); } const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : []; const hasApiServer = artifacts.some((artifact) => artifact?.path === 'api-server'); if (!hasApiServer) { console.error('[production-api-deploy] release-manifest.json 缺少 api-server artifact。'); process.exit(1); } if (requirePingora) { const hasPingora = artifacts.some((artifact) => artifact?.path === 'pingora-gateway'); if (!hasPingora) { console.error('[production-api-deploy] release-manifest.json 缺少 pingora-gateway artifact。'); process.exit(1); } } NODE } list_worker_services() { local pattern="$1" if [[ -z "${pattern}" ]]; then return 0 fi systemctl list-units --all --plain --no-legend "${pattern}" 2>/dev/null | awk '{print $1}' | sort -u } ensure_default_worker_service() { local pattern="$1" local default_service="genarrative-external-generation-worker@1.service" local template_service="genarrative-external-generation-worker@.service" local services=() if [[ -z "${pattern}" ]]; then return 0 fi if [[ "${pattern}" != "genarrative-external-generation-worker@*.service" ]]; then return 0 fi if ! systemctl cat "${template_service}" >/dev/null 2>&1; then echo "[production-api-deploy] 缺少外部生成 worker systemd 模板: ${template_service}" >&2 return 1 fi mapfile -t services < <(list_worker_services "${pattern}") if [[ "${#services[@]}" -gt 0 ]]; then return 0 fi echo "[production-api-deploy] 未发现外部生成 worker 实例,启用并启动默认实例: ${default_service}" systemctl enable --now "${default_service}" } restart_worker_services() { local pattern="$1" local services=() if [[ -z "${pattern}" ]]; then echo "[production-api-deploy] 跳过外部生成 worker 重启。" return 0 fi ensure_default_worker_service "${pattern}" mapfile -t services < <(list_worker_services "${pattern}") if [[ "${#services[@]}" -eq 0 ]]; then echo "[production-api-deploy] 未发现已加载的外部生成 worker 单元: ${pattern}" >&2 return 1 fi echo "[production-api-deploy] 重启外部生成 worker: ${services[*]}" systemctl restart "${services[@]}" } wait_for_worker_services() { local pattern="$1" local services=() local all_active if [[ -z "${pattern}" ]]; then return 0 fi mapfile -t services < <(list_worker_services "${pattern}") if [[ "${#services[@]}" -eq 0 ]]; then echo "[production-api-deploy] 外部生成 worker 单元不存在,发布失败: ${pattern}" >&2 return 1 fi echo "[production-api-deploy] 等待外部生成 worker active: ${services[*]}" for _ in {1..30}; do all_active=1 for service in "${services[@]}"; do if ! systemctl is-active --quiet "${service}"; then all_active=0 break fi done if [[ "${all_active}" -eq 1 ]]; then return 0 fi sleep 2 done systemctl --no-pager --full status "${services[@]}" || true echo "[production-api-deploy] 外部生成 worker 未在超时时间内进入 active,发布失败。" >&2 return 1 } ensure_worker_controller_service() { local service="$1" if [[ -z "${service}" ]]; then return 0 fi if ! systemctl cat "${service}" >/dev/null 2>&1; then echo "[production-api-deploy] 缺少外部生成 worker controller systemd 单元: ${service}" >&2 return 1 fi echo "[production-api-deploy] 启用并重启外部生成 worker controller: ${service}" systemctl enable "${service}" systemctl restart "${service}" } wait_for_worker_controller_service() { local service="$1" if [[ -z "${service}" ]]; then return 0 fi echo "[production-api-deploy] 等待外部生成 worker controller active: ${service}" for _ in {1..30}; do if systemctl is-active --quiet "${service}"; then return 0 fi sleep 2 done systemctl --no-pager --full status "${service}" || true echo "[production-api-deploy] 外部生成 worker controller 未在超时时间内进入 active,发布失败。" >&2 return 1 } SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" SOURCE_DIR="" VERSION="" RELEASE_ROOT="/opt/genarrative/releases" CURRENT_LINK="/opt/genarrative/current" SERVICE_NAME="genarrative-api.service" PINGORA_SERVICE_NAME="genarrative-pingora-gateway.service" WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service" WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service" HEALTH_URL="http://127.0.0.1:8082/readyz" API_ENV_FILE="/etc/genarrative/api-server.env" DATABASE="" SPACETIME_SERVER_URL="" DEPLOY_COMPLETED=0 PINGORA_INCLUDED=0 RELEASE_DIR="" STAGING_RELEASE_DIR="" while [[ $# -gt 0 ]]; do case "$1" in -h|--help) usage exit 0 ;; --source-dir) SOURCE_DIR="${2:?缺少 --source-dir 的值}" shift 2 ;; --version) VERSION="${2:?缺少 --version 的值}" shift 2 ;; --release-root) RELEASE_ROOT="${2:?缺少 --release-root 的值}" shift 2 ;; --current-link) CURRENT_LINK="${2:?缺少 --current-link 的值}" shift 2 ;; --service) SERVICE_NAME="${2:?缺少 --service 的值}" shift 2 ;; --pingora-service) PINGORA_SERVICE_NAME="${2:?缺少 --pingora-service 的值}" shift 2 ;; --worker-service-pattern) WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}" shift 2 ;; --no-worker-services) WORKER_SERVICE_PATTERN="" shift ;; --worker-controller-service) WORKER_CONTROLLER_SERVICE="${2:?缺少 --worker-controller-service 的值}" shift 2 ;; --no-worker-controller) WORKER_CONTROLLER_SERVICE="" shift ;; --health-url) HEALTH_URL="${2:?缺少 --health-url 的值}" shift 2 ;; --api-env-file) API_ENV_FILE="${2:?缺少 --api-env-file 的值}" shift 2 ;; --database) DATABASE="${2:?缺少 --database 的值}" shift 2 ;; --spacetime-server-url) SPACETIME_SERVER_URL="${2:?缺少 --spacetime-server-url 的值}" shift 2 ;; *) echo "[production-api-deploy] 未知参数: $1" >&2 usage >&2 exit 1 ;; esac done require_argument "${SOURCE_DIR}" "--source-dir" require_absolute_path "${RELEASE_ROOT}" "--release-root" require_absolute_path "${CURRENT_LINK}" "--current-link" require_absolute_path "${API_ENV_FILE}" "--api-env-file" if [[ -n "${DATABASE}" ]]; then validate_spacetime_database_name "${DATABASE}" fi if [[ ! -d "${SOURCE_DIR}" ]]; then echo "[production-api-deploy] 发布目录不存在: ${SOURCE_DIR}" >&2 exit 1 fi SOURCE_DIR="$(cd "${SOURCE_DIR}" && pwd)" VERSION="${VERSION:-$(basename "${SOURCE_DIR}")}" if [[ ! "${VERSION}" =~ ^[0-9A-Za-z][0-9A-Za-z._-]*$ ]]; then echo "[production-api-deploy] --version 必须以数字或字母开头,且只能包含数字、字母、点、下划线和短横线: ${VERSION}" >&2 exit 1 fi if [[ "${VERSION}" == "." || "${VERSION}" == ".." ]]; then echo "[production-api-deploy] --version 不能是点目录: ${VERSION}" >&2 exit 1 fi if [[ ! -f "${SOURCE_DIR}/api-server" || ! -f "${SOURCE_DIR}/api-server.sha256" ]]; then echo "[production-api-deploy] 缺少 api-server 或 api-server.sha256: ${SOURCE_DIR}" >&2 exit 1 fi RELEASE_DIR="${RELEASE_ROOT}/${VERSION}" STAGING_RELEASE_DIR="${RELEASE_ROOT}/.${VERSION}.staging.$$" if [[ -e "${RELEASE_DIR}" ]]; then echo "[production-api-deploy] 目标 release 已存在,拒绝覆盖或合并旧文件: ${RELEASE_DIR}" >&2 exit 1 fi if [[ -e "${STAGING_RELEASE_DIR}" ]]; then echo "[production-api-deploy] 临时 staging release 已存在: ${STAGING_RELEASE_DIR}" >&2 exit 1 fi if [[ -e "${CURRENT_LINK}" && ! -L "${CURRENT_LINK}" ]]; then echo "[production-api-deploy] current 链接路径已存在但不是符号链接,拒绝覆盖: ${CURRENT_LINK}" >&2 exit 1 fi cleanup_staging_release() { if [[ -n "${STAGING_RELEASE_DIR:-}" && -d "${STAGING_RELEASE_DIR}" ]]; then rm -rf "${STAGING_RELEASE_DIR}" fi } on_exit() { local exit_code=$? if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then cleanup_staging_release echo "[production-api-deploy] 部署失败,保持维护模式。" >&2 fi exit "${exit_code}" } trap on_exit EXIT bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}" echo "[production-api-deploy] 校验 api-server" ( cd "${SOURCE_DIR}" sha256sum -c api-server.sha256 if [[ -f pingora-gateway || -f pingora-gateway.sha256 ]]; then if [[ ! -f pingora-gateway || ! -f pingora-gateway.sha256 ]]; then echo "[production-api-deploy] pingora-gateway 与 pingora-gateway.sha256 必须同时存在。" >&2 exit 1 fi sha256sum -c pingora-gateway.sha256 fi ) if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then PINGORA_INCLUDED=1 fi validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "${PINGORA_INCLUDED}" mkdir -p "${RELEASE_ROOT}" mkdir "${STAGING_RELEASE_DIR}" RELEASE_CONTENT_DIR="${STAGING_RELEASE_DIR}" cp "${SOURCE_DIR}/api-server" "${RELEASE_CONTENT_DIR}/api-server" chmod +x "${RELEASE_CONTENT_DIR}/api-server" if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then cp "${SOURCE_DIR}/pingora-gateway" "${RELEASE_CONTENT_DIR}/pingora-gateway" chmod +x "${RELEASE_CONTENT_DIR}/pingora-gateway" echo "[production-api-deploy] 已复制 Pingora 影子网关;current 链接切换后将复核并重启已运行的 ${PINGORA_SERVICE_NAME}" fi BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" HEALTH_PATROL_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs" PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs" PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs" PINGORA_RELEASE_READINESS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs" PINGORA_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" PINGORA_ROLLBACK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" PINGORA_PREFLIGHT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" PINGORA_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs" PINGORA_CANARY_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs" PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" PINGORA_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/pingora" SYSTEMD_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/systemd" NGINX_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/nginx" ENV_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/env" mkdir -p "${RELEASE_CONTENT_DIR}/scripts" "${RELEASE_CONTENT_DIR}/scripts/deploy" "${RELEASE_CONTENT_DIR}/scripts/ops" "${RELEASE_CONTENT_DIR}/deploy" if [[ ! -f "${BACKUP_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 exit 1 fi cp "${BACKUP_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs" if [[ ! -f "${HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少生产健康巡检脚本: ${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs" >&2 exit 1 fi cp "${HEALTH_PATROL_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs" if [[ ! -f "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora current release 自审脚本: ${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs" >&2 exit 1 fi cp "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs" if [[ ! -f "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换状态快照脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" >&2 exit 1 fi cp "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据包脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" >&2 exit 1 fi cp "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" if [[ ! -f "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换命令证据脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" >&2 exit 1 fi cp "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据验真脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" >&2 exit 1 fi cp "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据根目录审计脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" >&2 exit 1 fi cp "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" if [[ ! -f "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少生产健康巡检 env 复核脚本: ${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs" >&2 exit 1 fi cp "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs" if [[ ! -f "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora release readiness 聚合门禁脚本: ${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs" >&2 exit 1 fi cp "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs" if [[ ! -f "${PINGORA_ENABLE_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连启用脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" >&2 exit 1 fi cp "${PINGORA_ENABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh" chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh" if [[ ! -f "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连回退脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" >&2 exit 1 fi cp "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh" chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh" if [[ ! -f "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora health patrol env 切换脚本: ${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" >&2 exit 1 fi cp "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" if [[ ! -f "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连预检脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" >&2 exit 1 fi cp "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs" if [[ ! -f "${PINGORA_LIVE_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连 live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs" >&2 exit 1 fi cp "${PINGORA_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs" if [[ ! -f "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora canary live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs" >&2 exit 1 fi cp "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs" if [[ ! -f "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora canary access log 对账脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" >&2 exit 1 fi cp "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" if [[ ! -d "${PINGORA_DEPLOY_DIR_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 配置目录: ${SOURCE_DIR}/deploy/pingora" >&2 exit 1 fi rm -rf "${RELEASE_CONTENT_DIR}/deploy/pingora" cp -R "${PINGORA_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/pingora" if [[ ! -d "${SYSTEMD_DEPLOY_DIR_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 systemd 配置目录: ${SOURCE_DIR}/deploy/systemd" >&2 exit 1 fi rm -rf "${RELEASE_CONTENT_DIR}/deploy/systemd" cp -R "${SYSTEMD_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/systemd" if [[ ! -d "${NGINX_DEPLOY_DIR_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Nginx 配置目录: ${SOURCE_DIR}/deploy/nginx" >&2 exit 1 fi rm -rf "${RELEASE_CONTENT_DIR}/deploy/nginx" cp -R "${NGINX_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/nginx" if [[ ! -d "${ENV_DEPLOY_DIR_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少环境变量示例目录: ${SOURCE_DIR}/deploy/env" >&2 exit 1 fi rm -rf "${RELEASE_CONTENT_DIR}/deploy/env" cp -R "${ENV_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/env" cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_CONTENT_DIR}/release-manifest.api-server.json" if [[ -n "${DATABASE}" ]]; then echo "[production-api-deploy] 写入 api-server SpacetimeDB database: ${DATABASE} -> ${API_ENV_FILE}" write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_DATABASE" "${DATABASE}" fi if [[ -n "${SPACETIME_SERVER_URL}" ]]; then echo "[production-api-deploy] 写入 api-server SpacetimeDB server: ${SPACETIME_SERVER_URL} -> ${API_ENV_FILE}" write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_SERVER_URL" "${SPACETIME_SERVER_URL}" fi ensure_runtime_env_and_dirs "${API_ENV_FILE}" mkdir -p "$(dirname "${CURRENT_LINK}")" if [[ -e "${RELEASE_DIR}" ]]; then echo "[production-api-deploy] 目标 release 在发布过程中出现,拒绝合并 staging: ${RELEASE_DIR}" >&2 exit 1 fi mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}" STAGING_RELEASE_DIR="" ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}" if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then restart_pingora_if_active "${PINGORA_SERVICE_NAME}" fi echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}" systemctl restart "${SERVICE_NAME}" restart_worker_services "${WORKER_SERVICE_PATTERN}" wait_for_worker_services "${WORKER_SERVICE_PATTERN}" ensure_worker_controller_service "${WORKER_CONTROLLER_SERVICE}" wait_for_worker_controller_service "${WORKER_CONTROLLER_SERVICE}" echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}" for _ in {1..30}; do if curl -fsS "${HEALTH_URL}" >/dev/null; then bash "${SCRIPT_DIR}/maintenance-off.sh" DEPLOY_COMPLETED=1 echo "[production-api-deploy] 完成: ${RELEASE_DIR}/api-server" exit 0 fi sleep 2 done echo "[production-api-deploy] readiness 检查超时: ${HEALTH_URL}" >&2 exit 1