kdletters
14f8a87ac5
API deploy 从随包安装外部生成 worker 模板和 controller 单元 安装 worker systemd 单元后自动 daemon-reload 补充 API release 与 deploy 动态门禁覆盖 worker systemd 模板
990 lines
39 KiB
Bash
990 lines
39 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
用法:
|
|
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
|
|
|
|
说明:
|
|
进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
|
|
默认同时重启外部生成 worker controller 和已加载的 worker 实例；未启用 worker 单元时会自动跳过。
|
|
若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。
|
|
若发布包包含 pingora-gateway，部署脚本会在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，再启动或重启 Pingora 影子服务并复核 active。
|
|
失败时保留维护模式。
|
|
EOF
|
|
}
|
|
|
|
require_argument() {
|
|
local value="$1"
|
|
local label="$2"
|
|
|
|
if [[ -z "${value}" ]]; then
|
|
echo "[production-api-deploy] 缺少参数: ${label}" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
require_absolute_path() {
|
|
local value="$1"
|
|
local label="$2"
|
|
|
|
if [[ ! "${value}" = /* ]]; then
|
|
echo "[production-api-deploy] ${label} 必须使用绝对路径: ${value}" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_spacetime_database_name() {
|
|
local database="$1"
|
|
|
|
if [[ ! "${database}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then
|
|
echo "[production-api-deploy] --database 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$，只能使用小写字母、数字，并用单个短横线分隔: ${database}" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
write_env_value() {
|
|
local file_path="$1"
|
|
local key="$2"
|
|
local value="$3"
|
|
|
|
local can_write_direct=0
|
|
|
|
if [[ -e "${file_path}" ]]; then
|
|
[[ -w "${file_path}" ]] && can_write_direct=1
|
|
else
|
|
mkdir -p "$(dirname "${file_path}")"
|
|
[[ -w "$(dirname "${file_path}")" ]] && can_write_direct=1
|
|
fi
|
|
|
|
# api-server 环境文件通常由 server-provision 以 root:root 0600 创建。
|
|
# 发布流水线可能以非 root Jenkins 用户运行，因此仅在不能直接写入时使用 sudo -n，避免因为 env 文件权限导致 API 发布中断。
|
|
if [[ "$(id -u)" -eq 0 || "${can_write_direct}" -eq 1 ]]; then
|
|
python3 - "${file_path}" "${key}" "${value}" <<'PY'
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
file_path = Path(sys.argv[1])
|
|
key = sys.argv[2]
|
|
value = sys.argv[3]
|
|
was_missing = not file_path.exists()
|
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
lines = []
|
|
if file_path.exists():
|
|
lines = file_path.read_text(encoding="utf-8").splitlines()
|
|
|
|
updated = False
|
|
next_lines = []
|
|
for line in lines:
|
|
if line.startswith(f"{key}="):
|
|
next_lines.append(f"{key}={value}")
|
|
updated = True
|
|
else:
|
|
next_lines.append(line)
|
|
|
|
if not updated:
|
|
next_lines.append(f"{key}={value}")
|
|
|
|
file_path.write_text("\n".join(next_lines) + "\n", encoding="utf-8")
|
|
if was_missing:
|
|
os.chmod(file_path, 0o600)
|
|
PY
|
|
else
|
|
if ! sudo -n true >/dev/null 2>&1; then
|
|
echo "[production-api-deploy] 当前用户无权写入 ${file_path}，且 sudo -n 不可用；请给部署用户配置免密写入该环境文件或以 root 执行发布。" >&2
|
|
exit 1
|
|
fi
|
|
sudo -n python3 - "${file_path}" "${key}" "${value}" <<'PY'
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
file_path = Path(sys.argv[1])
|
|
key = sys.argv[2]
|
|
value = sys.argv[3]
|
|
was_missing = not file_path.exists()
|
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
lines = []
|
|
if file_path.exists():
|
|
lines = file_path.read_text(encoding="utf-8").splitlines()
|
|
|
|
updated = False
|
|
next_lines = []
|
|
for line in lines:
|
|
if line.startswith(f"{key}="):
|
|
next_lines.append(f"{key}={value}")
|
|
updated = True
|
|
else:
|
|
next_lines.append(line)
|
|
|
|
if not updated:
|
|
next_lines.append(f"{key}={value}")
|
|
|
|
file_path.write_text("\n".join(next_lines) + "\n", encoding="utf-8")
|
|
if was_missing:
|
|
os.chmod(file_path, 0o600)
|
|
PY
|
|
fi
|
|
}
|
|
|
|
read_env_value() {
|
|
local file_path="$1"
|
|
local key="$2"
|
|
|
|
if [[ ! -f "${file_path}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local python_script='
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
path = Path(sys.argv[1])
|
|
key = sys.argv[2]
|
|
if not path.exists():
|
|
raise SystemExit(0)
|
|
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
|
line = raw_line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
current_key, value = line.split("=", 1)
|
|
if current_key == key:
|
|
value = value.strip()
|
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("\"", "'\''"):
|
|
value = value[1:-1]
|
|
print(value)
|
|
raise SystemExit(0)
|
|
'
|
|
|
|
if [[ -r "${file_path}" ]]; then
|
|
python3 -c "${python_script}" "${file_path}" "${key}"
|
|
else
|
|
if ! sudo -n true >/dev/null 2>&1; then
|
|
echo "[production-api-deploy] 当前用户无权读取 ${file_path}，且 sudo -n 不可用；无法检查运行态环境变量。" >&2
|
|
exit 1
|
|
fi
|
|
sudo -n python3 -c "${python_script}" "${file_path}" "${key}"
|
|
fi
|
|
}
|
|
|
|
ensure_env_value() {
|
|
local file_path="$1"
|
|
local key="$2"
|
|
local default_value="$3"
|
|
local current_value
|
|
|
|
current_value="$(read_env_value "${file_path}" "${key}")"
|
|
if [[ -n "${current_value}" ]]; then
|
|
return
|
|
fi
|
|
|
|
echo "[production-api-deploy] 补齐 api-server 环境变量: ${key} -> ${file_path}"
|
|
write_env_value "${file_path}" "${key}" "${default_value}"
|
|
}
|
|
|
|
run_privileged() {
|
|
if [[ "$(id -u)" -eq 0 ]]; then
|
|
"$@"
|
|
return
|
|
fi
|
|
if ! sudo -n true >/dev/null 2>&1; then
|
|
echo "[production-api-deploy] 当前用户不是 root，且 sudo -n 不可用；无法执行: $*" >&2
|
|
exit 1
|
|
fi
|
|
sudo -n "$@"
|
|
}
|
|
|
|
ensure_runtime_dir() {
|
|
local path="$1"
|
|
local mode="$2"
|
|
|
|
if [[ -z "${path}" ]]; then
|
|
return
|
|
fi
|
|
if [[ "${path}" != /* ]]; then
|
|
echo "[production-api-deploy] 运行态目录必须使用绝对路径，避免写入只读发布目录: ${path}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "[production-api-deploy] 确保运行态目录可写: ${path}"
|
|
run_privileged install -d -o genarrative -g genarrative -m "${mode}" "${path}"
|
|
}
|
|
|
|
ensure_runtime_env_and_dirs() {
|
|
local api_env_file="$1"
|
|
local tracking_enabled tracking_outbox_dir wallet_refund_enabled wallet_refund_outbox_dir
|
|
|
|
# 旧生产环境文件会被 server-provision 保留，不一定包含新增的运行态写入路径。
|
|
# 发布前只补缺省值，不覆盖线上已经定制过的目录或开关。
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS" "5000"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED" "true"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR" "/var/lib/genarrative/tracking-outbox"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE" "500"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS" "1000"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES" "268435456"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_ENABLED" "true"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_DIR" "/var/lib/genarrative/wallet-refund-outbox"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE" "100"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS" "1000"
|
|
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES" "67108864"
|
|
|
|
tracking_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED")"
|
|
tracking_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR")"
|
|
if [[ "$(printf "%s" "${tracking_enabled}" | tr '[:upper:]' '[:lower:]')" != "false" ]]; then
|
|
ensure_runtime_dir "${tracking_outbox_dir}" "0750"
|
|
fi
|
|
wallet_refund_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_ENABLED")"
|
|
wallet_refund_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_DIR")"
|
|
if [[ "$(printf "%s" "${wallet_refund_enabled}" | tr '[:upper:]' '[:lower:]')" != "false" ]]; then
|
|
ensure_runtime_dir "${wallet_refund_outbox_dir}" "0750"
|
|
fi
|
|
}
|
|
|
|
extract_pingora_env_files_from_unit() {
|
|
local service_name="$1"
|
|
local unit_content
|
|
|
|
if ! unit_content="$(systemctl cat "${service_name}")"; then
|
|
echo "[production-api-deploy] 无法读取 Pingora systemd 最终配置: ${service_name}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if printf "%s\n" "${unit_content}" | grep -Eq '^[[:space:]]*(AmbientCapabilities|CapabilityBoundingSet)=.*CAP_NET_BIND_SERVICE'; then
|
|
echo "[production-api-deploy] Pingora systemd 已包含 CAP_NET_BIND_SERVICE，疑似直连入口配置；API deploy 不会自动启动或重启直连服务: ${service_name}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf "%s\n" "${unit_content}" | while IFS= read -r raw_line; do
|
|
local line value token env_file
|
|
line="${raw_line#"${raw_line%%[![:space:]]*}"}"
|
|
[[ "${line}" == EnvironmentFile=* ]] || continue
|
|
value="${line#EnvironmentFile=}"
|
|
for token in ${value}; do
|
|
env_file="${token#-}"
|
|
env_file="${env_file%\"}"
|
|
env_file="${env_file#\"}"
|
|
env_file="${env_file%\'}"
|
|
env_file="${env_file#\'}"
|
|
[[ -n "${env_file}" ]] && printf "%s\n" "${env_file}"
|
|
done
|
|
done
|
|
}
|
|
|
|
find_pingora_gateway_env_file() {
|
|
local service_name="$1"
|
|
local env_file listen
|
|
|
|
while IFS= read -r env_file; do
|
|
if [[ "${env_file}" != /* ]]; then
|
|
echo "[production-api-deploy] Pingora EnvironmentFile 必须使用绝对路径: ${env_file}" >&2
|
|
exit 1
|
|
fi
|
|
listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")"
|
|
if [[ -n "${listen}" ]]; then
|
|
printf "%s\n" "${env_file}"
|
|
return
|
|
fi
|
|
done < <(extract_pingora_env_files_from_unit "${service_name}")
|
|
|
|
echo "[production-api-deploy] Pingora systemd 配置缺少包含 GENARRATIVE_PINGORA_GATEWAY_LISTEN 的 EnvironmentFile: ${service_name}" >&2
|
|
exit 1
|
|
}
|
|
|
|
require_pingora_shadow_env() {
|
|
local env_file="$1"
|
|
local listen tls_listen redirect_listen
|
|
|
|
listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")"
|
|
tls_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN")"
|
|
redirect_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN")"
|
|
|
|
if [[ "${listen}" != "127.0.0.1:18081" ]]; then
|
|
echo "[production-api-deploy] Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081，当前 GENARRATIVE_PINGORA_GATEWAY_LISTEN=${listen:-<empty>}" >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "${tls_listen}" ]]; then
|
|
echo "[production-api-deploy] Pingora 自动启动不允许启用 TLS_LISTEN，当前 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=${tls_listen}" >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "${redirect_listen}" ]]; then
|
|
echo "[production-api-deploy] Pingora 自动启动不允许启用 HTTP_REDIRECT_LISTEN，当前 GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=${redirect_listen}" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
check_pingora_shadow_service_config() {
|
|
local service_name="$1"
|
|
local env_file
|
|
|
|
env_file="$(find_pingora_gateway_env_file "${service_name}")"
|
|
require_pingora_shadow_env "${env_file}"
|
|
printf "%s\n" "${env_file}"
|
|
}
|
|
|
|
ensure_pingora_shadow_service() {
|
|
local service_name="$1"
|
|
local env_file="${2:-}"
|
|
|
|
if [[ -z "${env_file}" ]]; then
|
|
env_file="$(check_pingora_shadow_service_config "${service_name}")"
|
|
else
|
|
require_pingora_shadow_env "${env_file}"
|
|
fi
|
|
|
|
echo "[production-api-deploy] 发布包包含 Pingora，启动或重启 shadow 影子服务: ${service_name} (${env_file})"
|
|
systemctl restart "${service_name}"
|
|
if ! systemctl is-active --quiet "${service_name}"; then
|
|
echo "[production-api-deploy] Pingora shadow 影子服务启动或重启后不是 active: ${service_name}" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_release_manifest() {
|
|
local manifest_path="$1"
|
|
local require_pingora="$2"
|
|
|
|
if [[ ! -f "${manifest_path}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 release-manifest.json: ${manifest_path}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
node - "${manifest_path}" "${require_pingora}" <<'NODE'
|
|
const fs = require('fs');
|
|
|
|
const manifestPath = process.argv[2];
|
|
const requirePingora = process.argv[3] === '1';
|
|
let manifest;
|
|
try {
|
|
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
} catch (error) {
|
|
console.error(`[production-api-deploy] release-manifest.json 不是合法 JSON: ${error.message}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : [];
|
|
const hasApiServer = artifacts.some((artifact) => artifact?.path === 'api-server');
|
|
if (!hasApiServer) {
|
|
console.error('[production-api-deploy] release-manifest.json 缺少 api-server artifact。');
|
|
process.exit(1);
|
|
}
|
|
|
|
if (requirePingora) {
|
|
const hasPingora = artifacts.some((artifact) => artifact?.path === 'pingora-gateway');
|
|
if (!hasPingora) {
|
|
console.error('[production-api-deploy] release-manifest.json 缺少 pingora-gateway artifact。');
|
|
process.exit(1);
|
|
}
|
|
}
|
|
NODE
|
|
}
|
|
|
|
list_worker_services() {
|
|
local pattern="$1"
|
|
|
|
if [[ -z "${pattern}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
systemctl list-units --all --plain --no-legend "${pattern}" 2>/dev/null | awk '{print $1}' | sort -u
|
|
}
|
|
|
|
ensure_default_worker_service() {
|
|
local pattern="$1"
|
|
local default_service="genarrative-external-generation-worker@1.service"
|
|
local template_service="genarrative-external-generation-worker@.service"
|
|
local services=()
|
|
|
|
if [[ -z "${pattern}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ "${pattern}" != "genarrative-external-generation-worker@*.service" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if ! systemctl cat "${template_service}" >/dev/null 2>&1; then
|
|
echo "[production-api-deploy] 缺少外部生成 worker systemd 模板: ${template_service}" >&2
|
|
return 1
|
|
fi
|
|
|
|
mapfile -t services < <(list_worker_services "${pattern}")
|
|
if [[ "${#services[@]}" -gt 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
echo "[production-api-deploy] 未发现外部生成 worker 实例，启用并启动默认实例: ${default_service}"
|
|
systemctl enable --now "${default_service}"
|
|
}
|
|
|
|
install_release_systemd_unit() {
|
|
local source_path="$1"
|
|
local unit_name="$2"
|
|
local label="$3"
|
|
local unit_dir="${GENARRATIVE_SYSTEMD_UNIT_DIR:-/etc/systemd/system}"
|
|
|
|
if [[ ! -f "${source_path}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少${label}: ${source_path}" >&2
|
|
return 1
|
|
fi
|
|
if [[ "${unit_dir}" != /* ]]; then
|
|
echo "[production-api-deploy] systemd unit 目录必须使用绝对路径: ${unit_dir}" >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "[production-api-deploy] 安装${label}: ${unit_name}"
|
|
run_privileged install -d -m 0755 "${unit_dir}"
|
|
run_privileged install -m 0644 "${source_path}" "${unit_dir}/${unit_name}"
|
|
}
|
|
|
|
install_worker_systemd_units() {
|
|
local release_dir="$1"
|
|
local pattern="$2"
|
|
local controller_service="$3"
|
|
local installed_any=0
|
|
|
|
if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" ]]; then
|
|
install_release_systemd_unit \
|
|
"${release_dir}/deploy/systemd/genarrative-external-generation-worker@.service" \
|
|
"genarrative-external-generation-worker@.service" \
|
|
"外部生成 worker systemd 模板"
|
|
installed_any=1
|
|
fi
|
|
|
|
if [[ "${controller_service}" == "genarrative-external-generation-controller.service" ]]; then
|
|
install_release_systemd_unit \
|
|
"${release_dir}/deploy/systemd/genarrative-external-generation-controller.service" \
|
|
"genarrative-external-generation-controller.service" \
|
|
"外部生成 worker controller systemd 单元"
|
|
installed_any=1
|
|
fi
|
|
|
|
if [[ "${installed_any}" -eq 1 ]]; then
|
|
echo "[production-api-deploy] 重新加载 systemd unit。"
|
|
systemctl daemon-reload
|
|
fi
|
|
}
|
|
|
|
restart_worker_services() {
|
|
local pattern="$1"
|
|
local services=()
|
|
|
|
if [[ -z "${pattern}" ]]; then
|
|
echo "[production-api-deploy] 跳过外部生成 worker 重启。"
|
|
return 0
|
|
fi
|
|
|
|
ensure_default_worker_service "${pattern}"
|
|
mapfile -t services < <(list_worker_services "${pattern}")
|
|
if [[ "${#services[@]}" -eq 0 ]]; then
|
|
echo "[production-api-deploy] 未发现已加载的外部生成 worker 单元: ${pattern}" >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "[production-api-deploy] 重启外部生成 worker: ${services[*]}"
|
|
systemctl restart "${services[@]}"
|
|
}
|
|
|
|
wait_for_worker_services() {
|
|
local pattern="$1"
|
|
local services=()
|
|
local all_active
|
|
|
|
if [[ -z "${pattern}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
mapfile -t services < <(list_worker_services "${pattern}")
|
|
if [[ "${#services[@]}" -eq 0 ]]; then
|
|
echo "[production-api-deploy] 外部生成 worker 单元不存在，发布失败: ${pattern}" >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "[production-api-deploy] 等待外部生成 worker active: ${services[*]}"
|
|
for _ in {1..30}; do
|
|
all_active=1
|
|
for service in "${services[@]}"; do
|
|
if ! systemctl is-active --quiet "${service}"; then
|
|
all_active=0
|
|
break
|
|
fi
|
|
done
|
|
if [[ "${all_active}" -eq 1 ]]; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
systemctl --no-pager --full status "${services[@]}" || true
|
|
echo "[production-api-deploy] 外部生成 worker 未在超时时间内进入 active，发布失败。" >&2
|
|
return 1
|
|
}
|
|
|
|
ensure_worker_controller_service() {
|
|
local service="$1"
|
|
|
|
if [[ -z "${service}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if ! systemctl cat "${service}" >/dev/null 2>&1; then
|
|
echo "[production-api-deploy] 缺少外部生成 worker controller systemd 单元: ${service}" >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "[production-api-deploy] 启用并重启外部生成 worker controller: ${service}"
|
|
systemctl enable "${service}"
|
|
systemctl restart "${service}"
|
|
}
|
|
|
|
wait_for_worker_controller_service() {
|
|
local service="$1"
|
|
|
|
if [[ -z "${service}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
echo "[production-api-deploy] 等待外部生成 worker controller active: ${service}"
|
|
for _ in {1..30}; do
|
|
if systemctl is-active --quiet "${service}"; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
systemctl --no-pager --full status "${service}" || true
|
|
echo "[production-api-deploy] 外部生成 worker controller 未在超时时间内进入 active，发布失败。" >&2
|
|
return 1
|
|
}
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
SOURCE_DIR=""
|
|
VERSION=""
|
|
RELEASE_ROOT="/opt/genarrative/releases"
|
|
CURRENT_LINK="/opt/genarrative/current"
|
|
SERVICE_NAME="genarrative-api.service"
|
|
PINGORA_SERVICE_NAME="genarrative-pingora-gateway.service"
|
|
WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
|
|
WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service"
|
|
HEALTH_URL="http://127.0.0.1:8082/readyz"
|
|
API_ENV_FILE="/etc/genarrative/api-server.env"
|
|
DATABASE=""
|
|
SPACETIME_SERVER_URL=""
|
|
DEPLOY_COMPLETED=0
|
|
PINGORA_INCLUDED=0
|
|
RELEASE_DIR=""
|
|
STAGING_RELEASE_DIR=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
--source-dir)
|
|
SOURCE_DIR="${2:?缺少 --source-dir 的值}"
|
|
shift 2
|
|
;;
|
|
--version)
|
|
VERSION="${2:?缺少 --version 的值}"
|
|
shift 2
|
|
;;
|
|
--release-root)
|
|
RELEASE_ROOT="${2:?缺少 --release-root 的值}"
|
|
shift 2
|
|
;;
|
|
--current-link)
|
|
CURRENT_LINK="${2:?缺少 --current-link 的值}"
|
|
shift 2
|
|
;;
|
|
--service)
|
|
SERVICE_NAME="${2:?缺少 --service 的值}"
|
|
shift 2
|
|
;;
|
|
--pingora-service)
|
|
PINGORA_SERVICE_NAME="${2:?缺少 --pingora-service 的值}"
|
|
shift 2
|
|
;;
|
|
--worker-service-pattern)
|
|
WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}"
|
|
shift 2
|
|
;;
|
|
--no-worker-services)
|
|
WORKER_SERVICE_PATTERN=""
|
|
shift
|
|
;;
|
|
--worker-controller-service)
|
|
WORKER_CONTROLLER_SERVICE="${2:?缺少 --worker-controller-service 的值}"
|
|
shift 2
|
|
;;
|
|
--no-worker-controller)
|
|
WORKER_CONTROLLER_SERVICE=""
|
|
shift
|
|
;;
|
|
--health-url)
|
|
HEALTH_URL="${2:?缺少 --health-url 的值}"
|
|
shift 2
|
|
;;
|
|
--api-env-file)
|
|
API_ENV_FILE="${2:?缺少 --api-env-file 的值}"
|
|
shift 2
|
|
;;
|
|
--database)
|
|
DATABASE="${2:?缺少 --database 的值}"
|
|
shift 2
|
|
;;
|
|
--spacetime-server-url)
|
|
SPACETIME_SERVER_URL="${2:?缺少 --spacetime-server-url 的值}"
|
|
shift 2
|
|
;;
|
|
*)
|
|
echo "[production-api-deploy] 未知参数: $1" >&2
|
|
usage >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
require_argument "${SOURCE_DIR}" "--source-dir"
|
|
require_absolute_path "${RELEASE_ROOT}" "--release-root"
|
|
require_absolute_path "${CURRENT_LINK}" "--current-link"
|
|
require_absolute_path "${API_ENV_FILE}" "--api-env-file"
|
|
|
|
if [[ -n "${DATABASE}" ]]; then
|
|
validate_spacetime_database_name "${DATABASE}"
|
|
fi
|
|
|
|
if [[ ! -d "${SOURCE_DIR}" ]]; then
|
|
echo "[production-api-deploy] 发布目录不存在: ${SOURCE_DIR}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
SOURCE_DIR="$(cd "${SOURCE_DIR}" && pwd)"
|
|
VERSION="${VERSION:-$(basename "${SOURCE_DIR}")}"
|
|
|
|
if [[ ! "${VERSION}" =~ ^[0-9A-Za-z][0-9A-Za-z._-]*$ ]]; then
|
|
echo "[production-api-deploy] --version 必须以数字或字母开头，且只能包含数字、字母、点、下划线和短横线: ${VERSION}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${VERSION}" == "." || "${VERSION}" == ".." ]]; then
|
|
echo "[production-api-deploy] --version 不能是点目录: ${VERSION}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -f "${SOURCE_DIR}/api-server" || ! -f "${SOURCE_DIR}/api-server.sha256" ]]; then
|
|
echo "[production-api-deploy] 缺少 api-server 或 api-server.sha256: ${SOURCE_DIR}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
RELEASE_DIR="${RELEASE_ROOT}/${VERSION}"
|
|
STAGING_RELEASE_DIR="${RELEASE_ROOT}/.${VERSION}.staging.$$"
|
|
|
|
if [[ -e "${RELEASE_DIR}" ]]; then
|
|
echo "[production-api-deploy] 目标 release 已存在，拒绝覆盖或合并旧文件: ${RELEASE_DIR}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -e "${STAGING_RELEASE_DIR}" ]]; then
|
|
echo "[production-api-deploy] 临时 staging release 已存在: ${STAGING_RELEASE_DIR}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -e "${CURRENT_LINK}" && ! -L "${CURRENT_LINK}" ]]; then
|
|
echo "[production-api-deploy] current 链接路径已存在但不是符号链接，拒绝覆盖: ${CURRENT_LINK}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
cleanup_staging_release() {
|
|
if [[ -n "${STAGING_RELEASE_DIR:-}" && -d "${STAGING_RELEASE_DIR}" ]]; then
|
|
rm -rf "${STAGING_RELEASE_DIR}"
|
|
fi
|
|
}
|
|
|
|
on_exit() {
|
|
local exit_code=$?
|
|
if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
|
|
cleanup_staging_release
|
|
echo "[production-api-deploy] 部署失败，保持维护模式。" >&2
|
|
fi
|
|
exit "${exit_code}"
|
|
}
|
|
|
|
trap on_exit EXIT
|
|
|
|
bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}"
|
|
|
|
echo "[production-api-deploy] 校验 api-server"
|
|
(
|
|
cd "${SOURCE_DIR}"
|
|
sha256sum -c api-server.sha256
|
|
if [[ -f pingora-gateway || -f pingora-gateway.sha256 ]]; then
|
|
if [[ ! -f pingora-gateway || ! -f pingora-gateway.sha256 ]]; then
|
|
echo "[production-api-deploy] pingora-gateway 与 pingora-gateway.sha256 必须同时存在。" >&2
|
|
exit 1
|
|
fi
|
|
sha256sum -c pingora-gateway.sha256
|
|
fi
|
|
)
|
|
if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then
|
|
PINGORA_INCLUDED=1
|
|
fi
|
|
validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "${PINGORA_INCLUDED}"
|
|
|
|
mkdir -p "${RELEASE_ROOT}"
|
|
mkdir "${STAGING_RELEASE_DIR}"
|
|
RELEASE_CONTENT_DIR="${STAGING_RELEASE_DIR}"
|
|
cp "${SOURCE_DIR}/api-server" "${RELEASE_CONTENT_DIR}/api-server"
|
|
cp "${SOURCE_DIR}/api-server.sha256" "${RELEASE_CONTENT_DIR}/api-server.sha256"
|
|
chmod +x "${RELEASE_CONTENT_DIR}/api-server"
|
|
if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then
|
|
cp "${SOURCE_DIR}/pingora-gateway" "${RELEASE_CONTENT_DIR}/pingora-gateway"
|
|
cp "${SOURCE_DIR}/pingora-gateway.sha256" "${RELEASE_CONTENT_DIR}/pingora-gateway.sha256"
|
|
chmod +x "${RELEASE_CONTENT_DIR}/pingora-gateway"
|
|
echo "[production-api-deploy] 已复制 Pingora 影子网关；current 链接切换后将复核 shadow 配置并启动或重启 ${PINGORA_SERVICE_NAME}"
|
|
fi
|
|
|
|
BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
|
|
HEALTH_PATROL_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs"
|
|
PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs"
|
|
PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
|
|
PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs"
|
|
PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs"
|
|
PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs"
|
|
PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs"
|
|
PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs"
|
|
HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs"
|
|
PINGORA_RELEASE_READINESS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs"
|
|
PINGORA_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh"
|
|
PINGORA_ROLLBACK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh"
|
|
PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh"
|
|
PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh"
|
|
PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs"
|
|
PINGORA_GATEWAY_ENV_SHADOW_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs"
|
|
PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs"
|
|
PINGORA_PREFLIGHT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs"
|
|
PINGORA_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs"
|
|
PINGORA_CANARY_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs"
|
|
PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs"
|
|
PINGORA_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/pingora"
|
|
SYSTEMD_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/systemd"
|
|
NGINX_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/nginx"
|
|
ENV_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/env"
|
|
mkdir -p "${RELEASE_CONTENT_DIR}/scripts" "${RELEASE_CONTENT_DIR}/scripts/deploy" "${RELEASE_CONTENT_DIR}/scripts/ops" "${RELEASE_CONTENT_DIR}/deploy"
|
|
if [[ ! -f "${BACKUP_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${BACKUP_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs"
|
|
if [[ ! -f "${HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少生产健康巡检脚本: ${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${HEALTH_PATROL_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs"
|
|
if [[ ! -f "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora current release 自审脚本: ${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs"
|
|
if [[ ! -f "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连彩排状态脚本: ${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs"
|
|
if [[ ! -f "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换状态快照脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs"
|
|
if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据包脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs"
|
|
if [[ ! -f "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换命令证据脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs"
|
|
if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据验真脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs"
|
|
if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据根目录审计脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs"
|
|
if [[ ! -f "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少生产健康巡检 env 复核脚本: ${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs"
|
|
if [[ ! -f "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora release readiness 聚合门禁脚本: ${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs"
|
|
if [[ ! -f "${PINGORA_ENABLE_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连启用脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_ENABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh"
|
|
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh"
|
|
if [[ ! -f "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连回退脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh"
|
|
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh"
|
|
if [[ ! -f "${PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora realpath canary 启用脚本: ${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh"
|
|
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh"
|
|
if [[ ! -f "${PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora realpath canary 关闭脚本: ${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh"
|
|
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh"
|
|
if [[ ! -f "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora health patrol env 切换脚本: ${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs"
|
|
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs"
|
|
if [[ ! -f "${PINGORA_GATEWAY_ENV_SHADOW_SWITCH_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora gateway env shadow 切换脚本: ${SOURCE_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_GATEWAY_ENV_SHADOW_SWITCH_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs"
|
|
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs"
|
|
if [[ ! -f "${PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora TLS 证书同步脚本: ${SOURCE_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs"
|
|
chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs"
|
|
if [[ ! -f "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连预检脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs"
|
|
if [[ ! -f "${PINGORA_LIVE_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 直连 live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs"
|
|
if [[ ! -f "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora canary live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs"
|
|
if [[ ! -f "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora canary access log 对账脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" >&2
|
|
exit 1
|
|
fi
|
|
cp "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs"
|
|
chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs"
|
|
if [[ ! -d "${PINGORA_DEPLOY_DIR_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Pingora 配置目录: ${SOURCE_DIR}/deploy/pingora" >&2
|
|
exit 1
|
|
fi
|
|
rm -rf "${RELEASE_CONTENT_DIR}/deploy/pingora"
|
|
cp -R "${PINGORA_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/pingora"
|
|
if [[ ! -d "${SYSTEMD_DEPLOY_DIR_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 systemd 配置目录: ${SOURCE_DIR}/deploy/systemd" >&2
|
|
exit 1
|
|
fi
|
|
rm -rf "${RELEASE_CONTENT_DIR}/deploy/systemd"
|
|
cp -R "${SYSTEMD_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/systemd"
|
|
if [[ ! -d "${NGINX_DEPLOY_DIR_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少 Nginx 配置目录: ${SOURCE_DIR}/deploy/nginx" >&2
|
|
exit 1
|
|
fi
|
|
rm -rf "${RELEASE_CONTENT_DIR}/deploy/nginx"
|
|
cp -R "${NGINX_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/nginx"
|
|
if [[ ! -d "${ENV_DEPLOY_DIR_SOURCE}" ]]; then
|
|
echo "[production-api-deploy] 发布产物缺少环境变量示例目录: ${SOURCE_DIR}/deploy/env" >&2
|
|
exit 1
|
|
fi
|
|
rm -rf "${RELEASE_CONTENT_DIR}/deploy/env"
|
|
cp -R "${ENV_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/env"
|
|
|
|
cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_CONTENT_DIR}/release-manifest.api-server.json"
|
|
|
|
PINGORA_SHADOW_ENV_FILE=""
|
|
|
|
if [[ -n "${DATABASE}" ]]; then
|
|
echo "[production-api-deploy] 写入 api-server SpacetimeDB database: ${DATABASE} -> ${API_ENV_FILE}"
|
|
write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_DATABASE" "${DATABASE}"
|
|
fi
|
|
|
|
if [[ -n "${SPACETIME_SERVER_URL}" ]]; then
|
|
echo "[production-api-deploy] 写入 api-server SpacetimeDB server: ${SPACETIME_SERVER_URL} -> ${API_ENV_FILE}"
|
|
write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_SERVER_URL" "${SPACETIME_SERVER_URL}"
|
|
fi
|
|
|
|
ensure_runtime_env_and_dirs "${API_ENV_FILE}"
|
|
|
|
if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
|
|
PINGORA_SHADOW_ENV_FILE="$(check_pingora_shadow_service_config "${PINGORA_SERVICE_NAME}")"
|
|
fi
|
|
|
|
mkdir -p "$(dirname "${CURRENT_LINK}")"
|
|
if [[ -e "${RELEASE_DIR}" ]]; then
|
|
echo "[production-api-deploy] 目标 release 在发布过程中出现，拒绝合并 staging: ${RELEASE_DIR}" >&2
|
|
exit 1
|
|
fi
|
|
mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}"
|
|
STAGING_RELEASE_DIR=""
|
|
ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}"
|
|
|
|
if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
|
|
ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}"
|
|
fi
|
|
|
|
install_worker_systemd_units "${RELEASE_DIR}" "${WORKER_SERVICE_PATTERN}" "${WORKER_CONTROLLER_SERVICE}"
|
|
|
|
echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}"
|
|
systemctl restart "${SERVICE_NAME}"
|
|
restart_worker_services "${WORKER_SERVICE_PATTERN}"
|
|
wait_for_worker_services "${WORKER_SERVICE_PATTERN}"
|
|
ensure_worker_controller_service "${WORKER_CONTROLLER_SERVICE}"
|
|
wait_for_worker_controller_service "${WORKER_CONTROLLER_SERVICE}"
|
|
|
|
echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}"
|
|
for _ in {1..30}; do
|
|
if curl -fsS "${HEALTH_URL}" >/dev/null; then
|
|
bash "${SCRIPT_DIR}/maintenance-off.sh"
|
|
DEPLOY_COMPLETED=1
|
|
echo "[production-api-deploy] 完成: ${RELEASE_DIR}/api-server"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
echo "[production-api-deploy] readiness 检查超时: ${HEALTH_URL}" >&2
|
|
exit 1
|