787 lines
30 KiB
Bash
787 lines
30 KiB
Bash
#!/usr/bin/env bash
|
||
|
||
set -euo pipefail
|
||
umask 077
|
||
|
||
usage() {
|
||
cat <<'EOF'
|
||
用法:
|
||
./scripts/deploy/production-stdb-publish.sh --source-dir build/<version> --database <database> --migration-bootstrap-secret-file <protected-file> [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--api-health-url http://127.0.0.1:8082/healthz] [--api-readiness-timeout-seconds 60] [--keep-maintenance-mode] [--backup-mode async|sync|skip]
|
||
|
||
说明:
|
||
进入维护模式，校验 spacetime_module.wasm.sha256，并在生产实例本机执行 spacetime publish。
|
||
publish 固定使用 --delete-data=never 与 scoped --yes=migrate,break-clients；任何需要删除数据的 schema 冲突都会阻断发布。
|
||
默认使用 http://127.0.0.1:3101，避免与部署机本机 Git/Web 服务的 3000 端口冲突。
|
||
默认使用 /stdb 作为 spacetime CLI root-dir，并以 spacetimedb 用户发布，避免 root CLI 身份污染自托管实例。
|
||
发布时固定追加 --no-config，只使用显式参数，避免工作区或用户目录里的 spacetime 配置干扰目标。
|
||
async 模式会在 publish 前先做本地冷备份，再在 publish 完成后后台上传 OSS，避免低带宽上传阻塞部署。
|
||
如需强制等待备份完成并在失败时阻断 publish，传入 --backup-mode sync。
|
||
发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径，再重启并验活重启前 active 的服务。
|
||
--keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker，并在成功后保持维护态，交由后续 API deploy 恢复服务。
|
||
migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。
|
||
如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。
|
||
失败时保留维护模式。
|
||
EOF
|
||
}
|
||
|
||
require_argument() {
|
||
local value="$1"
|
||
local label="$2"
|
||
|
||
if [[ -z "${value}" ]]; then
|
||
echo "[production-stdb-publish] 缺少参数: ${label}" >&2
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
validate_spacetime_database_name() {
|
||
local database="$1"
|
||
|
||
if [[ ! "${database}" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then
|
||
echo "[production-stdb-publish] --database 必须匹配 ^[a-z0-9]+(-[a-z0-9]+)*$: ${database}" >&2
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||
SOURCE_DIR=""
|
||
DATABASE=""
|
||
SERVER_ALIAS="local"
|
||
SERVER_URL="http://127.0.0.1:3101"
|
||
SPACETIME_ROOT_DIR="/stdb"
|
||
RUN_AS_USER="spacetimedb"
|
||
MIGRATION_BOOTSTRAP_SECRET_FILE=""
|
||
API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-server.env}"
|
||
WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
|
||
KEEP_MAINTENANCE_MODE=0
|
||
BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}"
|
||
DEPLOY_COMPLETED=0
|
||
PUBLISH_TMP_DIR=""
|
||
ASYNC_BACKUP_STATUS_FILE=""
|
||
ASYNC_BACKUP_SCRIPT=""
|
||
ASYNC_BACKUP_ARCHIVE=""
|
||
ASYNC_BACKUP_MANIFEST=""
|
||
ASYNC_BACKUP_LOG=""
|
||
SPACETIME_READY_TIMEOUT_SECONDS="${GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS:-60}"
|
||
API_HEALTH_URL="${GENARRATIVE_STDB_PUBLISH_API_HEALTH_URL:-http://127.0.0.1:8082/healthz}"
|
||
API_READINESS_TIMEOUT_SECONDS="${GENARRATIVE_STDB_PUBLISH_API_READINESS_TIMEOUT_SECONDS:-60}"
|
||
API_WAS_ACTIVE=0
|
||
RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE="${GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE:-/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt}"
|
||
RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE="/var/lib/genarrative/spacetime/runtime-service-bootstrap-secret.txt"
|
||
|
||
run_privileged() {
|
||
if [[ "$(id -u)" -eq 0 ]]; then
|
||
"$@"
|
||
elif command -v sudo >/dev/null 2>&1; then
|
||
sudo -n "$@"
|
||
else
|
||
echo "[production-stdb-publish] 当前用户不是 root，且 sudo 不可用；无法完成受保护运行态文件或服务操作。" >&2
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
ensure_runtime_bootstrap_secret_env_file() {
|
||
local env_file="$1"
|
||
local required="$2"
|
||
|
||
if [[ ! -e "${env_file}" ]]; then
|
||
if [[ "${required}" == "true" ]]; then
|
||
echo "[production-stdb-publish] 运行时环境文件不存在，无法补齐 bootstrap secret FILE: ${env_file}" >&2
|
||
exit 1
|
||
fi
|
||
echo "[production-stdb-publish] worker 环境文件不存在，跳过 bootstrap secret FILE 补齐: ${env_file}"
|
||
return
|
||
fi
|
||
if [[ ! -f "${env_file}" || -L "${env_file}" ]]; then
|
||
echo "[production-stdb-publish] 运行时环境文件必须是普通文件且不能是符号链接: ${env_file}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
run_privileged python3 - "${env_file}" "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE}" <<'PY'
|
||
import os
|
||
import stat
|
||
import sys
|
||
import tempfile
|
||
from pathlib import Path
|
||
|
||
env_path = Path(sys.argv[1])
|
||
canonical_path = sys.argv[2]
|
||
file_key = "GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE"
|
||
direct_key = "GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET"
|
||
|
||
|
||
def normalize_value(value: str) -> str:
|
||
value = value.strip()
|
||
if len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'"):
|
||
value = value[1:-1]
|
||
return value.strip()
|
||
|
||
|
||
lines = env_path.read_text(encoding="utf-8").splitlines()
|
||
next_lines: list[str] = []
|
||
file_key_written = False
|
||
changed = False
|
||
|
||
for raw_line in lines:
|
||
stripped = raw_line.strip()
|
||
if not stripped or stripped.startswith("#") or "=" not in stripped:
|
||
next_lines.append(raw_line)
|
||
continue
|
||
|
||
key, value = stripped.split("=", 1)
|
||
if key == direct_key and normalize_value(value):
|
||
print(
|
||
f"[production-stdb-publish] {env_path} 不得保存 {direct_key} 明文；生产环境只允许使用 {file_key}。",
|
||
file=sys.stderr,
|
||
)
|
||
raise SystemExit(1)
|
||
if key != file_key:
|
||
next_lines.append(raw_line)
|
||
continue
|
||
|
||
current_value = normalize_value(value)
|
||
if current_value and current_value != canonical_path:
|
||
print(
|
||
f"[production-stdb-publish] {file_key} 必须使用固定路径 {canonical_path}: {env_path}",
|
||
file=sys.stderr,
|
||
)
|
||
raise SystemExit(1)
|
||
if file_key_written:
|
||
changed = True
|
||
continue
|
||
|
||
next_lines.append(f"{file_key}={canonical_path}")
|
||
file_key_written = True
|
||
changed = changed or raw_line != next_lines[-1]
|
||
|
||
if not file_key_written:
|
||
next_lines.append(f"{file_key}={canonical_path}")
|
||
changed = True
|
||
|
||
if changed:
|
||
metadata = env_path.stat()
|
||
fd, temp_name = tempfile.mkstemp(prefix=f".{env_path.name}.", dir=env_path.parent)
|
||
try:
|
||
os.fchmod(fd, stat.S_IMODE(metadata.st_mode))
|
||
os.fchown(fd, metadata.st_uid, metadata.st_gid)
|
||
with os.fdopen(fd, "w", encoding="utf-8") as temp_file:
|
||
temp_file.write("\n".join(next_lines) + "\n")
|
||
os.replace(temp_name, env_path)
|
||
finally:
|
||
if os.path.exists(temp_name):
|
||
os.unlink(temp_name)
|
||
PY
|
||
echo "[production-stdb-publish] 已确认 bootstrap secret FILE 配置: ${env_file}"
|
||
}
|
||
|
||
get_runtime_service_active_state() {
|
||
local service_name="$1"
|
||
local state=""
|
||
local exit_code=0
|
||
|
||
if state="$(run_privileged systemctl is-active "${service_name}")"; then
|
||
if [[ "${state}" != "active" ]]; then
|
||
echo "[production-stdb-publish] systemctl is-active 返回成功但状态异常: ${service_name}, state=${state}" >&2
|
||
return 1
|
||
fi
|
||
printf '%s\n' "${state}"
|
||
return 0
|
||
else
|
||
exit_code=$?
|
||
fi
|
||
|
||
if [[ "${exit_code}" -eq 3 && "${state}" =~ ^(inactive|failed|activating|deactivating|reloading|maintenance|refreshing)$ ]]; then
|
||
printf '%s\n' "${state}"
|
||
return 0
|
||
fi
|
||
|
||
echo "[production-stdb-publish] 查询运行时服务状态失败: ${service_name}, exit=${exit_code}, state=${state:-<empty>}" >&2
|
||
return 1
|
||
}
|
||
|
||
restart_runtime_service_and_require_active() {
|
||
local service_name="$1"
|
||
local state=""
|
||
|
||
echo "[production-stdb-publish] 重启运行时服务以加载新引导密钥: ${service_name}"
|
||
if ! run_privileged systemctl restart "${service_name}"; then
|
||
echo "[production-stdb-publish] 运行时服务重启失败: ${service_name}" >&2
|
||
return 1
|
||
fi
|
||
|
||
if ! state="$(get_runtime_service_active_state "${service_name}")"; then
|
||
return 1
|
||
fi
|
||
if [[ "${state}" != "active" ]]; then
|
||
echo "[production-stdb-publish] 运行时服务重启后未恢复 active: ${service_name}, state=${state}" >&2
|
||
return 1
|
||
fi
|
||
echo "[production-stdb-publish] 运行时服务重启后已恢复 active: ${service_name}"
|
||
}
|
||
|
||
restart_runtime_services_after_bootstrap_secret_install() {
|
||
local api_state=""
|
||
local controller_state=""
|
||
local worker_service=""
|
||
local worker_state=""
|
||
local worker_units_output=""
|
||
local list_units_exit_code=0
|
||
local -a active_worker_services=()
|
||
|
||
if ! api_state="$(get_runtime_service_active_state genarrative-api.service)"; then
|
||
return 1
|
||
fi
|
||
if ! controller_state="$(get_runtime_service_active_state genarrative-external-generation-controller.service)"; then
|
||
return 1
|
||
fi
|
||
|
||
if worker_units_output="$(
|
||
run_privileged systemctl list-units \
|
||
--type=service \
|
||
--state=active \
|
||
--no-legend \
|
||
--plain \
|
||
'genarrative-external-generation-worker@*.service'
|
||
)"; then
|
||
:
|
||
else
|
||
list_units_exit_code=$?
|
||
echo "[production-stdb-publish] 查询 active worker 服务失败，阻断退出维护模式: exit=${list_units_exit_code}" >&2
|
||
return 1
|
||
fi
|
||
|
||
while read -r worker_service _; do
|
||
if [[ "${worker_service}" =~ ^genarrative-external-generation-worker@[A-Za-z0-9_.@:-]+\.service$ ]]; then
|
||
active_worker_services+=("${worker_service}")
|
||
fi
|
||
done <<<"${worker_units_output}"
|
||
|
||
if [[ "${api_state}" == "active" ]]; then
|
||
API_WAS_ACTIVE=1
|
||
if ! restart_runtime_service_and_require_active genarrative-api.service; then
|
||
return 1
|
||
fi
|
||
fi
|
||
if [[ "${controller_state}" == "active" ]]; then
|
||
if ! restart_runtime_service_and_require_active genarrative-external-generation-controller.service; then
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
if [[ "${#active_worker_services[@]}" -gt 0 ]]; then
|
||
echo "[production-stdb-publish] 重启运行时 worker 以加载新引导密钥: ${active_worker_services[*]}"
|
||
if ! run_privileged systemctl restart "${active_worker_services[@]}"; then
|
||
echo "[production-stdb-publish] 运行时 worker 重启失败: ${active_worker_services[*]}" >&2
|
||
return 1
|
||
fi
|
||
for worker_service in "${active_worker_services[@]}"; do
|
||
if ! worker_state="$(get_runtime_service_active_state "${worker_service}")"; then
|
||
return 1
|
||
fi
|
||
if [[ "${worker_state}" != "active" ]]; then
|
||
echo "[production-stdb-publish] 运行时 worker 重启后未恢复 active: ${worker_service}, state=${worker_state}" >&2
|
||
return 1
|
||
fi
|
||
echo "[production-stdb-publish] 运行时 worker 重启后已恢复 active: ${worker_service}"
|
||
done
|
||
fi
|
||
}
|
||
|
||
stop_runtime_services_for_rollout_gate() {
|
||
local api_state=""
|
||
local controller_state=""
|
||
local worker_service=""
|
||
local worker_units_output=""
|
||
local -a services_to_stop=()
|
||
|
||
api_state="$(get_runtime_service_active_state genarrative-api.service)"
|
||
controller_state="$(get_runtime_service_active_state genarrative-external-generation-controller.service)"
|
||
if ! worker_units_output="$(
|
||
run_privileged systemctl list-units \
|
||
--type=service \
|
||
--state=active \
|
||
--no-legend \
|
||
--plain \
|
||
'genarrative-external-generation-worker@*.service'
|
||
)"; then
|
||
echo "[production-stdb-publish] 查询 active worker 服务失败，无法建立受控维护窗口。" >&2
|
||
return 1
|
||
fi
|
||
|
||
if [[ "${controller_state}" == "active" ]]; then
|
||
services_to_stop+=(genarrative-external-generation-controller.service)
|
||
fi
|
||
while read -r worker_service _; do
|
||
if [[ "${worker_service}" =~ ^genarrative-external-generation-worker@[A-Za-z0-9_.@:-]+\.service$ ]]; then
|
||
services_to_stop+=("${worker_service}")
|
||
fi
|
||
done <<<"${worker_units_output}"
|
||
if [[ "${api_state}" == "active" ]]; then
|
||
services_to_stop+=(genarrative-api.service)
|
||
fi
|
||
|
||
if [[ "${#services_to_stop[@]}" -eq 0 ]]; then
|
||
echo "[production-stdb-publish] 受控维护窗口开始前没有 active API/controller/worker。"
|
||
return 0
|
||
fi
|
||
|
||
echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}"
|
||
run_privileged systemctl stop "${services_to_stop[@]}"
|
||
for worker_service in "${services_to_stop[@]}"; do
|
||
if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then
|
||
echo "[production-stdb-publish] 运行时服务停止后仍为 active: ${worker_service}" >&2
|
||
return 1
|
||
fi
|
||
done
|
||
}
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
-h|--help)
|
||
usage
|
||
exit 0
|
||
;;
|
||
--source-dir)
|
||
SOURCE_DIR="${2:?缺少 --source-dir 的值}"
|
||
shift 2
|
||
;;
|
||
--database)
|
||
DATABASE="${2:?缺少 --database 的值}"
|
||
shift 2
|
||
;;
|
||
--server)
|
||
SERVER_ALIAS="${2:?缺少 --server 的值}"
|
||
SERVER_URL=""
|
||
shift 2
|
||
;;
|
||
--server-url)
|
||
SERVER_URL="${2:?缺少 --server-url 的值}"
|
||
shift 2
|
||
;;
|
||
--root-dir)
|
||
SPACETIME_ROOT_DIR="${2:?缺少 --root-dir 的值}"
|
||
shift 2
|
||
;;
|
||
--run-as-user)
|
||
RUN_AS_USER="${2:?缺少 --run-as-user 的值}"
|
||
shift 2
|
||
;;
|
||
--migration-bootstrap-secret-file)
|
||
MIGRATION_BOOTSTRAP_SECRET_FILE="${2:?缺少 --migration-bootstrap-secret-file 的值}"
|
||
shift 2
|
||
;;
|
||
--api-env-file)
|
||
API_ENV_FILE="${2:?缺少 --api-env-file 的值}"
|
||
shift 2
|
||
;;
|
||
--worker-env-file)
|
||
WORKER_ENV_FILE="${2:?缺少 --worker-env-file 的值}"
|
||
shift 2
|
||
;;
|
||
--api-health-url)
|
||
API_HEALTH_URL="${2:?缺少 --api-health-url 的值}"
|
||
shift 2
|
||
;;
|
||
--api-readiness-timeout-seconds)
|
||
API_READINESS_TIMEOUT_SECONDS="${2:?缺少 --api-readiness-timeout-seconds 的值}"
|
||
shift 2
|
||
;;
|
||
--keep-maintenance-mode)
|
||
KEEP_MAINTENANCE_MODE=1
|
||
shift
|
||
;;
|
||
--skip-backup)
|
||
BACKUP_MODE="skip"
|
||
shift
|
||
;;
|
||
--sync-backup)
|
||
BACKUP_MODE="sync"
|
||
shift
|
||
;;
|
||
--backup-mode)
|
||
BACKUP_MODE="${2:?缺少 --backup-mode 的值}"
|
||
shift 2
|
||
;;
|
||
*)
|
||
echo "[production-stdb-publish] 未知参数: $1" >&2
|
||
usage >&2
|
||
exit 1
|
||
;;
|
||
esac
|
||
done
|
||
|
||
require_argument "${SOURCE_DIR}" "--source-dir"
|
||
require_argument "${DATABASE}" "--database"
|
||
validate_spacetime_database_name "${DATABASE}"
|
||
|
||
if [[ ! "${SPACETIME_ROOT_DIR}" == /* || "${SPACETIME_ROOT_DIR}" == *".."* ]]; then
|
||
echo "[production-stdb-publish] --root-dir 必须是 Linux 绝对路径且不能包含 ..: ${SPACETIME_ROOT_DIR}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" != "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE}" ]]; then
|
||
echo "[production-stdb-publish] GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE 必须使用固定路径 ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_CANONICAL_FILE}: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do
|
||
if [[ "${runtime_env_file}" != /* || "${runtime_env_file}" == *".."* || "${runtime_env_file}" == "/" ]]; then
|
||
echo "[production-stdb-publish] 运行时环境文件必须是非根绝对路径且不能包含 ..: ${runtime_env_file}" >&2
|
||
exit 1
|
||
fi
|
||
done
|
||
|
||
if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then
|
||
echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ -n "${RUN_AS_USER}" && ! "${RUN_AS_USER}" =~ ^[A-Za-z_][A-Za-z0-9_-]*$ ]]; then
|
||
echo "[production-stdb-publish] --run-as-user 只能是本机用户名: ${RUN_AS_USER}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ ! "${SPACETIME_READY_TIMEOUT_SECONDS}" =~ ^[0-9]+$ || "${SPACETIME_READY_TIMEOUT_SECONDS}" -le 0 ]]; then
|
||
echo "[production-stdb-publish] GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS 必须是正整数: ${SPACETIME_READY_TIMEOUT_SECONDS}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ ! "${API_READINESS_TIMEOUT_SECONDS}" =~ ^[0-9]+$ || "${API_READINESS_TIMEOUT_SECONDS}" -le 0 ]]; then
|
||
echo "[production-stdb-publish] API readiness timeout 必须是正整数: ${API_READINESS_TIMEOUT_SECONDS}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ ! "${API_HEALTH_URL}" =~ ^http://(127\.0\.0\.1|localhost)(:[0-9]{1,5})?/healthz$ ]]; then
|
||
echo "[production-stdb-publish] API readiness 必须使用本机 HTTP /healthz: ${API_HEALTH_URL}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ ! -d "${SOURCE_DIR}" ]]; then
|
||
echo "[production-stdb-publish] 发布目录不存在: ${SOURCE_DIR}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
SOURCE_DIR="$(cd "${SOURCE_DIR}" && pwd)"
|
||
|
||
if [[ ! -f "${SOURCE_DIR}/spacetime_module.wasm" || ! -f "${SOURCE_DIR}/spacetime_module.wasm.sha256" || ! -f "${SOURCE_DIR}/release-manifest.json" ]]; then
|
||
echo "[production-stdb-publish] 缺少 spacetime_module.wasm、checksum 或 release-manifest.json: ${SOURCE_DIR}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ -z "${MIGRATION_BOOTSTRAP_SECRET_FILE}" || ! -f "${MIGRATION_BOOTSTRAP_SECRET_FILE}" || -L "${MIGRATION_BOOTSTRAP_SECRET_FILE}" || ! -r "${MIGRATION_BOOTSTRAP_SECRET_FILE}" ]]; then
|
||
echo "[production-stdb-publish] --migration-bootstrap-secret-file 必须是可读、非符号链接的受保护普通文件。" >&2
|
||
exit 1
|
||
fi
|
||
|
||
MIGRATION_BOOTSTRAP_SECRET="$(cat "${MIGRATION_BOOTSTRAP_SECRET_FILE}")"
|
||
MIGRATION_BOOTSTRAP_SECRET="${MIGRATION_BOOTSTRAP_SECRET%$'\r'}"
|
||
if [[ ! "${MIGRATION_BOOTSTRAP_SECRET}" =~ ^[0-9a-fA-F]{64}$ ]]; then
|
||
echo "[production-stdb-publish] 运行时服务身份引导密钥必须是 64 位十六进制高熵值。" >&2
|
||
exit 1
|
||
fi
|
||
MIGRATION_BOOTSTRAP_SECRET_DIGEST_LINE="$(printf '%s' "${MIGRATION_BOOTSTRAP_SECRET}" | sha256sum)"
|
||
MIGRATION_BOOTSTRAP_SECRET_SHA256="${MIGRATION_BOOTSTRAP_SECRET_DIGEST_LINE%% *}"
|
||
EXPECTED_MIGRATION_BOOTSTRAP_SECRET_SHA256="$(node -e 'const fs=require("node:fs"); const manifest=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); process.stdout.write(manifest.migration_bootstrap_secret_sha256 || "");' "${SOURCE_DIR}/release-manifest.json")"
|
||
if [[ ! "${EXPECTED_MIGRATION_BOOTSTRAP_SECRET_SHA256}" =~ ^[0-9a-f]{64}$ || "${MIGRATION_BOOTSTRAP_SECRET_SHA256}" != "${EXPECTED_MIGRATION_BOOTSTRAP_SECRET_SHA256}" ]]; then
|
||
echo "[production-stdb-publish] Secret File 与构建 WASM 的 bootstrap secret 摘要不一致。" >&2
|
||
exit 1
|
||
fi
|
||
unset MIGRATION_BOOTSTRAP_SECRET MIGRATION_BOOTSTRAP_SECRET_DIGEST_LINE
|
||
|
||
on_exit() {
|
||
local exit_code=$?
|
||
if [[ "${BACKUP_MODE}" == "async" && -n "${ASYNC_BACKUP_STATUS_FILE}" && -f "${ASYNC_BACKUP_STATUS_FILE}" ]]; then
|
||
start_async_backup_upload || true
|
||
fi
|
||
if [[ -n "${PUBLISH_TMP_DIR}" && -d "${PUBLISH_TMP_DIR}" ]]; then
|
||
rm -rf "${PUBLISH_TMP_DIR}"
|
||
fi
|
||
if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
|
||
echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2
|
||
fi
|
||
exit "${exit_code}"
|
||
}
|
||
|
||
trap on_exit EXIT
|
||
|
||
prepare_async_backup() {
|
||
local -a restart_service_args=()
|
||
ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
|
||
if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
|
||
ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
|
||
fi
|
||
if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
|
||
echo "[production-stdb-publish] 缺少数据库备份脚本: ${ASYNC_BACKUP_SCRIPT}" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
|
||
restart_service_args+=(--restart-service-after genarrative-api.service)
|
||
fi
|
||
|
||
ASYNC_BACKUP_STATUS_FILE="$(mktemp /tmp/genarrative-stdb-backup-status.XXXXXX.json)"
|
||
echo "[production-stdb-publish] publish 前生成本地冷备份，随后会异步上传 OSS"
|
||
node -- "${ASYNC_BACKUP_SCRIPT}" \
|
||
--env-file /etc/genarrative/api-server.env \
|
||
--data-dir "${SPACETIME_ROOT_DIR}" \
|
||
--database "${DATABASE}" \
|
||
--stop-service spacetimedb.service \
|
||
"${restart_service_args[@]}" \
|
||
--defer-upload \
|
||
--result-file "${ASYNC_BACKUP_STATUS_FILE}"
|
||
}
|
||
|
||
start_async_backup_upload() {
|
||
local log_dir=""
|
||
local node_binary=""
|
||
local unit_name=""
|
||
local unit_suffix=""
|
||
|
||
if [[ -z "${ASYNC_BACKUP_STATUS_FILE}" || ! -f "${ASYNC_BACKUP_STATUS_FILE}" ]]; then
|
||
echo "[production-stdb-publish] 警告：未找到可上传的本地备份状态文件，跳过异步上传" >&2
|
||
return 0
|
||
fi
|
||
|
||
ASYNC_BACKUP_ARCHIVE="$(node -e 'const fs=require("node:fs"); const p=process.argv[1]; const o=JSON.parse(fs.readFileSync(p,"utf8")); process.stdout.write(o.archivePath || "");' "${ASYNC_BACKUP_STATUS_FILE}")"
|
||
ASYNC_BACKUP_MANIFEST="$(node -e 'const fs=require("node:fs"); const p=process.argv[1]; const o=JSON.parse(fs.readFileSync(p,"utf8")); process.stdout.write(o.manifestPath || "");' "${ASYNC_BACKUP_STATUS_FILE}")"
|
||
if [[ -z "${ASYNC_BACKUP_ARCHIVE}" || -z "${ASYNC_BACKUP_MANIFEST}" ]]; then
|
||
echo "[production-stdb-publish] 警告：备份状态文件缺少 archivePath 或 manifestPath，跳过异步上传" >&2
|
||
return 0
|
||
fi
|
||
if [[ "${ASYNC_BACKUP_ARCHIVE}" != /* || ! -f "${ASYNC_BACKUP_ARCHIVE}" || -L "${ASYNC_BACKUP_ARCHIVE}" ]]; then
|
||
echo "[production-stdb-publish] 警告：异步上传归档必须是现存、非符号链接的普通绝对路径文件，保留状态文件等待处理: ${ASYNC_BACKUP_ARCHIVE}" >&2
|
||
return 1
|
||
fi
|
||
if [[ "${ASYNC_BACKUP_MANIFEST}" != /* || ! -f "${ASYNC_BACKUP_MANIFEST}" || -L "${ASYNC_BACKUP_MANIFEST}" ]]; then
|
||
echo "[production-stdb-publish] 警告：异步上传 manifest 必须是现存、非符号链接的普通绝对路径文件，保留状态文件等待处理: ${ASYNC_BACKUP_MANIFEST}" >&2
|
||
return 1
|
||
fi
|
||
if ! command -v systemd-run >/dev/null 2>&1; then
|
||
echo "[production-stdb-publish] 警告：systemd-run 不可用，无法启动独立上传服务；保留状态文件等待处理" >&2
|
||
return 1
|
||
fi
|
||
|
||
node_binary="$(command -v node || true)"
|
||
if [[ "${node_binary}" != /* || ! -x "${node_binary}" ]]; then
|
||
echo "[production-stdb-publish] 警告：未找到可供 systemd 服务执行的绝对 node 路径；保留状态文件等待处理" >&2
|
||
return 1
|
||
fi
|
||
|
||
log_dir="$(dirname "${ASYNC_BACKUP_ARCHIVE}")"
|
||
unit_suffix="$(date -u +%Y%m%dT%H%M%S%N)-$$-${RANDOM}"
|
||
unit_name="genarrative-stdb-backup-upload-${unit_suffix}.service"
|
||
if ! ASYNC_BACKUP_LOG="$(mktemp "${log_dir}/${DATABASE}-upload-${unit_suffix}.XXXXXX.log")"; then
|
||
echo "[production-stdb-publish] 警告：无法创建独立 OSS 上传日志，保留状态文件和本地归档等待处理" >&2
|
||
return 1
|
||
fi
|
||
if ! chmod 0600 "${ASYNC_BACKUP_LOG}"; then
|
||
echo "[production-stdb-publish] 警告：无法收紧独立 OSS 上传日志权限，保留状态文件和本地归档等待处理: ${ASYNC_BACKUP_LOG}" >&2
|
||
return 1
|
||
fi
|
||
|
||
echo "[production-stdb-publish] 通过独立 systemd 服务串行上传 deferred/pending 本地备份到 OSS: ${log_dir}"
|
||
if ! run_privileged systemd-run \
|
||
--no-ask-password \
|
||
--unit="${unit_name}" \
|
||
--description="Genarrative SpacetimeDB backup upload ${DATABASE}" \
|
||
--collect \
|
||
--service-type=exec \
|
||
--property="Restart=no" \
|
||
--property="UMask=0077" \
|
||
--property="StandardOutput=append:${ASYNC_BACKUP_LOG}" \
|
||
--property="StandardError=append:${ASYNC_BACKUP_LOG}" \
|
||
-- "${node_binary}" -- "${ASYNC_BACKUP_SCRIPT}" \
|
||
--env-file /etc/genarrative/api-server.env \
|
||
--database "${DATABASE}" \
|
||
--upload-deferred-dir "${log_dir}"; then
|
||
echo "[production-stdb-publish] 警告：独立 OSS 上传服务启动失败，保留状态文件和本地归档等待处理；启动日志: ${ASYNC_BACKUP_LOG}" >&2
|
||
return 1
|
||
fi
|
||
|
||
echo "[production-stdb-publish] OSS 上传服务已启动: ${unit_name}"
|
||
echo "[production-stdb-publish] OSS 上传日志: ${ASYNC_BACKUP_LOG}"
|
||
rm -f "${ASYNC_BACKUP_STATUS_FILE}"
|
||
ASYNC_BACKUP_STATUS_FILE=""
|
||
}
|
||
|
||
wait_for_spacetime_ready() {
|
||
if [[ -z "${SERVER_URL}" ]]; then
|
||
echo "[production-stdb-publish] 使用 server alias=${SERVER_ALIAS}，跳过 URL 健康检查等待"
|
||
return 0
|
||
fi
|
||
|
||
local ping_url="${SERVER_URL%/}/v1/ping"
|
||
local deadline=$((SECONDS + SPACETIME_READY_TIMEOUT_SECONDS))
|
||
local last_status=""
|
||
|
||
echo "[production-stdb-publish] 等待 SpacetimeDB 就绪: ${ping_url}，timeout=${SPACETIME_READY_TIMEOUT_SECONDS}s"
|
||
while (( SECONDS < deadline )); do
|
||
# curl 失败时通常表示服务尚未监听；不立即失败，等待冷备份恢复后的 systemd 启动完成。
|
||
if last_status="$(curl -fsS --max-time 2 "${ping_url}" 2>&1)"; then
|
||
echo "[production-stdb-publish] SpacetimeDB 已就绪: ${ping_url}"
|
||
return 0
|
||
fi
|
||
sleep 2
|
||
done
|
||
|
||
echo "[production-stdb-publish] SpacetimeDB 未在超时内就绪: ${ping_url}" >&2
|
||
if [[ -n "${last_status}" ]]; then
|
||
echo "[production-stdb-publish] 最后一次健康检查输出: ${last_status}" >&2
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
wait_for_api_healthz_ready() {
|
||
if [[ "${API_WAS_ACTIVE}" -ne 1 ]]; then
|
||
echo "[production-stdb-publish] API 重启前不是 active，跳过 /healthz readiness"
|
||
return 0
|
||
fi
|
||
|
||
local deadline=$((SECONDS + API_READINESS_TIMEOUT_SECONDS))
|
||
local last_status=""
|
||
|
||
echo "[production-stdb-publish] 等待 API /healthz readiness: ${API_HEALTH_URL}，timeout=${API_READINESS_TIMEOUT_SECONDS}s"
|
||
while (( SECONDS < deadline )); do
|
||
if last_status="$(curl -fsS --max-time 2 "${API_HEALTH_URL}" 2>&1)"; then
|
||
echo "[production-stdb-publish] API /healthz readiness 已通过: ${API_HEALTH_URL}"
|
||
return 0
|
||
fi
|
||
sleep 2
|
||
done
|
||
|
||
echo "[production-stdb-publish] API /healthz readiness 未在超时内通过: ${API_HEALTH_URL}" >&2
|
||
if [[ -n "${last_status}" ]]; then
|
||
echo "[production-stdb-publish] 最后一次 /healthz 检查输出: ${last_status}" >&2
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"
|
||
if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
|
||
stop_runtime_services_for_rollout_gate
|
||
fi
|
||
|
||
case "${BACKUP_MODE}" in
|
||
async)
|
||
prepare_async_backup
|
||
;;
|
||
sync)
|
||
SYNC_BACKUP_RESTART_SERVICE_ARGS=()
|
||
BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
|
||
if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
|
||
BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
|
||
fi
|
||
if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
|
||
echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2
|
||
exit 1
|
||
fi
|
||
if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
|
||
SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service)
|
||
fi
|
||
|
||
echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布"
|
||
node -- "${BACKUP_SCRIPT}" \
|
||
--env-file /etc/genarrative/api-server.env \
|
||
--data-dir "${SPACETIME_ROOT_DIR}" \
|
||
--database "${DATABASE}" \
|
||
--stop-service spacetimedb.service \
|
||
"${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}"
|
||
;;
|
||
skip)
|
||
echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份"
|
||
;;
|
||
esac
|
||
|
||
echo "[production-stdb-publish] 校验 wasm"
|
||
(
|
||
cd "${SOURCE_DIR}"
|
||
sha256sum -c spacetime_module.wasm.sha256
|
||
)
|
||
|
||
wait_for_spacetime_ready
|
||
|
||
PUBLISH_ARGS=(
|
||
--root-dir="${SPACETIME_ROOT_DIR}"
|
||
publish
|
||
"${DATABASE}"
|
||
--bin-path "${SOURCE_DIR}/spacetime_module.wasm"
|
||
--delete-data=never
|
||
--yes=migrate,break-clients
|
||
--no-config
|
||
)
|
||
|
||
if [[ -n "${SERVER_URL}" ]]; then
|
||
PUBLISH_ARGS+=(--server "${SERVER_URL}")
|
||
else
|
||
PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
|
||
fi
|
||
|
||
if [[ -n "${SERVER_URL}" ]]; then
|
||
echo "[production-stdb-publish] 发布 SpacetimeDB module: ${DATABASE} -> ${SERVER_URL}, root=${SPACETIME_ROOT_DIR}"
|
||
else
|
||
echo "[production-stdb-publish] 发布 SpacetimeDB module: ${DATABASE} -> ${SERVER_ALIAS}, root=${SPACETIME_ROOT_DIR}"
|
||
fi
|
||
|
||
if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then
|
||
if ! id "${RUN_AS_USER}" >/dev/null 2>&1; then
|
||
echo "[production-stdb-publish] 发布用户不存在: ${RUN_AS_USER}" >&2
|
||
exit 1
|
||
fi
|
||
PUBLISH_TMP_DIR="$(mktemp -d /tmp/genarrative-stdb-publish.XXXXXX)"
|
||
install -m 0644 "${SOURCE_DIR}/spacetime_module.wasm" "${PUBLISH_TMP_DIR}/spacetime_module.wasm"
|
||
chown -R "${RUN_AS_USER}:${RUN_AS_USER}" "${PUBLISH_TMP_DIR}"
|
||
PUBLISH_ARGS=(
|
||
--root-dir="${SPACETIME_ROOT_DIR}"
|
||
publish
|
||
"${DATABASE}"
|
||
--bin-path "${PUBLISH_TMP_DIR}/spacetime_module.wasm"
|
||
--delete-data=never
|
||
--yes=migrate,break-clients
|
||
--no-config
|
||
)
|
||
if [[ -n "${SERVER_URL}" ]]; then
|
||
PUBLISH_ARGS+=(--server "${SERVER_URL}")
|
||
else
|
||
PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
|
||
fi
|
||
runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}"
|
||
else
|
||
spacetime "${PUBLISH_ARGS[@]}"
|
||
fi
|
||
|
||
RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR="$(dirname "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}")"
|
||
if [[ -L "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR}" || -L "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" ]]; then
|
||
echo "[production-stdb-publish] 运行时服务身份引导密钥路径不能是符号链接: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}" >&2
|
||
exit 1
|
||
fi
|
||
if ! id genarrative >/dev/null 2>&1; then
|
||
echo "[production-stdb-publish] 缺少运行时服务用户 genarrative，无法安全安装引导密钥。" >&2
|
||
exit 1
|
||
fi
|
||
|
||
run_privileged install -d -o root -g genarrative -m 0750 "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_DIR}"
|
||
run_privileged install -o root -g genarrative -m 0440 \
|
||
"${MIGRATION_BOOTSTRAP_SECRET_FILE}" \
|
||
"${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
|
||
RUNTIME_SERVICE_BOOTSTRAP_SECRET_METADATA="$(run_privileged stat -c '%U:%G:%a' "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}")"
|
||
if [[ "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_METADATA}" != "root:genarrative:440" ]]; then
|
||
echo "[production-stdb-publish] 运行时服务身份引导密钥权限不符合 root:genarrative:0440: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_METADATA}" >&2
|
||
exit 1
|
||
fi
|
||
run_privileged runuser -u genarrative -- test -r "${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
|
||
echo "[production-stdb-publish] 已安装运行时服务身份引导密钥: ${RUNTIME_SERVICE_BOOTSTRAP_SECRET_FILE}"
|
||
ensure_runtime_bootstrap_secret_env_file "${API_ENV_FILE}" true
|
||
ensure_runtime_bootstrap_secret_env_file "${WORKER_ENV_FILE}" false
|
||
if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
|
||
echo "[production-stdb-publish] module 发布完成；按参数保持维护模式和旧运行时服务停止状态，等待人工维护与 API deploy。"
|
||
DEPLOY_COMPLETED=1
|
||
exit 0
|
||
fi
|
||
restart_runtime_services_after_bootstrap_secret_install
|
||
wait_for_api_healthz_ready
|
||
|
||
"${SCRIPT_DIR}/maintenance-off.sh"
|
||
DEPLOY_COMPLETED=1
|
||
|
||
echo "[production-stdb-publish] 完成"
|