合并 master 到 Pingora 网关分支

保留 Pingora 网关发布与巡检能力合入外部生成 worker 与 controller 运维链路对齐 SpacetimeDB 2.5.0 依赖锁定更新项目共享记忆与生产运维文档
2026-06-17 21:05:57 +08:00
parent 9eb4937afb 6984af782c
commit 27912fd020
523 changed files with 55530 additions and 16716 deletions
@@ -5,10 +5,11 @@ set -euo pipefail
 usage() {
  cat <<'EOF'
 用法:
-  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
+  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]

 说明:
  进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
+  默认同时重启外部生成 worker controller 和已加载的 worker 实例；未启用 worker 单元时会自动跳过。
  若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。
  若发布包包含 pingora-gateway，部署脚本会在 current 链接切换后仅对已 active 的 Pingora 影子服务执行 try-restart 并复核 active；未运行时不会主动拉起。
  失败时保留维护模式。
@@ -289,6 +290,136 @@ if (requirePingora) {
 NODE
 }

+list_worker_services() {
+  local pattern="$1"
+
+  if [[ -z "${pattern}" ]]; then
+    return 0
+  fi
+
+  systemctl list-units --all --plain --no-legend "${pattern}" 2>/dev/null | awk '{print $1}' | sort -u
+}
+
+ensure_default_worker_service() {
+  local pattern="$1"
+  local default_service="genarrative-external-generation-worker@1.service"
+  local template_service="genarrative-external-generation-worker@.service"
+  local services=()
+
+  if [[ -z "${pattern}" ]]; then
+    return 0
+  fi
+
+  if [[ "${pattern}" != "genarrative-external-generation-worker@*.service" ]]; then
+    return 0
+  fi
+
+  if ! systemctl cat "${template_service}" >/dev/null 2>&1; then
+    echo "[production-api-deploy] 缺少外部生成 worker systemd 模板: ${template_service}" >&2
+    return 1
+  fi
+
+  mapfile -t services < <(list_worker_services "${pattern}")
+  if [[ "${#services[@]}" -gt 0 ]]; then
+    return 0
+  fi
+
+  echo "[production-api-deploy] 未发现外部生成 worker 实例，启用并启动默认实例: ${default_service}"
+  systemctl enable --now "${default_service}"
+}
+
+restart_worker_services() {
+  local pattern="$1"
+  local services=()
+
+  if [[ -z "${pattern}" ]]; then
+    echo "[production-api-deploy] 跳过外部生成 worker 重启。"
+    return 0
+  fi
+
+  ensure_default_worker_service "${pattern}"
+  mapfile -t services < <(list_worker_services "${pattern}")
+  if [[ "${#services[@]}" -eq 0 ]]; then
+    echo "[production-api-deploy] 未发现已加载的外部生成 worker 单元: ${pattern}" >&2
+    return 1
+  fi
+
+  echo "[production-api-deploy] 重启外部生成 worker: ${services[*]}"
+  systemctl restart "${services[@]}"
+}
+
+wait_for_worker_services() {
+  local pattern="$1"
+  local services=()
+  local all_active
+
+  if [[ -z "${pattern}" ]]; then
+    return 0
+  fi
+
+  mapfile -t services < <(list_worker_services "${pattern}")
+  if [[ "${#services[@]}" -eq 0 ]]; then
+    echo "[production-api-deploy] 外部生成 worker 单元不存在，发布失败: ${pattern}" >&2
+    return 1
+  fi
+
+  echo "[production-api-deploy] 等待外部生成 worker active: ${services[*]}"
+  for _ in {1..30}; do
+    all_active=1
+    for service in "${services[@]}"; do
+      if ! systemctl is-active --quiet "${service}"; then
+        all_active=0
+        break
+      fi
+    done
+    if [[ "${all_active}" -eq 1 ]]; then
+      return 0
+    fi
+    sleep 2
+  done
+
+  systemctl --no-pager --full status "${services[@]}" || true
+  echo "[production-api-deploy] 外部生成 worker 未在超时时间内进入 active，发布失败。" >&2
+  return 1
+}
+
+ensure_worker_controller_service() {
+  local service="$1"
+
+  if [[ -z "${service}" ]]; then
+    return 0
+  fi
+
+  if ! systemctl cat "${service}" >/dev/null 2>&1; then
+    echo "[production-api-deploy] 缺少外部生成 worker controller systemd 单元: ${service}" >&2
+    return 1
+  fi
+
+  echo "[production-api-deploy] 启用并重启外部生成 worker controller: ${service}"
+  systemctl enable "${service}"
+  systemctl restart "${service}"
+}
+
+wait_for_worker_controller_service() {
+  local service="$1"
+
+  if [[ -z "${service}" ]]; then
+    return 0
+  fi
+
+  echo "[production-api-deploy] 等待外部生成 worker controller active: ${service}"
+  for _ in {1..30}; do
+    if systemctl is-active --quiet "${service}"; then
+      return 0
+    fi
+    sleep 2
+  done
+
+  systemctl --no-pager --full status "${service}" || true
+  echo "[production-api-deploy] 外部生成 worker controller 未在超时时间内进入 active，发布失败。" >&2
+  return 1
+}
+
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 SOURCE_DIR=""
 VERSION=""
@@ -296,6 +427,8 @@ RELEASE_ROOT="/opt/genarrative/releases"
 CURRENT_LINK="/opt/genarrative/current"
 SERVICE_NAME="genarrative-api.service"
 PINGORA_SERVICE_NAME="genarrative-pingora-gateway.service"
+WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
+WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service"
 HEALTH_URL="http://127.0.0.1:8082/readyz"
 API_ENV_FILE="/etc/genarrative/api-server.env"
 DATABASE=""
@@ -335,6 +468,22 @@ while [[ $# -gt 0 ]]; do
      PINGORA_SERVICE_NAME="${2:?缺少 --pingora-service 的值}"
      shift 2
      ;;
+    --worker-service-pattern)
+      WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}"
+      shift 2
+      ;;
+    --no-worker-services)
+      WORKER_SERVICE_PATTERN=""
+      shift
+      ;;
+    --worker-controller-service)
+      WORKER_CONTROLLER_SERVICE="${2:?缺少 --worker-controller-service 的值}"
+      shift 2
+      ;;
+    --no-worker-controller)
+      WORKER_CONTROLLER_SERVICE=""
+      shift
+      ;;
    --health-url)
      HEALTH_URL="${2:?缺少 --health-url 的值}"
      shift 2
@@ -634,6 +783,10 @@ fi

 echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}"
 systemctl restart "${SERVICE_NAME}"
+restart_worker_services "${WORKER_SERVICE_PATTERN}"
+wait_for_worker_services "${WORKER_SERVICE_PATTERN}"
+ensure_worker_controller_service "${WORKER_CONTROLLER_SERVICE}"
+wait_for_worker_controller_service "${WORKER_CONTROLLER_SERVICE}"

 echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}"
 for _ in {1..30}; do