feat: workerize external generation

scripts/check-spacetime-schema-guard.mjs

@@ -475,10 +475,16 @@ function loadBaseSources(baseRef) {
 
 function getChangedFiles(baseRef) {
   const diffOutput = tryGit(['diff', '--name-only', '-z', baseRef, '--']) ?? '';
-  const untrackedOutput =
+  const untrackedModuleOutput =
     tryGit(['ls-files', '--others', '--exclude-standard', '-z', moduleSrcRoot]) ?? '';
+  const untrackedBindingsOutput =
+    tryGit(['ls-files', '--others', '--exclude-standard', '-z', bindingsRoot]) ?? '';
   return new Set(
-    [...diffOutput.split(/\u0000/u), ...untrackedOutput.split(/\u0000/u)]
+    [
+      ...diffOutput.split(/\u0000/u),
+      ...untrackedModuleOutput.split(/\u0000/u),
+      ...untrackedBindingsOutput.split(/\u0000/u),
+    ]
       .map(normalizePath)
       .filter(Boolean),
   );

scripts/deploy/production-api-deploy.sh

@@ -5,10 +5,11 @@ set -euo pipefail
 usage() {
   cat <<'EOF'
 用法:
-  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
+  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
 
 说明:
   进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
+  默认同时重启已加载的外部生成 worker 实例；未启用 worker 单元时会自动跳过。
   若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。
   失败时保留维护模式。
 EOF
@@ -223,12 +224,106 @@ ensure_runtime_env_and_dirs() {
   fi
 }
 
+list_worker_services() {
+  local pattern="$1"
+
+  if [[ -z "${pattern}" ]]; then
+    return 0
+  fi
+
+  systemctl list-units --all --plain --no-legend "${pattern}" 2>/dev/null | awk '{print $1}' | sort -u
+}
+
+ensure_default_worker_service() {
+  local pattern="$1"
+  local default_service="genarrative-external-generation-worker@1.service"
+  local template_service="genarrative-external-generation-worker@.service"
+  local services=()
+
+  if [[ -z "${pattern}" ]]; then
+    return 0
+  fi
+
+  if [[ "${pattern}" != "genarrative-external-generation-worker@*.service" ]]; then
+    return 0
+  fi
+
+  if ! systemctl cat "${template_service}" >/dev/null 2>&1; then
+    echo "[production-api-deploy] 缺少外部生成 worker systemd 模板: ${template_service}" >&2
+    return 1
+  fi
+
+  mapfile -t services < <(list_worker_services "${pattern}")
+  if [[ "${#services[@]}" -gt 0 ]]; then
+    return 0
+  fi
+
+  echo "[production-api-deploy] 未发现外部生成 worker 实例，启用并启动默认实例: ${default_service}"
+  systemctl enable --now "${default_service}"
+}
+
+restart_worker_services() {
+  local pattern="$1"
+  local services=()
+
+  if [[ -z "${pattern}" ]]; then
+    echo "[production-api-deploy] 跳过外部生成 worker 重启。"
+    return 0
+  fi
+
+  ensure_default_worker_service "${pattern}"
+  mapfile -t services < <(list_worker_services "${pattern}")
+  if [[ "${#services[@]}" -eq 0 ]]; then
+    echo "[production-api-deploy] 未发现已加载的外部生成 worker 单元: ${pattern}" >&2
+    return 1
+  fi
+
+  echo "[production-api-deploy] 重启外部生成 worker: ${services[*]}"
+  systemctl restart "${services[@]}"
+}
+
+wait_for_worker_services() {
+  local pattern="$1"
+  local services=()
+  local all_active
+
+  if [[ -z "${pattern}" ]]; then
+    return 0
+  fi
+
+  mapfile -t services < <(list_worker_services "${pattern}")
+  if [[ "${#services[@]}" -eq 0 ]]; then
+    echo "[production-api-deploy] 外部生成 worker 单元不存在，发布失败: ${pattern}" >&2
+    return 1
+  fi
+
+  echo "[production-api-deploy] 等待外部生成 worker active: ${services[*]}"
+  for _ in {1..30}; do
+    all_active=1
+    for service in "${services[@]}"; do
+      if ! systemctl is-active --quiet "${service}"; then
+        all_active=0
+        break
+      fi
+    done
+    if [[ "${all_active}" -eq 1 ]]; then
+      return 0
+    fi
+    sleep 2
+  done
+
+  systemctl --no-pager --full status "${services[@]}" || true
+  echo "[production-api-deploy] 外部生成 worker 未在超时时间内进入 active，发布失败。" >&2
+  return 1
+}
+
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 SOURCE_DIR=""
 VERSION=""
 RELEASE_ROOT="/opt/genarrative/releases"
 CURRENT_LINK="/opt/genarrative/current"
 SERVICE_NAME="genarrative-api.service"
+WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
 HEALTH_URL="http://127.0.0.1:8082/readyz"
 API_ENV_FILE="/etc/genarrative/api-server.env"
 DATABASE=""
@@ -261,6 +356,14 @@ while [[ $# -gt 0 ]]; do
       SERVICE_NAME="${2:?缺少 --service 的值}"
       shift 2
       ;;
+    --worker-service-pattern)
+      WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}"
+      shift 2
+      ;;
+    --no-worker-services)
+      WORKER_SERVICE_PATTERN=""
+      shift
+      ;;
     --health-url)
       HEALTH_URL="${2:?缺少 --health-url 的值}"
       shift 2
@@ -362,6 +465,8 @@ ln -sfn "${RELEASE_DIR}" "${CURRENT_LINK}"
 
 echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}"
 systemctl restart "${SERVICE_NAME}"
+restart_worker_services "${WORKER_SERVICE_PATTERN}"
+wait_for_worker_services "${WORKER_SERVICE_PATTERN}"
 
 echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}"
 for _ in {1..30}; do

scripts/jenkins-server-provision.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}"
 SPACETIME_BIN_SOURCE="${SPACETIME_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/spacetime/spacetime}"
 OTELCOL_BIN_SOURCE="${OTELCOL_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/otelcol-contrib}"
+WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
 
 require_non_root_relative_path() {
   local label="$1"
@@ -417,6 +418,10 @@ render_api_env_example() {
     deploy/env/api-server.env.example
 }
 
+render_external_generation_worker_env_example() {
+  cat deploy/env/external-generation-worker.env.example
+}
+
 render_otelcol_service() {
   cat deploy/systemd/otelcol-contrib.service
 }
@@ -603,6 +608,18 @@ render_api_service() {
     deploy/systemd/genarrative-api.service
 }
 
+render_external_generation_worker_service() {
+  local current_escaped api_env_escaped worker_env_escaped
+  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
+  api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
+  worker_env_escaped="$(escape_sed_replacement "${WORKER_ENV_FILE}")"
+  sed \
+    -e "s|/opt/genarrative/current|${current_escaped}|g" \
+    -e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \
+    -e "s|/etc/genarrative/external-generation-worker.env|${worker_env_escaped}|g" \
+    deploy/systemd/genarrative-external-generation-worker@.service
+}
+
 render_database_backup_service() {
   local current_escaped env_escaped
   current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
@@ -615,6 +632,7 @@ render_database_backup_service() {
 
 require_path deploy/systemd/spacetimedb.service
 require_path deploy/systemd/genarrative-api.service
+require_path deploy/systemd/genarrative-external-generation-worker@.service
 require_path deploy/systemd/genarrative-database-backup.service
 require_path deploy/systemd/genarrative-database-backup.timer
 require_path deploy/systemd/otelcol-contrib.service
@@ -623,6 +641,7 @@ require_path deploy/nginx/genarrative.conf
 require_path deploy/nginx/genarrative-dev-http.conf
 require_path deploy/nginx/snippets/genarrative-maintenance.conf
 require_path deploy/env/api-server.env.example
+require_path deploy/env/external-generation-worker.env.example
 require_path scripts/deploy/maintenance-on.sh
 require_path scripts/deploy/maintenance-off.sh
 require_path scripts/deploy/maintenance-status.sh
@@ -665,15 +684,18 @@ sync_spacetime_install "${SPACETIME_ROOT}"
 
 spacetimedb_service="$(mktemp)"
 api_service="$(mktemp)"
+external_generation_worker_service="$(mktemp)"
 database_backup_service="$(mktemp)"
 render_spacetimedb_service >"${spacetimedb_service}"
 render_api_service >"${api_service}"
+render_external_generation_worker_service >"${external_generation_worker_service}"
 render_database_backup_service >"${database_backup_service}"
 install_file "${spacetimedb_service}" /etc/systemd/system/spacetimedb.service 0644
 install_file "${api_service}" /etc/systemd/system/genarrative-api.service 0644
+install_file "${external_generation_worker_service}" /etc/systemd/system/genarrative-external-generation-worker@.service 0644
 install_file "${database_backup_service}" /etc/systemd/system/genarrative-database-backup.service 0644
 install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644
-rm -f "${spacetimedb_service}" "${api_service}" "${database_backup_service}"
+rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${database_backup_service}"
 
 if [[ ! -f "${API_ENV_FILE}" ]]; then
   echo "+ create ${API_ENV_FILE} from example"
@@ -687,6 +709,17 @@ else
 fi
 ensure_api_runtime_env_defaults
 
+if [[ ! -f "${WORKER_ENV_FILE}" ]]; then
+  echo "+ create ${WORKER_ENV_FILE} from example"
+  if [[ "${DRY_RUN}" != "true" ]]; then
+    render_external_generation_worker_env_example >"${WORKER_ENV_FILE}"
+    chmod 0600 "${WORKER_ENV_FILE}"
+    chown root:root "${WORKER_ENV_FILE}"
+  fi
+else
+  echo "[server-provision] 已存在 worker 环境文件，保留不覆盖: ${WORKER_ENV_FILE}"
+fi
+
 if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
   sync_otelcol_install
   otelcol_service="$(mktemp)"
@@ -708,7 +741,7 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
   if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
     run_cmd systemctl enable otelcol-contrib.service
   fi
-  run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer
+  run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service
   if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
     run_cmd systemctl restart otelcol-contrib.service
   fi
@@ -717,8 +750,10 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
   ensure_spacetime_owner_client_token
   if [[ -x "${CURRENT_LINK}/api-server" ]]; then
     run_cmd systemctl restart genarrative-api.service
+    run_cmd systemctl enable --now genarrative-external-generation-worker@1.service
+    run_cmd systemctl restart genarrative-external-generation-worker@1.service
   else
-    echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server，跳过 api-server 首次启动。后续 API deploy 会重启服务。"
+    echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server，跳过 api-server 和外部生成 worker 首次启动。后续 API deploy 会启用并启动默认 worker 实例。"
   fi
 fi