完善外部生成Worker动态扩缩容

新增外部生成controller进程角色与systemd服务

补齐队列统计procedure与spacetime-client绑定

更新生产部署脚本、健康巡检和server provision的worker/controller口径

新增容器worker smoke脚本并同步运维文档与团队记忆

scripts/check-production-ops-guardrails.mjs

@@ -23,6 +23,46 @@ const checks = [
     includes: 'genarrative-health-patrol.timer',
     reason: 'Server-Provision 必须安装并启用健康巡检 timer。',
   },
+  {
+    file: 'scripts/jenkins-server-provision.sh',
+    includes: 'genarrative-external-generation-controller.service',
+    reason: 'Server-Provision 必须安装并启用外部生成 worker controller。',
+  },
+  {
+    file: 'scripts/jenkins-server-provision.sh',
+    includes: 'genarrative-external-generation-worker@1.service',
+    reason: 'Server-Provision 必须启用外部生成保底 worker 实例。',
+  },
+  {
+    file: 'scripts/deploy/production-api-deploy.sh',
+    includes: 'ensure_default_worker_service',
+    reason: 'API Deploy 必须在缺少 worker 实例时补启动默认外部生成 worker。',
+  },
+  {
+    file: 'scripts/deploy/production-api-deploy.sh',
+    includes: 'wait_for_worker_services',
+    reason: 'API Deploy 必须等待外部生成 worker 实例 active。',
+  },
+  {
+    file: 'scripts/deploy/production-api-deploy.sh',
+    includes: 'wait_for_worker_controller_service',
+    reason: 'API Deploy 必须重启并验活外部生成 worker controller。',
+  },
+  {
+    file: 'deploy/systemd/genarrative-external-generation-worker@.service',
+    includes: 'GENARRATIVE_PROCESS_ROLE=external-generation-worker',
+    reason: '外部生成 worker 模板必须作为独立 worker 进程角色运行。',
+  },
+  {
+    file: 'deploy/systemd/genarrative-external-generation-controller.service',
+    includes: 'GENARRATIVE_PROCESS_ROLE=external-generation-controller',
+    reason: '外部生成 worker controller 必须作为独立进程角色运行。',
+  },
+  {
+    file: 'scripts/ops/production-health-patrol.mjs',
+    includes: 'checkActiveWorkerInstances',
+    reason: '生产健康巡检必须检查至少一个外部生成 worker 实例 active。',
+  },
   {
     file: 'scripts/build-production-release.sh',
     includes: 'production-health-patrol.mjs',

scripts/deploy/production-api-deploy.sh

@@ -5,11 +5,11 @@ set -euo pipefail
 usage() {
   cat <<'EOF'
 用法:
-  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
+  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
 
 说明:
   进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
-  默认同时重启已加载的外部生成 worker 实例；未启用 worker 单元时会自动跳过。
+  默认同时重启外部生成 worker controller 和已加载的 worker 实例；未启用 worker 单元时会自动跳过。
   若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。
   失败时保留维护模式。
 EOF
@@ -317,6 +317,43 @@ wait_for_worker_services() {
   return 1
 }
 
+ensure_worker_controller_service() {
+  local service="$1"
+
+  if [[ -z "${service}" ]]; then
+    return 0
+  fi
+
+  if ! systemctl cat "${service}" >/dev/null 2>&1; then
+    echo "[production-api-deploy] 缺少外部生成 worker controller systemd 单元: ${service}" >&2
+    return 1
+  fi
+
+  echo "[production-api-deploy] 启用并重启外部生成 worker controller: ${service}"
+  systemctl enable "${service}"
+  systemctl restart "${service}"
+}
+
+wait_for_worker_controller_service() {
+  local service="$1"
+
+  if [[ -z "${service}" ]]; then
+    return 0
+  fi
+
+  echo "[production-api-deploy] 等待外部生成 worker controller active: ${service}"
+  for _ in {1..30}; do
+    if systemctl is-active --quiet "${service}"; then
+      return 0
+    fi
+    sleep 2
+  done
+
+  systemctl --no-pager --full status "${service}" || true
+  echo "[production-api-deploy] 外部生成 worker controller 未在超时时间内进入 active，发布失败。" >&2
+  return 1
+}
+
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 SOURCE_DIR=""
 VERSION=""
@@ -324,6 +361,7 @@ RELEASE_ROOT="/opt/genarrative/releases"
 CURRENT_LINK="/opt/genarrative/current"
 SERVICE_NAME="genarrative-api.service"
 WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
+WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service"
 HEALTH_URL="http://127.0.0.1:8082/readyz"
 API_ENV_FILE="/etc/genarrative/api-server.env"
 DATABASE=""
@@ -364,6 +402,14 @@ while [[ $# -gt 0 ]]; do
       WORKER_SERVICE_PATTERN=""
       shift
       ;;
+    --worker-controller-service)
+      WORKER_CONTROLLER_SERVICE="${2:?缺少 --worker-controller-service 的值}"
+      shift 2
+      ;;
+    --no-worker-controller)
+      WORKER_CONTROLLER_SERVICE=""
+      shift
+      ;;
     --health-url)
       HEALTH_URL="${2:?缺少 --health-url 的值}"
       shift 2
@@ -488,6 +534,8 @@ echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}"
 systemctl restart "${SERVICE_NAME}"
 restart_worker_services "${WORKER_SERVICE_PATTERN}"
 wait_for_worker_services "${WORKER_SERVICE_PATTERN}"
+ensure_worker_controller_service "${WORKER_CONTROLLER_SERVICE}"
+wait_for_worker_controller_service "${WORKER_CONTROLLER_SERVICE}"
 
 echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}"
 for _ in {1..30}; do

scripts/jenkins-server-provision.sh

@@ -5,6 +5,7 @@ PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}"
 SPACETIME_BIN_SOURCE="${SPACETIME_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/spacetime/spacetime}"
 OTELCOL_BIN_SOURCE="${OTELCOL_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/otelcol-contrib}"
 WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
+CONTROLLER_ENV_FILE="${CONTROLLER_ENV_FILE:-/etc/genarrative/external-generation-controller.env}"
 GENARRATIVE_OPENSSL_VERSION="${GENARRATIVE_OPENSSL_VERSION:-3.2.0}"
 GENARRATIVE_OPENSSL_PREFIX="${GENARRATIVE_OPENSSL_PREFIX:-/opt/genarrative/openssl-3.2.0}"
 GENARRATIVE_OPENSSL_SOURCE_URL="${GENARRATIVE_OPENSSL_SOURCE_URL:-https://github.com/openssl/openssl/releases/download/openssl-${GENARRATIVE_OPENSSL_VERSION}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz}"
@@ -542,6 +543,10 @@ render_external_generation_worker_env_example() {
   cat deploy/env/external-generation-worker.env.example
 }
 
+render_external_generation_controller_env_example() {
+  cat deploy/env/external-generation-controller.env.example
+}
+
 render_otelcol_service() {
   cat deploy/systemd/otelcol-contrib.service
 }
@@ -740,6 +745,18 @@ render_external_generation_worker_service() {
     deploy/systemd/genarrative-external-generation-worker@.service
 }
 
+render_external_generation_controller_service() {
+  local current_escaped api_env_escaped controller_env_escaped
+  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
+  api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
+  controller_env_escaped="$(escape_sed_replacement "${CONTROLLER_ENV_FILE}")"
+  sed \
+    -e "s|/opt/genarrative/current|${current_escaped}|g" \
+    -e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \
+    -e "s|/etc/genarrative/external-generation-controller.env|${controller_env_escaped}|g" \
+    deploy/systemd/genarrative-external-generation-controller.service
+}
+
 render_database_backup_service() {
   local current_escaped env_escaped
   current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
@@ -761,6 +778,7 @@ render_health_patrol_service() {
 require_path deploy/systemd/spacetimedb.service
 require_path deploy/systemd/genarrative-api.service
 require_path deploy/systemd/genarrative-external-generation-worker@.service
+require_path deploy/systemd/genarrative-external-generation-controller.service
 require_path deploy/systemd/genarrative-database-backup.service
 require_path deploy/systemd/genarrative-database-backup.timer
 require_path deploy/systemd/genarrative-health-patrol.service
@@ -772,6 +790,7 @@ require_path deploy/nginx/genarrative-dev-http.conf
 require_path deploy/nginx/snippets/genarrative-maintenance.conf
 require_path deploy/env/api-server.env.example
 require_path deploy/env/external-generation-worker.env.example
+require_path deploy/env/external-generation-controller.env.example
 require_path scripts/deploy/maintenance-on.sh
 require_path scripts/deploy/maintenance-off.sh
 require_path scripts/deploy/maintenance-status.sh
@@ -816,21 +835,24 @@ sync_spacetime_install "${SPACETIME_ROOT}"
 spacetimedb_service="$(mktemp)"
 api_service="$(mktemp)"
 external_generation_worker_service="$(mktemp)"
+external_generation_controller_service="$(mktemp)"
 database_backup_service="$(mktemp)"
 health_patrol_service="$(mktemp)"
 render_spacetimedb_service >"${spacetimedb_service}"
 render_api_service >"${api_service}"
 render_external_generation_worker_service >"${external_generation_worker_service}"
+render_external_generation_controller_service >"${external_generation_controller_service}"
 render_database_backup_service >"${database_backup_service}"
 render_health_patrol_service >"${health_patrol_service}"
 install_file "${spacetimedb_service}" /etc/systemd/system/spacetimedb.service 0644
 install_file "${api_service}" /etc/systemd/system/genarrative-api.service 0644
 install_file "${external_generation_worker_service}" /etc/systemd/system/genarrative-external-generation-worker@.service 0644
+install_file "${external_generation_controller_service}" /etc/systemd/system/genarrative-external-generation-controller.service 0644
 install_file "${database_backup_service}" /etc/systemd/system/genarrative-database-backup.service 0644
 install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644
 install_file "${health_patrol_service}" /etc/systemd/system/genarrative-health-patrol.service 0644
 install_file deploy/systemd/genarrative-health-patrol.timer /etc/systemd/system/genarrative-health-patrol.timer 0644
-rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${database_backup_service}" "${health_patrol_service}"
+rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${external_generation_controller_service}" "${database_backup_service}" "${health_patrol_service}"
 
 if [[ ! -f "${API_ENV_FILE}" ]]; then
   echo "+ create ${API_ENV_FILE} from example"
@@ -855,6 +877,17 @@ else
   echo "[server-provision] 已存在 worker 环境文件，保留不覆盖: ${WORKER_ENV_FILE}"
 fi
 
+if [[ ! -f "${CONTROLLER_ENV_FILE}" ]]; then
+  echo "+ create ${CONTROLLER_ENV_FILE} from example"
+  if [[ "${DRY_RUN}" != "true" ]]; then
+    render_external_generation_controller_env_example >"${CONTROLLER_ENV_FILE}"
+    chmod 0600 "${CONTROLLER_ENV_FILE}"
+    chown root:root "${CONTROLLER_ENV_FILE}"
+  fi
+else
+  echo "[server-provision] 已存在 controller 环境文件，保留不覆盖: ${CONTROLLER_ENV_FILE}"
+fi
+
 if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
   sync_otelcol_install
   otelcol_service="$(mktemp)"
@@ -876,7 +909,7 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
   if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
     run_cmd systemctl enable otelcol-contrib.service
   fi
-  run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-health-patrol.timer
+  run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service genarrative-health-patrol.timer
   if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
     run_cmd systemctl restart otelcol-contrib.service
   fi
@@ -887,8 +920,10 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
     run_cmd systemctl restart genarrative-api.service
     run_cmd systemctl enable --now genarrative-external-generation-worker@1.service
     run_cmd systemctl restart genarrative-external-generation-worker@1.service
+    run_cmd systemctl enable --now genarrative-external-generation-controller.service
+    run_cmd systemctl restart genarrative-external-generation-controller.service
   else
-    echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server，跳过 api-server 和外部生成 worker 首次启动。后续 API deploy 会启用并启动默认 worker 实例。"
+    echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server，跳过 api-server、外部生成 worker 和 controller 首次启动。后续 API deploy 会启用并启动默认 worker 与 controller。"
   fi
 fi
 

scripts/ops/production-health-patrol.mjs

@@ -20,9 +20,11 @@ const DEFAULT_PUBLIC_PATHS = [
 
 const DEFAULT_SERVICES = [
   'genarrative-api.service',
+  'genarrative-external-generation-controller.service',
   'spacetimedb.service',
   'nginx.service',
 ];
+const WORKER_SERVICE_PATTERN = 'genarrative-external-generation-worker@*.service';
 
 function usage() {
   console.log(`Usage:
@@ -216,6 +218,61 @@ async function checkService(serviceName, timeoutMs) {
   );
 }
 
+async function checkActiveWorkerInstances(config) {
+  const result = await runCommand(
+    'systemctl',
+    [
+      'list-units',
+      WORKER_SERVICE_PATTERN,
+      '--type=service',
+      '--state=active',
+      '--no-legend',
+      '--plain',
+      '--no-pager',
+    ],
+    config.timeoutMs,
+  );
+  if (result.code !== 0) {
+    return checkResult(
+      'service:external-generation-workers',
+      'CRITICAL',
+      '无法枚举外部生成 worker 实例',
+      {
+        command: result.command,
+        stderr: result.stderr.trim() || result.error,
+      },
+    );
+  }
+
+  const services = result.stdout
+    .split('\n')
+    .map((line) => line.trim().split(/\s+/u)[0])
+    .filter((service) =>
+      /^genarrative-external-generation-worker@.+\.service$/u.test(service),
+    );
+
+  if (services.length === 0) {
+    return checkResult(
+      'service:external-generation-workers',
+      'CRITICAL',
+      '没有 active 的外部生成 worker 实例',
+      {
+        command: result.command,
+      },
+    );
+  }
+
+  return checkResult(
+    'service:external-generation-workers',
+    'OK',
+    `${services.length} 个 worker active`,
+    {
+      command: result.command,
+      services,
+    },
+  );
+}
+
 function requestUrl(url, timeoutMs) {
   return new Promise((resolve) => {
     const startedAt = Date.now();
@@ -310,6 +367,10 @@ async function checkRecentJournal(config) {
     '-u',
     'genarrative-api.service',
     '-u',
+    'genarrative-external-generation-controller.service',
+    '-u',
+    WORKER_SERVICE_PATTERN,
+    '-u',
     'spacetimedb.service',
     '-u',
     'nginx.service',
@@ -426,6 +487,7 @@ async function main() {
   for (const serviceName of DEFAULT_SERVICES) {
     checks.push(await checkService(serviceName, config.timeoutMs));
   }
+  checks.push(await checkActiveWorkerInstances(config));
 
   checks.push(await checkHttp('api:/healthz', joinUrl(config.apiBaseUrl, '/healthz'), config));
   checks.push(await checkHttp('api:/readyz', joinUrl(config.apiBaseUrl, '/readyz'), config));