完善外部生成Worker动态扩缩容

新增外部生成controller进程角色与systemd服务

补齐队列统计procedure与spacetime-client绑定

更新生产部署脚本、健康巡检和server provision的worker/controller口径

新增容器worker smoke脚本并同步运维文档与团队记忆

scripts/jenkins-server-provision.sh

@@ -5,6 +5,7 @@ PROVISION_TOOLS_DIR="${PROVISION_TOOLS_DIR:-provision-tools}"
 SPACETIME_BIN_SOURCE="${SPACETIME_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/spacetime/spacetime}"
 OTELCOL_BIN_SOURCE="${OTELCOL_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/otelcol-contrib}"
 WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
+CONTROLLER_ENV_FILE="${CONTROLLER_ENV_FILE:-/etc/genarrative/external-generation-controller.env}"
 GENARRATIVE_OPENSSL_VERSION="${GENARRATIVE_OPENSSL_VERSION:-3.2.0}"
 GENARRATIVE_OPENSSL_PREFIX="${GENARRATIVE_OPENSSL_PREFIX:-/opt/genarrative/openssl-3.2.0}"
 GENARRATIVE_OPENSSL_SOURCE_URL="${GENARRATIVE_OPENSSL_SOURCE_URL:-https://github.com/openssl/openssl/releases/download/openssl-${GENARRATIVE_OPENSSL_VERSION}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz}"
@@ -542,6 +543,10 @@ render_external_generation_worker_env_example() {
   cat deploy/env/external-generation-worker.env.example
 }
 
+render_external_generation_controller_env_example() {
+  cat deploy/env/external-generation-controller.env.example
+}
+
 render_otelcol_service() {
   cat deploy/systemd/otelcol-contrib.service
 }
@@ -740,6 +745,18 @@ render_external_generation_worker_service() {
     deploy/systemd/genarrative-external-generation-worker@.service
 }
 
+render_external_generation_controller_service() {
+  local current_escaped api_env_escaped controller_env_escaped
+  current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
+  api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
+  controller_env_escaped="$(escape_sed_replacement "${CONTROLLER_ENV_FILE}")"
+  sed \
+    -e "s|/opt/genarrative/current|${current_escaped}|g" \
+    -e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \
+    -e "s|/etc/genarrative/external-generation-controller.env|${controller_env_escaped}|g" \
+    deploy/systemd/genarrative-external-generation-controller.service
+}
+
 render_database_backup_service() {
   local current_escaped env_escaped
   current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
@@ -761,6 +778,7 @@ render_health_patrol_service() {
 require_path deploy/systemd/spacetimedb.service
 require_path deploy/systemd/genarrative-api.service
 require_path deploy/systemd/genarrative-external-generation-worker@.service
+require_path deploy/systemd/genarrative-external-generation-controller.service
 require_path deploy/systemd/genarrative-database-backup.service
 require_path deploy/systemd/genarrative-database-backup.timer
 require_path deploy/systemd/genarrative-health-patrol.service
@@ -772,6 +790,7 @@ require_path deploy/nginx/genarrative-dev-http.conf
 require_path deploy/nginx/snippets/genarrative-maintenance.conf
 require_path deploy/env/api-server.env.example
 require_path deploy/env/external-generation-worker.env.example
+require_path deploy/env/external-generation-controller.env.example
 require_path scripts/deploy/maintenance-on.sh
 require_path scripts/deploy/maintenance-off.sh
 require_path scripts/deploy/maintenance-status.sh
@@ -816,21 +835,24 @@ sync_spacetime_install "${SPACETIME_ROOT}"
 spacetimedb_service="$(mktemp)"
 api_service="$(mktemp)"
 external_generation_worker_service="$(mktemp)"
+external_generation_controller_service="$(mktemp)"
 database_backup_service="$(mktemp)"
 health_patrol_service="$(mktemp)"
 render_spacetimedb_service >"${spacetimedb_service}"
 render_api_service >"${api_service}"
 render_external_generation_worker_service >"${external_generation_worker_service}"
+render_external_generation_controller_service >"${external_generation_controller_service}"
 render_database_backup_service >"${database_backup_service}"
 render_health_patrol_service >"${health_patrol_service}"
 install_file "${spacetimedb_service}" /etc/systemd/system/spacetimedb.service 0644
 install_file "${api_service}" /etc/systemd/system/genarrative-api.service 0644
 install_file "${external_generation_worker_service}" /etc/systemd/system/genarrative-external-generation-worker@.service 0644
+install_file "${external_generation_controller_service}" /etc/systemd/system/genarrative-external-generation-controller.service 0644
 install_file "${database_backup_service}" /etc/systemd/system/genarrative-database-backup.service 0644
 install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644
 install_file "${health_patrol_service}" /etc/systemd/system/genarrative-health-patrol.service 0644
 install_file deploy/systemd/genarrative-health-patrol.timer /etc/systemd/system/genarrative-health-patrol.timer 0644
-rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${database_backup_service}" "${health_patrol_service}"
+rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${external_generation_controller_service}" "${database_backup_service}" "${health_patrol_service}"
 
 if [[ ! -f "${API_ENV_FILE}" ]]; then
   echo "+ create ${API_ENV_FILE} from example"
@@ -855,6 +877,17 @@ else
   echo "[server-provision] 已存在 worker 环境文件，保留不覆盖: ${WORKER_ENV_FILE}"
 fi
 
+if [[ ! -f "${CONTROLLER_ENV_FILE}" ]]; then
+  echo "+ create ${CONTROLLER_ENV_FILE} from example"
+  if [[ "${DRY_RUN}" != "true" ]]; then
+    render_external_generation_controller_env_example >"${CONTROLLER_ENV_FILE}"
+    chmod 0600 "${CONTROLLER_ENV_FILE}"
+    chown root:root "${CONTROLLER_ENV_FILE}"
+  fi
+else
+  echo "[server-provision] 已存在 controller 环境文件，保留不覆盖: ${CONTROLLER_ENV_FILE}"
+fi
+
 if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
   sync_otelcol_install
   otelcol_service="$(mktemp)"
@@ -876,7 +909,7 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
   if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
     run_cmd systemctl enable otelcol-contrib.service
   fi
-  run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-health-patrol.timer
+  run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service genarrative-health-patrol.timer
   if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
     run_cmd systemctl restart otelcol-contrib.service
   fi
@@ -887,8 +920,10 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
     run_cmd systemctl restart genarrative-api.service
     run_cmd systemctl enable --now genarrative-external-generation-worker@1.service
     run_cmd systemctl restart genarrative-external-generation-worker@1.service
+    run_cmd systemctl enable --now genarrative-external-generation-controller.service
+    run_cmd systemctl restart genarrative-external-generation-controller.service
   else
-    echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server，跳过 api-server 和外部生成 worker 首次启动。后续 API deploy 会启用并启动默认 worker 实例。"
+    echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server，跳过 api-server、外部生成 worker 和 controller 首次启动。后续 API deploy 会启用并启动默认 worker 与 controller。"
   fi
 fi