修复生产发布 worker unit 路径渲染

按部署参数渲染 BgFilter、外部生成 worker 与 controller unit
补齐 controller 环境文件参数及 Jenkins 全链路透传
新增安装后 unit 内容与流水线参数回归门禁
同步生产运维文档和共享项目记忆

scripts/check-production-api-deploy.mjs

@@ -345,15 +345,92 @@ function assertDeployCopiesPingoraDirectReleaseDependencies() {
     ),
     'API deploy 必须把随包 BgFilter worker 单元安装到 systemd unit 目录。',
   );
+  const externalGenerationWorkerUnit = readFileSync(
+    path.join(
+      fixture.systemdUnitDir,
+      'genarrative-external-generation-worker@.service',
+    ),
+    'utf8',
+  );
+  const externalGenerationControllerUnit = readFileSync(
+    path.join(
+      fixture.systemdUnitDir,
+      'genarrative-external-generation-controller.service',
+    ),
+    'utf8',
+  );
   const bgfilterUnit = readFileSync(
     path.join(fixture.systemdUnitDir, 'genarrative-bgfilter-worker.service'),
     'utf8',
   );
+
+  for (const [label, unit] of [
+    ['外部生成 worker unit', externalGenerationWorkerUnit],
+    ['外部生成 controller unit', externalGenerationControllerUnit],
+    ['BgFilter worker unit', bgfilterUnit],
+  ]) {
+    assertIncludes(
+      unit,
+      `WorkingDirectory=${fixture.currentLink}`,
+      `${label} 必须使用本次部署的自定义 current link。`,
+    );
+    assertIncludes(
+      unit,
+      `${fixture.currentLink}/api-server`,
+      `${label} 必须启动本次部署的自定义 current link 下的二进制。`,
+    );
+    assertIncludes(
+      unit,
+      `EnvironmentFile=${fixture.apiEnvFile}`,
+      `${label} 必须加载本次部署指定的 API env。`,
+    );
+    assertNotIncludes(
+      unit,
+      '/opt/genarrative/current',
+      `${label} 安装后不得继续引用模板默认 current link。`,
+    );
+    assertNotIncludes(
+      unit,
+      '/etc/genarrative/api-server.env',
+      `${label} 安装后不得继续引用模板默认 API env。`,
+    );
+  }
+
+  assertIncludes(
+    externalGenerationWorkerUnit,
+    `EnvironmentFile=-${fixture.externalGenerationWorkerEnvFile}`,
+    '外部生成 worker unit 必须加载本次部署指定的专属 env。',
+  );
+  assertNotIncludes(
+    externalGenerationWorkerUnit,
+    '/etc/genarrative/external-generation-worker.env',
+    '外部生成 worker unit 安装后不得继续引用模板默认专属 env。',
+  );
+  assertIncludes(
+    externalGenerationControllerUnit,
+    `EnvironmentFile=-${fixture.externalGenerationControllerEnvFile}`,
+    '外部生成 controller unit 必须加载本次部署指定的专属 env。',
+  );
+  assertNotIncludes(
+    externalGenerationControllerUnit,
+    '/etc/genarrative/external-generation-controller.env',
+    '外部生成 controller unit 安装后不得继续引用模板默认专属 env。',
+  );
+  assertIncludes(
+    bgfilterUnit,
+    `EnvironmentFile=${fixture.bgfilterWorkerEnvFile}`,
+    'BgFilter unit 必须加载本次部署指定的专属 env。',
+  );
+  assertNotIncludes(
+    bgfilterUnit,
+    '/etc/genarrative/bgfilter-worker.env',
+    'BgFilter unit 安装后不得继续引用模板默认专属 env。',
+  );
   const sharedEnvIndex = bgfilterUnit.indexOf(
-    'EnvironmentFile=/etc/genarrative/api-server.env',
+    `EnvironmentFile=${fixture.apiEnvFile}`,
   );
   const dedicatedEnvIndex = bgfilterUnit.indexOf(
-    'EnvironmentFile=/etc/genarrative/bgfilter-worker.env',
+    `EnvironmentFile=${fixture.bgfilterWorkerEnvFile}`,
   );
   if (
     sharedEnvIndex < 0 ||
@@ -1905,6 +1982,11 @@ function prepareFixture(name) {
     'etc',
     'external-generation-worker.env',
   );
+  const externalGenerationControllerEnvFile = path.join(
+    root,
+    'etc',
+    'external-generation-controller.env',
+  );
   const bgfilterWorkerEnvFile = path.join(
     root,
     'etc',
@@ -1955,6 +2037,11 @@ function prepareFixture(name) {
     'GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY=2\n',
     'utf8',
   );
+  writeFileSync(
+    externalGenerationControllerEnvFile,
+    'GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MAX_WORKERS=4\n',
+    'utf8',
+  );
   writeFileSync(
     bgfilterWorkerEnvFile,
     [
@@ -2310,6 +2397,7 @@ function prepareFixture(name) {
     currentLink,
     apiEnvFile,
     externalGenerationWorkerEnvFile,
+    externalGenerationControllerEnvFile,
     bgfilterWorkerEnvFile,
     bgfilterTokenFile,
     pingoraEnvFile,
@@ -2386,6 +2474,8 @@ function runDeploy(fixture, options = {}) {
     options.apiEnvFile ?? fixture.apiEnvFile,
     '--worker-env-file',
     fixture.externalGenerationWorkerEnvFile,
+    '--controller-env-file',
+    fixture.externalGenerationControllerEnvFile,
     '--bgfilter-worker-env-file',
     fixture.bgfilterWorkerEnvFile,
     '--bgfilter-worker-health-url',
@@ -2506,6 +2596,12 @@ function assertIncludes(content, needle, reason) {
   }
 }
 
+function assertNotIncludes(content, needle, reason) {
+  if (content.includes(needle)) {
+    failures.push(`${reason} 不应包含: ${needle}`);
+  }
+}
+
 function assertPingoraEnvProductionDefaults(filePath, label) {
   if (!existsSync(filePath)) {
     return;

scripts/check-production-ops-guardrails.mjs

@@ -233,6 +233,12 @@ const checks = [
     reason:
       'API readiness 单次请求必须有超时，避免端口已建立但服务尚未响应时绕过重试上限无限挂起。',
   },
+  {
+    file: 'scripts/deploy/production-api-deploy.sh',
+    includes:
+      'on_exit() {\n  local exit_code=$?\n  cleanup_rendered_systemd_unit',
+    reason: 'API deploy 被中断时必须清理尚未安装完成的临时 systemd unit。',
+  },
   {
     file: 'jenkins/Jenkinsfile.production-api-deploy',
     includes:
@@ -244,6 +250,54 @@ const checks = [
     includes: 'maintenance_deploy_args+=(--keep-maintenance-mode)',
     reason: 'API Deploy Job 必须把保持维护参数传给发布产物内的部署脚本。',
   },
+  {
+    file: 'jenkins/Jenkinsfile.production-api-deploy',
+    includes:
+      "string(name: 'CONTROLLER_ENV_FILE', defaultValue: '/etc/genarrative/external-generation-controller.env'",
+    reason: 'API Deploy Job 必须暴露外部生成 controller env 路径。',
+  },
+  {
+    file: 'jenkins/Jenkinsfile.production-api-deploy',
+    includes:
+      "string(name: 'BGFILTER_WORKER_ENV_FILE', defaultValue: '/etc/genarrative/bgfilter-worker.env'",
+    reason: 'API Deploy Job 必须暴露 BgFilter worker env 路径。',
+  },
+  {
+    file: 'jenkins/Jenkinsfile.production-api-deploy',
+    includes:
+      '--controller-env-file "${CONTROLLER_ENV_FILE:-/etc/genarrative/external-generation-controller.env}"',
+    reason: 'API Deploy Job 必须把 controller env 路径传给发布脚本。',
+  },
+  {
+    file: 'jenkins/Jenkinsfile.production-api-deploy',
+    includes:
+      '--bgfilter-worker-env-file "${BGFILTER_WORKER_ENV_FILE:-/etc/genarrative/bgfilter-worker.env}"',
+    reason: 'API Deploy Job 必须把 BgFilter worker env 路径传给发布脚本。',
+  },
+  {
+    file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
+    includes:
+      "string(name: 'CONTROLLER_ENV_FILE', defaultValue: '/etc/genarrative/external-generation-controller.env'",
+    reason: 'Full Job 必须暴露外部生成 controller env 路径。',
+  },
+  {
+    file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
+    includes:
+      "string(name: 'BGFILTER_WORKER_ENV_FILE', defaultValue: '/etc/genarrative/bgfilter-worker.env'",
+    reason: 'Full Job 必须暴露 BgFilter worker env 路径。',
+  },
+  {
+    file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
+    includes:
+      "string(name: 'CONTROLLER_ENV_FILE', value: params.CONTROLLER_ENV_FILE ?: '/etc/genarrative/external-generation-controller.env')",
+    reason: 'Full Job 必须把 controller env 路径传给 API Deploy Job。',
+  },
+  {
+    file: 'jenkins/Jenkinsfile.production-full-build-and-deploy',
+    includes:
+      "string(name: 'BGFILTER_WORKER_ENV_FILE', value: params.BGFILTER_WORKER_ENV_FILE ?: '/etc/genarrative/bgfilter-worker.env')",
+    reason: 'Full Job 必须把 BgFilter worker env 路径传给 API Deploy Job。',
+  },
   {
     file: 'scripts/jenkins-server-provision.sh',
     includes: 'ensure_runtime_bootstrap_secret_file_env',

scripts/deploy/production-api-deploy.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 usage() {
   cat <<'EOF'
 用法:
-  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url http://127.0.0.1:8083/readyz] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode]
+  ./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url http://127.0.0.1:8083/readyz] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--controller-env-file /etc/genarrative/external-generation-controller.env] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode]
 
 说明:
   进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
@@ -35,6 +35,10 @@ require_absolute_path() {
     echo "[production-api-deploy] ${label} 必须使用绝对路径: ${value}" >&2
     exit 1
   fi
+  if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then
+    echo "[production-api-deploy] ${label} 不能包含换行符。" >&2
+    exit 1
+  fi
 }
 
 validate_spacetime_database_name() {
@@ -892,38 +896,103 @@ install_release_systemd_unit() {
   fi
 
   echo "[production-api-deploy] 安装${label}: ${unit_name}"
-  run_privileged install -d -m 0755 "${unit_dir}"
+  if ! run_privileged install -d -m 0755 "${unit_dir}"; then
+    return 1
+  fi
   run_privileged install -m 0644 "${source_path}" "${unit_dir}/${unit_name}"
 }
 
+render_and_install_release_systemd_unit() {
+  local source_path="$1"
+  local unit_name="$2"
+  local label="$3"
+  local rendered_path
+  local install_status=0
+  shift 3
+
+  if [[ ! -f "${source_path}" ]]; then
+    echo "[production-api-deploy] 发布产物缺少${label}: ${source_path}" >&2
+    return 1
+  fi
+  if [[ "$#" -eq 0 || $(( $# % 2 )) -ne 0 ]]; then
+    echo "[production-api-deploy] ${label} 缺少成对的模板路径和实际路径。" >&2
+    return 1
+  fi
+
+  RENDERED_SYSTEMD_UNIT_FILE="$(mktemp)"
+  rendered_path="${RENDERED_SYSTEMD_UNIT_FILE}"
+  if ! python3 - "${source_path}" "${rendered_path}" "$@" <<'PY'
+import re
+import sys
+from pathlib import Path
+
+source_path = Path(sys.argv[1])
+rendered_path = Path(sys.argv[2])
+replacements = sys.argv[3:]
+content = source_path.read_text(encoding="utf-8")
+replacement_map = dict(zip(replacements[::2], replacements[1::2]))
+
+for template_path in replacement_map:
+    if template_path not in content:
+        raise SystemExit(f"systemd unit 模板缺少占位路径: {template_path}")
+
+pattern = re.compile("|".join(re.escape(path) for path in replacement_map))
+content = pattern.sub(lambda match: replacement_map[match.group(0)], content)
+
+rendered_path.write_text(content, encoding="utf-8")
+PY
+  then
+    cleanup_rendered_systemd_unit
+    return 1
+  fi
+
+  install_release_systemd_unit "${rendered_path}" "${unit_name}" "${label}" || install_status=$?
+  cleanup_rendered_systemd_unit
+  return "${install_status}"
+}
+
 install_worker_systemd_units() {
   local release_dir="$1"
   local pattern="$2"
   local controller_service="$3"
   local bgfilter_service="$4"
+  local current_link="$5"
+  local api_env_file="$6"
+  local worker_env_file="$7"
+  local controller_env_file="$8"
+  local bgfilter_env_file="$9"
   local installed_any=0
 
   if [[ "${bgfilter_service}" == "genarrative-bgfilter-worker.service" ]]; then
-    install_release_systemd_unit \
+    render_and_install_release_systemd_unit \
       "${release_dir}/deploy/systemd/genarrative-bgfilter-worker.service" \
       "genarrative-bgfilter-worker.service" \
-      "BgFilter worker systemd 单元"
+      "BgFilter worker systemd 单元" \
+      "/opt/genarrative/current" "${current_link}" \
+      "/etc/genarrative/api-server.env" "${api_env_file}" \
+      "/etc/genarrative/bgfilter-worker.env" "${bgfilter_env_file}"
     installed_any=1
   fi
 
   if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" ]]; then
-    install_release_systemd_unit \
+    render_and_install_release_systemd_unit \
       "${release_dir}/deploy/systemd/genarrative-external-generation-worker@.service" \
       "genarrative-external-generation-worker@.service" \
-      "外部生成 worker systemd 模板"
+      "外部生成 worker systemd 模板" \
+      "/opt/genarrative/current" "${current_link}" \
+      "/etc/genarrative/api-server.env" "${api_env_file}" \
+      "/etc/genarrative/external-generation-worker.env" "${worker_env_file}"
     installed_any=1
   fi
 
   if [[ "${controller_service}" == "genarrative-external-generation-controller.service" ]]; then
-    install_release_systemd_unit \
+    render_and_install_release_systemd_unit \
       "${release_dir}/deploy/systemd/genarrative-external-generation-controller.service" \
       "genarrative-external-generation-controller.service" \
-      "外部生成 worker controller systemd 单元"
+      "外部生成 worker controller systemd 单元" \
+      "/opt/genarrative/current" "${current_link}" \
+      "/etc/genarrative/api-server.env" "${api_env_file}" \
+      "/etc/genarrative/external-generation-controller.env" "${controller_env_file}"
     installed_any=1
   fi
 
@@ -1066,6 +1135,7 @@ SERVICE_NAME="genarrative-api.service"
 PINGORA_SERVICE_NAME="genarrative-pingora-gateway.service"
 WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
 WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service"
+CONTROLLER_ENV_FILE="/etc/genarrative/external-generation-controller.env"
 BGFILTER_WORKER_SERVICE="genarrative-bgfilter-worker.service"
 BGFILTER_WORKER_HEALTH_URL="http://127.0.0.1:8083/readyz"
 BGFILTER_WORKER_ENV_FILE="/etc/genarrative/bgfilter-worker.env"
@@ -1083,6 +1153,7 @@ MAINTENANCE_FILE="${GENARRATIVE_MAINTENANCE_FILE:-/var/lib/genarrative/maintenan
 CURRENT_LINK_SWITCHED=0
 RELEASE_DIR=""
 STAGING_RELEASE_DIR=""
+RENDERED_SYSTEMD_UNIT_FILE=""
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -1134,6 +1205,10 @@ while [[ $# -gt 0 ]]; do
       WORKER_CONTROLLER_SERVICE="${2:?缺少 --worker-controller-service 的值}"
       shift 2
       ;;
+    --controller-env-file)
+      CONTROLLER_ENV_FILE="${2:?缺少 --controller-env-file 的值}"
+      shift 2
+      ;;
     --no-worker-controller)
       WORKER_CONTROLLER_SERVICE=""
       shift
@@ -1190,6 +1265,9 @@ require_absolute_path "${API_ENV_FILE}" "--api-env-file"
 if [[ -n "${WORKER_ENV_FILE}" ]]; then
   require_absolute_path "${WORKER_ENV_FILE}" "--worker-env-file"
 fi
+if [[ -n "${CONTROLLER_ENV_FILE}" ]]; then
+  require_absolute_path "${CONTROLLER_ENV_FILE}" "--controller-env-file"
+fi
 if [[ -n "${BGFILTER_WORKER_ENV_FILE}" ]]; then
   require_absolute_path "${BGFILTER_WORKER_ENV_FILE}" "--bgfilter-worker-env-file"
 fi
@@ -1245,8 +1323,16 @@ cleanup_staging_release() {
   fi
 }
 
+cleanup_rendered_systemd_unit() {
+  if [[ -n "${RENDERED_SYSTEMD_UNIT_FILE:-}" && -f "${RENDERED_SYSTEMD_UNIT_FILE}" ]]; then
+    rm -f "${RENDERED_SYSTEMD_UNIT_FILE}"
+  fi
+  RENDERED_SYSTEMD_UNIT_FILE=""
+}
+
 on_exit() {
   local exit_code=$?
+  cleanup_rendered_systemd_unit
   if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
     cleanup_staging_release
     if [[ "${MAINTENANCE_ENABLED_BY_DEPLOY}" -eq 1 && "${CURRENT_LINK_SWITCHED}" -ne 1 ]]; then
@@ -1553,7 +1639,16 @@ if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
   ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}"
 fi
 
-install_worker_systemd_units "${RELEASE_DIR}" "${WORKER_SERVICE_PATTERN}" "${WORKER_CONTROLLER_SERVICE}" "${BGFILTER_WORKER_SERVICE}"
+install_worker_systemd_units \
+  "${RELEASE_DIR}" \
+  "${WORKER_SERVICE_PATTERN}" \
+  "${WORKER_CONTROLLER_SERVICE}" \
+  "${BGFILTER_WORKER_SERVICE}" \
+  "${CURRENT_LINK}" \
+  "${API_ENV_FILE}" \
+  "${WORKER_ENV_FILE}" \
+  "${CONTROLLER_ENV_FILE}" \
+  "${BGFILTER_WORKER_ENV_FILE}"
 
 restart_and_wait_for_bgfilter_worker "${BGFILTER_WORKER_SERVICE}" "${BGFILTER_WORKER_HEALTH_URL}"