diff --git a/docs/technical/【后端架构】BgFilter受限资源调度方案-2026-07-21.md b/docs/technical/【后端架构】BgFilter受限资源调度方案-2026-07-21.md index 752a0871e..dfbd7a900 100644 --- a/docs/technical/【后端架构】BgFilter受限资源调度方案-2026-07-21.md +++ b/docs/technical/【后端架构】BgFilter受限资源调度方案-2026-07-21.md @@ -486,6 +486,7 @@ flat / complex 的每次 provider 失败审计都必须留在子 worker,保留 - worker unit 先加载共享 API env、再加载 worker 专属 env;父子有效 `N` 与 `est` 完全一致(两者都在共享 API env),flat / complex 统一熔断参数只由子 worker 配置和执行,cooldown 默认 `120s`。 - `external-generation-worker.env` 后加载时不得把内部 base URL、Token / Token 文件、connect timeout、`N`、`est`、OSS bucket 或 endpoint 覆盖为与共享 API env 不同的有效值;父侧必须把源对象写到子 worker 将要签名读取的同一 OSS 位置。外部生成 worker 可使用同 bucket 下权限等价或更小的独立 AK,不要求凭据文本相同。 - 父进程 `GENARRATIVE_BGFILTER_WORKER_BASE_URL`、子 worker `HOST / PORT` 和部署 readiness URL 必须指向同一个 `127.0.0.1:` endpoint;旧非空配置不能因为“无需补默认值”而绕过一致性检查。 +- deploy 安装的三个 worker systemd unit(bgfilter / worker@ / controller)必须先按 `--current-link` 与各 env 参数渲染再安装(与 provision 的 `render_*_service` 同语义),安装后的 unit 不得残留模板默认的 current 链接或 env 路径字面量;controller env 路径由 `--controller-env-file` 表达。默认参数下渲染输出与模板逐字节一致。 - `genarrative-bgfilter-worker.service` 必须保持 `TimeoutStopSec=900`,覆盖 `callBudgetMs` 排空上界(约 `321s`)与停止收口余量;停止时排队请求立即类型化失败,不参与排空。 - 发布目录切换前拒绝缺失、空、含内部空白、包含多个非空行、符号链接或权限错误的内部 Token 文件;允许文件末尾正常换行,父、子有效 Token 文件路径必须相同。 - 生产运行期巡检同时检查 `genarrative-bgfilter-worker.service` 为 active 且 worker `readyz` 成功(默认 `127.0.0.1:8083`),不能只依赖 systemd 自动重启。provision / deploy 的发布验活 URL 必须从已验证的 env `HOST/PORT` 派生,不得硬编码默认端口;若运维自定义 `GENARRATIVE_BGFILTER_WORKER_PORT`,必须同步覆盖巡检的 `GENARRATIVE_HEALTH_PATROL_BGFILTER_BASE_URL`(巡检是独立进程,不读取 worker env,默认值不会自动跟随)。 diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs index 3a6749730..d6ccd35d3 100644 --- a/scripts/check-production-api-deploy.mjs +++ b/scripts/check-production-api-deploy.mjs @@ -352,15 +352,68 @@ function assertDeployCopiesPingoraDirectReleaseDependencies() { ), 'API deploy 必须把随包 BgFilter worker 单元安装到 systemd unit 目录。', ); + // 安装的 unit 必须按本次部署参数渲染(fixture 全程使用自定义路径): + // 含 fixture 的 current 链接与 env 路径,不得残留模板默认字面量—— + // 否则自定义 --current-link/env 部署会让服务启动旧二进制或加载默认配置。 + const renderedUnits = [ + ['genarrative-bgfilter-worker.service', fixture.bgfilterWorkerEnvFile], + [ + 'genarrative-external-generation-worker@.service', + fixture.externalGenerationWorkerEnvFile, + ], + [ + 'genarrative-external-generation-controller.service', + fixture.controllerEnvFile, + ], + ]; + for (const [unitName, roleEnvFile] of renderedUnits) { + const unitContent = readFileSync( + path.join(fixture.systemdUnitDir, unitName), + 'utf8', + ); + assertIncludes( + unitContent, + `WorkingDirectory=${fixture.currentLink}`, + `${unitName} 的工作目录必须渲染为 --current-link。`, + ); + assertIncludes( + unitContent, + `${fixture.currentLink}/api-server`, + `${unitName} 的 ExecStart 必须指向 --current-link 下的二进制。`, + ); + assertIncludes( + unitContent, + `EnvironmentFile=${fixture.apiEnvFile}`, + `${unitName} 必须加载 --api-env-file 指定的共享 env。`, + ); + assertIncludes( + unitContent, + roleEnvFile, + `${unitName} 必须加载本角色 env 参数指定的文件。`, + ); + for (const templateDefault of [ + '/opt/genarrative/current', + '/etc/genarrative/api-server.env', + '/etc/genarrative/bgfilter-worker.env', + '/etc/genarrative/external-generation-worker.env', + '/etc/genarrative/external-generation-controller.env', + ]) { + if (unitContent.includes(templateDefault)) { + failures.push( + `${unitName} 安装后不得残留模板默认路径 ${templateDefault}:unit 必须按部署参数渲染。`, + ); + } + } + } const bgfilterUnit = readFileSync( path.join(fixture.systemdUnitDir, 'genarrative-bgfilter-worker.service'), 'utf8', ); const sharedEnvIndex = bgfilterUnit.indexOf( - 'EnvironmentFile=/etc/genarrative/api-server.env', + `EnvironmentFile=${fixture.apiEnvFile}`, ); const dedicatedEnvIndex = bgfilterUnit.indexOf( - 'EnvironmentFile=/etc/genarrative/bgfilter-worker.env', + `EnvironmentFile=${fixture.bgfilterWorkerEnvFile}`, ); if ( sharedEnvIndex < 0 || @@ -1998,6 +2051,11 @@ function prepareFixture(name) { 'etc', 'bgfilter-worker.env', ); + const controllerEnvFile = path.join( + root, + 'etc', + 'external-generation-controller.env', + ); const bgfilterTokenFile = path.join(root, 'etc', 'bgfilter-worker.token'); const pingoraEnvFile = path.join(root, 'etc', 'pingora-gateway.env'); const maintenanceFile = path.join(root, 'maintenance', 'enabled'); @@ -2043,6 +2101,11 @@ function prepareFixture(name) { 'GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY=2\n', 'utf8', ); + writeFileSync( + controllerEnvFile, + 'GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_PLACEHOLDER=1\n', + 'utf8', + ); writeFileSync( bgfilterWorkerEnvFile, [ @@ -2399,6 +2462,7 @@ function prepareFixture(name) { apiEnvFile, externalGenerationWorkerEnvFile, bgfilterWorkerEnvFile, + controllerEnvFile, bgfilterTokenFile, pingoraEnvFile, maintenanceFile, @@ -2476,6 +2540,8 @@ function runDeploy(fixture, options = {}) { fixture.externalGenerationWorkerEnvFile, '--bgfilter-worker-env-file', fixture.bgfilterWorkerEnvFile, + '--controller-env-file', + options.controllerEnvFile ?? fixture.controllerEnvFile, // 缺省不传 --bgfilter-worker-health-url,与真实 Jenkins 调用一致: // 让预检从已校验的 worker env HOST/PORT 派生 readiness URL。 ...(options.bgfilterWorkerHealthUrl === undefined diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index e8d000d6d..b374cd534 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -5,12 +5,13 @@ set -euo pipefail usage() { cat <<'EOF' 用法: - ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url ] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode] + ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url ] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--controller-env-file /etc/genarrative/external-generation-controller.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode] 说明: 进入维护模式,校验并发布 api-server 单文件,更新 current 链接,重启 systemd 服务并执行 readiness 检查。 默认先停止、启动并验活唯一 BgFilter worker,再重启 API、外部生成 worker controller 和已加载的 worker 实例。 --bgfilter-worker-health-url 缺省时从已校验的 worker env HOST/PORT 派生;显式传入时必须与父子 env 三方一致,否则预检失败。 + 随包 worker systemd unit 按 --current-link 与各 env 参数渲染后安装(与 provision 同语义),自定义路径不会被打回模板默认值;默认参数下渲染输出与模板一致。 若传入 --database,会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件,避免服务继续读取旧库。 若发布包包含 pingora-gateway,或传入 --require-pingora-gateway,部署脚本会要求 release manifest、二进制与 checksum 一致,再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置,启动或重启 Pingora 影子服务并复核 active。 默认在 readiness 通过后退出维护模式;传入 --keep-maintenance-mode 时保留维护文件,供人工验收后再恢复公网。 @@ -896,11 +897,21 @@ ensure_default_worker_service() { systemctl enable --now "${default_service}" } +escape_sed_replacement() { + printf "%s" "$1" | sed "s/[&|]/\\\\&/g" +} + +# 与 provision 的 render_*_service 同语义:unit 文件在仓库里是模板,安装前必须把 +# 默认 current 链接与 env 路径渲染为本次部署参数,自定义 --current-link/env 时 +# 不得让服务启动旧二进制或加载默认 env(默认参数下渲染输出与模板逐字节一致)。 install_release_systemd_unit() { local source_path="$1" local unit_name="$2" local label="$3" + shift 3 local unit_dir="${GENARRATIVE_SYSTEMD_UNIT_DIR:-/etc/systemd/system}" + local sed_args=() + local template_default replacement rendered if [[ ! -f "${source_path}" ]]; then echo "[production-api-deploy] 发布产物缺少${label}: ${source_path}" >&2 @@ -910,10 +921,27 @@ install_release_systemd_unit() { echo "[production-api-deploy] systemd unit 目录必须使用绝对路径: ${unit_dir}" >&2 return 1 fi + while [[ $# -gt 0 ]]; do + template_default="$1" + replacement="${2:?unit 渲染替换必须成对提供模板默认值与实际值}" + shift 2 + if [[ -z "${replacement}" ]]; then + echo "[production-api-deploy] ${label} 渲染值不能为空: ${template_default}" >&2 + return 1 + fi + sed_args+=(-e "s|${template_default}|$(escape_sed_replacement "${replacement}")|g") + done + rendered="$(mktemp)" + if [[ "${#sed_args[@]}" -gt 0 ]]; then + sed "${sed_args[@]}" "${source_path}" > "${rendered}" + else + cat "${source_path}" > "${rendered}" + fi echo "[production-api-deploy] 安装${label}: ${unit_name}" run_privileged install -d -m 0755 "${unit_dir}" - run_privileged install -m 0644 "${source_path}" "${unit_dir}/${unit_name}" + run_privileged install -m 0644 "${rendered}" "${unit_dir}/${unit_name}" + rm -f "${rendered}" } install_worker_systemd_units() { @@ -927,7 +955,10 @@ install_worker_systemd_units() { install_release_systemd_unit \ "${release_dir}/deploy/systemd/genarrative-bgfilter-worker.service" \ "genarrative-bgfilter-worker.service" \ - "BgFilter worker systemd 单元" + "BgFilter worker systemd 单元" \ + "/opt/genarrative/current" "${CURRENT_LINK}" \ + "/etc/genarrative/api-server.env" "${API_ENV_FILE}" \ + "/etc/genarrative/bgfilter-worker.env" "${BGFILTER_WORKER_ENV_FILE}" installed_any=1 fi @@ -935,7 +966,10 @@ install_worker_systemd_units() { install_release_systemd_unit \ "${release_dir}/deploy/systemd/genarrative-external-generation-worker@.service" \ "genarrative-external-generation-worker@.service" \ - "外部生成 worker systemd 模板" + "外部生成 worker systemd 模板" \ + "/opt/genarrative/current" "${CURRENT_LINK}" \ + "/etc/genarrative/api-server.env" "${API_ENV_FILE}" \ + "/etc/genarrative/external-generation-worker.env" "${WORKER_ENV_FILE}" installed_any=1 fi @@ -943,7 +977,10 @@ install_worker_systemd_units() { install_release_systemd_unit \ "${release_dir}/deploy/systemd/genarrative-external-generation-controller.service" \ "genarrative-external-generation-controller.service" \ - "外部生成 worker controller systemd 单元" + "外部生成 worker controller systemd 单元" \ + "/opt/genarrative/current" "${CURRENT_LINK}" \ + "/etc/genarrative/api-server.env" "${API_ENV_FILE}" \ + "/etc/genarrative/external-generation-controller.env" "${CONTROLLER_ENV_FILE}" installed_any=1 fi @@ -1093,6 +1130,8 @@ BGFILTER_WORKER_ENV_FILE="/etc/genarrative/bgfilter-worker.env" HEALTH_URL="http://127.0.0.1:8082/readyz" API_ENV_FILE="/etc/genarrative/api-server.env" WORKER_ENV_FILE="/etc/genarrative/external-generation-worker.env" +# controller unit 渲染专用:deploy 不读写该 env 内容,只保证安装的 unit 指向正确路径。 +CONTROLLER_ENV_FILE="/etc/genarrative/external-generation-controller.env" DATABASE="" SPACETIME_SERVER_URL="" DEPLOY_COMPLETED=0 @@ -1188,6 +1227,10 @@ while [[ $# -gt 0 ]]; do WORKER_ENV_FILE="${2:?缺少 --worker-env-file 的值}" shift 2 ;; + --controller-env-file) + CONTROLLER_ENV_FILE="${2:?缺少 --controller-env-file 的值}" + shift 2 + ;; --database) DATABASE="${2:?缺少 --database 的值}" shift 2 @@ -1214,6 +1257,9 @@ fi if [[ -n "${BGFILTER_WORKER_ENV_FILE}" ]]; then require_absolute_path "${BGFILTER_WORKER_ENV_FILE}" "--bgfilter-worker-env-file" fi +if [[ -n "${CONTROLLER_ENV_FILE}" ]]; then + require_absolute_path "${CONTROLLER_ENV_FILE}" "--controller-env-file" +fi if [[ -n "${DATABASE}" ]]; then validate_spacetime_database_name "${DATABASE}"