落地 BgFilter 单实例受限资源 Worker

新增私有 BgFilter worker,提供内部鉴权、并发限流、超时、顺序重试、flat 熔断和图片校验。
父生成流程改为通过内部二进制 HTTP 原地等待,并保留 flat 降级与 complex 失败语义。
接入本地开发、systemd、生产部署、Provision、健康巡检和配置漂移门禁。
补充动画、部署与运维测试、Linux fixture 隔离以及对应架构文档。
This commit is contained in:
2026-07-21 12:05:27 +00:00
parent a51b625101
commit 566f2f2194
32 changed files with 5570 additions and 1030 deletions
File diff suppressed because it is too large Load Diff
+36
View File
@@ -258,6 +258,42 @@ function assertApiReleaseContainsPingoraDirectDependencies() {
),
'API release 必须包含外部生成 worker controller systemd 单元。',
);
assertFileExists(
path.join(
releaseDir,
'deploy/systemd/genarrative-bgfilter-worker.service',
),
'API release 必须包含唯一 BgFilter worker systemd 单元。',
);
assertFileExists(
path.join(releaseDir, 'deploy/env/bgfilter-worker.env.example'),
'API release 必须包含 BgFilter worker env 示例。',
);
const bgfilterUnit = readFileSync(
path.join(
releaseDir,
'deploy/systemd/genarrative-bgfilter-worker.service',
),
'utf8',
);
const sharedEnvIndex = bgfilterUnit.indexOf(
'EnvironmentFile=/etc/genarrative/api-server.env',
);
const dedicatedEnvIndex = bgfilterUnit.indexOf(
'EnvironmentFile=/etc/genarrative/bgfilter-worker.env',
);
if (
sharedEnvIndex < 0 ||
dedicatedEnvIndex < 0 ||
sharedEnvIndex > dedicatedEnvIndex
) {
failures.push('API release 的 BgFilter unit 必须按共享 env → 专属 env 加载。');
}
assertIncludes(
bgfilterUnit,
'TimeoutStopSec=900',
'API release 的 BgFilter unit 必须给最多 600s 的内部请求预算留足优雅排空时间。',
);
assertFileExists(
path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'),
'API release 必须包含 Pingora env 示例。',
@@ -54,6 +54,11 @@ function assertPublicBaseUrlDefaultsToGatewayEntry() {
"process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n 'http://127.0.0.1'",
'publicBaseUrl 默认必须指向本机网关入口,不能回落到 API 直连端口。',
);
assertIncludes(
script,
"process.env.GENARRATIVE_HEALTH_PATROL_BGFILTER_BASE_URL ||\n 'http://127.0.0.1:8083'",
'BgFilter worker 巡检默认必须指向唯一实例的 loopback 端口。',
);
if (
script.includes(
'process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL',
@@ -85,6 +90,14 @@ async function assertNginxModeChecksNginxService() {
'systemctl is-active nginx.service',
'nginx gateway mode 必须检查 nginx.service。',
);
assertIncludes(
commandsLog,
'systemctl is-active genarrative-bgfilter-worker.service',
'生产巡检必须检查唯一 BgFilter worker service。',
);
if (!payload.checks.some((check) => check.name === 'bgfilter:/readyz')) {
failures.push('生产巡检必须探测 BgFilter worker /readyz。');
}
if (commandsLog.includes('genarrative-pingora-gateway.service')) {
failures.push(
'nginx gateway mode 不应要求 Pingora gateway service active。',
@@ -369,6 +382,8 @@ async function runPatrol(fixture, args) {
'scripts/ops/production-health-patrol.mjs',
'--api-base-url',
fixture.baseUrl,
'--bgfilter-base-url',
fixture.baseUrl,
'--spacetime-base-url',
fixture.baseUrl,
'--public-base-url',
@@ -1647,6 +1647,50 @@ const checks = [
includes: 'genarrative-external-generation-worker@1.service',
reason: 'Server-Provision 必须启用外部生成保底 worker 实例。',
},
{
file: 'deploy/systemd/genarrative-bgfilter-worker.service',
includes: 'TimeoutStopSec=900',
reason:
'BgFilter worker 必须给最多 600s 的内部请求预算留足优雅排空时间,不能沿用 systemd 默认停止窗口。',
},
{
file: 'scripts/jenkins-server-provision.sh',
includes: 'validate_no_bgfilter_internal_token_plaintext',
reason:
'Server-Provision 必须拒绝 API 或 BgFilter worker env 保存内部 Token 明文。',
},
{
file: 'scripts/jenkins-server-provision.sh',
includes:
'for env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"; do',
reason:
'Server-Provision 必须同时拒绝 external-generation-worker.env 保存 BgFilter 内部 Token 明文。',
},
{
file: 'scripts/jenkins-server-provision.sh',
includes:
'validate_bgfilter_env_file_alignment "${WORKER_ENV_FILE}" "外部生成 worker env" "false"',
reason:
'Server-Provision 启动外部生成 worker 前必须拒绝 BgFilter URL、Token 文件、timeout 与 OSS 位置漂移。',
},
{
file: 'scripts/jenkins-server-provision.sh',
includes: 'validate_bgfilter_loopback_endpoint_alignment',
reason:
'Server-Provision 启动 BgFilter worker 前必须确认父 base URL 与子 listener 指向同一 loopback endpoint。',
},
{
file: 'scripts/jenkins-server-provision.sh',
includes: 'BgFilter 内部 Token 文件不得为空或只包含空白字符',
reason:
'Server-Provision 必须拒绝仅含空白字符的 BgFilter 内部 Token 文件。',
},
{
file: 'scripts/jenkins-server-provision.sh',
includes: "root:genarrative:440",
reason:
'Server-Provision 必须复核 BgFilter 内部 Token 文件的 owner、group 与 0440 权限。',
},
{
file: 'scripts/deploy/production-api-deploy.sh',
includes: 'ensure_default_worker_service',
+339 -5
View File
@@ -5,11 +5,11 @@ set -euo pipefail
usage() {
cat <<'EOF'
用法:
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode]
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url http://127.0.0.1:8083/readyz] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode]
说明:
进入维护模式,校验并发布 api-server 单文件,更新 current 链接,重启 systemd 服务并执行 readiness 检查。
默认同时重启外部生成 worker controller 和已加载的 worker 实例;未启用 worker 单元时会自动跳过。
默认先停止、启动并验活唯一 BgFilter worker,再重启 API、外部生成 worker controller 和已加载的 worker 实例。
若传入 --database,会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件,避免服务继续读取旧库。
若发布包包含 pingora-gateway,或传入 --require-pingora-gateway,部署脚本会要求 release manifest、二进制与 checksum 一致,再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置,启动或重启 Pingora 影子服务并复核 active。
默认在 readiness 通过后退出维护模式;传入 --keep-maintenance-mode 时保留维护文件,供人工验收后再恢复公网。
@@ -175,6 +175,81 @@ if matched_value is not None:
fi
}
env_contains_nonempty_assignment() {
local file_path="$1"
local key="$2"
if [[ ! -f "${file_path}" ]]; then
return 1
fi
local python_script='
import sys
from pathlib import Path
path = Path(sys.argv[1])
key = sys.argv[2]
for raw_line in path.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
current_key, value = line.split("=", 1)
if current_key.strip() != key:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("\"", chr(39)):
value = value[1:-1]
if value.strip():
raise SystemExit(0)
raise SystemExit(1)
'
if [[ -r "${file_path}" ]]; then
python3 -c "${python_script}" "${file_path}" "${key}"
else
if ! sudo -n true >/dev/null 2>&1; then
echo "[production-api-deploy] 当前用户无权读取 ${file_path},且 sudo -n 不可用;无法检查运行态环境变量。" >&2
exit 1
fi
sudo -n python3 -c "${python_script}" "${file_path}" "${key}"
fi
}
env_has_assignment() {
local file_path="$1"
local key="$2"
if [[ ! -f "${file_path}" ]]; then
return 1
fi
local python_script='
import sys
from pathlib import Path
path = Path(sys.argv[1])
key = sys.argv[2]
for raw_line in path.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
current_key, _ = line.split("=", 1)
if current_key.strip() == key:
raise SystemExit(0)
raise SystemExit(1)
'
if [[ -r "${file_path}" ]]; then
python3 -c "${python_script}" "${file_path}" "${key}"
else
if ! sudo -n true >/dev/null 2>&1; then
echo "[production-api-deploy] 当前用户无权读取 ${file_path},且 sudo -n 不可用;无法检查运行态环境变量。" >&2
exit 1
fi
sudo -n python3 -c "${python_script}" "${file_path}" "${key}"
fi
}
ensure_env_value() {
local file_path="$1"
local key="$2"
@@ -303,10 +378,11 @@ ensure_runtime_env_and_dirs() {
ensure_env_value_migrates_old_default "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL" "http://127.0.0.1:8083"
ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE" "/etc/genarrative/secrets/bgfilter-worker.token"
ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS" "2000"
ensure_runtime_bootstrap_secret_file_env "${api_env_file}"
ensure_env_value_migrates_old_default "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS" "45000" "180000"
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
tracking_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED")"
tracking_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR")"
@@ -351,6 +427,186 @@ ensure_worker_runtime_env_defaults() {
ensure_runtime_bootstrap_secret_file_env "${worker_env_file}"
}
ensure_bgfilter_worker_runtime_env_defaults() {
local bgfilter_env_file="$1"
if [[ -z "${bgfilter_env_file}" ]]; then
return
fi
if [[ ! -f "${bgfilter_env_file}" ]]; then
echo "[production-api-deploy] BgFilter worker 环境文件不存在: ${bgfilter_env_file}" >&2
return 1
fi
ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_HOST" "127.0.0.1"
ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_PORT" "8083"
ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
}
validate_bgfilter_shared_runtime_env() {
local api_env_file="$1"
local request_timeout_ms connect_timeout_ms
request_timeout_ms="$(read_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS")"
if [[ ! "${request_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
echo "[production-api-deploy] GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${api_env_file}" >&2
return 1
fi
connect_timeout_ms="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS")"
if [[ ! "${connect_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${api_env_file}" >&2
return 1
fi
}
validate_bgfilter_worker_capacity() {
local bgfilter_env_file="$1"
local concurrency max_requests
concurrency="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY")"
max_requests="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS")"
if [[ ! "${concurrency}" =~ ^[1-9][0-9]{0,8}$ ]]; then
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_CONCURRENCY 必须是正整数: ${bgfilter_env_file}" >&2
return 1
fi
if [[ ! "${max_requests}" =~ ^[1-9][0-9]{0,8}$ ]]; then
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS 必须是正整数: ${bgfilter_env_file}" >&2
return 1
fi
if (( 10#${max_requests} < 10#${concurrency} )); then
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS 必须大于或等于 CONCURRENCY: ${bgfilter_env_file}" >&2
return 1
fi
}
validate_bgfilter_worker_shared_env_alignment() {
local api_env_file="$1"
local bgfilter_env_file="$2"
local key shared_value dedicated_value
for key in \
GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS \
GENARRATIVE_EDITOR_BGFILTER_BASE_URL \
GENARRATIVE_EDITOR_BGFILTER_TOKEN \
GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE \
ALIYUN_OSS_BUCKET \
ALIYUN_OSS_ENDPOINT \
ALIYUN_OSS_ACCESS_KEY_ID \
ALIYUN_OSS_ACCESS_KEY_SECRET \
ALIYUN_OSS_READ_EXPIRE_SECONDS; do
if ! env_has_assignment "${bgfilter_env_file}" "${key}"; then
continue
fi
dedicated_value="$(read_env_value "${bgfilter_env_file}" "${key}")"
shared_value="$(read_env_value "${api_env_file}" "${key}")"
if [[ "${dedicated_value}" != "${shared_value}" ]]; then
echo "[production-api-deploy] BgFilter 专属 env 中的共享配置与 API env 不一致: ${key};请迁移到 ${api_env_file} 并从 ${bgfilter_env_file} 删除重复项。" >&2
return 1
fi
done
}
validate_external_generation_worker_bgfilter_env_alignment() {
local api_env_file="$1"
local worker_env_file="$2"
local key shared_value worker_value
if [[ -z "${worker_env_file}" || ! -f "${worker_env_file}" ]]; then
return 0
fi
for key in \
GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS \
GENARRATIVE_BGFILTER_WORKER_BASE_URL \
GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE \
GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS \
ALIYUN_OSS_BUCKET \
ALIYUN_OSS_ENDPOINT; do
if ! env_has_assignment "${worker_env_file}" "${key}"; then
continue
fi
worker_value="$(read_env_value "${worker_env_file}" "${key}")"
shared_value="$(read_env_value "${api_env_file}" "${key}")"
if [[ "${worker_value}" != "${shared_value}" ]]; then
echo "[production-api-deploy] 外部生成 worker env 中的 BgFilter 共享配置与 API env 不一致: ${key};${worker_env_file} 会在 systemd 中后加载并覆盖父侧有效值。" >&2
return 1
fi
done
}
validate_bgfilter_loopback_endpoint_alignment() {
local api_env_file="$1"
local bgfilter_env_file="$2"
local health_url="$3"
local base_url host port expected_base_url expected_health_url
base_url="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL")"
host="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_HOST")"
port="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_PORT")"
if [[ "${host}" != "127.0.0.1" ]]; then
echo "[production-api-deploy] BgFilter worker 首版必须监听 127.0.0.1,当前 GENARRATIVE_BGFILTER_WORKER_HOST=${host:-<empty>}: ${bgfilter_env_file}" >&2
return 1
fi
if [[ ! "${port}" =~ ^[1-9][0-9]{0,4}$ ]] || (( 10#${port} > 65535 )); then
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_PORT 必须是 1-65535 的有效端口: ${bgfilter_env_file}" >&2
return 1
fi
expected_base_url="http://${host}:${port}"
if [[ "${base_url%/}" != "${expected_base_url}" ]]; then
echo "[production-api-deploy] 父进程 GENARRATIVE_BGFILTER_WORKER_BASE_URL 必须与 BgFilter worker 有效监听地址一致: expected=${expected_base_url}, actual=${base_url:-<empty>}" >&2
return 1
fi
expected_health_url="${expected_base_url}/readyz"
if [[ "${health_url}" != "${expected_health_url}" ]]; then
echo "[production-api-deploy] --bgfilter-worker-health-url 必须与父进程 base URL 和子 worker listener 指向同一 loopback endpoint: expected=${expected_health_url}, actual=${health_url:-<empty>}" >&2
return 1
fi
}
validate_no_bgfilter_internal_token_plaintext() {
local env_file
for env_file in "$@"; do
if [[ -z "${env_file}" ]]; then
continue
fi
if env_contains_nonempty_assignment "${env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN"; then
echo "[production-api-deploy] ${env_file} 不得保存 GENARRATIVE_BGFILTER_INTERNAL_TOKEN 明文;生产环境只允许使用 GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE。" >&2
return 1
fi
done
}
validate_bgfilter_internal_token_file() {
local api_env_file="$1"
local token_file token_metadata
token_file="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE")"
if [[ -z "${token_file}" || "${token_file}" != /* ]]; then
echo "[production-api-deploy] GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE 必须指向绝对路径: ${api_env_file}" >&2
return 1
fi
if [[ -L "${token_file}" || ! -f "${token_file}" || ! -s "${token_file}" ]]; then
echo "[production-api-deploy] BgFilter 内部 Token 必须是非空普通文件且不能是符号链接: ${token_file}" >&2
return 1
fi
if ! run_privileged grep -q '[^[:space:]]' -- "${token_file}"; then
echo "[production-api-deploy] BgFilter 内部 Token 文件必须至少包含一个非空白字符: ${token_file}" >&2
return 1
fi
token_metadata="$(run_privileged stat -c '%U:%G:%a' -- "${token_file}")"
if [[ "${token_metadata}" != "root:genarrative:440" ]]; then
echo "[production-api-deploy] BgFilter 内部 Token 权限必须为 root:genarrative 0440,且必须是普通文件: ${token_file} (${token_metadata})" >&2
return 1
fi
}
extract_pingora_env_files_from_unit() {
local service_name="$1"
local unit_content
@@ -572,8 +828,17 @@ install_worker_systemd_units() {
local release_dir="$1"
local pattern="$2"
local controller_service="$3"
local bgfilter_service="$4"
local installed_any=0
if [[ "${bgfilter_service}" == "genarrative-bgfilter-worker.service" ]]; then
install_release_systemd_unit \
"${release_dir}/deploy/systemd/genarrative-bgfilter-worker.service" \
"genarrative-bgfilter-worker.service" \
"BgFilter worker systemd 单元"
installed_any=1
fi
if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" ]]; then
install_release_systemd_unit \
"${release_dir}/deploy/systemd/genarrative-external-generation-worker@.service" \
@@ -688,6 +953,38 @@ wait_for_worker_controller_service() {
return 1
}
restart_and_wait_for_bgfilter_worker() {
local service="$1"
local health_url="$2"
if [[ -z "${service}" ]]; then
echo "[production-api-deploy] 跳过 BgFilter worker 启动。"
return 0
fi
if ! systemctl cat "${service}" >/dev/null 2>&1; then
echo "[production-api-deploy] 缺少 BgFilter worker systemd 单元: ${service}" >&2
return 1
fi
echo "[production-api-deploy] 停止旧 BgFilter worker 并等待在途请求排空: ${service}"
systemctl stop "${service}"
systemctl enable "${service}"
echo "[production-api-deploy] 启动唯一 BgFilter worker: ${service}"
systemctl start "${service}"
for _ in {1..30}; do
if systemctl is-active --quiet "${service}" && curl -fsS --max-time 2 "${health_url}" >/dev/null; then
echo "[production-api-deploy] BgFilter worker readiness 通过: ${health_url}"
return 0
fi
sleep 2
done
systemctl --no-pager --full status "${service}" || true
echo "[production-api-deploy] BgFilter worker readiness 检查超时: ${health_url}" >&2
return 1
}
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
SOURCE_DIR=""
VERSION=""
@@ -697,6 +994,9 @@ SERVICE_NAME="genarrative-api.service"
PINGORA_SERVICE_NAME="genarrative-pingora-gateway.service"
WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service"
BGFILTER_WORKER_SERVICE="genarrative-bgfilter-worker.service"
BGFILTER_WORKER_HEALTH_URL="http://127.0.0.1:8083/readyz"
BGFILTER_WORKER_ENV_FILE="/etc/genarrative/bgfilter-worker.env"
HEALTH_URL="http://127.0.0.1:8082/readyz"
API_ENV_FILE="/etc/genarrative/api-server.env"
WORKER_ENV_FILE="/etc/genarrative/external-generation-worker.env"
@@ -766,6 +1066,23 @@ while [[ $# -gt 0 ]]; do
WORKER_CONTROLLER_SERVICE=""
shift
;;
--bgfilter-worker-service)
BGFILTER_WORKER_SERVICE="${2:?缺少 --bgfilter-worker-service 的值}"
shift 2
;;
--bgfilter-worker-health-url)
BGFILTER_WORKER_HEALTH_URL="${2:?缺少 --bgfilter-worker-health-url 的值}"
shift 2
;;
--bgfilter-worker-env-file)
BGFILTER_WORKER_ENV_FILE="${2:?缺少 --bgfilter-worker-env-file 的值}"
shift 2
;;
--no-bgfilter-worker)
BGFILTER_WORKER_SERVICE=""
BGFILTER_WORKER_ENV_FILE=""
shift
;;
--health-url)
HEALTH_URL="${2:?缺少 --health-url 的值}"
shift 2
@@ -801,6 +1118,9 @@ require_absolute_path "${API_ENV_FILE}" "--api-env-file"
if [[ -n "${WORKER_ENV_FILE}" ]]; then
require_absolute_path "${WORKER_ENV_FILE}" "--worker-env-file"
fi
if [[ -n "${BGFILTER_WORKER_ENV_FILE}" ]]; then
require_absolute_path "${BGFILTER_WORKER_ENV_FILE}" "--bgfilter-worker-env-file"
fi
if [[ -n "${DATABASE}" ]]; then
validate_spacetime_database_name "${DATABASE}"
@@ -1127,8 +1447,20 @@ if [[ -n "${SPACETIME_SERVER_URL}" ]]; then
fi
ensure_runtime_env_and_dirs "${API_ENV_FILE}"
validate_bgfilter_shared_runtime_env "${API_ENV_FILE}"
validate_real_wechat_pay_refund_reconciliation "${API_ENV_FILE}"
ensure_worker_runtime_env_defaults "${WORKER_ENV_FILE}"
ensure_bgfilter_worker_runtime_env_defaults "${BGFILTER_WORKER_ENV_FILE}"
validate_external_generation_worker_bgfilter_env_alignment "${API_ENV_FILE}" "${WORKER_ENV_FILE}"
validate_no_bgfilter_internal_token_plaintext "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"
if [[ -n "${BGFILTER_WORKER_SERVICE}" ]]; then
validate_bgfilter_internal_token_file "${API_ENV_FILE}"
validate_bgfilter_loopback_endpoint_alignment "${API_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}" "${BGFILTER_WORKER_HEALTH_URL}"
fi
if [[ -n "${BGFILTER_WORKER_ENV_FILE}" ]]; then
validate_bgfilter_worker_capacity "${BGFILTER_WORKER_ENV_FILE}"
validate_bgfilter_worker_shared_env_alignment "${API_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"
fi
migrate_legacy_editor_generation_pricing_override "${CURRENT_LINK}"
if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
@@ -1149,7 +1481,9 @@ if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}"
fi
install_worker_systemd_units "${RELEASE_DIR}" "${WORKER_SERVICE_PATTERN}" "${WORKER_CONTROLLER_SERVICE}"
install_worker_systemd_units "${RELEASE_DIR}" "${WORKER_SERVICE_PATTERN}" "${WORKER_CONTROLLER_SERVICE}" "${BGFILTER_WORKER_SERVICE}"
restart_and_wait_for_bgfilter_worker "${BGFILTER_WORKER_SERVICE}" "${BGFILTER_WORKER_HEALTH_URL}"
echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}"
systemctl restart "${SERVICE_NAME}"
+4 -2
View File
@@ -55,8 +55,8 @@ export function parsePortRangeSpec(value) {
throw new Error(`端口段无效: ${spec},端口必须在 1024-65535 且起始不大于结束`);
}
if (end - start + 1 < 4) {
throw new Error(`端口段至少需要 4 个端口: ${spec}`);
if (end - start + 1 < 5) {
throw new Error(`端口段至少需要 5 个端口: ${spec}`);
}
return {start, end, label: `${start}-${end}`};
@@ -118,6 +118,7 @@ export function mapDevPortsToPortRange(portRange) {
apiPort: normalizedRange.start + 1,
spacetimePort: normalizedRange.start + 2,
adminWebPort: normalizedRange.start + 3,
bgfilterWorkerPort: normalizedRange.start + 4,
range: normalizedRange,
};
}
@@ -569,6 +570,7 @@ export async function resolveDevStackPorts(config) {
['api', config.api],
['web', config.web],
['adminWeb', config.adminWeb],
['bgfilterWorker', config.bgfilterWorker],
].filter(([, portConfig]) => Boolean(portConfig));
const result = {};
+7 -2
View File
@@ -47,7 +47,7 @@ async function reserveConsecutivePorts() {
}
describe('dev stack port utils', () => {
it('解析端口段并映射到四个 dev 端口', () => {
it('解析端口段并映射到五个 dev 端口', () => {
expect(parsePortRangeSpec('10000-10099')).toEqual({
start: 10000,
end: 10099,
@@ -58,7 +58,11 @@ describe('dev stack port utils', () => {
apiPort: 10001,
spacetimePort: 10002,
adminWebPort: 10003,
bgfilterWorkerPort: 10004,
});
expect(() => parsePortRangeSpec('10000-10003')).toThrow(
'端口段至少需要 5 个端口',
);
});
it('使用端口可用性检查为被占用端口寻找后续可用端口', async () => {
@@ -112,9 +116,10 @@ describe('dev stack port utils', () => {
api: {host: '127.0.0.1', preferredPort: 0},
web: {host: '127.0.0.1', preferredPort: 0},
adminWeb: {host: '127.0.0.1', preferredPort: 0},
bgfilterWorker: {host: '127.0.0.1', preferredPort: 0},
});
expect(new Set(Object.values(resolvedPorts)).size).toBe(4);
expect(new Set(Object.values(resolvedPorts)).size).toBe(5);
});
it('端口段内会一直漂移到段尾,不会被默认 200 次尝试截断', async () => {
+335 -23
View File
File diff suppressed because it is too large Load Diff
+172 -1
View File
@@ -19,6 +19,7 @@ import {
assertReusableSpacetimeProcessVersionMatchesWorkspace,
assertSpacetimeToolVersionMatchesWorkspace,
buildApiServerProcessEnv,
buildBgfilterWorkerProcessEnv,
buildDevStackSnapshot,
buildFrontendProcessEnv,
buildLocalRustProcessEnv,
@@ -85,6 +86,26 @@ describe('dev scheduler argument routing', () => {
expect(runner.resolveFrontendApiTarget()).toBe('http://127.0.0.1:8090');
});
test('独立 BgFilter worker 命令解析内部监听地址', () => {
const { command, explicitOptions, options } = parseArgs(
[
'bgfilter-worker',
'--bgfilter-worker-host',
'127.0.0.2',
'--bgfilter-worker-port',
'18083',
],
{},
);
expect(command).toBe('bgfilter-worker');
expect(explicitOptions).toEqual(
new Set(['bgfilterWorkerHost', 'bgfilterWorkerPort']),
);
expect(options.bgfilterWorkerHost).toBe('127.0.0.2');
expect(options.bgfilterWorkerPort).toBe(18083);
});
test('单独 dev:web 未显式指定 api 参数时沿用已有 Rust target', () => {
const testEnv = {
RUST_SERVER_TARGET: 'http://127.0.0.1:3100',
@@ -129,7 +150,7 @@ describe('dev scheduler argument routing', () => {
);
});
linuxTest('Linux 启动时按系统级端口段映射四个 dev 端口', async () => {
linuxTest('Linux 启动时按系统级端口段映射五个 dev 端口', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'genarrative-dev-port-range-'));
try {
const { command, explicitOptions, options } = parseArgs([], {
@@ -156,7 +177,9 @@ describe('dev scheduler argument routing', () => {
expect(runner.options.apiPort).toBe(22001);
expect(runner.options.spacetimePort).toBe(22002);
expect(runner.options.adminWebPort).toBe(22003);
expect(runner.options.bgfilterWorkerPort).toBe(22004);
expect(runner.state.apiTarget).toBe('http://127.0.0.1:22001');
expect(runner.state.bgfilterWorkerTarget).toBe('http://127.0.0.1:22004');
expect(runner.state.spacetimeServer).toBe('http://127.0.0.1:22002');
} finally {
rmSync(tempDir, { recursive: true, force: true });
@@ -196,6 +219,7 @@ describe('dev scheduler argument routing', () => {
expect(runner.options.apiPort).toBe(22001);
expect(runner.options.spacetimePort).toBe(22002);
expect(runner.options.adminWebPort).toBe(22003);
expect(runner.options.bgfilterWorkerPort).toBe(22004);
} finally {
rmSync(tempDir, { recursive: true, force: true });
}
@@ -233,6 +257,7 @@ describe('dev scheduler argument routing', () => {
expect(runner.options.apiPort).toBe(8082);
expect(runner.options.spacetimePort).toBe(3101);
expect(runner.options.adminWebPort).toBe(3102);
expect(runner.options.bgfilterWorkerPort).toBe(8083);
} finally {
if (originalPlatform) {
Object.defineProperty(process, 'platform', originalPlatform);
@@ -282,6 +307,45 @@ describe('dev scheduler api-server env', () => {
expect(env.GENARRATIVE_PROCESS_ROLE).toBe('api');
});
test('父 API 与独立 BgFilter worker 共享实际 URL 和内部 token', () => {
const { options } = parseArgs([], {});
options.bgfilterWorkerPort = 18083;
const state = {
spacetimeServer: 'http://127.0.0.1:3199',
bgfilterWorkerTarget: 'http://127.0.0.1:18083',
};
const internalToken = 'local-bgfilter-token';
const apiEnv = buildApiServerProcessEnv({
baseEnv: {},
options,
state,
bgfilterInternalToken: internalToken,
processRole: 'all',
});
const workerEnv = buildBgfilterWorkerProcessEnv({
baseEnv: {},
options,
state,
bgfilterInternalToken: internalToken,
});
expect(apiEnv.GENARRATIVE_PROCESS_ROLE).toBe('all');
expect(workerEnv.GENARRATIVE_PROCESS_ROLE).toBe('bgfilter-worker');
expect(apiEnv.GENARRATIVE_BGFILTER_WORKER_BASE_URL).toBe(
state.bgfilterWorkerTarget,
);
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_BASE_URL).toBe(
state.bgfilterWorkerTarget,
);
expect(apiEnv.GENARRATIVE_BGFILTER_INTERNAL_TOKEN).toBe(internalToken);
expect(workerEnv.GENARRATIVE_BGFILTER_INTERNAL_TOKEN).toBe(internalToken);
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_HOST).toBe('127.0.0.1');
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_PORT).toBe('18083');
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_CONCURRENCY).toBe('4');
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS).toBe('128');
});
test('Windows 本地 dev 自动注入已安装的 FFmpeg 路径', () => {
const tempDir = mkdtempSync(join(tmpdir(), 'genarrative-ffmpeg-'));
try {
@@ -339,6 +403,100 @@ describe('dev scheduler api-server env', () => {
});
});
describe('dev scheduler Rust service orchestration', () => {
test('Rust 双进程重启时先全部停止,再先 ready BgFilter、后 ready API', async () => {
const { explicitOptions, options } = parseArgs([], {});
const runner = new DevRunner(options, {}, explicitOptions);
const events: string[] = [];
runner.command = 'all';
runner.windowsApiServerCleanupCompleted = true;
runner.services = new Map([
[
'api-server',
{
stop: async () => events.push('stop-api'),
start: async () => events.push('start-api'),
},
],
[
'bgfilter-worker',
{
stop: async () => events.push('stop-bgfilter'),
start: async () => events.push('start-bgfilter'),
},
],
]);
vi.spyOn(runner, 'waitForBgfilterWorker').mockImplementation(async () => {
events.push('ready-bgfilter');
});
vi.spyOn(runner, 'waitForApiServer').mockImplementation(async () => {
events.push('ready-api');
});
await runner.restartRustServicePair();
expect(events).toEqual([
'stop-api',
'stop-bgfilter',
'start-bgfilter',
'ready-bgfilter',
'start-api',
'ready-api',
]);
});
test('dev:api-server 安全自动带起同 runner 的 BgFilter worker', async () => {
const { explicitOptions, options } = parseArgs(['api-server'], {});
const runner = new DevRunner(options, {}, explicitOptions);
const startPair = vi
.spyOn(runner, 'startRustServicePair')
.mockResolvedValue(undefined);
const startWatchers = vi
.spyOn(runner, 'startWatchers')
.mockImplementation(() => {});
await runner.startCommand('api-server');
expect(startPair).toHaveBeenCalledOnce();
expect(startWatchers).toHaveBeenCalledWith([
'api-server',
'bgfilter-worker',
]);
});
test('完整栈只为两个 Rust 角色创建一套组合 watcher', () => {
const { explicitOptions, options } = parseArgs(['--watch'], {});
const runner = new DevRunner(options, {}, explicitOptions);
runner.command = 'all';
runner.registerServices();
try {
runner.startWatchers(['api-server', 'bgfilter-worker']);
expect(runner.watchers).toHaveLength(1);
} finally {
for (const watcher of runner.watchers) {
watcher.close();
}
runner.watchers = [];
}
});
test('BgFilter worker 在 readiness 前退出时立即失败', async () => {
const { explicitOptions, options } = parseArgs([], {});
const runner = new DevRunner(options, {}, explicitOptions);
runner.services = new Map([
['bgfilter-worker', { runtime: { status: 'failed' } }],
]);
globalThis.fetch = vi.fn(async () => ({
status: 503,
})) as unknown as typeof fetch;
await expect(runner.waitForBgfilterWorker()).rejects.toThrow(
'bgfilter-worker 在 readiness 前退出',
);
});
});
describe('dev scheduler local worker cleanup', () => {
const expected = {
expectedDatabase: 'xushi-p4wfr',
@@ -471,6 +629,8 @@ describe('dev scheduler stack state file', () => {
options: {
apiHost: '127.0.0.1',
apiPort: 8090,
bgfilterWorkerHost: '127.0.0.1',
bgfilterWorkerPort: 8091,
webHost: '0.0.0.0',
webPort: 3010,
adminWebHost: '127.0.0.1',
@@ -483,6 +643,7 @@ describe('dev scheduler stack state file', () => {
},
state: {
apiTarget: 'http://127.0.0.1:8090',
bgfilterWorkerTarget: 'http://127.0.0.1:8091',
adminWebTargetHost: '127.0.0.1',
spacetimeServer: 'http://127.0.0.1:3120',
},
@@ -527,6 +688,12 @@ describe('dev scheduler stack state file', () => {
port: 8090,
url: 'http://127.0.0.1:8090',
});
expect(snapshot.services['bgfilter-worker']).toMatchObject({
status: 'idle',
pid: null,
port: 8091,
url: 'http://127.0.0.1:8091',
});
});
});
@@ -1258,6 +1425,8 @@ spacetimedb tool version 2.6.0; spacetimedb-lib version 2.6.0;
'migration-secret-hash',
GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET:
'runtime-secret',
GENARRATIVE_BGFILTER_INTERNAL_TOKEN: 'bgfilter-token',
GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE: 'bgfilter-token-file',
SAFE_VALUE: 'kept',
},
{ RUST_SERVER_TARGET: 'http://127.0.0.1:8082' },
@@ -1273,6 +1442,8 @@ spacetimedb tool version 2.6.0; spacetimedb-lib version 2.6.0;
expect(env).not.toHaveProperty(
'GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET',
);
expect(env).not.toHaveProperty('GENARRATIVE_BGFILTER_INTERNAL_TOKEN');
expect(env).not.toHaveProperty('GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE');
expect(env.SAFE_VALUE).toBe('kept');
});
+346 -6
View File
@@ -6,6 +6,7 @@ SPACETIME_BIN_SOURCE="${SPACETIME_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/spacetime/s
OTELCOL_BIN_SOURCE="${OTELCOL_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/otelcol-contrib}"
WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
CONTROLLER_ENV_FILE="${CONTROLLER_ENV_FILE:-/etc/genarrative/external-generation-controller.env}"
BGFILTER_WORKER_ENV_FILE="${BGFILTER_WORKER_ENV_FILE:-/etc/genarrative/bgfilter-worker.env}"
GENARRATIVE_OPENSSL_VERSION="${GENARRATIVE_OPENSSL_VERSION:-3.2.0}"
GENARRATIVE_OPENSSL_PREFIX="${GENARRATIVE_OPENSSL_PREFIX:-/opt/genarrative/openssl-3.2.0}"
GENARRATIVE_OPENSSL_SOURCE_URL="${GENARRATIVE_OPENSSL_SOURCE_URL:-https://github.com/openssl/openssl/releases/download/openssl-${GENARRATIVE_OPENSSL_VERSION}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz}"
@@ -408,6 +409,101 @@ read_env_value() {
done <"${file}"
}
env_contains_nonempty_assignment() {
local file="$1"
local key="$2"
local line value first_char last_char
if [[ ! -f "${file}" ]]; then
return 1
fi
while IFS= read -r line || [[ -n "${line}" ]]; do
line="${line#"${line%%[![:space:]]*}"}"
if [[ -z "${line}" || "${line}" == \#* || "${line}" != "${key}="* ]]; then
continue
fi
value="${line#*=}"
value="${value%$'\r'}"
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
if [[ ${#value} -ge 2 ]]; then
first_char="${value:0:1}"
last_char="${value: -1}"
if [[ "${first_char}" == "${last_char}" && ( "${first_char}" == '"' || "${first_char}" == "'" ) ]]; then
value="${value:1:${#value}-2}"
fi
fi
if [[ "${value}" =~ [^[:space:]] ]]; then
return 0
fi
done <"${file}"
return 1
}
env_has_assignment() {
local file="$1"
local key="$2"
local line current_key
if [[ ! -f "${file}" ]]; then
return 1
fi
while IFS= read -r line || [[ -n "${line}" ]]; do
line="${line%$'\r'}"
line="${line#"${line%%[![:space:]]*}"}"
if [[ -z "${line}" || "${line}" == \#* || "${line}" != *"="* ]]; then
continue
fi
current_key="${line%%=*}"
current_key="${current_key%"${current_key##*[![:space:]]}"}"
if [[ "${current_key}" == "${key}" ]]; then
return 0
fi
done <"${file}"
return 1
}
read_effective_env_value() {
local file="$1"
local key="$2"
local line current_key value first_char last_char matched_value="" found="false"
if [[ ! -f "${file}" ]]; then
return
fi
while IFS= read -r line || [[ -n "${line}" ]]; do
line="${line%$'\r'}"
line="${line#"${line%%[![:space:]]*}"}"
if [[ -z "${line}" || "${line}" == \#* || "${line}" != *"="* ]]; then
continue
fi
current_key="${line%%=*}"
current_key="${current_key%"${current_key##*[![:space:]]}"}"
if [[ "${current_key}" != "${key}" ]]; then
continue
fi
value="${line#*=}"
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
if [[ ${#value} -ge 2 ]]; then
first_char="${value:0:1}"
last_char="${value: -1}"
if [[ "${first_char}" == "${last_char}" && ( "${first_char}" == '"' || "${first_char}" == "'" ) ]]; then
value="${value:1:${#value}-2}"
fi
fi
matched_value="${value}"
found="true"
done <"${file}"
if [[ "${found}" == "true" ]]; then
printf "%s" "${matched_value}"
fi
}
write_env_value() {
local file="$1"
local key="$2"
@@ -525,10 +621,120 @@ ensure_api_runtime_env_defaults() {
ensure_env_value_migrates_old_default "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL" "http://127.0.0.1:8083"
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE" "/etc/genarrative/secrets/bgfilter-worker.token"
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS" "2000"
ensure_runtime_bootstrap_secret_file_env "${API_ENV_FILE}"
ensure_env_value_migrates_old_default "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS" "45000" "180000"
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
}
validate_bgfilter_shared_runtime_env() {
local request_timeout_ms connect_timeout_ms
if [[ "${DRY_RUN}" == "true" ]]; then
echo "+ validate shared BgFilter provider attempt timeout in ${API_ENV_FILE}"
return
fi
request_timeout_ms="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS")"
if [[ ! "${request_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
echo "[server-provision] GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${API_ENV_FILE}" >&2
exit 1
fi
connect_timeout_ms="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS")"
if [[ ! "${connect_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
echo "[server-provision] GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${API_ENV_FILE}" >&2
exit 1
fi
}
validate_no_bgfilter_internal_token_plaintext() {
local env_file
if [[ "${DRY_RUN}" == "true" ]]; then
echo "+ reject non-empty GENARRATIVE_BGFILTER_INTERNAL_TOKEN in ${API_ENV_FILE}, ${WORKER_ENV_FILE}, and ${BGFILTER_WORKER_ENV_FILE}"
return
fi
for env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"; do
if env_contains_nonempty_assignment "${env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN"; then
echo "[server-provision] ${env_file} 不得保存 GENARRATIVE_BGFILTER_INTERNAL_TOKEN 明文;生产环境只允许使用 GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE。" >&2
exit 1
fi
done
}
validate_bgfilter_env_file_alignment() {
local env_file="$1"
local label="$2"
local include_provider_credentials="$3"
local key shared_value dedicated_value
local -a shared_keys=(
GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS
GENARRATIVE_BGFILTER_WORKER_BASE_URL
GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE
GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS
ALIYUN_OSS_BUCKET
ALIYUN_OSS_ENDPOINT
)
if [[ "${DRY_RUN}" == "true" ]]; then
echo "+ validate ${label} BgFilter shared configuration alignment with ${API_ENV_FILE}"
return
fi
if [[ ! -f "${env_file}" ]]; then
echo "[server-provision] ${label} 不存在,无法检查 BgFilter 共享配置: ${env_file}" >&2
exit 1
fi
if [[ "${include_provider_credentials}" == "true" ]]; then
shared_keys+=(
GENARRATIVE_EDITOR_BGFILTER_BASE_URL
GENARRATIVE_EDITOR_BGFILTER_TOKEN
ALIYUN_OSS_ACCESS_KEY_ID
ALIYUN_OSS_ACCESS_KEY_SECRET
ALIYUN_OSS_READ_EXPIRE_SECONDS
)
fi
for key in "${shared_keys[@]}"; do
if ! env_has_assignment "${env_file}" "${key}"; then
continue
fi
dedicated_value="$(read_effective_env_value "${env_file}" "${key}")"
shared_value="$(read_effective_env_value "${API_ENV_FILE}" "${key}")"
if [[ "${dedicated_value}" != "${shared_value}" ]]; then
echo "[server-provision] ${label} 中的 BgFilter 共享配置与 API env 不一致: ${key};后加载 env 会覆盖进程有效值。" >&2
exit 1
fi
done
}
validate_bgfilter_loopback_endpoint_alignment() {
local base_url host port expected_base_url
if [[ "${DRY_RUN}" == "true" ]]; then
echo "+ validate BgFilter parent base URL and child listener alignment"
return
fi
base_url="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL")"
host="$(read_effective_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_HOST")"
port="$(read_effective_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_PORT")"
if [[ "${host}" != "127.0.0.1" ]]; then
echo "[server-provision] BgFilter worker 首版必须监听 127.0.0.1,当前 GENARRATIVE_BGFILTER_WORKER_HOST=${host:-<empty>}" >&2
exit 1
fi
if [[ ! "${port}" =~ ^[1-9][0-9]{0,4}$ ]] || (( 10#${port} > 65535 )); then
echo "[server-provision] GENARRATIVE_BGFILTER_WORKER_PORT 必须是 1-65535 的有效端口: ${BGFILTER_WORKER_ENV_FILE}" >&2
exit 1
fi
expected_base_url="http://${host}:${port}"
if [[ "${base_url%/}" != "${expected_base_url}" ]]; then
echo "[server-provision] 父进程 GENARRATIVE_BGFILTER_WORKER_BASE_URL 必须与 BgFilter worker 有效监听地址一致: expected=${expected_base_url}, actual=${base_url:-<empty>}" >&2
exit 1
fi
}
ensure_worker_runtime_env_defaults() {
@@ -547,6 +753,80 @@ ensure_worker_runtime_env_defaults() {
ensure_runtime_bootstrap_secret_file_env "${WORKER_ENV_FILE}"
}
ensure_bgfilter_worker_runtime_env_defaults() {
if [[ "${DRY_RUN}" == "true" ]]; then
echo "+ ensure BgFilter worker runtime env defaults in ${BGFILTER_WORKER_ENV_FILE}"
return
fi
if [[ ! -f "${BGFILTER_WORKER_ENV_FILE}" ]]; then
echo "[server-provision] BgFilter worker 环境文件不存在,无法补齐运行态变量: ${BGFILTER_WORKER_ENV_FILE}" >&2
exit 1
fi
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_HOST" "127.0.0.1"
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_PORT" "8083"
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY" "4"
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS" "128"
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
}
ensure_bgfilter_internal_token_file() {
local token_file="/etc/genarrative/secrets/bgfilter-worker.token"
local token_dir="/etc/genarrative/secrets"
local openssl_bin="${GENARRATIVE_OPENSSL_PREFIX}/bin/openssl"
local configured_token_file temporary_file token_metadata
if [[ "${DRY_RUN}" == "true" ]]; then
echo "+ ensure shared BgFilter internal token file ${token_file} (root:genarrative 0440)"
return
fi
configured_token_file="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE")"
if [[ "${configured_token_file}" != "${token_file}" ]]; then
echo "[server-provision] GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE 必须与 Provision 管理路径一致: ${token_file}" >&2
exit 1
fi
if [[ -L "${token_dir}" || -L "${token_file}" ]]; then
echo "[server-provision] BgFilter 内部 Token 目录和文件不能是符号链接: ${token_file}" >&2
exit 1
fi
if [[ -e "${token_file}" && ! -f "${token_file}" ]]; then
echo "[server-provision] BgFilter 内部 Token 必须是普通文件: ${token_file}" >&2
exit 1
fi
install -d -o root -g genarrative -m 0750 "${token_dir}"
if [[ ! -f "${token_file}" ]]; then
temporary_file="$(mktemp "${token_dir}/.bgfilter-worker.token.XXXXXX")"
if ! "${openssl_bin}" rand -hex 32 >"${temporary_file}"; then
rm -f "${temporary_file}"
echo "[server-provision] 生成 BgFilter 内部 Token 失败。" >&2
exit 1
fi
if ! grep -q '[^[:space:]]' -- "${temporary_file}"; then
rm -f "${temporary_file}"
echo "[server-provision] 生成的 BgFilter 内部 Token 不得为空或只包含空白字符。" >&2
exit 1
fi
chown root:genarrative "${temporary_file}"
chmod 0440 "${temporary_file}"
mv -T "${temporary_file}" "${token_file}"
echo "[server-provision] 已生成 BgFilter 内部 Token 文件: ${token_file}"
else
if ! grep -q '[^[:space:]]' -- "${token_file}"; then
echo "[server-provision] BgFilter 内部 Token 文件不得为空或只包含空白字符: ${token_file}" >&2
exit 1
fi
chown root:genarrative "${token_file}"
chmod 0440 "${token_file}"
echo "[server-provision] BgFilter 内部 Token 文件已存在,保留内容并收紧权限。"
fi
token_metadata="$(stat -c '%U:%G:%a' -- "${token_file}")"
if [[ "${token_metadata}" != "root:genarrative:440" ]]; then
echo "[server-provision] BgFilter 内部 Token 权限必须为 root:genarrative 0440: ${token_file} (${token_metadata})" >&2
exit 1
fi
}
parse_json_string_field() {
local json="$1"
local key="$2"
@@ -684,6 +964,10 @@ render_external_generation_controller_env_example() {
cat deploy/env/external-generation-controller.env.example
}
render_bgfilter_worker_env_example() {
cat deploy/env/bgfilter-worker.env.example
}
render_otelcol_service() {
cat deploy/systemd/otelcol-contrib.service
}
@@ -927,6 +1211,35 @@ render_external_generation_controller_service() {
deploy/systemd/genarrative-external-generation-controller.service
}
render_bgfilter_worker_service() {
local current_escaped api_env_escaped bgfilter_env_escaped
current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
bgfilter_env_escaped="$(escape_sed_replacement "${BGFILTER_WORKER_ENV_FILE}")"
sed \
-e "s|/opt/genarrative/current|${current_escaped}|g" \
-e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \
-e "s|/etc/genarrative/bgfilter-worker.env|${bgfilter_env_escaped}|g" \
deploy/systemd/genarrative-bgfilter-worker.service
}
wait_for_bgfilter_worker_service() {
if [[ "${DRY_RUN}" == "true" ]]; then
echo "+ curl -fsS --max-time 2 http://127.0.0.1:8083/readyz"
return
fi
echo "[server-provision] 等待 BgFilter worker readiness。"
for _ in {1..30}; do
if systemctl is-active --quiet genarrative-bgfilter-worker.service && curl -fsS --max-time 2 http://127.0.0.1:8083/readyz >/dev/null; then
return
fi
sleep 2
done
systemctl --no-pager --full status genarrative-bgfilter-worker.service || true
echo "[server-provision] BgFilter worker 未在超时时间内通过 readiness。" >&2
exit 1
}
render_database_backup_service() {
local current_escaped env_escaped
current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
@@ -995,6 +1308,7 @@ require_path deploy/systemd/spacetimedb.service
require_path deploy/systemd/genarrative-api.service
require_path deploy/systemd/genarrative-external-generation-worker@.service
require_path deploy/systemd/genarrative-external-generation-controller.service
require_path deploy/systemd/genarrative-bgfilter-worker.service
require_path deploy/systemd/genarrative-database-backup.service
require_path deploy/systemd/genarrative-database-backup-files-history.conf
require_path deploy/systemd/genarrative-database-backup.timer
@@ -1013,6 +1327,7 @@ require_path deploy/logrotate/genarrative-pingora-gateway
require_path deploy/env/api-server.env.example
require_path deploy/env/external-generation-worker.env.example
require_path deploy/env/external-generation-controller.env.example
require_path deploy/env/bgfilter-worker.env.example
require_path scripts/deploy/maintenance-on.sh
require_path scripts/deploy/maintenance-off.sh
require_path scripts/deploy/maintenance-status.sh
@@ -1026,7 +1341,7 @@ echo "[server-provision] target=${DEPLOY_TARGET}, dry_run=${DRY_RUN}, nginx_conf
run_cmd id
require_root_for_real_provision
install_nginx_brotli_modules
run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /etc/genarrative/pingora /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/wallet-refund-outbox /var/lib/genarrative/editor-generation-pricing /var/lib/genarrative/spacetime /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol /var/log/genarrative
run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /etc/genarrative/pingora /etc/genarrative/secrets /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/wallet-refund-outbox /var/lib/genarrative/editor-generation-pricing /var/lib/genarrative/spacetime /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol /var/log/genarrative
if ! id spacetimedb >/dev/null 2>&1; then
run_cmd useradd --system --home-dir "${SPACETIME_ROOT}" --shell /usr/sbin/nologin spacetimedb
@@ -1069,18 +1384,21 @@ spacetimedb_service="$(mktemp)"
api_service="$(mktemp)"
external_generation_worker_service="$(mktemp)"
external_generation_controller_service="$(mktemp)"
bgfilter_worker_service="$(mktemp)"
database_backup_service="$(mktemp)"
health_patrol_service="$(mktemp)"
render_spacetimedb_service >"${spacetimedb_service}"
render_api_service >"${api_service}"
render_external_generation_worker_service >"${external_generation_worker_service}"
render_external_generation_controller_service >"${external_generation_controller_service}"
render_bgfilter_worker_service >"${bgfilter_worker_service}"
render_database_backup_service >"${database_backup_service}"
render_health_patrol_service >"${health_patrol_service}"
install_file "${spacetimedb_service}" /etc/systemd/system/spacetimedb.service 0644
install_file "${api_service}" /etc/systemd/system/genarrative-api.service 0644
install_file "${external_generation_worker_service}" /etc/systemd/system/genarrative-external-generation-worker@.service 0644
install_file "${external_generation_controller_service}" /etc/systemd/system/genarrative-external-generation-controller.service 0644
install_file "${bgfilter_worker_service}" /etc/systemd/system/genarrative-bgfilter-worker.service 0644
install_file "${database_backup_service}" /etc/systemd/system/genarrative-database-backup.service 0644
install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644
install_file "${health_patrol_service}" /etc/systemd/system/genarrative-health-patrol.service 0644
@@ -1088,7 +1406,7 @@ install_file deploy/systemd/genarrative-health-patrol.timer /etc/systemd/system/
install_file deploy/systemd/genarrative-pingora-gateway.service /etc/systemd/system/genarrative-pingora-gateway.service 0644
install_file deploy/systemd/genarrative-pingora-gateway-direct-entry.conf /etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf 0644
install_file deploy/logrotate/genarrative-pingora-gateway /etc/logrotate.d/genarrative-pingora-gateway 0644
rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${external_generation_controller_service}" "${database_backup_service}" "${health_patrol_service}"
rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${external_generation_controller_service}" "${bgfilter_worker_service}" "${database_backup_service}" "${health_patrol_service}"
if [[ ! -f "${API_ENV_FILE}" ]]; then
echo "+ create ${API_ENV_FILE} from example"
@@ -1101,6 +1419,7 @@ else
echo "[server-provision] 已存在环境文件,保留不覆盖: ${API_ENV_FILE}"
fi
ensure_api_runtime_env_defaults
validate_bgfilter_shared_runtime_env
configure_database_backup_profile
if [[ ! -f "${WORKER_ENV_FILE}" ]]; then
@@ -1115,6 +1434,23 @@ else
fi
ensure_worker_runtime_env_defaults
if [[ ! -f "${BGFILTER_WORKER_ENV_FILE}" ]]; then
echo "+ create ${BGFILTER_WORKER_ENV_FILE} from example"
if [[ "${DRY_RUN}" != "true" ]]; then
render_bgfilter_worker_env_example >"${BGFILTER_WORKER_ENV_FILE}"
chmod 0600 "${BGFILTER_WORKER_ENV_FILE}"
chown root:root "${BGFILTER_WORKER_ENV_FILE}"
fi
else
echo "[server-provision] 已存在 BgFilter worker 环境文件,保留不覆盖: ${BGFILTER_WORKER_ENV_FILE}"
fi
ensure_bgfilter_worker_runtime_env_defaults
validate_bgfilter_env_file_alignment "${WORKER_ENV_FILE}" "外部生成 worker env" "false"
validate_bgfilter_env_file_alignment "${BGFILTER_WORKER_ENV_FILE}" "BgFilter 专属 env" "true"
validate_bgfilter_loopback_endpoint_alignment
validate_no_bgfilter_internal_token_plaintext
ensure_bgfilter_internal_token_file
if [[ ! -f "${CONTROLLER_ENV_FILE}" ]]; then
echo "+ create ${CONTROLLER_ENV_FILE} from example"
if [[ "${DRY_RUN}" != "true" ]]; then
@@ -1149,7 +1485,7 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
run_cmd systemctl enable otelcol-contrib.service
fi
stamp_database_backup_timer_now
run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service genarrative-health-patrol.timer
run_cmd systemctl enable spacetimedb.service genarrative-bgfilter-worker.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service genarrative-health-patrol.timer
run_cmd systemctl start genarrative-database-backup.timer
if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
run_cmd systemctl restart otelcol-contrib.service
@@ -1158,13 +1494,17 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
wait_for_spacetimedb_service
ensure_spacetime_owner_client_token
if [[ -x "${CURRENT_LINK}/api-server" ]]; then
run_cmd systemctl enable genarrative-bgfilter-worker.service
run_cmd systemctl stop genarrative-bgfilter-worker.service
run_cmd systemctl start genarrative-bgfilter-worker.service
wait_for_bgfilter_worker_service
run_cmd systemctl restart genarrative-api.service
run_cmd systemctl enable --now genarrative-external-generation-worker@1.service
run_cmd systemctl restart genarrative-external-generation-worker@1.service
run_cmd systemctl enable --now genarrative-external-generation-controller.service
run_cmd systemctl restart genarrative-external-generation-controller.service
else
echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server,跳过 api-server、外部生成 worker 和 controller 首次启动。后续 API deploy 会启用并启动默认 worker 与 controller。"
echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server,跳过 BgFilter worker、api-server、外部生成 worker 和 controller 首次启动。后续 API deploy 会按顺序启动。"
fi
fi
+16
View File
@@ -18,12 +18,14 @@ const DEFAULT_PUBLIC_PATHS = [
const DEFAULT_SERVICES = [
'genarrative-api.service',
'genarrative-bgfilter-worker.service',
'genarrative-external-generation-controller.service',
'spacetimedb.service',
'nginx.service',
];
const PINGORA_DIRECT_SERVICES = [
'genarrative-api.service',
'genarrative-bgfilter-worker.service',
'genarrative-external-generation-controller.service',
'spacetimedb.service',
'genarrative-pingora-gateway.service',
@@ -37,6 +39,7 @@ function usage() {
Options:
--api-base-url <url> API direct base URL, default http://127.0.0.1:8082
--bgfilter-base-url <url> BgFilter worker base URL, default http://127.0.0.1:8083
--spacetime-base-url <url> SpacetimeDB base URL, default http://127.0.0.1:3101
--public-base-url <url> Nginx/public base URL, default http://127.0.0.1
--public-host <host> Optional public Host header, useful when probing 127.0.0.1
@@ -88,6 +91,9 @@ function parseArgs(argv) {
apiBaseUrl:
process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL ||
'http://127.0.0.1:8082',
bgfilterBaseUrl:
process.env.GENARRATIVE_HEALTH_PATROL_BGFILTER_BASE_URL ||
'http://127.0.0.1:8083',
spacetimeBaseUrl:
process.env.GENARRATIVE_HEALTH_PATROL_SPACETIME_BASE_URL ||
'http://127.0.0.1:3101',
@@ -131,6 +137,9 @@ function parseArgs(argv) {
case '--api-base-url':
config.apiBaseUrl = requireValue(argv, ++index, arg);
break;
case '--bgfilter-base-url':
config.bgfilterBaseUrl = requireValue(argv, ++index, arg);
break;
case '--spacetime-base-url':
config.spacetimeBaseUrl = requireValue(argv, ++index, arg);
break;
@@ -693,6 +702,13 @@ async function main() {
config,
),
);
checks.push(
await checkHttp(
'bgfilter:/readyz',
joinUrl(config.bgfilterBaseUrl, '/readyz'),
config,
),
);
checks.push(
await checkHttp(
'spacetimedb:/v1/ping',