落地 BgFilter 单实例受限资源 Worker
新增私有 BgFilter worker,提供内部鉴权、并发限流、超时、顺序重试、flat 熔断和图片校验。 父生成流程改为通过内部二进制 HTTP 原地等待,并保留 flat 降级与 complex 失败语义。 接入本地开发、systemd、生产部署、Provision、健康巡检和配置漂移门禁。 补充动画、部署与运维测试、Linux fixture 隔离以及对应架构文档。
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -258,6 +258,42 @@ function assertApiReleaseContainsPingoraDirectDependencies() {
|
||||
),
|
||||
'API release 必须包含外部生成 worker controller systemd 单元。',
|
||||
);
|
||||
assertFileExists(
|
||||
path.join(
|
||||
releaseDir,
|
||||
'deploy/systemd/genarrative-bgfilter-worker.service',
|
||||
),
|
||||
'API release 必须包含唯一 BgFilter worker systemd 单元。',
|
||||
);
|
||||
assertFileExists(
|
||||
path.join(releaseDir, 'deploy/env/bgfilter-worker.env.example'),
|
||||
'API release 必须包含 BgFilter worker env 示例。',
|
||||
);
|
||||
const bgfilterUnit = readFileSync(
|
||||
path.join(
|
||||
releaseDir,
|
||||
'deploy/systemd/genarrative-bgfilter-worker.service',
|
||||
),
|
||||
'utf8',
|
||||
);
|
||||
const sharedEnvIndex = bgfilterUnit.indexOf(
|
||||
'EnvironmentFile=/etc/genarrative/api-server.env',
|
||||
);
|
||||
const dedicatedEnvIndex = bgfilterUnit.indexOf(
|
||||
'EnvironmentFile=/etc/genarrative/bgfilter-worker.env',
|
||||
);
|
||||
if (
|
||||
sharedEnvIndex < 0 ||
|
||||
dedicatedEnvIndex < 0 ||
|
||||
sharedEnvIndex > dedicatedEnvIndex
|
||||
) {
|
||||
failures.push('API release 的 BgFilter unit 必须按共享 env → 专属 env 加载。');
|
||||
}
|
||||
assertIncludes(
|
||||
bgfilterUnit,
|
||||
'TimeoutStopSec=900',
|
||||
'API release 的 BgFilter unit 必须给最多 600s 的内部请求预算留足优雅排空时间。',
|
||||
);
|
||||
assertFileExists(
|
||||
path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'),
|
||||
'API release 必须包含 Pingora env 示例。',
|
||||
|
||||
@@ -54,6 +54,11 @@ function assertPublicBaseUrlDefaultsToGatewayEntry() {
|
||||
"process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n 'http://127.0.0.1'",
|
||||
'publicBaseUrl 默认必须指向本机网关入口,不能回落到 API 直连端口。',
|
||||
);
|
||||
assertIncludes(
|
||||
script,
|
||||
"process.env.GENARRATIVE_HEALTH_PATROL_BGFILTER_BASE_URL ||\n 'http://127.0.0.1:8083'",
|
||||
'BgFilter worker 巡检默认必须指向唯一实例的 loopback 端口。',
|
||||
);
|
||||
if (
|
||||
script.includes(
|
||||
'process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL',
|
||||
@@ -85,6 +90,14 @@ async function assertNginxModeChecksNginxService() {
|
||||
'systemctl is-active nginx.service',
|
||||
'nginx gateway mode 必须检查 nginx.service。',
|
||||
);
|
||||
assertIncludes(
|
||||
commandsLog,
|
||||
'systemctl is-active genarrative-bgfilter-worker.service',
|
||||
'生产巡检必须检查唯一 BgFilter worker service。',
|
||||
);
|
||||
if (!payload.checks.some((check) => check.name === 'bgfilter:/readyz')) {
|
||||
failures.push('生产巡检必须探测 BgFilter worker /readyz。');
|
||||
}
|
||||
if (commandsLog.includes('genarrative-pingora-gateway.service')) {
|
||||
failures.push(
|
||||
'nginx gateway mode 不应要求 Pingora gateway service active。',
|
||||
@@ -369,6 +382,8 @@ async function runPatrol(fixture, args) {
|
||||
'scripts/ops/production-health-patrol.mjs',
|
||||
'--api-base-url',
|
||||
fixture.baseUrl,
|
||||
'--bgfilter-base-url',
|
||||
fixture.baseUrl,
|
||||
'--spacetime-base-url',
|
||||
fixture.baseUrl,
|
||||
'--public-base-url',
|
||||
|
||||
@@ -1647,6 +1647,50 @@ const checks = [
|
||||
includes: 'genarrative-external-generation-worker@1.service',
|
||||
reason: 'Server-Provision 必须启用外部生成保底 worker 实例。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-bgfilter-worker.service',
|
||||
includes: 'TimeoutStopSec=900',
|
||||
reason:
|
||||
'BgFilter worker 必须给最多 600s 的内部请求预算留足优雅排空时间,不能沿用 systemd 默认停止窗口。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/jenkins-server-provision.sh',
|
||||
includes: 'validate_no_bgfilter_internal_token_plaintext',
|
||||
reason:
|
||||
'Server-Provision 必须拒绝 API 或 BgFilter worker env 保存内部 Token 明文。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/jenkins-server-provision.sh',
|
||||
includes:
|
||||
'for env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"; do',
|
||||
reason:
|
||||
'Server-Provision 必须同时拒绝 external-generation-worker.env 保存 BgFilter 内部 Token 明文。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/jenkins-server-provision.sh',
|
||||
includes:
|
||||
'validate_bgfilter_env_file_alignment "${WORKER_ENV_FILE}" "外部生成 worker env" "false"',
|
||||
reason:
|
||||
'Server-Provision 启动外部生成 worker 前必须拒绝 BgFilter URL、Token 文件、timeout 与 OSS 位置漂移。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/jenkins-server-provision.sh',
|
||||
includes: 'validate_bgfilter_loopback_endpoint_alignment',
|
||||
reason:
|
||||
'Server-Provision 启动 BgFilter worker 前必须确认父 base URL 与子 listener 指向同一 loopback endpoint。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/jenkins-server-provision.sh',
|
||||
includes: 'BgFilter 内部 Token 文件不得为空或只包含空白字符',
|
||||
reason:
|
||||
'Server-Provision 必须拒绝仅含空白字符的 BgFilter 内部 Token 文件。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/jenkins-server-provision.sh',
|
||||
includes: "root:genarrative:440",
|
||||
reason:
|
||||
'Server-Provision 必须复核 BgFilter 内部 Token 文件的 owner、group 与 0440 权限。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-api-deploy.sh',
|
||||
includes: 'ensure_default_worker_service',
|
||||
|
||||
@@ -5,11 +5,11 @@ set -euo pipefail
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法:
|
||||
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode]
|
||||
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url http://127.0.0.1:8083/readyz] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode]
|
||||
|
||||
说明:
|
||||
进入维护模式,校验并发布 api-server 单文件,更新 current 链接,重启 systemd 服务并执行 readiness 检查。
|
||||
默认同时重启外部生成 worker controller 和已加载的 worker 实例;未启用 worker 单元时会自动跳过。
|
||||
默认先停止、启动并验活唯一 BgFilter worker,再重启 API、外部生成 worker controller 和已加载的 worker 实例。
|
||||
若传入 --database,会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件,避免服务继续读取旧库。
|
||||
若发布包包含 pingora-gateway,或传入 --require-pingora-gateway,部署脚本会要求 release manifest、二进制与 checksum 一致,再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置,启动或重启 Pingora 影子服务并复核 active。
|
||||
默认在 readiness 通过后退出维护模式;传入 --keep-maintenance-mode 时保留维护文件,供人工验收后再恢复公网。
|
||||
@@ -175,6 +175,81 @@ if matched_value is not None:
|
||||
fi
|
||||
}
|
||||
|
||||
env_contains_nonempty_assignment() {
|
||||
local file_path="$1"
|
||||
local key="$2"
|
||||
|
||||
if [[ ! -f "${file_path}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local python_script='
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
key = sys.argv[2]
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
current_key, value = line.split("=", 1)
|
||||
if current_key.strip() != key:
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("\"", chr(39)):
|
||||
value = value[1:-1]
|
||||
if value.strip():
|
||||
raise SystemExit(0)
|
||||
raise SystemExit(1)
|
||||
'
|
||||
|
||||
if [[ -r "${file_path}" ]]; then
|
||||
python3 -c "${python_script}" "${file_path}" "${key}"
|
||||
else
|
||||
if ! sudo -n true >/dev/null 2>&1; then
|
||||
echo "[production-api-deploy] 当前用户无权读取 ${file_path},且 sudo -n 不可用;无法检查运行态环境变量。" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo -n python3 -c "${python_script}" "${file_path}" "${key}"
|
||||
fi
|
||||
}
|
||||
|
||||
env_has_assignment() {
|
||||
local file_path="$1"
|
||||
local key="$2"
|
||||
|
||||
if [[ ! -f "${file_path}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local python_script='
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
key = sys.argv[2]
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
current_key, _ = line.split("=", 1)
|
||||
if current_key.strip() == key:
|
||||
raise SystemExit(0)
|
||||
raise SystemExit(1)
|
||||
'
|
||||
|
||||
if [[ -r "${file_path}" ]]; then
|
||||
python3 -c "${python_script}" "${file_path}" "${key}"
|
||||
else
|
||||
if ! sudo -n true >/dev/null 2>&1; then
|
||||
echo "[production-api-deploy] 当前用户无权读取 ${file_path},且 sudo -n 不可用;无法检查运行态环境变量。" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo -n python3 -c "${python_script}" "${file_path}" "${key}"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_env_value() {
|
||||
local file_path="$1"
|
||||
local key="$2"
|
||||
@@ -303,10 +378,11 @@ ensure_runtime_env_and_dirs() {
|
||||
ensure_env_value_migrates_old_default "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL" "http://127.0.0.1:8083"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE" "/etc/genarrative/secrets/bgfilter-worker.token"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS" "2000"
|
||||
ensure_runtime_bootstrap_secret_file_env "${api_env_file}"
|
||||
ensure_env_value_migrates_old_default "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS" "45000" "180000"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
|
||||
|
||||
tracking_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED")"
|
||||
tracking_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR")"
|
||||
@@ -351,6 +427,186 @@ ensure_worker_runtime_env_defaults() {
|
||||
ensure_runtime_bootstrap_secret_file_env "${worker_env_file}"
|
||||
}
|
||||
|
||||
ensure_bgfilter_worker_runtime_env_defaults() {
|
||||
local bgfilter_env_file="$1"
|
||||
|
||||
if [[ -z "${bgfilter_env_file}" ]]; then
|
||||
return
|
||||
fi
|
||||
if [[ ! -f "${bgfilter_env_file}" ]]; then
|
||||
echo "[production-api-deploy] BgFilter worker 环境文件不存在: ${bgfilter_env_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_HOST" "127.0.0.1"
|
||||
ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_PORT" "8083"
|
||||
ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
|
||||
ensure_env_value "${bgfilter_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
|
||||
}
|
||||
|
||||
validate_bgfilter_shared_runtime_env() {
|
||||
local api_env_file="$1"
|
||||
local request_timeout_ms connect_timeout_ms
|
||||
|
||||
request_timeout_ms="$(read_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS")"
|
||||
if [[ ! "${request_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "[production-api-deploy] GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${api_env_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
connect_timeout_ms="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS")"
|
||||
if [[ ! "${connect_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${api_env_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_bgfilter_worker_capacity() {
|
||||
local bgfilter_env_file="$1"
|
||||
local concurrency max_requests
|
||||
|
||||
concurrency="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY")"
|
||||
max_requests="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS")"
|
||||
if [[ ! "${concurrency}" =~ ^[1-9][0-9]{0,8}$ ]]; then
|
||||
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_CONCURRENCY 必须是正整数: ${bgfilter_env_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! "${max_requests}" =~ ^[1-9][0-9]{0,8}$ ]]; then
|
||||
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS 必须是正整数: ${bgfilter_env_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
if (( 10#${max_requests} < 10#${concurrency} )); then
|
||||
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS 必须大于或等于 CONCURRENCY: ${bgfilter_env_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_bgfilter_worker_shared_env_alignment() {
|
||||
local api_env_file="$1"
|
||||
local bgfilter_env_file="$2"
|
||||
local key shared_value dedicated_value
|
||||
|
||||
for key in \
|
||||
GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS \
|
||||
GENARRATIVE_EDITOR_BGFILTER_BASE_URL \
|
||||
GENARRATIVE_EDITOR_BGFILTER_TOKEN \
|
||||
GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE \
|
||||
ALIYUN_OSS_BUCKET \
|
||||
ALIYUN_OSS_ENDPOINT \
|
||||
ALIYUN_OSS_ACCESS_KEY_ID \
|
||||
ALIYUN_OSS_ACCESS_KEY_SECRET \
|
||||
ALIYUN_OSS_READ_EXPIRE_SECONDS; do
|
||||
if ! env_has_assignment "${bgfilter_env_file}" "${key}"; then
|
||||
continue
|
||||
fi
|
||||
dedicated_value="$(read_env_value "${bgfilter_env_file}" "${key}")"
|
||||
shared_value="$(read_env_value "${api_env_file}" "${key}")"
|
||||
if [[ "${dedicated_value}" != "${shared_value}" ]]; then
|
||||
echo "[production-api-deploy] BgFilter 专属 env 中的共享配置与 API env 不一致: ${key};请迁移到 ${api_env_file} 并从 ${bgfilter_env_file} 删除重复项。" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
validate_external_generation_worker_bgfilter_env_alignment() {
|
||||
local api_env_file="$1"
|
||||
local worker_env_file="$2"
|
||||
local key shared_value worker_value
|
||||
|
||||
if [[ -z "${worker_env_file}" || ! -f "${worker_env_file}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for key in \
|
||||
GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS \
|
||||
GENARRATIVE_BGFILTER_WORKER_BASE_URL \
|
||||
GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE \
|
||||
GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS \
|
||||
ALIYUN_OSS_BUCKET \
|
||||
ALIYUN_OSS_ENDPOINT; do
|
||||
if ! env_has_assignment "${worker_env_file}" "${key}"; then
|
||||
continue
|
||||
fi
|
||||
worker_value="$(read_env_value "${worker_env_file}" "${key}")"
|
||||
shared_value="$(read_env_value "${api_env_file}" "${key}")"
|
||||
if [[ "${worker_value}" != "${shared_value}" ]]; then
|
||||
echo "[production-api-deploy] 外部生成 worker env 中的 BgFilter 共享配置与 API env 不一致: ${key};${worker_env_file} 会在 systemd 中后加载并覆盖父侧有效值。" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
validate_bgfilter_loopback_endpoint_alignment() {
|
||||
local api_env_file="$1"
|
||||
local bgfilter_env_file="$2"
|
||||
local health_url="$3"
|
||||
local base_url host port expected_base_url expected_health_url
|
||||
|
||||
base_url="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL")"
|
||||
host="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_HOST")"
|
||||
port="$(read_env_value "${bgfilter_env_file}" "GENARRATIVE_BGFILTER_WORKER_PORT")"
|
||||
|
||||
if [[ "${host}" != "127.0.0.1" ]]; then
|
||||
echo "[production-api-deploy] BgFilter worker 首版必须监听 127.0.0.1,当前 GENARRATIVE_BGFILTER_WORKER_HOST=${host:-<empty>}: ${bgfilter_env_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! "${port}" =~ ^[1-9][0-9]{0,4}$ ]] || (( 10#${port} > 65535 )); then
|
||||
echo "[production-api-deploy] GENARRATIVE_BGFILTER_WORKER_PORT 必须是 1-65535 的有效端口: ${bgfilter_env_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
expected_base_url="http://${host}:${port}"
|
||||
if [[ "${base_url%/}" != "${expected_base_url}" ]]; then
|
||||
echo "[production-api-deploy] 父进程 GENARRATIVE_BGFILTER_WORKER_BASE_URL 必须与 BgFilter worker 有效监听地址一致: expected=${expected_base_url}, actual=${base_url:-<empty>}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
expected_health_url="${expected_base_url}/readyz"
|
||||
if [[ "${health_url}" != "${expected_health_url}" ]]; then
|
||||
echo "[production-api-deploy] --bgfilter-worker-health-url 必须与父进程 base URL 和子 worker listener 指向同一 loopback endpoint: expected=${expected_health_url}, actual=${health_url:-<empty>}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_no_bgfilter_internal_token_plaintext() {
|
||||
local env_file
|
||||
|
||||
for env_file in "$@"; do
|
||||
if [[ -z "${env_file}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if env_contains_nonempty_assignment "${env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN"; then
|
||||
echo "[production-api-deploy] ${env_file} 不得保存 GENARRATIVE_BGFILTER_INTERNAL_TOKEN 明文;生产环境只允许使用 GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE。" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
validate_bgfilter_internal_token_file() {
|
||||
local api_env_file="$1"
|
||||
local token_file token_metadata
|
||||
|
||||
token_file="$(read_env_value "${api_env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE")"
|
||||
if [[ -z "${token_file}" || "${token_file}" != /* ]]; then
|
||||
echo "[production-api-deploy] GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE 必须指向绝对路径: ${api_env_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -L "${token_file}" || ! -f "${token_file}" || ! -s "${token_file}" ]]; then
|
||||
echo "[production-api-deploy] BgFilter 内部 Token 必须是非空普通文件且不能是符号链接: ${token_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! run_privileged grep -q '[^[:space:]]' -- "${token_file}"; then
|
||||
echo "[production-api-deploy] BgFilter 内部 Token 文件必须至少包含一个非空白字符: ${token_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
token_metadata="$(run_privileged stat -c '%U:%G:%a' -- "${token_file}")"
|
||||
if [[ "${token_metadata}" != "root:genarrative:440" ]]; then
|
||||
echo "[production-api-deploy] BgFilter 内部 Token 权限必须为 root:genarrative 0440,且必须是普通文件: ${token_file} (${token_metadata})" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
extract_pingora_env_files_from_unit() {
|
||||
local service_name="$1"
|
||||
local unit_content
|
||||
@@ -572,8 +828,17 @@ install_worker_systemd_units() {
|
||||
local release_dir="$1"
|
||||
local pattern="$2"
|
||||
local controller_service="$3"
|
||||
local bgfilter_service="$4"
|
||||
local installed_any=0
|
||||
|
||||
if [[ "${bgfilter_service}" == "genarrative-bgfilter-worker.service" ]]; then
|
||||
install_release_systemd_unit \
|
||||
"${release_dir}/deploy/systemd/genarrative-bgfilter-worker.service" \
|
||||
"genarrative-bgfilter-worker.service" \
|
||||
"BgFilter worker systemd 单元"
|
||||
installed_any=1
|
||||
fi
|
||||
|
||||
if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" ]]; then
|
||||
install_release_systemd_unit \
|
||||
"${release_dir}/deploy/systemd/genarrative-external-generation-worker@.service" \
|
||||
@@ -688,6 +953,38 @@ wait_for_worker_controller_service() {
|
||||
return 1
|
||||
}
|
||||
|
||||
restart_and_wait_for_bgfilter_worker() {
|
||||
local service="$1"
|
||||
local health_url="$2"
|
||||
|
||||
if [[ -z "${service}" ]]; then
|
||||
echo "[production-api-deploy] 跳过 BgFilter worker 启动。"
|
||||
return 0
|
||||
fi
|
||||
if ! systemctl cat "${service}" >/dev/null 2>&1; then
|
||||
echo "[production-api-deploy] 缺少 BgFilter worker systemd 单元: ${service}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "[production-api-deploy] 停止旧 BgFilter worker 并等待在途请求排空: ${service}"
|
||||
systemctl stop "${service}"
|
||||
systemctl enable "${service}"
|
||||
echo "[production-api-deploy] 启动唯一 BgFilter worker: ${service}"
|
||||
systemctl start "${service}"
|
||||
|
||||
for _ in {1..30}; do
|
||||
if systemctl is-active --quiet "${service}" && curl -fsS --max-time 2 "${health_url}" >/dev/null; then
|
||||
echo "[production-api-deploy] BgFilter worker readiness 通过: ${health_url}"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
systemctl --no-pager --full status "${service}" || true
|
||||
echo "[production-api-deploy] BgFilter worker readiness 检查超时: ${health_url}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SOURCE_DIR=""
|
||||
VERSION=""
|
||||
@@ -697,6 +994,9 @@ SERVICE_NAME="genarrative-api.service"
|
||||
PINGORA_SERVICE_NAME="genarrative-pingora-gateway.service"
|
||||
WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
|
||||
WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service"
|
||||
BGFILTER_WORKER_SERVICE="genarrative-bgfilter-worker.service"
|
||||
BGFILTER_WORKER_HEALTH_URL="http://127.0.0.1:8083/readyz"
|
||||
BGFILTER_WORKER_ENV_FILE="/etc/genarrative/bgfilter-worker.env"
|
||||
HEALTH_URL="http://127.0.0.1:8082/readyz"
|
||||
API_ENV_FILE="/etc/genarrative/api-server.env"
|
||||
WORKER_ENV_FILE="/etc/genarrative/external-generation-worker.env"
|
||||
@@ -766,6 +1066,23 @@ while [[ $# -gt 0 ]]; do
|
||||
WORKER_CONTROLLER_SERVICE=""
|
||||
shift
|
||||
;;
|
||||
--bgfilter-worker-service)
|
||||
BGFILTER_WORKER_SERVICE="${2:?缺少 --bgfilter-worker-service 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--bgfilter-worker-health-url)
|
||||
BGFILTER_WORKER_HEALTH_URL="${2:?缺少 --bgfilter-worker-health-url 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--bgfilter-worker-env-file)
|
||||
BGFILTER_WORKER_ENV_FILE="${2:?缺少 --bgfilter-worker-env-file 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--no-bgfilter-worker)
|
||||
BGFILTER_WORKER_SERVICE=""
|
||||
BGFILTER_WORKER_ENV_FILE=""
|
||||
shift
|
||||
;;
|
||||
--health-url)
|
||||
HEALTH_URL="${2:?缺少 --health-url 的值}"
|
||||
shift 2
|
||||
@@ -801,6 +1118,9 @@ require_absolute_path "${API_ENV_FILE}" "--api-env-file"
|
||||
if [[ -n "${WORKER_ENV_FILE}" ]]; then
|
||||
require_absolute_path "${WORKER_ENV_FILE}" "--worker-env-file"
|
||||
fi
|
||||
if [[ -n "${BGFILTER_WORKER_ENV_FILE}" ]]; then
|
||||
require_absolute_path "${BGFILTER_WORKER_ENV_FILE}" "--bgfilter-worker-env-file"
|
||||
fi
|
||||
|
||||
if [[ -n "${DATABASE}" ]]; then
|
||||
validate_spacetime_database_name "${DATABASE}"
|
||||
@@ -1127,8 +1447,20 @@ if [[ -n "${SPACETIME_SERVER_URL}" ]]; then
|
||||
fi
|
||||
|
||||
ensure_runtime_env_and_dirs "${API_ENV_FILE}"
|
||||
validate_bgfilter_shared_runtime_env "${API_ENV_FILE}"
|
||||
validate_real_wechat_pay_refund_reconciliation "${API_ENV_FILE}"
|
||||
ensure_worker_runtime_env_defaults "${WORKER_ENV_FILE}"
|
||||
ensure_bgfilter_worker_runtime_env_defaults "${BGFILTER_WORKER_ENV_FILE}"
|
||||
validate_external_generation_worker_bgfilter_env_alignment "${API_ENV_FILE}" "${WORKER_ENV_FILE}"
|
||||
validate_no_bgfilter_internal_token_plaintext "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"
|
||||
if [[ -n "${BGFILTER_WORKER_SERVICE}" ]]; then
|
||||
validate_bgfilter_internal_token_file "${API_ENV_FILE}"
|
||||
validate_bgfilter_loopback_endpoint_alignment "${API_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}" "${BGFILTER_WORKER_HEALTH_URL}"
|
||||
fi
|
||||
if [[ -n "${BGFILTER_WORKER_ENV_FILE}" ]]; then
|
||||
validate_bgfilter_worker_capacity "${BGFILTER_WORKER_ENV_FILE}"
|
||||
validate_bgfilter_worker_shared_env_alignment "${API_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"
|
||||
fi
|
||||
migrate_legacy_editor_generation_pricing_override "${CURRENT_LINK}"
|
||||
|
||||
if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
|
||||
@@ -1149,7 +1481,9 @@ if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
|
||||
ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}"
|
||||
fi
|
||||
|
||||
install_worker_systemd_units "${RELEASE_DIR}" "${WORKER_SERVICE_PATTERN}" "${WORKER_CONTROLLER_SERVICE}"
|
||||
install_worker_systemd_units "${RELEASE_DIR}" "${WORKER_SERVICE_PATTERN}" "${WORKER_CONTROLLER_SERVICE}" "${BGFILTER_WORKER_SERVICE}"
|
||||
|
||||
restart_and_wait_for_bgfilter_worker "${BGFILTER_WORKER_SERVICE}" "${BGFILTER_WORKER_HEALTH_URL}"
|
||||
|
||||
echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}"
|
||||
systemctl restart "${SERVICE_NAME}"
|
||||
|
||||
@@ -55,8 +55,8 @@ export function parsePortRangeSpec(value) {
|
||||
throw new Error(`端口段无效: ${spec},端口必须在 1024-65535 且起始不大于结束`);
|
||||
}
|
||||
|
||||
if (end - start + 1 < 4) {
|
||||
throw new Error(`端口段至少需要 4 个端口: ${spec}`);
|
||||
if (end - start + 1 < 5) {
|
||||
throw new Error(`端口段至少需要 5 个端口: ${spec}`);
|
||||
}
|
||||
|
||||
return {start, end, label: `${start}-${end}`};
|
||||
@@ -118,6 +118,7 @@ export function mapDevPortsToPortRange(portRange) {
|
||||
apiPort: normalizedRange.start + 1,
|
||||
spacetimePort: normalizedRange.start + 2,
|
||||
adminWebPort: normalizedRange.start + 3,
|
||||
bgfilterWorkerPort: normalizedRange.start + 4,
|
||||
range: normalizedRange,
|
||||
};
|
||||
}
|
||||
@@ -569,6 +570,7 @@ export async function resolveDevStackPorts(config) {
|
||||
['api', config.api],
|
||||
['web', config.web],
|
||||
['adminWeb', config.adminWeb],
|
||||
['bgfilterWorker', config.bgfilterWorker],
|
||||
].filter(([, portConfig]) => Boolean(portConfig));
|
||||
const result = {};
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ async function reserveConsecutivePorts() {
|
||||
}
|
||||
|
||||
describe('dev stack port utils', () => {
|
||||
it('解析端口段并映射到四个 dev 端口', () => {
|
||||
it('解析端口段并映射到五个 dev 端口', () => {
|
||||
expect(parsePortRangeSpec('10000-10099')).toEqual({
|
||||
start: 10000,
|
||||
end: 10099,
|
||||
@@ -58,7 +58,11 @@ describe('dev stack port utils', () => {
|
||||
apiPort: 10001,
|
||||
spacetimePort: 10002,
|
||||
adminWebPort: 10003,
|
||||
bgfilterWorkerPort: 10004,
|
||||
});
|
||||
expect(() => parsePortRangeSpec('10000-10003')).toThrow(
|
||||
'端口段至少需要 5 个端口',
|
||||
);
|
||||
});
|
||||
|
||||
it('使用端口可用性检查为被占用端口寻找后续可用端口', async () => {
|
||||
@@ -112,9 +116,10 @@ describe('dev stack port utils', () => {
|
||||
api: {host: '127.0.0.1', preferredPort: 0},
|
||||
web: {host: '127.0.0.1', preferredPort: 0},
|
||||
adminWeb: {host: '127.0.0.1', preferredPort: 0},
|
||||
bgfilterWorker: {host: '127.0.0.1', preferredPort: 0},
|
||||
});
|
||||
|
||||
expect(new Set(Object.values(resolvedPorts)).size).toBe(4);
|
||||
expect(new Set(Object.values(resolvedPorts)).size).toBe(5);
|
||||
});
|
||||
|
||||
it('端口段内会一直漂移到段尾,不会被默认 200 次尝试截断', async () => {
|
||||
|
||||
+335
-23
File diff suppressed because it is too large
Load Diff
+172
-1
@@ -19,6 +19,7 @@ import {
|
||||
assertReusableSpacetimeProcessVersionMatchesWorkspace,
|
||||
assertSpacetimeToolVersionMatchesWorkspace,
|
||||
buildApiServerProcessEnv,
|
||||
buildBgfilterWorkerProcessEnv,
|
||||
buildDevStackSnapshot,
|
||||
buildFrontendProcessEnv,
|
||||
buildLocalRustProcessEnv,
|
||||
@@ -85,6 +86,26 @@ describe('dev scheduler argument routing', () => {
|
||||
expect(runner.resolveFrontendApiTarget()).toBe('http://127.0.0.1:8090');
|
||||
});
|
||||
|
||||
test('独立 BgFilter worker 命令解析内部监听地址', () => {
|
||||
const { command, explicitOptions, options } = parseArgs(
|
||||
[
|
||||
'bgfilter-worker',
|
||||
'--bgfilter-worker-host',
|
||||
'127.0.0.2',
|
||||
'--bgfilter-worker-port',
|
||||
'18083',
|
||||
],
|
||||
{},
|
||||
);
|
||||
|
||||
expect(command).toBe('bgfilter-worker');
|
||||
expect(explicitOptions).toEqual(
|
||||
new Set(['bgfilterWorkerHost', 'bgfilterWorkerPort']),
|
||||
);
|
||||
expect(options.bgfilterWorkerHost).toBe('127.0.0.2');
|
||||
expect(options.bgfilterWorkerPort).toBe(18083);
|
||||
});
|
||||
|
||||
test('单独 dev:web 未显式指定 api 参数时沿用已有 Rust target', () => {
|
||||
const testEnv = {
|
||||
RUST_SERVER_TARGET: 'http://127.0.0.1:3100',
|
||||
@@ -129,7 +150,7 @@ describe('dev scheduler argument routing', () => {
|
||||
);
|
||||
});
|
||||
|
||||
linuxTest('Linux 启动时按系统级端口段映射四个 dev 端口', async () => {
|
||||
linuxTest('Linux 启动时按系统级端口段映射五个 dev 端口', async () => {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), 'genarrative-dev-port-range-'));
|
||||
try {
|
||||
const { command, explicitOptions, options } = parseArgs([], {
|
||||
@@ -156,7 +177,9 @@ describe('dev scheduler argument routing', () => {
|
||||
expect(runner.options.apiPort).toBe(22001);
|
||||
expect(runner.options.spacetimePort).toBe(22002);
|
||||
expect(runner.options.adminWebPort).toBe(22003);
|
||||
expect(runner.options.bgfilterWorkerPort).toBe(22004);
|
||||
expect(runner.state.apiTarget).toBe('http://127.0.0.1:22001');
|
||||
expect(runner.state.bgfilterWorkerTarget).toBe('http://127.0.0.1:22004');
|
||||
expect(runner.state.spacetimeServer).toBe('http://127.0.0.1:22002');
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
@@ -196,6 +219,7 @@ describe('dev scheduler argument routing', () => {
|
||||
expect(runner.options.apiPort).toBe(22001);
|
||||
expect(runner.options.spacetimePort).toBe(22002);
|
||||
expect(runner.options.adminWebPort).toBe(22003);
|
||||
expect(runner.options.bgfilterWorkerPort).toBe(22004);
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
@@ -233,6 +257,7 @@ describe('dev scheduler argument routing', () => {
|
||||
expect(runner.options.apiPort).toBe(8082);
|
||||
expect(runner.options.spacetimePort).toBe(3101);
|
||||
expect(runner.options.adminWebPort).toBe(3102);
|
||||
expect(runner.options.bgfilterWorkerPort).toBe(8083);
|
||||
} finally {
|
||||
if (originalPlatform) {
|
||||
Object.defineProperty(process, 'platform', originalPlatform);
|
||||
@@ -282,6 +307,45 @@ describe('dev scheduler api-server env', () => {
|
||||
expect(env.GENARRATIVE_PROCESS_ROLE).toBe('api');
|
||||
});
|
||||
|
||||
test('父 API 与独立 BgFilter worker 共享实际 URL 和内部 token', () => {
|
||||
const { options } = parseArgs([], {});
|
||||
options.bgfilterWorkerPort = 18083;
|
||||
const state = {
|
||||
spacetimeServer: 'http://127.0.0.1:3199',
|
||||
bgfilterWorkerTarget: 'http://127.0.0.1:18083',
|
||||
};
|
||||
const internalToken = 'local-bgfilter-token';
|
||||
|
||||
const apiEnv = buildApiServerProcessEnv({
|
||||
baseEnv: {},
|
||||
options,
|
||||
state,
|
||||
bgfilterInternalToken: internalToken,
|
||||
processRole: 'all',
|
||||
});
|
||||
const workerEnv = buildBgfilterWorkerProcessEnv({
|
||||
baseEnv: {},
|
||||
options,
|
||||
state,
|
||||
bgfilterInternalToken: internalToken,
|
||||
});
|
||||
|
||||
expect(apiEnv.GENARRATIVE_PROCESS_ROLE).toBe('all');
|
||||
expect(workerEnv.GENARRATIVE_PROCESS_ROLE).toBe('bgfilter-worker');
|
||||
expect(apiEnv.GENARRATIVE_BGFILTER_WORKER_BASE_URL).toBe(
|
||||
state.bgfilterWorkerTarget,
|
||||
);
|
||||
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_BASE_URL).toBe(
|
||||
state.bgfilterWorkerTarget,
|
||||
);
|
||||
expect(apiEnv.GENARRATIVE_BGFILTER_INTERNAL_TOKEN).toBe(internalToken);
|
||||
expect(workerEnv.GENARRATIVE_BGFILTER_INTERNAL_TOKEN).toBe(internalToken);
|
||||
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_HOST).toBe('127.0.0.1');
|
||||
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_PORT).toBe('18083');
|
||||
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_CONCURRENCY).toBe('4');
|
||||
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS).toBe('128');
|
||||
});
|
||||
|
||||
test('Windows 本地 dev 自动注入已安装的 FFmpeg 路径', () => {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), 'genarrative-ffmpeg-'));
|
||||
try {
|
||||
@@ -339,6 +403,100 @@ describe('dev scheduler api-server env', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('dev scheduler Rust service orchestration', () => {
|
||||
test('Rust 双进程重启时先全部停止,再先 ready BgFilter、后 ready API', async () => {
|
||||
const { explicitOptions, options } = parseArgs([], {});
|
||||
const runner = new DevRunner(options, {}, explicitOptions);
|
||||
const events: string[] = [];
|
||||
runner.command = 'all';
|
||||
runner.windowsApiServerCleanupCompleted = true;
|
||||
runner.services = new Map([
|
||||
[
|
||||
'api-server',
|
||||
{
|
||||
stop: async () => events.push('stop-api'),
|
||||
start: async () => events.push('start-api'),
|
||||
},
|
||||
],
|
||||
[
|
||||
'bgfilter-worker',
|
||||
{
|
||||
stop: async () => events.push('stop-bgfilter'),
|
||||
start: async () => events.push('start-bgfilter'),
|
||||
},
|
||||
],
|
||||
]);
|
||||
vi.spyOn(runner, 'waitForBgfilterWorker').mockImplementation(async () => {
|
||||
events.push('ready-bgfilter');
|
||||
});
|
||||
vi.spyOn(runner, 'waitForApiServer').mockImplementation(async () => {
|
||||
events.push('ready-api');
|
||||
});
|
||||
|
||||
await runner.restartRustServicePair();
|
||||
|
||||
expect(events).toEqual([
|
||||
'stop-api',
|
||||
'stop-bgfilter',
|
||||
'start-bgfilter',
|
||||
'ready-bgfilter',
|
||||
'start-api',
|
||||
'ready-api',
|
||||
]);
|
||||
});
|
||||
|
||||
test('dev:api-server 安全自动带起同 runner 的 BgFilter worker', async () => {
|
||||
const { explicitOptions, options } = parseArgs(['api-server'], {});
|
||||
const runner = new DevRunner(options, {}, explicitOptions);
|
||||
const startPair = vi
|
||||
.spyOn(runner, 'startRustServicePair')
|
||||
.mockResolvedValue(undefined);
|
||||
const startWatchers = vi
|
||||
.spyOn(runner, 'startWatchers')
|
||||
.mockImplementation(() => {});
|
||||
|
||||
await runner.startCommand('api-server');
|
||||
|
||||
expect(startPair).toHaveBeenCalledOnce();
|
||||
expect(startWatchers).toHaveBeenCalledWith([
|
||||
'api-server',
|
||||
'bgfilter-worker',
|
||||
]);
|
||||
});
|
||||
|
||||
test('完整栈只为两个 Rust 角色创建一套组合 watcher', () => {
|
||||
const { explicitOptions, options } = parseArgs(['--watch'], {});
|
||||
const runner = new DevRunner(options, {}, explicitOptions);
|
||||
runner.command = 'all';
|
||||
runner.registerServices();
|
||||
|
||||
try {
|
||||
runner.startWatchers(['api-server', 'bgfilter-worker']);
|
||||
expect(runner.watchers).toHaveLength(1);
|
||||
} finally {
|
||||
for (const watcher of runner.watchers) {
|
||||
watcher.close();
|
||||
}
|
||||
runner.watchers = [];
|
||||
}
|
||||
});
|
||||
|
||||
test('BgFilter worker 在 readiness 前退出时立即失败', async () => {
|
||||
const { explicitOptions, options } = parseArgs([], {});
|
||||
const runner = new DevRunner(options, {}, explicitOptions);
|
||||
runner.services = new Map([
|
||||
['bgfilter-worker', { runtime: { status: 'failed' } }],
|
||||
]);
|
||||
globalThis.fetch = vi.fn(async () => ({
|
||||
status: 503,
|
||||
})) as unknown as typeof fetch;
|
||||
|
||||
await expect(runner.waitForBgfilterWorker()).rejects.toThrow(
|
||||
'bgfilter-worker 在 readiness 前退出',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dev scheduler local worker cleanup', () => {
|
||||
const expected = {
|
||||
expectedDatabase: 'xushi-p4wfr',
|
||||
@@ -471,6 +629,8 @@ describe('dev scheduler stack state file', () => {
|
||||
options: {
|
||||
apiHost: '127.0.0.1',
|
||||
apiPort: 8090,
|
||||
bgfilterWorkerHost: '127.0.0.1',
|
||||
bgfilterWorkerPort: 8091,
|
||||
webHost: '0.0.0.0',
|
||||
webPort: 3010,
|
||||
adminWebHost: '127.0.0.1',
|
||||
@@ -483,6 +643,7 @@ describe('dev scheduler stack state file', () => {
|
||||
},
|
||||
state: {
|
||||
apiTarget: 'http://127.0.0.1:8090',
|
||||
bgfilterWorkerTarget: 'http://127.0.0.1:8091',
|
||||
adminWebTargetHost: '127.0.0.1',
|
||||
spacetimeServer: 'http://127.0.0.1:3120',
|
||||
},
|
||||
@@ -527,6 +688,12 @@ describe('dev scheduler stack state file', () => {
|
||||
port: 8090,
|
||||
url: 'http://127.0.0.1:8090',
|
||||
});
|
||||
expect(snapshot.services['bgfilter-worker']).toMatchObject({
|
||||
status: 'idle',
|
||||
pid: null,
|
||||
port: 8091,
|
||||
url: 'http://127.0.0.1:8091',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1258,6 +1425,8 @@ spacetimedb tool version 2.6.0; spacetimedb-lib version 2.6.0;
|
||||
'migration-secret-hash',
|
||||
GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET:
|
||||
'runtime-secret',
|
||||
GENARRATIVE_BGFILTER_INTERNAL_TOKEN: 'bgfilter-token',
|
||||
GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE: 'bgfilter-token-file',
|
||||
SAFE_VALUE: 'kept',
|
||||
},
|
||||
{ RUST_SERVER_TARGET: 'http://127.0.0.1:8082' },
|
||||
@@ -1273,6 +1442,8 @@ spacetimedb tool version 2.6.0; spacetimedb-lib version 2.6.0;
|
||||
expect(env).not.toHaveProperty(
|
||||
'GENARRATIVE_SPACETIME_RUNTIME_SERVICE_BOOTSTRAP_SECRET',
|
||||
);
|
||||
expect(env).not.toHaveProperty('GENARRATIVE_BGFILTER_INTERNAL_TOKEN');
|
||||
expect(env).not.toHaveProperty('GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE');
|
||||
expect(env.SAFE_VALUE).toBe('kept');
|
||||
});
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ SPACETIME_BIN_SOURCE="${SPACETIME_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/spacetime/s
|
||||
OTELCOL_BIN_SOURCE="${OTELCOL_BIN_SOURCE:-${PROVISION_TOOLS_DIR}/otelcol-contrib}"
|
||||
WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
|
||||
CONTROLLER_ENV_FILE="${CONTROLLER_ENV_FILE:-/etc/genarrative/external-generation-controller.env}"
|
||||
BGFILTER_WORKER_ENV_FILE="${BGFILTER_WORKER_ENV_FILE:-/etc/genarrative/bgfilter-worker.env}"
|
||||
GENARRATIVE_OPENSSL_VERSION="${GENARRATIVE_OPENSSL_VERSION:-3.2.0}"
|
||||
GENARRATIVE_OPENSSL_PREFIX="${GENARRATIVE_OPENSSL_PREFIX:-/opt/genarrative/openssl-3.2.0}"
|
||||
GENARRATIVE_OPENSSL_SOURCE_URL="${GENARRATIVE_OPENSSL_SOURCE_URL:-https://github.com/openssl/openssl/releases/download/openssl-${GENARRATIVE_OPENSSL_VERSION}/openssl-${GENARRATIVE_OPENSSL_VERSION}.tar.gz}"
|
||||
@@ -408,6 +409,101 @@ read_env_value() {
|
||||
done <"${file}"
|
||||
}
|
||||
|
||||
env_contains_nonempty_assignment() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
local line value first_char last_char
|
||||
|
||||
if [[ ! -f "${file}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
while IFS= read -r line || [[ -n "${line}" ]]; do
|
||||
line="${line#"${line%%[![:space:]]*}"}"
|
||||
if [[ -z "${line}" || "${line}" == \#* || "${line}" != "${key}="* ]]; then
|
||||
continue
|
||||
fi
|
||||
value="${line#*=}"
|
||||
value="${value%$'\r'}"
|
||||
value="${value#"${value%%[![:space:]]*}"}"
|
||||
value="${value%"${value##*[![:space:]]}"}"
|
||||
if [[ ${#value} -ge 2 ]]; then
|
||||
first_char="${value:0:1}"
|
||||
last_char="${value: -1}"
|
||||
if [[ "${first_char}" == "${last_char}" && ( "${first_char}" == '"' || "${first_char}" == "'" ) ]]; then
|
||||
value="${value:1:${#value}-2}"
|
||||
fi
|
||||
fi
|
||||
if [[ "${value}" =~ [^[:space:]] ]]; then
|
||||
return 0
|
||||
fi
|
||||
done <"${file}"
|
||||
return 1
|
||||
}
|
||||
|
||||
env_has_assignment() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
local line current_key
|
||||
|
||||
if [[ ! -f "${file}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
while IFS= read -r line || [[ -n "${line}" ]]; do
|
||||
line="${line%$'\r'}"
|
||||
line="${line#"${line%%[![:space:]]*}"}"
|
||||
if [[ -z "${line}" || "${line}" == \#* || "${line}" != *"="* ]]; then
|
||||
continue
|
||||
fi
|
||||
current_key="${line%%=*}"
|
||||
current_key="${current_key%"${current_key##*[![:space:]]}"}"
|
||||
if [[ "${current_key}" == "${key}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done <"${file}"
|
||||
return 1
|
||||
}
|
||||
|
||||
read_effective_env_value() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
local line current_key value first_char last_char matched_value="" found="false"
|
||||
|
||||
if [[ ! -f "${file}" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
while IFS= read -r line || [[ -n "${line}" ]]; do
|
||||
line="${line%$'\r'}"
|
||||
line="${line#"${line%%[![:space:]]*}"}"
|
||||
if [[ -z "${line}" || "${line}" == \#* || "${line}" != *"="* ]]; then
|
||||
continue
|
||||
fi
|
||||
current_key="${line%%=*}"
|
||||
current_key="${current_key%"${current_key##*[![:space:]]}"}"
|
||||
if [[ "${current_key}" != "${key}" ]]; then
|
||||
continue
|
||||
fi
|
||||
value="${line#*=}"
|
||||
value="${value#"${value%%[![:space:]]*}"}"
|
||||
value="${value%"${value##*[![:space:]]}"}"
|
||||
if [[ ${#value} -ge 2 ]]; then
|
||||
first_char="${value:0:1}"
|
||||
last_char="${value: -1}"
|
||||
if [[ "${first_char}" == "${last_char}" && ( "${first_char}" == '"' || "${first_char}" == "'" ) ]]; then
|
||||
value="${value:1:${#value}-2}"
|
||||
fi
|
||||
fi
|
||||
matched_value="${value}"
|
||||
found="true"
|
||||
done <"${file}"
|
||||
|
||||
if [[ "${found}" == "true" ]]; then
|
||||
printf "%s" "${matched_value}"
|
||||
fi
|
||||
}
|
||||
|
||||
write_env_value() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
@@ -525,10 +621,120 @@ ensure_api_runtime_env_defaults() {
|
||||
ensure_env_value_migrates_old_default "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL" "http://127.0.0.1:8083"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE" "/etc/genarrative/secrets/bgfilter-worker.token"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS" "2000"
|
||||
ensure_runtime_bootstrap_secret_file_env "${API_ENV_FILE}"
|
||||
ensure_env_value_migrates_old_default "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS" "45000" "180000"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
|
||||
}
|
||||
|
||||
validate_bgfilter_shared_runtime_env() {
|
||||
local request_timeout_ms connect_timeout_ms
|
||||
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ validate shared BgFilter provider attempt timeout in ${API_ENV_FILE}"
|
||||
return
|
||||
fi
|
||||
|
||||
request_timeout_ms="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS")"
|
||||
if [[ ! "${request_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "[server-provision] GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${API_ENV_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
connect_timeout_ms="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS")"
|
||||
if [[ ! "${connect_timeout_ms}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "[server-provision] GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS 必须在共享 API env 中配置为正整数毫秒: ${API_ENV_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
validate_no_bgfilter_internal_token_plaintext() {
|
||||
local env_file
|
||||
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ reject non-empty GENARRATIVE_BGFILTER_INTERNAL_TOKEN in ${API_ENV_FILE}, ${WORKER_ENV_FILE}, and ${BGFILTER_WORKER_ENV_FILE}"
|
||||
return
|
||||
fi
|
||||
|
||||
for env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${BGFILTER_WORKER_ENV_FILE}"; do
|
||||
if env_contains_nonempty_assignment "${env_file}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN"; then
|
||||
echo "[server-provision] ${env_file} 不得保存 GENARRATIVE_BGFILTER_INTERNAL_TOKEN 明文;生产环境只允许使用 GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE。" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
validate_bgfilter_env_file_alignment() {
|
||||
local env_file="$1"
|
||||
local label="$2"
|
||||
local include_provider_credentials="$3"
|
||||
local key shared_value dedicated_value
|
||||
local -a shared_keys=(
|
||||
GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS
|
||||
GENARRATIVE_BGFILTER_WORKER_BASE_URL
|
||||
GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE
|
||||
GENARRATIVE_BGFILTER_WORKER_CONNECT_TIMEOUT_MS
|
||||
ALIYUN_OSS_BUCKET
|
||||
ALIYUN_OSS_ENDPOINT
|
||||
)
|
||||
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ validate ${label} BgFilter shared configuration alignment with ${API_ENV_FILE}"
|
||||
return
|
||||
fi
|
||||
if [[ ! -f "${env_file}" ]]; then
|
||||
echo "[server-provision] ${label} 不存在,无法检查 BgFilter 共享配置: ${env_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${include_provider_credentials}" == "true" ]]; then
|
||||
shared_keys+=(
|
||||
GENARRATIVE_EDITOR_BGFILTER_BASE_URL
|
||||
GENARRATIVE_EDITOR_BGFILTER_TOKEN
|
||||
ALIYUN_OSS_ACCESS_KEY_ID
|
||||
ALIYUN_OSS_ACCESS_KEY_SECRET
|
||||
ALIYUN_OSS_READ_EXPIRE_SECONDS
|
||||
)
|
||||
fi
|
||||
|
||||
for key in "${shared_keys[@]}"; do
|
||||
if ! env_has_assignment "${env_file}" "${key}"; then
|
||||
continue
|
||||
fi
|
||||
dedicated_value="$(read_effective_env_value "${env_file}" "${key}")"
|
||||
shared_value="$(read_effective_env_value "${API_ENV_FILE}" "${key}")"
|
||||
if [[ "${dedicated_value}" != "${shared_value}" ]]; then
|
||||
echo "[server-provision] ${label} 中的 BgFilter 共享配置与 API env 不一致: ${key};后加载 env 会覆盖进程有效值。" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
validate_bgfilter_loopback_endpoint_alignment() {
|
||||
local base_url host port expected_base_url
|
||||
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ validate BgFilter parent base URL and child listener alignment"
|
||||
return
|
||||
fi
|
||||
|
||||
base_url="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_BASE_URL")"
|
||||
host="$(read_effective_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_HOST")"
|
||||
port="$(read_effective_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_PORT")"
|
||||
if [[ "${host}" != "127.0.0.1" ]]; then
|
||||
echo "[server-provision] BgFilter worker 首版必须监听 127.0.0.1,当前 GENARRATIVE_BGFILTER_WORKER_HOST=${host:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! "${port}" =~ ^[1-9][0-9]{0,4}$ ]] || (( 10#${port} > 65535 )); then
|
||||
echo "[server-provision] GENARRATIVE_BGFILTER_WORKER_PORT 必须是 1-65535 的有效端口: ${BGFILTER_WORKER_ENV_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
expected_base_url="http://${host}:${port}"
|
||||
if [[ "${base_url%/}" != "${expected_base_url}" ]]; then
|
||||
echo "[server-provision] 父进程 GENARRATIVE_BGFILTER_WORKER_BASE_URL 必须与 BgFilter worker 有效监听地址一致: expected=${expected_base_url}, actual=${base_url:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_worker_runtime_env_defaults() {
|
||||
@@ -547,6 +753,80 @@ ensure_worker_runtime_env_defaults() {
|
||||
ensure_runtime_bootstrap_secret_file_env "${WORKER_ENV_FILE}"
|
||||
}
|
||||
|
||||
ensure_bgfilter_worker_runtime_env_defaults() {
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ ensure BgFilter worker runtime env defaults in ${BGFILTER_WORKER_ENV_FILE}"
|
||||
return
|
||||
fi
|
||||
if [[ ! -f "${BGFILTER_WORKER_ENV_FILE}" ]]; then
|
||||
echo "[server-provision] BgFilter worker 环境文件不存在,无法补齐运行态变量: ${BGFILTER_WORKER_ENV_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_HOST" "127.0.0.1"
|
||||
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_PORT" "8083"
|
||||
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_CONCURRENCY" "4"
|
||||
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS" "128"
|
||||
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
|
||||
ensure_env_value "${BGFILTER_WORKER_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
|
||||
}
|
||||
|
||||
ensure_bgfilter_internal_token_file() {
|
||||
local token_file="/etc/genarrative/secrets/bgfilter-worker.token"
|
||||
local token_dir="/etc/genarrative/secrets"
|
||||
local openssl_bin="${GENARRATIVE_OPENSSL_PREFIX}/bin/openssl"
|
||||
local configured_token_file temporary_file token_metadata
|
||||
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ ensure shared BgFilter internal token file ${token_file} (root:genarrative 0440)"
|
||||
return
|
||||
fi
|
||||
configured_token_file="$(read_effective_env_value "${API_ENV_FILE}" "GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE")"
|
||||
if [[ "${configured_token_file}" != "${token_file}" ]]; then
|
||||
echo "[server-provision] GENARRATIVE_BGFILTER_INTERNAL_TOKEN_FILE 必须与 Provision 管理路径一致: ${token_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -L "${token_dir}" || -L "${token_file}" ]]; then
|
||||
echo "[server-provision] BgFilter 内部 Token 目录和文件不能是符号链接: ${token_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -e "${token_file}" && ! -f "${token_file}" ]]; then
|
||||
echo "[server-provision] BgFilter 内部 Token 必须是普通文件: ${token_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
install -d -o root -g genarrative -m 0750 "${token_dir}"
|
||||
if [[ ! -f "${token_file}" ]]; then
|
||||
temporary_file="$(mktemp "${token_dir}/.bgfilter-worker.token.XXXXXX")"
|
||||
if ! "${openssl_bin}" rand -hex 32 >"${temporary_file}"; then
|
||||
rm -f "${temporary_file}"
|
||||
echo "[server-provision] 生成 BgFilter 内部 Token 失败。" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q '[^[:space:]]' -- "${temporary_file}"; then
|
||||
rm -f "${temporary_file}"
|
||||
echo "[server-provision] 生成的 BgFilter 内部 Token 不得为空或只包含空白字符。" >&2
|
||||
exit 1
|
||||
fi
|
||||
chown root:genarrative "${temporary_file}"
|
||||
chmod 0440 "${temporary_file}"
|
||||
mv -T "${temporary_file}" "${token_file}"
|
||||
echo "[server-provision] 已生成 BgFilter 内部 Token 文件: ${token_file}"
|
||||
else
|
||||
if ! grep -q '[^[:space:]]' -- "${token_file}"; then
|
||||
echo "[server-provision] BgFilter 内部 Token 文件不得为空或只包含空白字符: ${token_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
chown root:genarrative "${token_file}"
|
||||
chmod 0440 "${token_file}"
|
||||
echo "[server-provision] BgFilter 内部 Token 文件已存在,保留内容并收紧权限。"
|
||||
fi
|
||||
token_metadata="$(stat -c '%U:%G:%a' -- "${token_file}")"
|
||||
if [[ "${token_metadata}" != "root:genarrative:440" ]]; then
|
||||
echo "[server-provision] BgFilter 内部 Token 权限必须为 root:genarrative 0440: ${token_file} (${token_metadata})" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
parse_json_string_field() {
|
||||
local json="$1"
|
||||
local key="$2"
|
||||
@@ -684,6 +964,10 @@ render_external_generation_controller_env_example() {
|
||||
cat deploy/env/external-generation-controller.env.example
|
||||
}
|
||||
|
||||
render_bgfilter_worker_env_example() {
|
||||
cat deploy/env/bgfilter-worker.env.example
|
||||
}
|
||||
|
||||
render_otelcol_service() {
|
||||
cat deploy/systemd/otelcol-contrib.service
|
||||
}
|
||||
@@ -927,6 +1211,35 @@ render_external_generation_controller_service() {
|
||||
deploy/systemd/genarrative-external-generation-controller.service
|
||||
}
|
||||
|
||||
render_bgfilter_worker_service() {
|
||||
local current_escaped api_env_escaped bgfilter_env_escaped
|
||||
current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
|
||||
api_env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
|
||||
bgfilter_env_escaped="$(escape_sed_replacement "${BGFILTER_WORKER_ENV_FILE}")"
|
||||
sed \
|
||||
-e "s|/opt/genarrative/current|${current_escaped}|g" \
|
||||
-e "s|/etc/genarrative/api-server.env|${api_env_escaped}|g" \
|
||||
-e "s|/etc/genarrative/bgfilter-worker.env|${bgfilter_env_escaped}|g" \
|
||||
deploy/systemd/genarrative-bgfilter-worker.service
|
||||
}
|
||||
|
||||
wait_for_bgfilter_worker_service() {
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ curl -fsS --max-time 2 http://127.0.0.1:8083/readyz"
|
||||
return
|
||||
fi
|
||||
echo "[server-provision] 等待 BgFilter worker readiness。"
|
||||
for _ in {1..30}; do
|
||||
if systemctl is-active --quiet genarrative-bgfilter-worker.service && curl -fsS --max-time 2 http://127.0.0.1:8083/readyz >/dev/null; then
|
||||
return
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
systemctl --no-pager --full status genarrative-bgfilter-worker.service || true
|
||||
echo "[server-provision] BgFilter worker 未在超时时间内通过 readiness。" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
render_database_backup_service() {
|
||||
local current_escaped env_escaped
|
||||
current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
|
||||
@@ -995,6 +1308,7 @@ require_path deploy/systemd/spacetimedb.service
|
||||
require_path deploy/systemd/genarrative-api.service
|
||||
require_path deploy/systemd/genarrative-external-generation-worker@.service
|
||||
require_path deploy/systemd/genarrative-external-generation-controller.service
|
||||
require_path deploy/systemd/genarrative-bgfilter-worker.service
|
||||
require_path deploy/systemd/genarrative-database-backup.service
|
||||
require_path deploy/systemd/genarrative-database-backup-files-history.conf
|
||||
require_path deploy/systemd/genarrative-database-backup.timer
|
||||
@@ -1013,6 +1327,7 @@ require_path deploy/logrotate/genarrative-pingora-gateway
|
||||
require_path deploy/env/api-server.env.example
|
||||
require_path deploy/env/external-generation-worker.env.example
|
||||
require_path deploy/env/external-generation-controller.env.example
|
||||
require_path deploy/env/bgfilter-worker.env.example
|
||||
require_path scripts/deploy/maintenance-on.sh
|
||||
require_path scripts/deploy/maintenance-off.sh
|
||||
require_path scripts/deploy/maintenance-status.sh
|
||||
@@ -1026,7 +1341,7 @@ echo "[server-provision] target=${DEPLOY_TARGET}, dry_run=${DRY_RUN}, nginx_conf
|
||||
run_cmd id
|
||||
require_root_for_real_provision
|
||||
install_nginx_brotli_modules
|
||||
run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /etc/genarrative/pingora /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/wallet-refund-outbox /var/lib/genarrative/editor-generation-pricing /var/lib/genarrative/spacetime /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol /var/log/genarrative
|
||||
run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /etc/genarrative/pingora /etc/genarrative/secrets /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/wallet-refund-outbox /var/lib/genarrative/editor-generation-pricing /var/lib/genarrative/spacetime /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol /var/log/genarrative
|
||||
|
||||
if ! id spacetimedb >/dev/null 2>&1; then
|
||||
run_cmd useradd --system --home-dir "${SPACETIME_ROOT}" --shell /usr/sbin/nologin spacetimedb
|
||||
@@ -1069,18 +1384,21 @@ spacetimedb_service="$(mktemp)"
|
||||
api_service="$(mktemp)"
|
||||
external_generation_worker_service="$(mktemp)"
|
||||
external_generation_controller_service="$(mktemp)"
|
||||
bgfilter_worker_service="$(mktemp)"
|
||||
database_backup_service="$(mktemp)"
|
||||
health_patrol_service="$(mktemp)"
|
||||
render_spacetimedb_service >"${spacetimedb_service}"
|
||||
render_api_service >"${api_service}"
|
||||
render_external_generation_worker_service >"${external_generation_worker_service}"
|
||||
render_external_generation_controller_service >"${external_generation_controller_service}"
|
||||
render_bgfilter_worker_service >"${bgfilter_worker_service}"
|
||||
render_database_backup_service >"${database_backup_service}"
|
||||
render_health_patrol_service >"${health_patrol_service}"
|
||||
install_file "${spacetimedb_service}" /etc/systemd/system/spacetimedb.service 0644
|
||||
install_file "${api_service}" /etc/systemd/system/genarrative-api.service 0644
|
||||
install_file "${external_generation_worker_service}" /etc/systemd/system/genarrative-external-generation-worker@.service 0644
|
||||
install_file "${external_generation_controller_service}" /etc/systemd/system/genarrative-external-generation-controller.service 0644
|
||||
install_file "${bgfilter_worker_service}" /etc/systemd/system/genarrative-bgfilter-worker.service 0644
|
||||
install_file "${database_backup_service}" /etc/systemd/system/genarrative-database-backup.service 0644
|
||||
install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644
|
||||
install_file "${health_patrol_service}" /etc/systemd/system/genarrative-health-patrol.service 0644
|
||||
@@ -1088,7 +1406,7 @@ install_file deploy/systemd/genarrative-health-patrol.timer /etc/systemd/system/
|
||||
install_file deploy/systemd/genarrative-pingora-gateway.service /etc/systemd/system/genarrative-pingora-gateway.service 0644
|
||||
install_file deploy/systemd/genarrative-pingora-gateway-direct-entry.conf /etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf 0644
|
||||
install_file deploy/logrotate/genarrative-pingora-gateway /etc/logrotate.d/genarrative-pingora-gateway 0644
|
||||
rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${external_generation_controller_service}" "${database_backup_service}" "${health_patrol_service}"
|
||||
rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${external_generation_controller_service}" "${bgfilter_worker_service}" "${database_backup_service}" "${health_patrol_service}"
|
||||
|
||||
if [[ ! -f "${API_ENV_FILE}" ]]; then
|
||||
echo "+ create ${API_ENV_FILE} from example"
|
||||
@@ -1101,6 +1419,7 @@ else
|
||||
echo "[server-provision] 已存在环境文件,保留不覆盖: ${API_ENV_FILE}"
|
||||
fi
|
||||
ensure_api_runtime_env_defaults
|
||||
validate_bgfilter_shared_runtime_env
|
||||
configure_database_backup_profile
|
||||
|
||||
if [[ ! -f "${WORKER_ENV_FILE}" ]]; then
|
||||
@@ -1115,6 +1434,23 @@ else
|
||||
fi
|
||||
ensure_worker_runtime_env_defaults
|
||||
|
||||
if [[ ! -f "${BGFILTER_WORKER_ENV_FILE}" ]]; then
|
||||
echo "+ create ${BGFILTER_WORKER_ENV_FILE} from example"
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
render_bgfilter_worker_env_example >"${BGFILTER_WORKER_ENV_FILE}"
|
||||
chmod 0600 "${BGFILTER_WORKER_ENV_FILE}"
|
||||
chown root:root "${BGFILTER_WORKER_ENV_FILE}"
|
||||
fi
|
||||
else
|
||||
echo "[server-provision] 已存在 BgFilter worker 环境文件,保留不覆盖: ${BGFILTER_WORKER_ENV_FILE}"
|
||||
fi
|
||||
ensure_bgfilter_worker_runtime_env_defaults
|
||||
validate_bgfilter_env_file_alignment "${WORKER_ENV_FILE}" "外部生成 worker env" "false"
|
||||
validate_bgfilter_env_file_alignment "${BGFILTER_WORKER_ENV_FILE}" "BgFilter 专属 env" "true"
|
||||
validate_bgfilter_loopback_endpoint_alignment
|
||||
validate_no_bgfilter_internal_token_plaintext
|
||||
ensure_bgfilter_internal_token_file
|
||||
|
||||
if [[ ! -f "${CONTROLLER_ENV_FILE}" ]]; then
|
||||
echo "+ create ${CONTROLLER_ENV_FILE} from example"
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
@@ -1149,7 +1485,7 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
|
||||
run_cmd systemctl enable otelcol-contrib.service
|
||||
fi
|
||||
stamp_database_backup_timer_now
|
||||
run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service genarrative-health-patrol.timer
|
||||
run_cmd systemctl enable spacetimedb.service genarrative-bgfilter-worker.service genarrative-api.service genarrative-database-backup.timer genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service genarrative-health-patrol.timer
|
||||
run_cmd systemctl start genarrative-database-backup.timer
|
||||
if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then
|
||||
run_cmd systemctl restart otelcol-contrib.service
|
||||
@@ -1158,13 +1494,17 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then
|
||||
wait_for_spacetimedb_service
|
||||
ensure_spacetime_owner_client_token
|
||||
if [[ -x "${CURRENT_LINK}/api-server" ]]; then
|
||||
run_cmd systemctl enable genarrative-bgfilter-worker.service
|
||||
run_cmd systemctl stop genarrative-bgfilter-worker.service
|
||||
run_cmd systemctl start genarrative-bgfilter-worker.service
|
||||
wait_for_bgfilter_worker_service
|
||||
run_cmd systemctl restart genarrative-api.service
|
||||
run_cmd systemctl enable --now genarrative-external-generation-worker@1.service
|
||||
run_cmd systemctl restart genarrative-external-generation-worker@1.service
|
||||
run_cmd systemctl enable --now genarrative-external-generation-controller.service
|
||||
run_cmd systemctl restart genarrative-external-generation-controller.service
|
||||
else
|
||||
echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server,跳过 api-server、外部生成 worker 和 controller 首次启动。后续 API deploy 会启用并启动默认 worker 与 controller。"
|
||||
echo "[server-provision] 尚未发现 ${CURRENT_LINK}/api-server,跳过 BgFilter worker、api-server、外部生成 worker 和 controller 首次启动。后续 API deploy 会按顺序启动。"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@@ -18,12 +18,14 @@ const DEFAULT_PUBLIC_PATHS = [
|
||||
|
||||
const DEFAULT_SERVICES = [
|
||||
'genarrative-api.service',
|
||||
'genarrative-bgfilter-worker.service',
|
||||
'genarrative-external-generation-controller.service',
|
||||
'spacetimedb.service',
|
||||
'nginx.service',
|
||||
];
|
||||
const PINGORA_DIRECT_SERVICES = [
|
||||
'genarrative-api.service',
|
||||
'genarrative-bgfilter-worker.service',
|
||||
'genarrative-external-generation-controller.service',
|
||||
'spacetimedb.service',
|
||||
'genarrative-pingora-gateway.service',
|
||||
@@ -37,6 +39,7 @@ function usage() {
|
||||
|
||||
Options:
|
||||
--api-base-url <url> API direct base URL, default http://127.0.0.1:8082
|
||||
--bgfilter-base-url <url> BgFilter worker base URL, default http://127.0.0.1:8083
|
||||
--spacetime-base-url <url> SpacetimeDB base URL, default http://127.0.0.1:3101
|
||||
--public-base-url <url> Nginx/public base URL, default http://127.0.0.1
|
||||
--public-host <host> Optional public Host header, useful when probing 127.0.0.1
|
||||
@@ -88,6 +91,9 @@ function parseArgs(argv) {
|
||||
apiBaseUrl:
|
||||
process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL ||
|
||||
'http://127.0.0.1:8082',
|
||||
bgfilterBaseUrl:
|
||||
process.env.GENARRATIVE_HEALTH_PATROL_BGFILTER_BASE_URL ||
|
||||
'http://127.0.0.1:8083',
|
||||
spacetimeBaseUrl:
|
||||
process.env.GENARRATIVE_HEALTH_PATROL_SPACETIME_BASE_URL ||
|
||||
'http://127.0.0.1:3101',
|
||||
@@ -131,6 +137,9 @@ function parseArgs(argv) {
|
||||
case '--api-base-url':
|
||||
config.apiBaseUrl = requireValue(argv, ++index, arg);
|
||||
break;
|
||||
case '--bgfilter-base-url':
|
||||
config.bgfilterBaseUrl = requireValue(argv, ++index, arg);
|
||||
break;
|
||||
case '--spacetime-base-url':
|
||||
config.spacetimeBaseUrl = requireValue(argv, ++index, arg);
|
||||
break;
|
||||
@@ -693,6 +702,13 @@ async function main() {
|
||||
config,
|
||||
),
|
||||
);
|
||||
checks.push(
|
||||
await checkHttp(
|
||||
'bgfilter:/readyz',
|
||||
joinUrl(config.bgfilterBaseUrl, '/readyz'),
|
||||
config,
|
||||
),
|
||||
);
|
||||
checks.push(
|
||||
await checkHttp(
|
||||
'spacetimedb:/v1/ping',
|
||||
|
||||
Reference in New Issue
Block a user