收紧外部生成任务租约和超时保护
缩短 worker lease 并新增普通和长任务执行预算配置 让 worker 超过执行预算后写失败并释放槽位 降低 BgFilter 默认超时并加入连续失败熔断兜底 补充资产计费 future 取消时的退款补偿保护 同步生产 env 示例、部署脚本和运维文档
This commit is contained in:
@@ -15,7 +15,9 @@ GENARRATIVE_EXTERNAL_GENERATION_MODE=queue
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID=
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY=2
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS=2000
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=3600
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=600
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS=900
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS=1800
|
||||
GENARRATIVE_API_MAX_CONCURRENT_REQUESTS=512
|
||||
GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS=320
|
||||
GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS=64
|
||||
@@ -30,6 +32,9 @@ GENARRATIVE_WALLET_REFUND_OUTBOX_DIR=/var/lib/genarrative/wallet-refund-outbox
|
||||
GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE=100
|
||||
GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS=1000
|
||||
GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES=67108864
|
||||
GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS=45000
|
||||
GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3
|
||||
GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=300
|
||||
|
||||
GENARRATIVE_OTEL_ENABLED=true
|
||||
OTEL_SERVICE_NAME=genarrative-api
|
||||
|
||||
Vendored
+6
-1
@@ -14,7 +14,9 @@ GENARRATIVE_EXTERNAL_GENERATION_MODE=queue
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID=
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY=2
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS=2000
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=3600
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=600
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS=900
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS=1800
|
||||
GENARRATIVE_API_MAX_CONCURRENT_REQUESTS=512
|
||||
GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS=320
|
||||
GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS=64
|
||||
@@ -30,6 +32,9 @@ GENARRATIVE_WALLET_REFUND_OUTBOX_DIR=/var/lib/genarrative/wallet-refund-outbox
|
||||
GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE=100
|
||||
GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS=1000
|
||||
GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES=67108864
|
||||
GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS=45000
|
||||
GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3
|
||||
GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=300
|
||||
GENARRATIVE_OTEL_ENABLED=true
|
||||
OTEL_SERVICE_NAME=genarrative-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318
|
||||
|
||||
+3
-1
@@ -8,6 +8,8 @@ GENARRATIVE_SPACETIME_POOL_SIZE=1
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY=2
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS=2000
|
||||
# 单次 lease 会由 worker 自动续租；该值覆盖心跳抖动窗口即可。
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=3600
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=600
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS=900
|
||||
GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS=1800
|
||||
GENARRATIVE_API_LOG=info,tower_http=info
|
||||
OTEL_SERVICE_NAME=genarrative-external-generation-worker
|
||||
|
||||
@@ -43,7 +43,7 @@
|
||||
## 2026-07-03 图片画布生成纯色背景资产接入 BgFilter
|
||||
|
||||
- 背景：独立 BgFilter 服务已部署在 image host，并提供 `POST /bgfilter/remove-background`，支持显式 `screen_color` 和 `seg_model`。手动去背景已有独立 BiRefNet BFF，不能把两个服务的配置或语义混在一起。
|
||||
- 决策：角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取在保存带纯色背景源图后，统一调用 BgFilter 生成透明 PNG；请求 multipart 字段为 `file`、`screen_color=<screenColor>` 和 `seg_model=<segModel>`，前端用户路径固定提交 `segModel=birefnet` 且不展示抠图模型选择；`anime-seg` 路径保留为后端可识别的内部能力但不对用户可见。BgFilter 使用独立配置 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL`、`GENARRATIVE_EDITOR_BGFILTER_TOKEN`、`GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS`，默认 base URL 为 `http://58.87.105.82/bgfilter`，token 未配置时复用 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN`。手动 `POST /api/editor/images/background-removals` 继续使用独立 BiRefNet 配置 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL`，不受 BgFilter 影响。BgFilter 参数里的 `seg_model=birefnet` 只表示 BgFilter 内部分割后端，不等于手动去背景的独立 BiRefNet 服务。若 BgFilter 失败，api-server 对这些标准纯色背景生成图使用本地 `editor_green_screen` 兜底。角色动作抽帧仍保留 legacy `#00FF00` 与本地 `editor_green_screen` 透明化。
|
||||
- 决策：角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取在保存带纯色背景源图后，统一调用 BgFilter 生成透明 PNG；请求 multipart 字段为 `file`、`screen_color=<screenColor>` 和 `seg_model=<segModel>`，前端用户路径固定提交 `segModel=birefnet` 且不展示抠图模型选择；`anime-seg` 路径保留为后端可识别的内部能力但不对用户可见。BgFilter 使用独立配置 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL`、`GENARRATIVE_EDITOR_BGFILTER_TOKEN`、`GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS`，默认 base URL 为 `http://58.87.105.82/bgfilter`，默认请求超时 `45000ms`，token 未配置时复用 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN`。手动 `POST /api/editor/images/background-removals` 继续使用独立 BiRefNet 配置 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL`，不受 BgFilter 影响。BgFilter 参数里的 `seg_model=birefnet` 只表示 BgFilter 内部分割后端，不等于手动去背景的独立 BiRefNet 服务。若 BgFilter 失败，api-server 对这些标准纯色背景生成图使用本地 `editor_green_screen` 兜底；连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD`（默认 `3`）后，`GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS`（默认 `300`）内直接本地兜底。角色动作抽帧仍保留 legacy `#00FF00` 与本地 `editor_green_screen` 透明化。
|
||||
- 影响范围：`server-rs/crates/api-server/src/config.rs`、`server-rs/crates/api-server/src/editor_project.rs`、图片画布 MVP 文档和角色形象生成设计文档。
|
||||
- 验证方式：运行 `cargo test -p api-server config::tests::from_env_reads_editor_bgfilter_settings_and_reuses_background_token editor_project::tests::editor_canvas_screen_background_generation_uses_bgfilter_postprocess --manifest-path server-rs/Cargo.toml`、`cargo check -p api-server --manifest-path server-rs/Cargo.toml`、`npm run check:encoding` 和 `git diff --check`。
|
||||
- 关联文档：`docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md`、`docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md`。
|
||||
@@ -1215,6 +1215,7 @@
|
||||
- 2026-06-07 追加：`GENARRATIVE_EXTERNAL_GENERATION_MODE` 使用 `queue|inline` 显式策略；生产和容器扩缩容验证保持 `queue`。本地开发若需要同步等待结果，应通过 `.env.local` 或本机环境显式配置为 `inline`，由 HTTP handler 复用同一 worker executor 直接返回 `completed`，不创建 `external_generation_job`，不支持 worker 动态扩缩容；脚本不得硬编码该策略。拼图写回 guard 字段改为可选，queue 路径仍必须完整校验 `job_id + worker_id + lease_token`；inline 路径只允许三项同时为空，半空 guard 仍拒绝。
|
||||
- 2026-06-11 追加：生产新增固定 `external-generation-controller` 进程角色和 `genarrative-external-generation-controller.service`。controller 只读取 `get_external_generation_queue_stats_and_return` 队列统计并管理 `genarrative-external-generation-worker@N.service`，不监听 HTTP、不执行外部生成任务；默认保留 `@1`，按 `claimable_pending + running_active + expired_running` 计算目标实例数，上限由 `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MAX_WORKERS` 控制，缩容需要连续空闲轮数且每轮只停最高编号一个实例。
|
||||
- 2026-07-08 追加：生产 worker/controller 作为轻量 SpacetimeDB 客户端运行，专属 env 示例默认 `GENARRATIVE_SPACETIME_POOL_SIZE=1`；非 HTTP 角色只保留 `external_generation_job` 队列窄订阅作为响应式唤醒信号，实际抢占和扩缩容判断仍走 SpacetimeDB procedure，且不再订阅 API 读模型连接池。worker/controller poll interval 只作为订阅失效、漏事件和 lease 过期这类时间条件的兜底，不作为正常领取任务的主路径。
|
||||
- 2026-07-08 追加：worker lease 默认从 `3600s` 收短到 `600s`，普通 job 执行预算默认 `900s`，视频 / 角色动作等长 job 默认 `1800s`；预算到期后当前 worker 结束本次尝试、写失败 / 重试状态并释放 worker 槽位，若 SpacetimeDB 当时不可写则最多等到较短 lease 过期后重新领取，避免无进展续租无限延长。资产计费层对已扣费但 future 被取消的外部生成操作做异步补偿退款，避免超时取消绕过退款 outbox。
|
||||
- 影响范围：`server-rs/crates/spacetime-module/src/external_generation.rs`、`server-rs/crates/spacetime-client/src/external_generation.rs`、`server-rs/crates/api-server/src/external_generation_worker.rs`、`server-rs/crates/api-server/src/external_generation_worker_controller.rs`、`deploy/systemd/genarrative-external-generation-worker@.service`、`deploy/systemd/genarrative-external-generation-controller.service`、`deploy/env/external-generation-controller.env.example`、`scripts/deploy/production-api-deploy.sh`、`scripts/jenkins-server-provision.sh`、拼图 `compile_puzzle_draft`、拼图 `generate_puzzle_images`、拼图 `generate_puzzle_ui_background`、生产 env 模板和运维文档。
|
||||
- 验证方式：`npm run spacetime:generate`、`npm run check:spacetime-schema`、`npm run check:server-rs-ddd`、`cargo check -p api-server --manifest-path server-rs/Cargo.toml`，并在 queue 模式下用 `GENARRATIVE_PROCESS_ROLE=all npm run dev` smoke 至少一次 queued -> worker 完成链路；本地 inline 排查只确认不创建 `external_generation_job`。
|
||||
- 关联文档：`docs/technical/【后端架构】外部生成Worker化方案-2026-06-03.md`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`、`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`。
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -112,7 +112,9 @@ worker 配置：
|
||||
- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_ID`：实例 ID；未配置时用 hostname/pid 派生。
|
||||
- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_CONCURRENCY`：单进程并发领取/执行数量。
|
||||
- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS`：空队列轮询间隔。
|
||||
- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS`：任务 lease 时长；worker 会按约三分之一 lease、最长 30 秒的间隔续租。该值应覆盖一次心跳网络抖动窗口，不需要大于完整外部生成链路耗时。
|
||||
- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS`：任务 lease 时长，默认 `600`；worker 会按约三分之一 lease、最长 30 秒的间隔续租。该值应覆盖一次心跳网络抖动窗口，不需要大于完整外部生成链路耗时。
|
||||
- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS`：普通外部生成 job 的执行预算，默认 `900`。超过预算后当前 worker 停止当前尝试、写入失败 / 重试状态并释放 worker 槽位，避免任务长期保持 `running_active`；若业务 future 已在计费操作内被取消，计费层会按外部生成 job id 异步补偿退款。
|
||||
- `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS`：视频、角色动作等长耗时 job 的执行预算，默认 `1800`。
|
||||
|
||||
controller 配置：
|
||||
|
||||
@@ -124,7 +126,7 @@ controller 配置：
|
||||
- `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_SERVICE_TEMPLATE`：systemd worker 模板，默认 `genarrative-external-generation-worker@{}.service`。
|
||||
- `GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_DRY_RUN`：只记录决策不执行 systemctl，默认 `false`。
|
||||
|
||||
动态缩扩容方式：生产默认由 `deploy/systemd/genarrative-external-generation-controller.service` 启动 `GENARRATIVE_PROCESS_ROLE=external-generation-controller`，controller 读取 `get_external_generation_queue_stats_and_return` 后对 `genarrative-external-generation-worker@N.service` 执行精确 `systemctl start/stop`；无需改变 HTTP 进程数。controller 只操作 `@1..@MAX` 中的缺口或最高编号多余实例，保留 `@1` 作为保底 worker。缩容或发布重启 worker 时，进程收到 SIGINT/SIGTERM 后会停止 claim 新任务并等待当前任务完成；若进程被硬杀、机器断电或超过 systemd `TimeoutStopSec`，未完成任务会在 lease 过期后被其它 worker 重新领取。容器链路已有独立 `external-generation-worker` compose service；扩 worker 必须扩这个 worker service，不能只扩 `api-server` HTTP service。
|
||||
动态缩扩容方式：生产默认由 `deploy/systemd/genarrative-external-generation-controller.service` 启动 `GENARRATIVE_PROCESS_ROLE=external-generation-controller`，controller 读取 `get_external_generation_queue_stats_and_return` 后对 `genarrative-external-generation-worker@N.service` 执行精确 `systemctl start/stop`；无需改变 HTTP 进程数。controller 只操作 `@1..@MAX` 中的缺口或最高编号多余实例，保留 `@1` 作为保底 worker。缩容或发布重启 worker 时，进程收到 SIGINT/SIGTERM 后会停止 claim 新任务并等待当前任务完成；若进程被硬杀、机器断电或超过 systemd `TimeoutStopSec`，未完成任务会在 lease 过期后被其它 worker 重新领取。若 worker 内业务 future 长时间无返回，执行预算到期后会结束当前尝试并释放槽位；如果当时 SpacetimeDB 写回失败，任务也会按较短 lease 进入可重领窗口，避免无进展续租无限延长。容器链路已有独立 `external-generation-worker` compose service；扩 worker 必须扩这个 worker service，不能只扩 `api-server` HTTP service。
|
||||
|
||||
## 已接入的拼图纵切
|
||||
|
||||
|
||||
@@ -220,7 +220,7 @@ npm run check:server-rs-ddd
|
||||
|
||||
- LLM：通用 LLM 门面继续使用 `GENARRATIVE_LLM_*`；创意 Agent `gpt-5.4-mini` Chat Completions 文本链路已于 2026-06 从 APIMart 迁移到 VectorEngine，使用 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` 构造 OpenAI-compatible client，`api-server` 会把未带 `/v1` 的 VectorEngine base URL 规范化到 `/v1` 后请求 `/chat/completions`。通用 `/api/llm/chat/completions` 代理使用 `GENARRATIVE_LLM_PROVIDER=openai-compatible`、`GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1`、`GENARRATIVE_LLM_MODEL=gpt-5.4-mini`；未单独配置 `GENARRATIVE_LLM_API_KEY` 时可复用 `VECTOR_ENGINE_API_KEY`。`APIMART_BASE_URL` / `APIMART_API_KEY` 只作为历史残留，不再作为创意 Agent gpt-5.4-mini 客户端来源；后续排障时优先确认 VectorEngine `/v1/models`、`/v1/chat/completions` 和 `/v1/responses` 可用性。
|
||||
- 图片生成：VectorEngine `gpt-image-2` 图片 provider 归属 `platform-image`，密钥只在后端环境变量中；`api-server` 内的 `openai_image_generation.rs` 只是兼容调用面和外部失败审计桥接，不再承载 provider 协议实现。实际外部生成运行记录统一落 `tracking_event`，`event_key = external_generation_run`，metadata 记录开始 / 结束时间、耗时、状态、成功标记、失败原因、provider task id 和结果摘要，不再写回过时的 `ai_task`。DashScope 只按仍在使用的历史能力单独处理，不作为 GPT-image-2 兜底。VectorEngine `/v1/images/generations` 和 `/v1/images/edits` 上游 POST 使用 `libcurl` 发送；`reqwest` 只保留给参考图 URL 下载和响应中图片 URL 下载。`/v1/images/edits` 的 multipart 参考图必须作为 libcurl 文件上传 part 发送，字段名为 `image`，实现上使用 `Form::buffer(file_name, bytes)` 并设置 `Content-Type`；不能只用 `contents(...).filename(...)`，否则上游会把请求转码为缺少图片并返回 `image is required`。`request_send` 阶段的 curl timeout / connect error 按可重试传输错误处理，最多尝试 5 次，并使用指数退避加短抖动；排障时优先看 `attempt`、`max_attempts`、`retry_delay_ms`、`reference_image_bytes_total` 和 `request_params`，不要把 `SendRequest` 当成上游业务错误。
|
||||
- 编辑器抠图服务：手动 `POST /api/editor/images/background-removals` 继续代理独立 BiRefNet 服务，配置为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL`、`GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN` 和 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS`。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的生成后纯色背景透明化改走独立 BgFilter 服务，配置为 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL`、`GENARRATIVE_EDITOR_BGFILTER_TOKEN` 和 `GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS`，默认 base URL 为 `http://58.87.105.82/bgfilter`，token 未配置时复用 BiRefNet token。BgFilter 请求必须显式传 `screen_color=<screenColor>` 和 `seg_model=<segModel>`；前端用户路径不展示抠图模型选择并固定提交默认 `birefnet`，后端仍识别内部保留的 `anime-seg`，其中 `birefnet` 只表示 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。角色动作抽帧仍沿用 legacy `#00FF00` 和本地 `editor_green_screen` 透明化。
|
||||
- 编辑器抠图服务：手动 `POST /api/editor/images/background-removals` 继续代理独立 BiRefNet 服务，配置为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL`、`GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN` 和 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS`。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的生成后纯色背景透明化改走独立 BgFilter 服务，配置为 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL`、`GENARRATIVE_EDITOR_BGFILTER_TOKEN` 和 `GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS`，默认 base URL 为 `http://58.87.105.82/bgfilter`，默认请求超时为 `45000ms`，token 未配置时复用 BiRefNet token。BgFilter 请求必须显式传 `screen_color=<screenColor>` 和 `seg_model=<segModel>`；前端用户路径不展示抠图模型选择并固定提交默认 `birefnet`，后端仍识别内部保留的 `anime-seg`，其中 `birefnet` 只表示 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。BgFilter 连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD`（默认 `3`）后，会在 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS`（默认 `300`）内直接使用本地 `editor_green_screen` 兜底，避免上游故障占住 worker。角色动作抽帧仍沿用 legacy `#00FF00` 和本地 `editor_green_screen` 透明化。
|
||||
- Match3D 物品 sheet：关卡整图完成后走 VectorEngine `/v1/images/edits` multipart `image`，模型为 `gpt-image-2`，`2K 1:1` 输出 `10*10` spritesheet；物品 sheet prompt 固定要求单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景，后端上传 OSS 前必须把绿幕扣成透明 PNG，并把透明整图写入 `itemSpritesheetImageSrc/itemSpritesheetImageObjectKey`。后端优先按透明 alpha 连通域从该 sheet 识别真实素材矩形并持久化 20 个物品、每个 5 个形态；识别数量不足时才回退 `10*10` 固定网格。通用系列素材图集的行列索引按每行 2 个物品计算，必须落在 `1..=10`，难度只决定运行态加载 3 / 9 / 15 / 20 种。
|
||||
- Match3D UI spritesheet 和背景派生图：关卡整图作为参考图并发生成 `1K 1:1` UI spritesheet 与 `1K 9:16` 背景图，模型均为 `gpt-image-2`。UI spritesheet prompt 固定要求单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景，后端上传 OSS 前必须把绿幕扣成透明 PNG；背景图必须合成为全画幅不透明 PNG。
|
||||
- Match3D 1:1 容器 UI：VectorEngine `/v1/images/edits` multipart 参考图。该容器参考图是后端生图协议输入，必须通过 `include_bytes!` 随 `api-server` 编译进二进制，避免 API 单独发布或运行目录缺少 `public/` 时生成失败。
|
||||
|
||||
@@ -59,6 +59,10 @@ Windows 本地如果已在 `%LOCALAPPDATA%\Genarrative\ffmpeg\bin` 安装 FFmpeg
|
||||
|
||||
生产拆分角色时，`external-generation-worker` 和 `external-generation-controller` 的专属 env 示例会把 `GENARRATIVE_SPACETIME_POOL_SIZE` 覆盖为 `1`；非 HTTP 角色只保留 `external_generation_job` 队列窄订阅作为响应式唤醒信号，实际抢占和扩缩容判断仍走 SpacetimeDB procedure，且不再订阅 API 读模型连接池。`GENARRATIVE_EXTERNAL_GENERATION_WORKER_POLL_INTERVAL_MS` 与 controller poll interval 只作为订阅失效、漏事件和 lease 过期这类时间条件的兜底，不作为正常领取任务的主路径。
|
||||
|
||||
生产 worker 默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS=600`，只覆盖 worker 心跳抖动和短暂断连窗口，不再把 lease 当成完整任务时长；默认 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS=900`，角色动画 / 视频类长任务使用 `GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS=1800`。worker 在单次尝试超过执行预算后会停止当前尝试、写入失败 / 重试状态并释放 worker 槽位；如果 SpacetimeDB 当时不可写，当前租约最多再保留到 lease 过期，之后任务重新变为可领取。生产部署和 provision 脚本会给 `/etc/genarrative/api-server.env` 与 `/etc/genarrative/external-generation-worker.env` 补齐这些变量；已有自定义值不覆盖，只会把历史旧默认 `3600` 迁移为 `600`。
|
||||
|
||||
图片画布角色图、图标素材和 UI 素材提取在绿色 / 蓝色幕布去背景时优先调用 BgFilter；默认 `GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS=45000`，连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3` 后熔断 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=300` 秒，熔断期直接走本地幕布色去背景兜底。修改这些变量后需要重启对应 `api-server` / worker 进程；排查时先从 worker 启动日志确认 lease 和 job timeout，再看 `editor_bgfilter_request_start`、`editor_bgfilter_fallback_to_local_screen_background_removal` 与 `editor_bgfilter_circuit_open_fallback_to_local_screen_background_removal` 日志。
|
||||
|
||||
`我的` 页签或排障面板展示队列等待时，只读取 BFF 队列接口：`GET /api/runtime/external-generation/queue-overview` 查看当前用户可见队列概览，`GET /api/runtime/external-generation/jobs/{jobId}` 查看单 job 状态。生成页 / 进度页不承接队列概览，只展示当前玩法业务进度；队列接口只提供等待 / 运行 / 失败 / 完成状态补充，最终草稿、作品和结果页仍要轮询对应玩法 session/detail 接口收敛到 ready 或 failed；不要直接查询 `external_generation_job` private table，也不要把 worker 内部 payload 暴露到前端。
|
||||
|
||||
需要验证“更新 API 不停 worker”和“worker 是否持续消费队列”时，优先使用隔离容器 smoke：`npm run container:worker-smoke -- smoke`。该脚本生成 gitignored 的 `deploy/container/worker-smoke/api-server.env`，启动独立 compose project 与独立 SpacetimeDB，发布当前 `spacetime-module` 后写入 `worker_smoke_unsupported` 测试 job；预期 worker claim 后执行 unsupported 失败分支，再执行 API-only recreate 并确认 worker 容器 ID 不变，最后再次入队验证 API 更新后队列仍可消费。`external_generation_job` 是 private table，脚本通过 worker 日志确认 job_id 被消费，不用 CLI SQL 查询私表。该 smoke 不读取 `.env.local`，也不依赖真实 VectorEngine / OSS 密钥；真实生图链路联调再在本地私有 env 中补齐 provider 配置。worker-smoke 默认把本机 `spacetime` CLI 打成轻量 SpacetimeDB 镜像，避免本机首次 smoke 依赖官方大镜像下载。若容器内 Cargo 拉取 crates.io 依赖不稳定，可用 `npm run container:worker-smoke -- smoke --local-binary` 让容器内 Cargo 复用本机 Cargo 缓存构建当前二进制，再打入 Debian bookworm smoke runtime 临时镜像；可用 `GENARRATIVE_WORKER_SMOKE_LOCAL_BASE_IMAGE` 覆盖运行时基础镜像；若隔离端口或库数据需要重建，追加 `--force`。完成 queue 链路验证时，还要用队列概览 BFF 和单 job 状态接口确认 job 从 queued/running 收敛，并用对应玩法 session/detail 接口确认业务状态同步完成。
|
||||
|
||||
@@ -5,7 +5,7 @@ set -euo pipefail
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法:
|
||||
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
|
||||
./scripts/deploy/production-api-deploy.sh --source-dir build/<version> [--version <version>] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101]
|
||||
|
||||
说明:
|
||||
进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。
|
||||
@@ -183,10 +183,30 @@ ensure_env_value() {
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[production-api-deploy] 补齐 api-server 环境变量: ${key} -> ${file_path}"
|
||||
echo "[production-api-deploy] 补齐运行态环境变量: ${key} -> ${file_path}"
|
||||
write_env_value "${file_path}" "${key}" "${default_value}"
|
||||
}
|
||||
|
||||
ensure_env_value_migrates_old_default() {
|
||||
local file_path="$1"
|
||||
local key="$2"
|
||||
local old_default="$3"
|
||||
local new_default="$4"
|
||||
local current_value
|
||||
|
||||
current_value="$(read_env_value "${file_path}" "${key}")"
|
||||
if [[ -z "${current_value}" ]]; then
|
||||
ensure_env_value "${file_path}" "${key}" "${new_default}"
|
||||
return
|
||||
fi
|
||||
if [[ "${current_value}" != "${old_default}" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[production-api-deploy] 迁移运行态环境变量旧默认值: ${key} ${old_default} -> ${new_default} (${file_path})"
|
||||
write_env_value "${file_path}" "${key}" "${new_default}"
|
||||
}
|
||||
|
||||
run_privileged() {
|
||||
if [[ "$(id -u)" -eq 0 ]]; then
|
||||
"$@"
|
||||
@@ -232,6 +252,12 @@ ensure_runtime_env_and_dirs() {
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE" "100"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS" "1000"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES" "67108864"
|
||||
ensure_env_value_migrates_old_default "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS" "45000"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
|
||||
ensure_env_value "${api_env_file}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
|
||||
|
||||
tracking_enabled="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_ENABLED")"
|
||||
tracking_outbox_dir="$(read_env_value "${api_env_file}" "GENARRATIVE_TRACKING_OUTBOX_DIR")"
|
||||
@@ -245,6 +271,22 @@ ensure_runtime_env_and_dirs() {
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_worker_runtime_env_defaults() {
|
||||
local worker_env_file="$1"
|
||||
|
||||
if [[ -z "${worker_env_file}" ]]; then
|
||||
return
|
||||
fi
|
||||
if [[ ! -f "${worker_env_file}" ]]; then
|
||||
echo "[production-api-deploy] worker 环境文件不存在，跳过运行态默认值补齐: ${worker_env_file}"
|
||||
return
|
||||
fi
|
||||
|
||||
ensure_env_value_migrates_old_default "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${worker_env_file}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
}
|
||||
|
||||
extract_pingora_env_files_from_unit() {
|
||||
local service_name="$1"
|
||||
local unit_content
|
||||
@@ -585,6 +627,7 @@ WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service"
|
||||
WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service"
|
||||
HEALTH_URL="http://127.0.0.1:8082/readyz"
|
||||
API_ENV_FILE="/etc/genarrative/api-server.env"
|
||||
WORKER_ENV_FILE="/etc/genarrative/external-generation-worker.env"
|
||||
DATABASE=""
|
||||
SPACETIME_SERVER_URL=""
|
||||
DEPLOY_COMPLETED=0
|
||||
@@ -653,6 +696,10 @@ while [[ $# -gt 0 ]]; do
|
||||
API_ENV_FILE="${2:?缺少 --api-env-file 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--worker-env-file)
|
||||
WORKER_ENV_FILE="${2:?缺少 --worker-env-file 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--database)
|
||||
DATABASE="${2:?缺少 --database 的值}"
|
||||
shift 2
|
||||
@@ -673,6 +720,9 @@ require_argument "${SOURCE_DIR}" "--source-dir"
|
||||
require_absolute_path "${RELEASE_ROOT}" "--release-root"
|
||||
require_absolute_path "${CURRENT_LINK}" "--current-link"
|
||||
require_absolute_path "${API_ENV_FILE}" "--api-env-file"
|
||||
if [[ -n "${WORKER_ENV_FILE}" ]]; then
|
||||
require_absolute_path "${WORKER_ENV_FILE}" "--worker-env-file"
|
||||
fi
|
||||
|
||||
if [[ -n "${DATABASE}" ]]; then
|
||||
validate_spacetime_database_name "${DATABASE}"
|
||||
@@ -979,6 +1029,7 @@ if [[ -n "${SPACETIME_SERVER_URL}" ]]; then
|
||||
fi
|
||||
|
||||
ensure_runtime_env_and_dirs "${API_ENV_FILE}"
|
||||
ensure_worker_runtime_env_defaults "${WORKER_ENV_FILE}"
|
||||
|
||||
if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then
|
||||
PINGORA_SHADOW_ENV_FILE="$(check_pingora_shadow_service_config "${PINGORA_SERVICE_NAME}")"
|
||||
|
||||
@@ -427,12 +427,34 @@ ensure_env_value() {
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[server-provision] 补齐 api-server 环境变量: ${key} -> ${file}"
|
||||
echo "[server-provision] 补齐运行态环境变量: ${key} -> ${file}"
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
write_env_value "${file}" "${key}" "${default_value}"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_env_value_migrates_old_default() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
local old_default="$3"
|
||||
local new_default="$4"
|
||||
local current_value
|
||||
|
||||
current_value="$(read_env_value "${file}" "${key}")"
|
||||
if [[ -z "${current_value}" ]]; then
|
||||
ensure_env_value "${file}" "${key}" "${new_default}"
|
||||
return
|
||||
fi
|
||||
if [[ "${current_value}" != "${old_default}" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[server-provision] 迁移运行态环境变量旧默认值: ${key} ${old_default} -> ${new_default} (${file})"
|
||||
if [[ "${DRY_RUN}" != "true" ]]; then
|
||||
write_env_value "${file}" "${key}" "${new_default}"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_api_runtime_env_defaults() {
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ ensure api-server runtime env defaults in ${API_ENV_FILE}"
|
||||
@@ -454,6 +476,27 @@ ensure_api_runtime_env_defaults() {
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_BATCH_SIZE" "100"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_FLUSH_INTERVAL_MS" "1000"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_WALLET_REFUND_OUTBOX_MAX_BYTES" "67108864"
|
||||
ensure_env_value_migrates_old_default "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS" "45000"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD" "3"
|
||||
ensure_env_value "${API_ENV_FILE}" "GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS" "300"
|
||||
}
|
||||
|
||||
ensure_worker_runtime_env_defaults() {
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo "+ ensure external-generation worker runtime env defaults in ${WORKER_ENV_FILE}"
|
||||
return
|
||||
fi
|
||||
if [[ ! -f "${WORKER_ENV_FILE}" ]]; then
|
||||
echo "[server-provision] worker 环境文件不存在，无法补齐运行态变量: ${WORKER_ENV_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ensure_env_value_migrates_old_default "${WORKER_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS" "3600" "600"
|
||||
ensure_env_value "${WORKER_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS" "900"
|
||||
ensure_env_value "${WORKER_ENV_FILE}" "GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS" "1800"
|
||||
}
|
||||
|
||||
parse_json_string_field() {
|
||||
@@ -964,6 +1007,7 @@ if [[ ! -f "${WORKER_ENV_FILE}" ]]; then
|
||||
else
|
||||
echo "[server-provision] 已存在 worker 环境文件，保留不覆盖: ${WORKER_ENV_FILE}"
|
||||
fi
|
||||
ensure_worker_runtime_env_defaults
|
||||
|
||||
if [[ ! -f "${CONTROLLER_ENV_FILE}" ]]; then
|
||||
echo "+ create ${CONTROLLER_ENV_FILE} from example"
|
||||
|
||||
@@ -65,24 +65,127 @@ where
|
||||
let points_consumed =
|
||||
consume_asset_operation_points(state, owner_user_id, asset_kind, asset_id, points_cost)
|
||||
.await?;
|
||||
let mut refund_on_drop = AssetOperationRefundOnDrop::new(
|
||||
state.clone(),
|
||||
owner_user_id,
|
||||
asset_kind,
|
||||
asset_id,
|
||||
points_cost,
|
||||
points_consumed,
|
||||
current_external_generation_job_id(),
|
||||
);
|
||||
match operation.await {
|
||||
Ok(value) => Ok(value),
|
||||
Ok(value) => {
|
||||
refund_on_drop.disarm();
|
||||
Ok(value)
|
||||
}
|
||||
Err(error) => {
|
||||
if points_consumed && should_refund_asset_operation_error(&error) {
|
||||
refund_asset_operation_points(
|
||||
state,
|
||||
owner_user_id,
|
||||
asset_kind,
|
||||
asset_id,
|
||||
points_cost,
|
||||
)
|
||||
.await;
|
||||
refund_on_drop.refund_now().await;
|
||||
} else {
|
||||
refund_on_drop.disarm();
|
||||
}
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct AssetOperationRefundOnDrop {
|
||||
state: AppState,
|
||||
owner_user_id: String,
|
||||
asset_kind: String,
|
||||
asset_id: String,
|
||||
points_cost: u64,
|
||||
external_generation_job_id: Option<String>,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
impl AssetOperationRefundOnDrop {
|
||||
fn new(
|
||||
state: AppState,
|
||||
owner_user_id: &str,
|
||||
asset_kind: &str,
|
||||
asset_id: &str,
|
||||
points_cost: u64,
|
||||
points_consumed: bool,
|
||||
external_generation_job_id: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
state,
|
||||
owner_user_id: owner_user_id.to_string(),
|
||||
asset_kind: asset_kind.to_string(),
|
||||
asset_id: asset_id.to_string(),
|
||||
points_cost,
|
||||
external_generation_job_id,
|
||||
active: points_consumed,
|
||||
}
|
||||
}
|
||||
|
||||
fn disarm(&mut self) {
|
||||
self.active = false;
|
||||
}
|
||||
|
||||
async fn refund_now(&mut self) {
|
||||
if !self.active {
|
||||
return;
|
||||
}
|
||||
refund_asset_operation_points_with_job_id(
|
||||
self.state.clone(),
|
||||
self.owner_user_id.clone(),
|
||||
self.asset_kind.clone(),
|
||||
self.asset_id.clone(),
|
||||
self.points_cost,
|
||||
self.external_generation_job_id.clone(),
|
||||
)
|
||||
.await;
|
||||
self.active = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AssetOperationRefundOnDrop {
|
||||
fn drop(&mut self) {
|
||||
if !self.active {
|
||||
return;
|
||||
}
|
||||
let state = self.state.clone();
|
||||
let owner_user_id = self.owner_user_id.clone();
|
||||
let asset_kind = self.asset_kind.clone();
|
||||
let asset_id = self.asset_id.clone();
|
||||
let points_cost = self.points_cost;
|
||||
let external_generation_job_id = self.external_generation_job_id.clone();
|
||||
let Ok(handle) = tokio::runtime::Handle::try_current() else {
|
||||
tracing::error!(
|
||||
owner_user_id,
|
||||
asset_kind,
|
||||
asset_id,
|
||||
external_generation_job_id,
|
||||
points_cost,
|
||||
"资产操作 future 被取消，但当前没有 Tokio runtime，无法异步补偿退款"
|
||||
);
|
||||
return;
|
||||
};
|
||||
handle.spawn(async move {
|
||||
tracing::warn!(
|
||||
owner_user_id,
|
||||
asset_kind,
|
||||
asset_id,
|
||||
external_generation_job_id,
|
||||
points_cost,
|
||||
"资产操作 future 被取消，异步补偿退款"
|
||||
);
|
||||
refund_asset_operation_points_with_job_id(
|
||||
state,
|
||||
owner_user_id,
|
||||
asset_kind,
|
||||
asset_id,
|
||||
points_cost,
|
||||
external_generation_job_id,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn should_refund_asset_operation_error(error: &AppError) -> bool {
|
||||
let message = error.body_text();
|
||||
// 中文注释：worker lease guard 拒绝表示当前进程已失去队列写权限；
|
||||
@@ -129,25 +232,24 @@ async fn consume_asset_operation_points(
|
||||
}
|
||||
}
|
||||
|
||||
/// 外部生成或发布 mutation 失败后补偿退款；立即退款失败会进入 outbox，避免覆盖原始业务错误。
|
||||
async fn refund_asset_operation_points(
|
||||
state: &AppState,
|
||||
owner_user_id: &str,
|
||||
asset_kind: &str,
|
||||
asset_id: &str,
|
||||
async fn refund_asset_operation_points_with_job_id(
|
||||
state: AppState,
|
||||
owner_user_id: String,
|
||||
asset_kind: String,
|
||||
asset_id: String,
|
||||
points_cost: u64,
|
||||
external_generation_job_id: Option<String>,
|
||||
) {
|
||||
let ledger_id = format!(
|
||||
"asset_operation_refund:{}:{}:{}",
|
||||
owner_user_id, asset_kind, asset_id
|
||||
);
|
||||
let created_at_micros = current_utc_micros();
|
||||
let external_generation_job_id = current_external_generation_job_id();
|
||||
let metadata_json = wallet_metadata_json(external_generation_job_id.as_deref());
|
||||
if let Err(error) = state
|
||||
.spacetime_client()
|
||||
.refund_profile_wallet_points_with_metadata(
|
||||
owner_user_id.to_string(),
|
||||
owner_user_id.clone(),
|
||||
points_cost,
|
||||
ledger_id.clone(),
|
||||
created_at_micros,
|
||||
@@ -159,12 +261,12 @@ async fn refund_asset_operation_points(
|
||||
if let Some(outbox) = state.wallet_refund_outbox() {
|
||||
match outbox
|
||||
.enqueue(WalletRefundOutboxRecord {
|
||||
owner_user_id: owner_user_id.to_string(),
|
||||
owner_user_id: owner_user_id.clone(),
|
||||
amount: points_cost,
|
||||
ledger_id: ledger_id.clone(),
|
||||
created_at_micros,
|
||||
asset_kind: asset_kind.to_string(),
|
||||
asset_id: asset_id.to_string(),
|
||||
asset_kind: asset_kind.clone(),
|
||||
asset_id: asset_id.clone(),
|
||||
external_generation_job_id: external_generation_job_id.clone(),
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -13,11 +13,16 @@ use platform_speech::{
|
||||
const DEFAULT_INTERNAL_API_SECRET: &str = "genarrative-dev-internal-bridge";
|
||||
const SPACETIME_LOCAL_CONFIG_FILE: &str = "spacetime.local.json";
|
||||
const DEFAULT_SPACETIME_HEALTH_CHECK_TIMEOUT_SECONDS: u64 = 2;
|
||||
const DEFAULT_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS: u64 = 600;
|
||||
const DEFAULT_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS: u64 = 900;
|
||||
const DEFAULT_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS: u64 = 1_800;
|
||||
pub(crate) const DEFAULT_VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS: u64 = 1_000_000;
|
||||
const DEFAULT_EDITOR_BACKGROUND_REMOVAL_BASE_URL: &str = "http://58.87.105.82";
|
||||
const DEFAULT_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS: u64 = 120_000;
|
||||
const DEFAULT_EDITOR_BGFILTER_BASE_URL: &str = "http://58.87.105.82/bgfilter";
|
||||
const DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS: u64 = 120_000;
|
||||
const DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS: u64 = 45_000;
|
||||
const DEFAULT_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD: u32 = 3;
|
||||
const DEFAULT_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS: u64 = 300;
|
||||
|
||||
// 集中管理 api-server 的启动配置，避免入口层直接散落环境变量解析逻辑。
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -32,6 +37,8 @@ pub struct AppConfig {
|
||||
pub external_generation_worker_concurrency: usize,
|
||||
pub external_generation_worker_poll_interval: Duration,
|
||||
pub external_generation_worker_lease: Duration,
|
||||
pub external_generation_worker_job_timeout: Duration,
|
||||
pub external_generation_worker_long_job_timeout: Duration,
|
||||
pub external_generation_controller_min_workers: usize,
|
||||
pub external_generation_controller_max_workers: usize,
|
||||
pub external_generation_controller_target_jobs_per_worker: usize,
|
||||
@@ -61,6 +68,8 @@ pub struct AppConfig {
|
||||
pub editor_bgfilter_base_url: String,
|
||||
pub editor_bgfilter_token: Option<String>,
|
||||
pub editor_bgfilter_request_timeout_ms: u64,
|
||||
pub editor_bgfilter_circuit_failure_threshold: u32,
|
||||
pub editor_bgfilter_circuit_cooldown: Duration,
|
||||
pub image_editor_agent_sidebar_enabled: bool,
|
||||
pub log_filter: String,
|
||||
pub otel_enabled: bool,
|
||||
@@ -258,7 +267,15 @@ impl Default for AppConfig {
|
||||
external_generation_worker_id: default_external_generation_worker_id(),
|
||||
external_generation_worker_concurrency: 2,
|
||||
external_generation_worker_poll_interval: Duration::from_millis(2_000),
|
||||
external_generation_worker_lease: Duration::from_secs(3_600),
|
||||
external_generation_worker_lease: Duration::from_secs(
|
||||
DEFAULT_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS,
|
||||
),
|
||||
external_generation_worker_job_timeout: Duration::from_secs(
|
||||
DEFAULT_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS,
|
||||
),
|
||||
external_generation_worker_long_job_timeout: Duration::from_secs(
|
||||
DEFAULT_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS,
|
||||
),
|
||||
external_generation_controller_min_workers: 1,
|
||||
external_generation_controller_max_workers: 8,
|
||||
external_generation_controller_target_jobs_per_worker: 2,
|
||||
@@ -292,6 +309,11 @@ impl Default for AppConfig {
|
||||
editor_bgfilter_base_url: DEFAULT_EDITOR_BGFILTER_BASE_URL.to_string(),
|
||||
editor_bgfilter_token: None,
|
||||
editor_bgfilter_request_timeout_ms: DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS,
|
||||
editor_bgfilter_circuit_failure_threshold:
|
||||
DEFAULT_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD,
|
||||
editor_bgfilter_circuit_cooldown: Duration::from_secs(
|
||||
DEFAULT_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS,
|
||||
),
|
||||
image_editor_agent_sidebar_enabled: false,
|
||||
log_filter: "info,tower_http=info".to_string(),
|
||||
otel_enabled: false,
|
||||
@@ -496,6 +518,16 @@ impl AppConfig {
|
||||
{
|
||||
config.editor_bgfilter_request_timeout_ms = timeout_ms;
|
||||
}
|
||||
if let Some(threshold) =
|
||||
read_first_u32_env(&["GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD"])
|
||||
{
|
||||
config.editor_bgfilter_circuit_failure_threshold = threshold;
|
||||
}
|
||||
if let Some(cooldown_seconds) = read_first_duration_seconds_env(&[
|
||||
"GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS",
|
||||
]) {
|
||||
config.editor_bgfilter_circuit_cooldown = Duration::from_secs(cooldown_seconds.max(1));
|
||||
}
|
||||
if let Some(enabled) =
|
||||
read_first_bool_env(&["GENARRATIVE_ENABLE_IMAGE_EDITOR_AGENT_SIDEBAR"])
|
||||
{
|
||||
@@ -538,6 +570,18 @@ impl AppConfig {
|
||||
]) {
|
||||
config.external_generation_worker_lease = Duration::from_secs(lease_seconds.max(1));
|
||||
}
|
||||
if let Some(timeout_seconds) = read_first_duration_seconds_env(&[
|
||||
"GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS",
|
||||
]) {
|
||||
config.external_generation_worker_job_timeout =
|
||||
Duration::from_secs(timeout_seconds.max(1));
|
||||
}
|
||||
if let Some(timeout_seconds) = read_first_duration_seconds_env(&[
|
||||
"GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS",
|
||||
]) {
|
||||
config.external_generation_worker_long_job_timeout =
|
||||
Duration::from_secs(timeout_seconds.max(1));
|
||||
}
|
||||
if let Some(min_workers) =
|
||||
read_first_usize_env(&["GENARRATIVE_EXTERNAL_GENERATION_CONTROLLER_MIN_WORKERS"])
|
||||
{
|
||||
@@ -1503,9 +1547,14 @@ mod tests {
|
||||
use super::{
|
||||
AppConfig, DEFAULT_EDITOR_BACKGROUND_REMOVAL_BASE_URL,
|
||||
DEFAULT_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS, DEFAULT_EDITOR_BGFILTER_BASE_URL,
|
||||
DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS, DEFAULT_VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS,
|
||||
ExternalGenerationMode, LlmProvider, ProcessRole, parse_bool,
|
||||
parse_external_generation_mode, parse_process_role,
|
||||
DEFAULT_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS,
|
||||
DEFAULT_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD,
|
||||
DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS,
|
||||
DEFAULT_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS,
|
||||
DEFAULT_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS,
|
||||
DEFAULT_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS,
|
||||
DEFAULT_VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS, ExternalGenerationMode, LlmProvider,
|
||||
ProcessRole, parse_bool, parse_external_generation_mode, parse_process_role,
|
||||
};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
|
||||
@@ -1536,7 +1585,27 @@ mod tests {
|
||||
config.editor_bgfilter_request_timeout_ms,
|
||||
DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS
|
||||
);
|
||||
assert_eq!(
|
||||
config.editor_bgfilter_circuit_failure_threshold,
|
||||
DEFAULT_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD
|
||||
);
|
||||
assert_eq!(
|
||||
config.editor_bgfilter_circuit_cooldown.as_secs(),
|
||||
DEFAULT_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS
|
||||
);
|
||||
assert!(config.editor_bgfilter_token.is_none());
|
||||
assert_eq!(
|
||||
config.external_generation_worker_lease.as_secs(),
|
||||
DEFAULT_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS
|
||||
);
|
||||
assert_eq!(
|
||||
config.external_generation_worker_job_timeout.as_secs(),
|
||||
DEFAULT_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS
|
||||
);
|
||||
assert_eq!(
|
||||
config.external_generation_worker_long_job_timeout.as_secs(),
|
||||
DEFAULT_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS
|
||||
);
|
||||
assert!(config.ark_character_video_base_url.is_empty());
|
||||
assert_eq!(config.hyper3d_base_url, "https://api.hyper3d.com/api/v2");
|
||||
assert!(config.ark_character_video_model.is_empty());
|
||||
@@ -2263,6 +2332,87 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_env_reads_editor_bgfilter_circuit_settings() {
|
||||
let _guard = ENV_LOCK
|
||||
.get_or_init(|| Mutex::new(()))
|
||||
.lock()
|
||||
.expect("env lock should not poison");
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD");
|
||||
std::env::remove_var("GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS");
|
||||
std::env::set_var("GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD", "4");
|
||||
std::env::set_var("GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS", "90");
|
||||
}
|
||||
|
||||
let config = AppConfig::from_env();
|
||||
assert_eq!(config.editor_bgfilter_circuit_failure_threshold, 4);
|
||||
assert_eq!(config.editor_bgfilter_circuit_cooldown.as_secs(), 90);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD");
|
||||
std::env::remove_var("GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_env_allows_zero_editor_bgfilter_circuit_threshold() {
|
||||
let _guard = ENV_LOCK
|
||||
.get_or_init(|| Mutex::new(()))
|
||||
.lock()
|
||||
.expect("env lock should not poison");
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD");
|
||||
std::env::set_var("GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD", "0");
|
||||
}
|
||||
|
||||
let config = AppConfig::from_env();
|
||||
assert_eq!(config.editor_bgfilter_circuit_failure_threshold, 0);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_env_reads_external_generation_worker_timeout_settings() {
|
||||
let _guard = ENV_LOCK
|
||||
.get_or_init(|| Mutex::new(()))
|
||||
.lock()
|
||||
.expect("env lock should not poison");
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS");
|
||||
std::env::remove_var("GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS");
|
||||
std::env::remove_var("GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS");
|
||||
std::env::set_var("GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS", "5m");
|
||||
std::env::set_var(
|
||||
"GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS",
|
||||
"12m",
|
||||
);
|
||||
std::env::set_var(
|
||||
"GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS",
|
||||
"30m",
|
||||
);
|
||||
}
|
||||
|
||||
let config = AppConfig::from_env();
|
||||
assert_eq!(config.external_generation_worker_lease.as_secs(), 300);
|
||||
assert_eq!(config.external_generation_worker_job_timeout.as_secs(), 720);
|
||||
assert_eq!(
|
||||
config.external_generation_worker_long_job_timeout.as_secs(),
|
||||
1_800
|
||||
);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("GENARRATIVE_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS");
|
||||
std::env::remove_var("GENARRATIVE_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS");
|
||||
std::env::remove_var("GENARRATIVE_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_env_reads_image_editor_agent_sidebar_runtime_flag() {
|
||||
let _guard = ENV_LOCK
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
use std::{borrow::Cow, collections::BTreeMap, io::Cursor, time::Instant};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::BTreeMap,
|
||||
io::Cursor,
|
||||
sync::{Mutex, OnceLock},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
@@ -113,6 +119,14 @@ const EDITOR_BGFILTER_SEG_MODEL_ANIME_SEG: &str = "anime-seg";
|
||||
const EDITOR_PUBLICATION_MATERIAL_ASSET_KIND: &str = "editor_publication_material";
|
||||
const EDITOR_LEGACY_INLINE_IMAGE_ASSET_KIND: &str = "editor_legacy_inline_image";
|
||||
|
||||
static EDITOR_BGFILTER_CIRCUIT: OnceLock<Mutex<EditorBgfilterCircuitState>> = OnceLock::new();
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct EditorBgfilterCircuitState {
|
||||
consecutive_failures: u32,
|
||||
open_until: Option<Instant>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct EditorProjectCreateRequest {
|
||||
@@ -2205,6 +2219,17 @@ async fn remove_editor_generated_screen_background_with_bgfilter(
|
||||
screen_color: EditorScreenBackgroundColor,
|
||||
seg_model: &str,
|
||||
) -> Result<DownloadedOpenAiImage, AppError> {
|
||||
if let Some(remaining) = editor_bgfilter_circuit_open_remaining(state) {
|
||||
tracing::warn!(
|
||||
provider = "bgfilter",
|
||||
screen_color = screen_color.hex,
|
||||
seg_model,
|
||||
cooldown_ms = remaining.as_millis() as u64,
|
||||
"editor_bgfilter_circuit_open_fallback_to_local_screen_background_removal"
|
||||
);
|
||||
return remove_editor_generated_green_screen_background(image, screen_color);
|
||||
}
|
||||
|
||||
match request_editor_generated_screen_background_with_bgfilter(
|
||||
state,
|
||||
image,
|
||||
@@ -2213,8 +2238,12 @@ async fn remove_editor_generated_screen_background_with_bgfilter(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(image) => Ok(image),
|
||||
Ok(image) => {
|
||||
record_editor_bgfilter_success();
|
||||
Ok(image)
|
||||
}
|
||||
Err(error) => {
|
||||
record_editor_bgfilter_failure(state);
|
||||
tracing::warn!(
|
||||
provider = "bgfilter",
|
||||
screen_color = screen_color.hex,
|
||||
@@ -2228,6 +2257,57 @@ async fn remove_editor_generated_screen_background_with_bgfilter(
|
||||
}
|
||||
}
|
||||
|
||||
fn editor_bgfilter_circuit_open_remaining(state: &AppState) -> Option<Duration> {
|
||||
let threshold = state.config.editor_bgfilter_circuit_failure_threshold;
|
||||
if threshold == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut circuit = editor_bgfilter_circuit_state().lock().ok()?;
|
||||
let open_until = circuit.open_until?;
|
||||
let now = Instant::now();
|
||||
if open_until > now {
|
||||
return Some(open_until.duration_since(now));
|
||||
}
|
||||
|
||||
circuit.open_until = None;
|
||||
circuit.consecutive_failures = 0;
|
||||
None
|
||||
}
|
||||
|
||||
fn record_editor_bgfilter_success() {
|
||||
if let Ok(mut circuit) = editor_bgfilter_circuit_state().lock() {
|
||||
circuit.consecutive_failures = 0;
|
||||
circuit.open_until = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn record_editor_bgfilter_failure(state: &AppState) {
|
||||
let threshold = state.config.editor_bgfilter_circuit_failure_threshold;
|
||||
if threshold == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let cooldown = state.config.editor_bgfilter_circuit_cooldown;
|
||||
if let Ok(mut circuit) = editor_bgfilter_circuit_state().lock() {
|
||||
circuit.consecutive_failures = circuit.consecutive_failures.saturating_add(1);
|
||||
if circuit.consecutive_failures >= threshold {
|
||||
circuit.open_until = Some(Instant::now() + cooldown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn editor_bgfilter_circuit_state() -> &'static Mutex<EditorBgfilterCircuitState> {
|
||||
EDITOR_BGFILTER_CIRCUIT.get_or_init(|| Mutex::new(EditorBgfilterCircuitState::default()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn reset_editor_bgfilter_circuit_for_tests() {
|
||||
if let Ok(mut circuit) = editor_bgfilter_circuit_state().lock() {
|
||||
*circuit = EditorBgfilterCircuitState::default();
|
||||
}
|
||||
}
|
||||
|
||||
async fn request_editor_generated_screen_background_with_bgfilter(
|
||||
state: &AppState,
|
||||
image: &DownloadedOpenAiImage,
|
||||
@@ -7141,6 +7221,29 @@ mod tests {
|
||||
assert!(parse_editor_bgfilter_seg_model(Some("u2net")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn editor_bgfilter_circuit_opens_after_consecutive_failures_and_resets_on_success() {
|
||||
reset_editor_bgfilter_circuit_for_tests();
|
||||
let state = AppState::new(AppConfig {
|
||||
editor_bgfilter_circuit_failure_threshold: 2,
|
||||
editor_bgfilter_circuit_cooldown: Duration::from_secs(60),
|
||||
..AppConfig::default()
|
||||
})
|
||||
.expect("state should build");
|
||||
|
||||
assert!(editor_bgfilter_circuit_open_remaining(&state).is_none());
|
||||
|
||||
record_editor_bgfilter_failure(&state);
|
||||
assert!(editor_bgfilter_circuit_open_remaining(&state).is_none());
|
||||
|
||||
record_editor_bgfilter_failure(&state);
|
||||
assert!(editor_bgfilter_circuit_open_remaining(&state).is_some());
|
||||
|
||||
record_editor_bgfilter_success();
|
||||
assert!(editor_bgfilter_circuit_open_remaining(&state).is_none());
|
||||
reset_editor_bgfilter_circuit_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn editor_image_and_spritesheet_generation_are_billable() {
|
||||
let source = include_str!("editor_project.rs");
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::{
|
||||
character_animation_assets::{
|
||||
generate_editor_character_animation_for_owner, generate_editor_video_for_owner,
|
||||
},
|
||||
config::AppConfig,
|
||||
editor_generation_queue::{
|
||||
EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, EDITOR_BACKGROUND_REMOVAL_JOB_KIND,
|
||||
EDITOR_CHARACTER_ANIMATION_GENERATION_JOB_KIND,
|
||||
@@ -77,6 +78,14 @@ pub(crate) async fn run_external_generation_worker(state: AppState) -> Result<()
|
||||
concurrency,
|
||||
poll_interval_ms = poll_interval.as_millis(),
|
||||
lease_seconds = lease.as_secs(),
|
||||
job_timeout_seconds = state
|
||||
.config
|
||||
.external_generation_worker_job_timeout
|
||||
.as_secs(),
|
||||
long_job_timeout_seconds = state
|
||||
.config
|
||||
.external_generation_worker_long_job_timeout
|
||||
.as_secs(),
|
||||
"external generation worker 已启动"
|
||||
);
|
||||
|
||||
@@ -296,6 +305,7 @@ async fn process_external_generation_job(
|
||||
job: ExternalGenerationJobRecord,
|
||||
) -> Result<(), String> {
|
||||
let heartbeat_interval = external_generation_worker_heartbeat_interval(lease);
|
||||
let job_timeout = external_generation_worker_job_timeout(&state.config, job.job_kind.as_str());
|
||||
let work = with_external_generation_billing_task_id(
|
||||
job.job_id.clone(),
|
||||
process_external_generation_job_once(state.clone(), worker_id.clone(), job.clone()),
|
||||
@@ -303,11 +313,24 @@ async fn process_external_generation_job(
|
||||
tokio::pin!(work);
|
||||
let heartbeat = sleep(heartbeat_interval);
|
||||
tokio::pin!(heartbeat);
|
||||
let job_deadline = sleep(job_timeout);
|
||||
tokio::pin!(job_deadline);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut work => return result,
|
||||
_ = &mut job_deadline => {
|
||||
let message = external_generation_worker_timeout_message(&job, job_timeout);
|
||||
warn!(
|
||||
job_id = %job.job_id,
|
||||
job_kind = %job.job_kind,
|
||||
timeout_seconds = job_timeout.as_secs(),
|
||||
"external generation worker 任务超过执行预算，停止当前尝试并释放 worker 槽位"
|
||||
);
|
||||
fail_job(&state, &worker_id, &job, message.clone()).await?;
|
||||
return Err(message);
|
||||
}
|
||||
_ = &mut heartbeat => {
|
||||
renew_job_lease(&state, &worker_id, &job, lease).await?;
|
||||
heartbeat.as_mut().reset(Instant::now() + heartbeat_interval);
|
||||
@@ -1075,6 +1098,27 @@ fn external_generation_worker_heartbeat_interval(lease: Duration) -> Duration {
|
||||
Duration::from_millis(heartbeat_millis)
|
||||
}
|
||||
|
||||
fn external_generation_worker_job_timeout(config: &AppConfig, job_kind: &str) -> Duration {
|
||||
match job_kind {
|
||||
EDITOR_CHARACTER_ANIMATION_GENERATION_JOB_KIND | EDITOR_VIDEO_GENERATION_JOB_KIND => {
|
||||
config.external_generation_worker_long_job_timeout
|
||||
}
|
||||
_ => config.external_generation_worker_job_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
fn external_generation_worker_timeout_message(
|
||||
job: &ExternalGenerationJobRecord,
|
||||
timeout: Duration,
|
||||
) -> String {
|
||||
format!(
|
||||
"外部生成任务 {}（{}）超过 worker 执行预算 {} 秒，已停止当前尝试",
|
||||
job.job_id,
|
||||
job.job_kind,
|
||||
timeout.as_secs()
|
||||
)
|
||||
}
|
||||
|
||||
fn current_utc_micros() -> i64 {
|
||||
offset_datetime_to_unix_micros(time::OffsetDateTime::now_utc())
|
||||
}
|
||||
@@ -1133,6 +1177,42 @@ mod tests {
|
||||
assert_eq!(message, "VECTOR_ENGINE_API_KEY 未配置");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_job_timeout_uses_long_budget_for_video_jobs() {
|
||||
let config = AppConfig {
|
||||
external_generation_worker_job_timeout: Duration::from_secs(60),
|
||||
external_generation_worker_long_job_timeout: Duration::from_secs(600),
|
||||
..AppConfig::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
external_generation_worker_job_timeout(&config, EDITOR_IMAGE_GENERATION_JOB_KIND),
|
||||
Duration::from_secs(60)
|
||||
);
|
||||
assert_eq!(
|
||||
external_generation_worker_job_timeout(
|
||||
&config,
|
||||
EDITOR_CHARACTER_ANIMATION_GENERATION_JOB_KIND
|
||||
),
|
||||
Duration::from_secs(600)
|
||||
);
|
||||
assert_eq!(
|
||||
external_generation_worker_job_timeout(&config, EDITOR_VIDEO_GENERATION_JOB_KIND),
|
||||
Duration::from_secs(600)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_timeout_message_mentions_job_and_budget() {
|
||||
let job = external_generation_job_record_fixture(Some("lease-1"));
|
||||
|
||||
let message = external_generation_worker_timeout_message(&job, Duration::from_secs(90));
|
||||
|
||||
assert!(message.contains("extgen-1"));
|
||||
assert!(message.contains(PUZZLE_GENERATE_IMAGES_JOB_KIND));
|
||||
assert!(message.contains("90 秒"));
|
||||
}
|
||||
|
||||
fn external_generation_job_record_fixture(
|
||||
lease_token: Option<&str>,
|
||||
) -> ExternalGenerationJobRecord {
|
||||
|
||||
Reference in New Issue
Block a user