From d7d98acfd4312e1f19f8e2b2fed38389f4a562c4 Mon Sep 17 00:00:00 2001 From: kdletters Date: Sun, 5 Jul 2026 17:42:27 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8E=A5=E5=85=A5VectorEngine=E7=94=BB?= =?UTF-8?q?=E5=B8=83Agent=E8=A7=84=E5=88=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 切换画布 Agent 与通用 LLM 代理到 VectorEngine gpt-5.4-mini。 为画布 Agent 注入上一轮生成结果并默认承接上一张图编辑。 补齐规范图工具分流、规范展板 prompt 要求和 JSON 残片回复兜底。 同步前后端契约、测试脚本、环境示例和项目文档。 --- .env.example | 22 +- deploy/container/api-server.env.example | 4 +- deploy/env/api-server.env.example | 8 +- .../shared-memory/decision-log.md | 15 +- docs/project-memory/shared-memory/pitfalls.md | 2 +- ...】server-rs与SpacetimeDB数据契约-2026-05-15.md | 2 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 4 +- .../【编辑器】画布Agent对话面板-2026-07-03.md | 10 +- packages/shared/src/contracts/editorAgent.ts | 1 + scripts/container-worker-smoke.mjs | 4 +- scripts/test-ve-llm.mjs | 16 +- server-rs/crates/api-server/src/config.rs | 28 +- .../crates/api-server/src/editor_agent.rs | 718 +++++++++++++++++- server-rs/crates/api-server/src/llm.rs | 23 +- server-rs/crates/api-server/src/state.rs | 9 +- .../src/apimart_gpt5_adapter.rs | 4 +- .../shared-contracts/src/editor_agent.rs | 2 + .../useEditorAgentConversation.ts | 3 + src/services/llmClient.test.ts | 62 +- src/services/llmClient.ts | 124 ++- 20 files changed, 945 insertions(+), 116 deletions(-) diff --git a/.env.example b/.env.example index 4d8a13877..df656b892 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,12 @@ # Server-side OpenAI-compatible LLM endpoint base URL. -LLM_BASE_URL="https://ark.cn-beijing.volces.com/api/v3" +LLM_BASE_URL="https://api.vectorengine.cn/v1" # Server-side API key used by the local Vite proxy. -# Recommended: set `LLM_API_KEY` or `ARK_API_KEY`. +# Recommended: set `LLM_API_KEY` locally, or use `VECTOR_ENGINE_API_KEY` +# through the Rust api-server proxy. # Legacy compatibility: `VITE_LLM_API_KEY` is still supported by the proxy, # but it should not be relied on by browser code. -LLM_API_KEY="YOUR_API_KEY" +LLM_API_KEY="" # Optional frontend override for the local proxy path. VITE_LLM_PROXY_BASE_URL="/api/llm" @@ -116,7 +117,11 @@ WECHAT_MINIPROGRAM_GENERATION_RESULT_TEMPLATE_ID="m5z7BkkBhJGbcH0cdDeHaeRU2tViDE WECHAT_MINIPROGRAM_SUBSCRIBE_MESSAGE_STATE="formal" # Model name for chat completions. -VITE_LLM_MODEL="doubao-1-5-pro-32k-character-250715" +VITE_LLM_MODEL="gpt-5.4-mini" +GENARRATIVE_LLM_PROVIDER="openai-compatible" +GENARRATIVE_LLM_BASE_URL="https://api.vectorengine.cn/v1" +GENARRATIVE_LLM_API_KEY="" +GENARRATIVE_LLM_MODEL="gpt-5.4-mini" # Optional: enable upstream web search for RPG story text generation. RPG_LLM_WEB_SEARCH_ENABLED="true" @@ -125,13 +130,8 @@ RPG_LLM_WEB_SEARCH_ENABLED="true" DASHSCOPE_BASE_URL="https://dashscope.aliyuncs.com/api/v1" DASHSCOPE_API_KEY="YOUR_DASHSCOPE_API_KEY" -# APIMart Responses config for creative-agent text/multimodal understanding. -APIMART_BASE_URL="https://api.apimart.ai/v1" -APIMART_API_KEY="YOUR_APIMART_API_KEY" -APIMART_IMAGE_REQUEST_TIMEOUT_MS="180000" - -# VectorEngine GPT-image-2 / Gemini image generation config. -VECTOR_ENGINE_BASE_URL="https://api.vectorengine.ai" +# VectorEngine LLM and GPT-image-2 / Gemini image generation config. +VECTOR_ENGINE_BASE_URL="https://api.vectorengine.cn" VECTOR_ENGINE_API_KEY="" VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS="1000000" diff --git a/deploy/container/api-server.env.example b/deploy/container/api-server.env.example index 6bf7a53d4..f46cb36a1 100644 --- a/deploy/container/api-server.env.example +++ b/deploy/container/api-server.env.example @@ -49,8 +49,8 @@ GENARRATIVE_SPACETIME_POOL_SIZE=8 GENARRATIVE_SPACETIME_PROCEDURE_TIMEOUT_SECONDS=45 GENARRATIVE_LLM_PROVIDER=openai-compatible -GENARRATIVE_LLM_BASE_URL= +GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1 GENARRATIVE_LLM_API_KEY= -GENARRATIVE_LLM_MODEL= +GENARRATIVE_LLM_MODEL=gpt-5.4-mini WECHAT_MINIPROGRAM_MESSAGE_TOKEN= WECHAT_MINIPROGRAM_MESSAGE_ENCODING_AES_KEY= diff --git a/deploy/env/api-server.env.example b/deploy/env/api-server.env.example index dc0b399c2..a0ac06244 100644 --- a/deploy/env/api-server.env.example +++ b/deploy/env/api-server.env.example @@ -57,16 +57,12 @@ GENARRATIVE_SPACETIME_POOL_SIZE=8 GENARRATIVE_SPACETIME_PROCEDURE_TIMEOUT_SECONDS=45 GENARRATIVE_LLM_PROVIDER=openai-compatible -GENARRATIVE_LLM_BASE_URL= +GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1 GENARRATIVE_LLM_API_KEY= -GENARRATIVE_LLM_MODEL= +GENARRATIVE_LLM_MODEL=gpt-5.4-mini GENARRATIVE_RPG_LLM_WEB_SEARCH_ENABLED=false GENARRATIVE_CREATION_AGENT_LLM_WEB_SEARCH_ENABLED=false -APIMART_BASE_URL= -APIMART_API_KEY= -APIMART_IMAGE_REQUEST_TIMEOUT_MS=180000 - VECTOR_ENGINE_BASE_URL=https://api.vectorengine.cn VECTOR_ENGINE_API_KEY= VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS=1000000 diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index fda2ea979..5e6fe14c2 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -2126,7 +2126,7 @@ ## 2026-05-09 GPT-image-2 图片生成统一迁移到 VectorEngine - 背景：仓库内 RPG、拼图、方洞和本地模板脚本的 GPT-image-2 生图此前依赖 APIMart 图片网关；团队要求参考 VectorEngine Apifox `api-448710071`，后续不再使用 APIMart 执行 GPT-image-2 图片生成。 -- 决策：所有 GPT-image-2 无参考图生图请求统一走 VectorEngine `POST /v1/images/generations`，有参考图请求走 `POST /v1/images/edits` multipart，基础配置读取 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` / `VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS`，上游模型使用 `gpt-image-2`，请求体不再携带 `official_fallback`。APIMart 只保留给创意 Agent 的 `gpt-5` Responses 文本/多模态链路。 +- 决策：所有 GPT-image-2 无参考图生图请求统一走 VectorEngine `POST /v1/images/generations`，有参考图请求走 `POST /v1/images/edits` multipart，基础配置读取 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` / `VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS`，上游模型使用 `gpt-image-2`，请求体不再携带 `official_fallback`。当时 APIMart 仍保留给创意 Agent 的 `gpt-5` Responses 文本/多模态链路；该文本链路已被 2026-07-05 VectorEngine Chat Completions `gpt-5.4-mini` 决策覆盖。 - 影响范围：`api-server` 共享图片 helper、拼图图片生成、角色主图、RPG 场景图、开局 CG 故事板、方洞视觉资产、生产环境示例、gpt-image-2 本地 skill 和相关技术文档。 - 验证方式：执行 `npm run check:encoding`、`cargo test -p api-server openai_image --manifest-path server-rs/Cargo.toml`、`cargo test -p api-server puzzle --manifest-path server-rs/Cargo.toml`、`cargo test -p api-server custom_world_ai --manifest-path server-rs/Cargo.toml`、`cargo test -p api-server character_visual --manifest-path server-rs/Cargo.toml`，并用 `npm run dev:api-server` + `/healthz` 做后端 smoke。 - 关联文档：`docs/technical/VECTOR_ENGINE_GPT_IMAGE_2_GENERATION_2026-05-09.md`、`docs/technical/API_SERVER_EXTERNAL_SERVICE_ENV_CONFIG_2026-05-07.md`。 @@ -2149,7 +2149,7 @@ ## 2026-05-08 APIMart 接口统一携带 `official_fallback` -> 2026-05-09 追认：本决策中的图片生成部分已被“GPT-image-2 图片生成统一迁移到 VectorEngine”覆盖；当前只保留 APIMart `gpt-5` Responses 文本/多模态链路继续显式携带 `official_fallback`。 +> 2026-05-09 追认：本决策中的图片生成部分已被“GPT-image-2 图片生成统一迁移到 VectorEngine”覆盖；2026-07-05 后 APIMart `gpt-5` Responses 文本/多模态链路也被 VectorEngine Chat Completions `gpt-5.4-mini` 覆盖，不再携带 `official_fallback`。 - 背景：APIMart 的图片生成和 Responses 接口在仓库内分散于 `api-server`、`platform-llm` 和本地 skill 脚本，若只修单点，容易出现不同入口的上游请求体不一致。 - 决策：凡是仓库内调用 APIMart 的 OpenAI 兼容接口，请求体统一携带 `official_fallback: true`；其中图片生成请求直接固定写入，`platform-llm` 的 APIMart GPT-5 client 通过显式开关开启，不默认扩散到 Ark 等其它 provider。 @@ -2304,7 +2304,7 @@ ## 2026-05-05 creative-agent Task E API / SSE facade 已落地 - 背景：Phase 1 需要先把创意 Agent 的 HTTP/SSE 门面接入 Rust `api-server`，用于前端工作区调用和拼图模板确认闭环。 -- 决策：`api-server` 挂载 `/api/runtime/creative-agent/*` 六个鉴权路由；creative session 在 Task D 表未收口前暂存在 `api-server` 运行态并按 authenticated user 校验 owner；未确认模板前不创建拼图 session，`confirm-template` 后才通过既有 `spacetime-client` 创建/编译 `puzzle_agent_session`；`gpt-5` 请求只从 `APIMART_BASE_URL` / `APIMART_API_KEY` 构造专用 Responses client，不复用通用 `GENARRATIVE_LLM_API_KEY`。 +- 决策：`api-server` 挂载 `/api/runtime/creative-agent/*` 六个鉴权路由；creative session 在 Task D 表未收口前暂存在 `api-server` 运行态并按 authenticated user 校验 owner；未确认模板前不创建拼图 session，`confirm-template` 后才通过既有 `spacetime-client` 创建/编译 `puzzle_agent_session`；当时 `gpt-5` 请求只从 `APIMART_BASE_URL` / `APIMART_API_KEY` 构造专用 Responses client，不复用通用 `GENARRATIVE_LLM_API_KEY`。该 LLM 来源已被 2026-07-05 VectorEngine Chat Completions `gpt-5.4-mini` 决策覆盖。 - 影响范围：`server-rs/crates/api-server/src/creative_agent.rs`、`creative_agent_sse.rs`、`app.rs`、`state.rs`、`module-puzzle` creative template/tool、Phase 1 PRD。 - 验证方式：`cargo check -p api-server`、`cargo test -p module-puzzle creative`、`cargo test -p api-server creative_agent`、`npm run dev:api-server` 后检查 `/healthz`、`POST /api/runtime/creative-agent/sessions`、`POST /api/runtime/creative-agent/sessions/{sessionId}/messages/stream`。 - 关联文档：`docs/prd/CREATIVE_INTERACTIVE_AGENT_PHASE1_LANGCHAIN_RUST_PUZZLE_LOOP_PRD_2026-05-05.md`。 @@ -3821,3 +3821,12 @@ - 决策：兑换码使用校验中，公共码继续按 `max_uses` 控制单用户可兑次数；唯一码和私有码改为同一 `code + user_id` 只能成功兑换一次。私有码仍先校验 `allowed_user_ids`，命中允许名单后再按该用户历史使用次数拒绝重复兑换；`global_used_count` 只保留为统计字段，不再作为唯一码 / 私有码的兑换阻断条件。 - 影响范围：`module-runtime::validate_runtime_profile_redeem_code_usage`、`profile_redeem_code_usage` 计次语义。 - 验证方式：`cargo test -p module-runtime --manifest-path server-rs/Cargo.toml runtime_profile_redeem_code_usage_validation_matches_modes`、`npm run check:encoding`、`git diff --check`。 + +## 2026-07-05 VectorEngine LLM 默认使用 `gpt-5.4-mini` + +- 背景：VectorEngine Apifox `api-349239079` 暴露 OpenAI-compatible `POST /v1/chat/completions`；创意 Agent 和通用 LLM 代理需要统一到 VectorEngine 文本服务，并将默认文本模型切换为 `gpt-5.4-mini`。 +- 决策：创意 Agent 的 `CREATIVE_AGENT_GPT5_MODEL` 固定为 `gpt-5.4-mini`，协议切到 Chat Completions，不再携带旧 APIMart `official_fallback` 字段；画布 Agent 侧边栏聊天规划请求也复用该模型和 Chat Completions 协议，不再显式使用 `gpt-4o` / Responses。通用 `/api/llm/chat/completions` 代理使用 `GENARRATIVE_LLM_PROVIDER=openai-compatible`、`GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1`、`GENARRATIVE_LLM_MODEL=gpt-5.4-mini`。未单独配置 `GENARRATIVE_LLM_API_KEY` 时，api-server 可复用 `VECTOR_ENGINE_API_KEY`；前端 LLM 客户端必须兼容 OpenAI `choices`、api-server raw `{content}` 和项目 envelope `{ok,data:{content}}` 三种非流式响应，以及 OpenAI SSE delta 和 api-server `event: delta` 两种流式响应。 +- 决策补充：画布 Agent 的 planning prompt 必须自动注入上一条已完成生成结果的 `latestGeneratedImage`，来源为上一轮 generation 的 `summary` / `toolName` / `resourceId` / `objectKey` 等轻量摘要。用户用「这张」「刚才那个」「上一张」「把衣服换成……」等方式指代上一张图或继续编辑时，规划默认调用 `edit_image` 并引用该结果；不能因为本轮没有手动附件而退回 `generate_image`。 +- 决策补充：画布 Agent 侧边栏的“规范图 / 视觉规范图 / 风格规范图 / 素材规范展板”是 Agent 规划 prompt 和 function-calling 工具选择约束，不是侧边栏 UI 说明文案。此类请求默认走 `generate_image`，prompt 必须要求规范展板包含统一视角、线条粗细、色卡、材质、阴影、圆角、状态层级、尺寸标注等视觉规范元素；角色规范图若是规范展板也走 `generate_image`，只有实际角色立绘才走 `generate_character`，多个图标素材 / 图集才走 `generate_icon_spritesheet`。 +- 影响范围：`server-rs/crates/platform-agent`、`server-rs/crates/api-server/src/config.rs`、`src/services/llmClient.ts`、`.env.example`、`deploy/env/api-server.env.example`、`scripts/test-ve-llm.mjs`。 +- 验证方式：`npm run test -- src/services/llmClient.test.ts`、`cargo test -p api-server --manifest-path server-rs/Cargo.toml from_env_reads_non_public_models_and_urls app_state_builds_creative_agent_gpt5_client_from_vector_engine_settings llm_chat_completions editor_agent_llm_request_uses_vector_engine_chat_model`、`cargo test -p platform-agent --manifest-path server-rs/Cargo.toml`、`npm run check:encoding`、`git diff --check`。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 354022d23..818d914bc 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -1370,7 +1370,7 @@ ## GPT-image-2 不再读 APIMart 图片配置 - 现象：配置了 `APIMART_BASE_URL` / `APIMART_API_KEY` 后，RPG、拼图或方洞的 GPT-image-2 生图仍返回缺配置，或请求体里还出现 `official_fallback` / `image_urls`。 -- 原因：2026-05-21 后 GPT-image-2 图片生成按 VectorEngine 创建/编辑接口分流，APIMart 只保留给创意 Agent 的 `gpt-5` Responses 文本/多模态链路。 +- 原因：2026-05-21 后 GPT-image-2 图片生成按 VectorEngine 创建/编辑接口分流；2026-07-05 后创意 Agent 文本链路也改为 VectorEngine Chat Completions `gpt-5.4-mini`，APIMart 不再作为当前创意 Agent 来源。 - 处理：为图片生成配置 `VECTOR_ENGINE_BASE_URL=https://api.vectorengine.ai`、`VECTOR_ENGINE_API_KEY`、`VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS`；排查请求体时确认无参考图路径为 `/v1/images/generations`、有参考图路径为 `/v1/images/edits`，模型为 `gpt-image-2`。 - 验证：运行 `cargo test -p api-server openai_image --manifest-path server-rs/Cargo.toml` 和相关玩法图片生成测试；真实联调只在本地私密环境放置 VectorEngine key。 - 关联：`docs/technical/VECTOR_ENGINE_GPT_IMAGE_2_GENERATION_2026-05-09.md`、`server-rs/crates/api-server/src/openai_image_generation.rs`。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 94ee0617d..8dab2f039 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -216,7 +216,7 @@ npm run check:server-rs-ddd ## 外部服务与资产 -- LLM：通用 LLM 门面继续使用 `GENARRATIVE_LLM_*`；创意 Agent `gpt-5` Responses / Chat Completions 文本链路已于 2026-06 从 APIMart 迁移到 VectorEngine，使用 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` 构造 OpenAI-compatible client，`api-server` 会把未带 `/v1` 的 VectorEngine base URL 规范化到 `/v1` 后请求 `/responses`。`APIMART_BASE_URL` / `APIMART_API_KEY` 只作为历史残留，不再作为创意 Agent gpt-5 客户端来源；后续排障时优先确认 VectorEngine `/v1/models`、`/v1/chat/completions` 和 `/v1/responses` 可用性。 +- LLM：通用 LLM 门面继续使用 `GENARRATIVE_LLM_*`；创意 Agent `gpt-5.4-mini` Chat Completions 文本链路已于 2026-06 从 APIMart 迁移到 VectorEngine，使用 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` 构造 OpenAI-compatible client，`api-server` 会把未带 `/v1` 的 VectorEngine base URL 规范化到 `/v1` 后请求 `/chat/completions`。通用 `/api/llm/chat/completions` 代理使用 `GENARRATIVE_LLM_PROVIDER=openai-compatible`、`GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1`、`GENARRATIVE_LLM_MODEL=gpt-5.4-mini`；未单独配置 `GENARRATIVE_LLM_API_KEY` 时可复用 `VECTOR_ENGINE_API_KEY`。`APIMART_BASE_URL` / `APIMART_API_KEY` 只作为历史残留，不再作为创意 Agent gpt-5.4-mini 客户端来源；后续排障时优先确认 VectorEngine `/v1/models`、`/v1/chat/completions` 和 `/v1/responses` 可用性。 - 图片生成：VectorEngine `gpt-image-2` 图片 provider 归属 `platform-image`，密钥只在后端环境变量中；`api-server` 内的 `openai_image_generation.rs` 只是兼容调用面和外部失败审计桥接，不再承载 provider 协议实现。实际外部生成运行记录统一落 `tracking_event`，`event_key = external_generation_run`，metadata 记录开始 / 结束时间、耗时、状态、成功标记、失败原因、provider task id 和结果摘要，不再写回过时的 `ai_task`。DashScope 只按仍在使用的历史能力单独处理，不作为 GPT-image-2 兜底。VectorEngine `/v1/images/generations` 和 `/v1/images/edits` 上游 POST 使用 `libcurl` 发送；`reqwest` 只保留给参考图 URL 下载和响应中图片 URL 下载。`/v1/images/edits` 的 multipart 参考图必须作为 libcurl 文件上传 part 发送，字段名为 `image`，实现上使用 `Form::buffer(file_name, bytes)` 并设置 `Content-Type`；不能只用 `contents(...).filename(...)`，否则上游会把请求转码为缺少图片并返回 `image is required`。`request_send` 阶段的 curl timeout / connect error 按可重试传输错误处理，最多尝试 5 次，并使用指数退避加短抖动；排障时优先看 `attempt`、`max_attempts`、`retry_delay_ms`、`reference_image_bytes_total` 和 `request_params`，不要把 `SendRequest` 当成上游业务错误。 - Match3D 物品 sheet：关卡整图完成后走 VectorEngine `/v1/images/edits` multipart `image`，模型为 `gpt-image-2`，`2K 1:1` 输出 `10*10` spritesheet；物品 sheet prompt 固定要求单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景，后端上传 OSS 前必须把绿幕扣成透明 PNG，并把透明整图写入 `itemSpritesheetImageSrc/itemSpritesheetImageObjectKey`。后端优先按透明 alpha 连通域从该 sheet 识别真实素材矩形并持久化 20 个物品、每个 5 个形态；识别数量不足时才回退 `10*10` 固定网格。通用系列素材图集的行列索引按每行 2 个物品计算，必须落在 `1..=10`，难度只决定运行态加载 3 / 9 / 15 / 20 种。 - Match3D UI spritesheet 和背景派生图：关卡整图作为参考图并发生成 `1K 1:1` UI spritesheet 与 `1K 9:16` 背景图，模型均为 `gpt-image-2`。UI spritesheet prompt 固定要求单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景，后端上传 OSS 前必须把绿幕扣成透明 PNG；背景图必须合成为全画幅不透明 PNG。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index aaa645b70..94d2eadc6 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -455,13 +455,13 @@ OpenTelemetry 现阶段默认开启 OTLP traces / metrics / logs，但本地日 结构化创作 / RPG 的 Responses JSON 链路默认不打开 `web_search`；本地和生产如需联网增强，必须显式配置 `GENARRATIVE_RPG_LLM_WEB_SEARCH_ENABLED=true` 或 `GENARRATIVE_CREATION_AGENT_LLM_WEB_SEARCH_ENABLED=true`。如果上游未开通工具，Responses 可能先吐自然语言再返回 `ToolNotOpen`，这类报错应按工具不可用排查，不要先当成 JSON 解析 bug。 -创意 Agent `gpt-5` 文本链路已从 APIMart 切到 VectorEngine：`api-server` 读取 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` 构造 OpenAI-compatible LLM client，并自动补齐 `/v1` 前缀用于 Responses 协议。排查或切换密钥后，可在本地运行： +创意 Agent `gpt-5.4-mini` 文本链路已从 APIMart 切到 VectorEngine：`api-server` 读取 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` 构造 OpenAI-compatible LLM client，并自动补齐 `/v1` 前缀后请求 `/chat/completions`。通用 `/api/llm/chat/completions` 代理使用 `GENARRATIVE_LLM_PROVIDER=openai-compatible`、`GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1`、`GENARRATIVE_LLM_MODEL=gpt-5.4-mini`，未单独配置 `GENARRATIVE_LLM_API_KEY` 时可复用 `VECTOR_ENGINE_API_KEY`。排查或切换密钥后，可在本地运行： ```bash node scripts/test-ve-llm.mjs ``` -该脚本读取仓库根目录 `.env.secrets.local` 中的 `VECTOR_ENGINE_BASE_URL` 和 `VECTOR_ENGINE_API_KEY`，依次探测 `/v1/models`、`/v1/chat/completions`、`/v1/responses`、`gpt-5` Chat Completions 和基础 JSON 输出能力；脚本只输出 HTTP 状态、耗时、模型和截断摘要，不应打印密钥。若 `.env.secrets.local` 不存在，先补本地 secrets 文件再运行，不要把 secrets 提交进仓库。 +该脚本读取仓库根目录 `.env.secrets.local` 中的 `VECTOR_ENGINE_BASE_URL` 和 `VECTOR_ENGINE_API_KEY`，依次探测 `/v1/models`、`/v1/chat/completions`、`/v1/responses`、`gpt-5.4-mini` Chat Completions 和基础 JSON 输出能力；脚本只输出 HTTP 状态、耗时、模型和截断摘要，不应打印密钥。若 `.env.secrets.local` 不存在，先补本地 secrets 文件再运行，不要把 secrets 提交进仓库。 ### 手机验证码短信 diff --git a/docs/【编辑器】画布Agent对话面板-2026-07-03.md b/docs/【编辑器】画布Agent对话面板-2026-07-03.md index d86c9d27b..632598beb 100644 --- a/docs/【编辑器】画布Agent对话面板-2026-07-03.md +++ b/docs/【编辑器】画布Agent对话面板-2026-07-03.md @@ -21,6 +21,9 @@ | 生成 UI 设计图 | 既有 UI 设计图生成入口对应接口 | - 意图解析与工具编排在后端 api-server，前端只渲染状态，不承接业务规则。 +- 下面的工具选择口径属于 Agent 规划 prompt / function-calling 约束，不是侧边栏 UI 说明文案；侧边栏面板不展示这些规则解释。 +- 用户要求“规范图 / 视觉规范图 / 风格规范图 / 素材规范展板”时，规划默认选择 `generate_image`，并在 prompt 中明确要求生成规范展板，包含统一视角、线条粗细、色卡、材质、阴影、圆角、状态层级、尺寸标注等可落地的视觉规范元素。 +- 用户要求“角色规范图”且语义是角色的规范展板、风格展板或设定板时，仍走 `generate_image`，不要误分流到 `generate_character`；只有实际生成角色立绘、角色主形象或角色视觉资产时才走 `generate_character`。用户要求多个图标素材、图集或 spritesheet 时才走 `generate_icon_spritesheet`。 - 所有生成必须走 `execute_billable_asset_operation_with_cost` 与模型定价配置，禁止绕过定价收口。 - 视频 / 音频 / 图层操作等其余画板功能第一期不进入对话工具面，仍走现有面板。 @@ -67,9 +70,12 @@ ## LLM 与计费 -- 编排复用 `creative_agent_gpt5_client` 的 LLM 接入配置（同 provider/env，独立用途标识），但画布 Agent 的轻量规划请求显式使用 `gpt-4o`，避免把简单 JSON 意图识别交给容易长 reasoning 的 gpt-5；function-calling 注册五类工具。 +- 编排复用 `creative_agent_gpt5_client` 的 LLM 接入配置（同 provider/env，独立用途标识），画布 Agent 规划请求固定使用 VectorEngine `gpt-5.4-mini` Chat Completions；function-calling 注册五类工具。 - 每个用户回合必须由 LLM 返回结构化计划；LLM 未配置、请求失败或返回格式不可解析时，后端写入明确错误消息，不使用本地关键词或“收到：...”回显兜底。 -- 画布 Agent 规划请求使用 Responses API、1024 `max_output_tokens` 和 60 秒 Agent 专用请求超时；生成图片/编辑图片仍走对应生成工具和模型计费。 +- 规划 prompt 必须自动带入上一条已完成生成结果的 `latestGeneratedImage` 引用，内容只包含上一轮 generation 的 `summary` / `toolName` / `resourceId` / `objectKey` / `assetObjectId` 等轻量元数据，不把私有签名 URL 或大图内容塞进 prompt。 +- 用户使用「这张」「刚才那个」「上一张」「把衣服换成……」等方式指代或编辑上一张结果图时，LLM 默认选择 `edit_image` 并引用 `latestGeneratedImage` 作为源图；除非用户明确要求全新生成，否则不能因为本轮没有重新上传附件而降级为 `generate_image`。 +- 规划 prompt 必须显式区分“规范展板”和“实际素材产出”：规范图、视觉规范图、风格规范图、素材规范展板、角色规范图等规范展板请求走 `generate_image`，并补齐统一视角、线条粗细、色卡、材质、阴影、圆角、状态层级、尺寸标注等要求；实际角色立绘才走 `generate_character`，多个图标素材 / 图集才走 `generate_icon_spritesheet`。 +- 画布 Agent 规划请求使用 Chat Completions、1024 `max_tokens` 和 60 秒 Agent 专用请求超时；生成图片/编辑图片仍走对应生成工具和模型计费。 - **对话回合免费**（聊天、分析回复不扣泥点），仅 Agent 实际触发生成工具时按对应模型定价扣泥点。 - 工具调用前后端校验泥点余额；不足时该次生成失败并在对话中以明确错误气泡告知，对话本身可继续。 diff --git a/packages/shared/src/contracts/editorAgent.ts b/packages/shared/src/contracts/editorAgent.ts index 3476bfbc8..a7daaeb33 100644 --- a/packages/shared/src/contracts/editorAgent.ts +++ b/packages/shared/src/contracts/editorAgent.ts @@ -59,6 +59,7 @@ export type EditorAgentGenerationStatus = 'generating' | 'completed' | 'failed'; export interface EditorAgentGenerationRecord { toolCallId: string; toolName: EditorAgentToolName; + summary?: string | null; taskId: string | null; status: EditorAgentGenerationStatus; model: string | null; diff --git a/scripts/container-worker-smoke.mjs b/scripts/container-worker-smoke.mjs index cf5988b0d..261e58288 100644 --- a/scripts/container-worker-smoke.mjs +++ b/scripts/container-worker-smoke.mjs @@ -464,9 +464,9 @@ GENARRATIVE_SPACETIME_POOL_SIZE=2 GENARRATIVE_SPACETIME_PROCEDURE_TIMEOUT_SECONDS=15 GENARRATIVE_LLM_PROVIDER=openai-compatible -GENARRATIVE_LLM_BASE_URL= +GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1 GENARRATIVE_LLM_API_KEY= -GENARRATIVE_LLM_MODEL= +GENARRATIVE_LLM_MODEL=gpt-5.4-mini VECTOR_ENGINE_BASE_URL= VECTOR_ENGINE_API_KEY= ALIYUN_OSS_BUCKET= diff --git a/scripts/test-ve-llm.mjs b/scripts/test-ve-llm.mjs index 764a5e55a..d2e65f06a 100644 --- a/scripts/test-ve-llm.mjs +++ b/scripts/test-ve-llm.mjs @@ -83,32 +83,32 @@ const tests = [ method: 'POST', path: '/v1/chat/completions', body: { - model: 'gpt-4o', + model: 'gpt-5.4-mini', messages: [{ role: 'user', content: '回复 ok，不要解释' }], max_tokens: 10, }, }, - // 3. Responses - Apimart 当前使用的协议 + // 3. Responses - 仅作兼容探测，creative_agent 默认走 Chat Completions { name: 'POST /v1/responses (Responses)', method: 'POST', path: '/v1/responses', body: { - model: 'gpt-4o', + model: 'gpt-5.4-mini', input: [ { role: 'user', content: [{ type: 'input_text', text: '回复 ok，不要解释' }] }, ], }, }, - // 4. 测试 gpt-5 (creative_agent 模型) + // 4. 测试 gpt-5.4-mini (creative_agent 模型) { - name: 'POST /v1/chat/completions (gpt-5, Chat)', + name: 'POST /v1/chat/completions (gpt-5.4-mini, Chat)', method: 'POST', path: '/v1/chat/completions', body: { - model: 'gpt-5', + model: 'gpt-5.4-mini', messages: [{ role: 'user', content: '回复 ok' }], max_tokens: 10, }, @@ -120,7 +120,7 @@ const tests = [ method: 'POST', path: '/v1/chat/completions', body: { - model: 'gpt-4o', + model: 'gpt-5.4-mini', messages: [ { role: 'system', content: '你是抓大鹅游戏编辑，只返回 JSON。' }, { role: 'user', content: '题材：水果。请生成 JSON：{"gameName":"水果切切乐","items":[{"name":"苹果","itemSize":"中"},{"name":"西瓜","itemSize":"大"}]}' }, @@ -155,7 +155,7 @@ console.log(`=== 结果: ${pass}/${tests.length} 通过, ${fail}/${tests.length} // 结论 if (pass >= 3) { console.log('\n✅ VectorEngine 支持 LLM 文本调用，可替代 Apimart。'); - console.log(' 将 .env.secrets.local 中 APIMART_BASE_URL 改为 VectorEngine 地址即可。'); + console.log(' 将 .env.secrets.local 中 VECTOR_ENGINE_BASE_URL / VECTOR_ENGINE_API_KEY 配好即可。'); } else if (pass <= 1) { console.log('\n❌ VectorEngine 不支持 LLM 文本调用。'); } else { diff --git a/server-rs/crates/api-server/src/config.rs b/server-rs/crates/api-server/src/config.rs index f50289e2e..fa03dc2a6 100644 --- a/server-rs/crates/api-server/src/config.rs +++ b/server-rs/crates/api-server/src/config.rs @@ -940,8 +940,21 @@ impl AppConfig { config.llm_base_url = DEFAULT_ARK_BASE_URL.to_string(); } - config.llm_api_key = - read_first_non_empty_env(&["GENARRATIVE_LLM_API_KEY", "LLM_API_KEY", "ARK_API_KEY"]); + config.llm_api_key = if config.llm_provider == LlmProvider::OpenAiCompatible { + read_first_non_empty_env(&[ + "GENARRATIVE_LLM_API_KEY", + "LLM_API_KEY", + "VECTOR_ENGINE_API_KEY", + "ARK_API_KEY", + ]) + } else { + read_first_non_empty_env(&[ + "GENARRATIVE_LLM_API_KEY", + "LLM_API_KEY", + "ARK_API_KEY", + "VECTOR_ENGINE_API_KEY", + ]) + }; if let Some(llm_model) = read_first_non_empty_env(&["GENARRATIVE_LLM_MODEL", "LLM_MODEL", "VITE_LLM_MODEL"]) @@ -1629,10 +1642,14 @@ mod tests { unsafe { std::env::remove_var("GENARRATIVE_LLM_PROVIDER"); std::env::remove_var("GENARRATIVE_LLM_BASE_URL"); + std::env::remove_var("GENARRATIVE_LLM_API_KEY"); + std::env::remove_var("LLM_API_KEY"); + std::env::remove_var("ARK_API_KEY"); std::env::remove_var("GENARRATIVE_LLM_MODEL"); std::env::remove_var("APIMART_BASE_URL"); std::env::remove_var("APIMART_IMAGE_REQUEST_TIMEOUT_MS"); std::env::remove_var("VECTOR_ENGINE_BASE_URL"); + std::env::remove_var("VECTOR_ENGINE_API_KEY"); std::env::remove_var("VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS"); std::env::remove_var("HYPER3D_BASE_URL"); std::env::remove_var("DASHSCOPE_SCENE_IMAGE_MODEL"); @@ -1648,7 +1665,9 @@ mod tests { std::env::set_var("GENARRATIVE_LLM_MODEL", "internal-text-model"); std::env::set_var("APIMART_BASE_URL", "https://responses.internal.example/v1"); std::env::set_var("APIMART_IMAGE_REQUEST_TIMEOUT_MS", "190000"); + std::env::set_var("ARK_API_KEY", "ark-key"); std::env::set_var("VECTOR_ENGINE_BASE_URL", "https://vector.internal.example"); + std::env::set_var("VECTOR_ENGINE_API_KEY", "vector-engine-key"); std::env::set_var("VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS", "210000"); std::env::set_var("HYPER3D_BASE_URL", "https://model.internal.example/api/v2"); std::env::set_var("DASHSCOPE_SCENE_IMAGE_MODEL", "scene-model"); @@ -1664,6 +1683,7 @@ mod tests { let config = AppConfig::from_env(); assert_eq!(config.llm_provider, LlmProvider::OpenAiCompatible); assert_eq!(config.llm_base_url, "https://llm.internal.example/v1"); + assert_eq!(config.llm_api_key.as_deref(), Some("vector-engine-key")); assert_eq!(config.llm_model, "internal-text-model"); // assert_eq!( // config.apimart_base_url, @@ -1694,10 +1714,14 @@ mod tests { unsafe { std::env::remove_var("GENARRATIVE_LLM_PROVIDER"); std::env::remove_var("GENARRATIVE_LLM_BASE_URL"); + std::env::remove_var("GENARRATIVE_LLM_API_KEY"); + std::env::remove_var("LLM_API_KEY"); + std::env::remove_var("ARK_API_KEY"); std::env::remove_var("GENARRATIVE_LLM_MODEL"); std::env::remove_var("APIMART_BASE_URL"); std::env::remove_var("APIMART_IMAGE_REQUEST_TIMEOUT_MS"); std::env::remove_var("VECTOR_ENGINE_BASE_URL"); + std::env::remove_var("VECTOR_ENGINE_API_KEY"); std::env::remove_var("VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS"); std::env::remove_var("HYPER3D_BASE_URL"); std::env::remove_var("DASHSCOPE_SCENE_IMAGE_MODEL"); diff --git a/server-rs/crates/api-server/src/editor_agent.rs b/server-rs/crates/api-server/src/editor_agent.rs index 76fa1077c..1739eb0f6 100644 --- a/server-rs/crates/api-server/src/editor_agent.rs +++ b/server-rs/crates/api-server/src/editor_agent.rs @@ -65,7 +65,7 @@ const EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES: usize = 2 * 1024 * 1024; const EDITOR_AGENT_MESSAGES_READ_EXPIRE_SECONDS: u64 = 60; const EDITOR_AGENT_LLM_ATTACHMENT_URL_EXPIRE_SECONDS: u64 = 300; const EDITOR_AGENT_LLM_MAX_HISTORY_MESSAGES: usize = 12; -const EDITOR_AGENT_LLM_PLANNING_MODEL: &str = "gpt-4o"; +const EDITOR_AGENT_LLM_PLANNING_MODEL: &str = platform_agent::CREATIVE_AGENT_GPT5_MODEL; const EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS: u32 = 1024; const EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS: u64 = 60_000; const EDITOR_AGENT_TOOL_CALL_ID_PREFIX: &str = "editor-agent-tool"; @@ -262,6 +262,7 @@ pub async fn stream_editor_agent_message( }; document.messages.push(user_message.clone()); write_messages_document(&state, &conversation, &document).await?; + let latest_generation_reference = latest_editor_agent_generated_image_reference(&document); let assistant_message_id = build_prefixed_uuid_id(EDITOR_AGENT_MESSAGE_ID_PREFIX); let stream_state = state.clone(); @@ -269,6 +270,7 @@ pub async fn stream_editor_agent_message( let stream_title = was_empty.then(|| derive_conversation_title(user_message.text.as_str())); let stream_assistant_message_id = assistant_message_id.clone(); let stream_user_message = user_message.clone(); + let stream_latest_generation_reference = latest_generation_reference.clone(); let stream_request_context = request_context.clone(); let stream_conversation_lock_guard = conversation_lock_guard; @@ -291,6 +293,7 @@ pub async fn stream_editor_agent_message( &stream_conversation, &document, &stream_user_message, + stream_latest_generation_reference.as_ref(), ) .await { Ok(turn_plan) => turn_plan, @@ -373,6 +376,7 @@ pub async fn stream_editor_agent_message( &stream_conversation, &stream_user_message, &tool_call_id, + stream_latest_generation_reference.as_ref(), tool_call, ) .await @@ -413,6 +417,7 @@ pub async fn stream_editor_agent_message( generation_records.push(EditorAgentGenerationRecord { tool_call_id: tool_call_id.clone(), tool_name: error.tool_name, + summary: tool_summary.clone(), task_id: None, status: EditorAgentGenerationStatus::Failed, model: error_model.clone(), @@ -426,7 +431,7 @@ pub async fn stream_editor_agent_message( message_id: stream_assistant_message_id.clone(), tool_call_id, tool_name: error.tool_name, - summary: Some(editor_agent_tool_default_summary(error.tool_name)), + summary: tool_summary, task_id: None, model: error_model, status: Some(EditorAgentGenerationStatus::Failed), @@ -968,6 +973,22 @@ struct EditorAgentToolExecution { summary: String, } +#[derive(Clone, Debug)] +struct EditorAgentGeneratedImageReference { + source_message_id: String, + tool_call_id: String, + tool_name: EditorAgentToolName, + summary: Option, + model: Option, + resource_id: Option, + object_key: Option, + asset_object_id: Option, + image_src: String, + thumbnail_src: Option, + width: Option, + height: Option, +} + #[derive(Debug)] struct EditorAgentToolExecutionError { tool_name: EditorAgentToolName, @@ -1022,17 +1043,28 @@ async fn plan_editor_agent_turn( conversation: &EditorAgentConversationRecord, document: &EditorAgentConversationMessagesDocument, user_message: &EditorAgentMessage, + latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, ) -> Result { let Some(llm_client) = state.creative_agent_gpt5_client() else { return Err(EditorAgentPlanningError::LlmUnavailable); }; - let request = build_editor_agent_llm_request(state, conversation, document, user_message).await; + let request = build_editor_agent_llm_request( + state, + conversation, + document, + user_message, + latest_generation_reference, + ) + .await; let response = llm_client .request_text(request) .await .map_err(EditorAgentPlanningError::LlmRequestFailed)?; parse_editor_agent_turn_plan(response.content.as_str()) + .map(|plan| { + apply_previous_generation_edit_default(plan, user_message, latest_generation_reference) + }) .ok_or(EditorAgentPlanningError::InvalidLlmResponse) } @@ -1041,21 +1073,35 @@ async fn build_editor_agent_llm_request( conversation: &EditorAgentConversationRecord, document: &EditorAgentConversationMessagesDocument, user_message: &EditorAgentMessage, + latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, ) -> LlmTextRequest { let mut user_parts = vec![LlmMessageContentPart::InputText { - text: build_editor_agent_llm_user_prompt(conversation, document, user_message), + text: build_editor_agent_llm_user_prompt( + conversation, + document, + user_message, + latest_generation_reference, + ), }]; for attachment in &user_message.attachments { if let Some(image_url) = sign_editor_agent_attachment_image_url(state, attachment) { user_parts.push(LlmMessageContentPart::InputImage { image_url }); } } + if user_message.attachments.is_empty() + && editor_agent_text_mentions_previous_generation(user_message.text.as_str()) + { + if let Some(reference) = latest_generation_reference { + if let Some(image_url) = sign_editor_agent_generated_image_url(state, reference) { + user_parts.push(LlmMessageContentPart::InputImage { image_url }); + } + } + } LlmTextRequest::new(vec![ LlmMessage::system(editor_agent_llm_system_prompt()), LlmMessage::user_multimodal(user_parts), ]) - .with_responses_api() .with_model(EDITOR_AGENT_LLM_PLANNING_MODEL) .with_max_tokens(EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS) .with_request_timeout_ms(EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS) @@ -1067,7 +1113,20 @@ fn editor_agent_llm_system_prompt() -> String { "必须只输出一个 JSON 对象，不要输出 Markdown、解释或额外文本。", "JSON 结构：{\"replyText\":\"给用户看的中文回复\",\"toolCall\":null 或 {\"toolName\":\"generate_image|edit_image|generate_character|generate_icon_spritesheet|generate_ui_design\",\"prompt\":\"生成或修改提示词\",\"summary\":\"短动作摘要\",\"model\":null,\"aspectRatio\":null,\"imageSize\":\"1K\",\"iconDescriptions\":[\"图标描述\"]}}。", "只有用户明确要求生成、重绘、修改、画 UI、出角色或图标时才给 toolCall；普通咨询、评价、解释时 toolCall 为 null。", - "edit_image 和 generate_icon_spritesheet 必须依赖用户附件；没有附件时不要调用这两个工具，在 replyText 中提示先选择参考图。", + "工具说明：", + "generate_image：从文字生成一张全新图片，适合新场景、新物体、新插画、新背景、规范图/视觉规范图/风格规范图/素材规范展板；不要用于修改上一张图或附件图。prompt 必须写完整画面、主体、风格、构图和背景。", + "edit_image：修改已有图片，适合换衣服、改颜色、替换背景、局部重绘、保持主体/构图/姿势不变的编辑请求；必须使用 latestAttachments 或 latestGeneratedImage 作为源图。", + "generate_character：生成新的角色形象、人物立绘或角色设定图；如果用户是在改上一张角色图的服装、颜色、表情、姿势或背景，应改用 edit_image。", + "generate_icon_spritesheet：生成一组图标素材或图标图集，适合用户明确要多个 icon / 图标 / spritesheet；必须有 latestAttachments 作为图标规范或风格参考，并填写 iconDescriptions。", + "generate_ui_design：生成一张完整 UI 设计图或界面稿，适合 HUD、弹窗、面板、按钮组合和整页界面；不要用于提取图标、拆素材或修改上一张图。", + "规范图要求：用户要求生成规范图/视觉规范图/风格规范图/素材规范展板时，当前使用 generate_image；prompt 必须明确这是规范展板，并写入统一视角、线条粗细、描边、填充风格、材质、阴影、圆角、状态层级、色卡/色号、尺寸标注和排版层级。", + "角色规范图/角色美术视觉规范设定图属于规范展板时使用 generate_image，prompt 要包含头身比例、标准立绘、动作帧样例、服饰配饰分层和专属角色色卡；只有用户要生成单个全新角色形象/立绘/普通角色设定图时才使用 generate_character。", + "图标规范图/图标视觉规范展板属于规范展板时使用 generate_image；只有用户明确要生成多个图标成品、图标素材图集或 spritesheet，并提供图标规范/风格参考附件时才使用 generate_icon_spritesheet。", + "UI 规范图/组件规范展板如果是规范展板而非完整可用界面稿，也使用 generate_image；完整界面稿/HUD/弹窗/面板才使用 generate_ui_design。", + "工具选择优先级：明确修改/指代已有图 => edit_image；规范图/视觉规范图/风格规范图/素材规范展板 => generate_image；明确新角色且不是规范展板或修改已有图 => generate_character；多个图标成品/图标图集 => generate_icon_spritesheet；完整界面稿 => generate_ui_design；其他全新图片 => generate_image。", + "edit_image 必须依赖 latestAttachments 或 latestGeneratedImage；generate_icon_spritesheet 必须依赖 latestAttachments。", + "当 latestGeneratedImage 存在，且用户说“这张/刚才那个/上一张/把衣服换成/改成/换成/修改上一张图”等指代或修改上一轮结果的话，必须选择 edit_image，prompt 保留用户修改要求，不要要求用户重新选择参考图，也不要降级成 generate_image。", + "没有 latestAttachments 且没有 latestGeneratedImage 时，不要调用 edit_image 或 generate_icon_spritesheet，应在 replyText 中提示先选择参考图。", "generate_ui_design 表示生成一张 UI 设计图，不是提取 UI 素材。", "对话回复要简短，不能承诺免费生成；生成工具由后端按模型定价扣泥点。", ] @@ -1078,6 +1137,7 @@ fn build_editor_agent_llm_user_prompt( conversation: &EditorAgentConversationRecord, document: &EditorAgentConversationMessagesDocument, user_message: &EditorAgentMessage, + latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, ) -> String { let history = document .messages @@ -1094,6 +1154,11 @@ fn build_editor_agent_llm_user_prompt( "text": message.text, "attachmentCount": message.attachments.len(), "generationCount": message.generations.len(), + "generations": message + .generations + .iter() + .map(editor_agent_generation_context_json) + .collect::>(), }) }) .collect::>(); @@ -1117,11 +1182,53 @@ fn build_editor_agent_llm_user_prompt( "projectId": conversation.project_id, "latestUserText": user_message.text, "latestAttachments": attachments, + "latestGeneratedImage": latest_generation_reference + .map(editor_agent_generated_image_reference_context_json), "recentMessages": history, }) .to_string() } +fn editor_agent_generation_context_json(generation: &EditorAgentGenerationRecord) -> Value { + json!({ + "toolCallId": generation.tool_call_id, + "toolName": generation.tool_name, + "summary": generation.summary, + "status": generation.status, + "model": generation.model, + "images": generation.images.iter().take(4).map(|image| json!({ + "resourceId": image.resource_id, + "objectKey": image.object_key, + "assetObjectId": image.asset_object_id, + "imageSrc": image.image_src, + "thumbnailSrc": image.thumbnail_src, + "width": image.width, + "height": image.height, + "hasObjectKey": image.object_key.as_ref().is_some_and(|value| !value.trim().is_empty()), + })).collect::>(), + }) +} + +fn editor_agent_generated_image_reference_context_json( + reference: &EditorAgentGeneratedImageReference, +) -> Value { + json!({ + "sourceMessageId": reference.source_message_id, + "toolCallId": reference.tool_call_id, + "toolName": reference.tool_name, + "summary": reference.summary, + "model": reference.model, + "resourceId": reference.resource_id, + "objectKey": reference.object_key, + "assetObjectId": reference.asset_object_id, + "imageSrc": reference.image_src, + "thumbnailSrc": reference.thumbnail_src, + "width": reference.width, + "height": reference.height, + "hasObjectKey": reference.object_key.as_ref().is_some_and(|value| !value.trim().is_empty()), + }) +} + fn sign_editor_agent_attachment_image_url( state: &AppState, attachment: &EditorAgentAttachmentRef, @@ -1141,10 +1248,201 @@ fn sign_editor_agent_attachment_image_url( .map(|signed| signed.signed_url) } +fn sign_editor_agent_generated_image_url( + state: &AppState, + reference: &EditorAgentGeneratedImageReference, +) -> Option { + let object_key = reference + .object_key + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty())?; + state + .oss_client()? + .sign_get_object_url(OssSignedGetObjectUrlRequest { + object_key: object_key.trim_start_matches('/').to_string(), + expire_seconds: Some(EDITOR_AGENT_LLM_ATTACHMENT_URL_EXPIRE_SECONDS), + }) + .ok() + .map(|signed| signed.signed_url) +} + +fn latest_editor_agent_generated_image_reference( + document: &EditorAgentConversationMessagesDocument, +) -> Option { + document.messages.iter().rev().find_map(|message| { + message + .generations + .iter() + .rev() + .filter(|generation| generation.status == EditorAgentGenerationStatus::Completed) + .find_map(|generation| { + generation.images.iter().rev().find_map(|image| { + let has_reference_source = image + .object_key + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + || image + .resource_id + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + || !image.image_src.trim().is_empty(); + has_reference_source.then(|| EditorAgentGeneratedImageReference { + source_message_id: message.id.clone(), + tool_call_id: generation.tool_call_id.clone(), + tool_name: generation.tool_name, + summary: generation.summary.clone(), + model: generation.model.clone(), + resource_id: image.resource_id.clone(), + object_key: image.object_key.clone(), + asset_object_id: image.asset_object_id.clone(), + image_src: image.image_src.clone(), + thumbnail_src: image.thumbnail_src.clone(), + width: image.width, + height: image.height, + }) + }) + }) + }) +} + +fn editor_agent_generated_reference_source( + reference: &EditorAgentGeneratedImageReference, +) -> Option { + reference + .object_key + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(|value| value.trim_start_matches('/').to_string()) + .or_else(|| { + reference + .resource_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + }) + .or_else(|| { + reference + .image_src + .trim() + .strip_prefix('/') + .map(str::to_string) + .filter(|value| !value.is_empty()) + }) +} + +fn editor_agent_text_mentions_previous_generation(text: &str) -> bool { + let normalized = text.trim().to_lowercase(); + contains_any( + normalized.as_str(), + &[ + "这张", + "这个图", + "这幅", + "这张图", + "刚才", + "刚刚", + "刚才那个", + "刚才那张", + "上一张", + "上张", + "上一幅", + "上一个", + "前一张", + "上次生成", + "刚生成", + "latest image", + "previous image", + "last image", + ], + ) +} + +fn editor_agent_text_requests_previous_generation_edit(text: &str) -> bool { + let normalized = text.trim().to_lowercase(); + if editor_agent_text_mentions_previous_generation(normalized.as_str()) + && contains_any( + normalized.as_str(), + &[ + "修改", "改成", "换成", "替换", "重绘", "编辑", "优化", "去掉", "加上", "增加", + "变成", "调整", "换一", "换掉", "remove", "change", "edit", "redraw", + ], + ) + { + return true; + } + + let has_visual_target = contains_any( + normalized.as_str(), + &[ + "衣服", "服装", "发型", "头发", "背景", "颜色", "色调", "表情", "姿势", "主体", "风格", + "脸", "眼睛", "帽子", "鞋", "裙子", "外套", + ], + ); + let has_edit_verb = contains_any( + normalized.as_str(), + &[ + "换", "改", "变", "加", "去", "删", "替换", "调整", "change", "edit", "remove", + ], + ); + has_visual_target && has_edit_verb +} + +fn apply_previous_generation_edit_default( + mut plan: EditorAgentTurnPlan, + user_message: &EditorAgentMessage, + latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, +) -> EditorAgentTurnPlan { + if latest_generation_reference.is_none() + || !user_message.attachments.is_empty() + || !editor_agent_text_requests_previous_generation_edit(user_message.text.as_str()) + { + return plan; + } + + let should_force_edit = plan + .tool_call + .as_ref() + .is_none_or(|tool_call| tool_call.tool_name != EditorAgentToolName::EditImage); + if !should_force_edit { + return plan; + } + + let original_tool = plan.tool_call.take(); + let prompt = original_tool + .as_ref() + .map(|tool_call| tool_call.prompt.clone()) + .unwrap_or_else(|| user_message.text.trim().to_string()); + let summary = original_tool + .as_ref() + .and_then(|tool_call| tool_call.summary.clone()) + .or_else(|| Some("修改上一张图".to_string())); + let model = original_tool + .as_ref() + .and_then(|tool_call| tool_call.model.clone()); + let image_size = original_tool + .as_ref() + .and_then(|tool_call| tool_call.image_size.clone()) + .or_else(|| Some("1K".to_string())); + + plan.reply_text = "我会基于上一张生成图按你的要求修改。".to_string(); + plan.tool_call = Some(EditorAgentToolCallPlan { + tool_name: EditorAgentToolName::EditImage, + prompt, + summary, + model, + aspect_ratio: None, + image_size, + icon_descriptions: Vec::new(), + }); + plan +} + fn parse_editor_agent_turn_plan(raw_text: &str) -> Option { let json_text = extract_editor_agent_json_object(raw_text)?; let raw: EditorAgentRawTurnPlan = serde_json::from_str(json_text).ok()?; - let reply_text = normalize_optional_string(raw.reply_text)?; let tool_call = match raw.tool_call { Some(tool) => { let prompt = normalize_optional_string(tool.prompt)?; @@ -1165,6 +1463,7 @@ fn parse_editor_agent_turn_plan(raw_text: &str) -> Option { } None => None, }; + let reply_text = normalize_editor_agent_reply_text(raw.reply_text, tool_call.as_ref())?; Some(EditorAgentTurnPlan { reply_text, @@ -1186,6 +1485,42 @@ fn extract_editor_agent_json_object(raw_text: &str) -> Option<&str> { (start <= end).then_some(&without_fence[start..=end]) } +fn normalize_editor_agent_reply_text( + reply_text: Option, + tool_call: Option<&EditorAgentToolCallPlan>, +) -> Option { + let reply_text = normalize_optional_string(reply_text)?; + if !editor_agent_reply_text_looks_structural(reply_text.as_str()) { + return Some(reply_text); + } + tool_call.map(editor_agent_tool_call_reply_text) +} + +fn editor_agent_reply_text_looks_structural(text: &str) -> bool { + let trimmed = text.trim(); + if trimmed.is_empty() { + return true; + } + let lower = trimmed.to_ascii_lowercase(); + if matches!(lower.as_str(), "null" | "undefined") { + return true; + } + trimmed.len() <= 2 + && trimmed + .chars() + .all(|character| matches!(character, '{' | '}' | '[' | ']' | '"' | ':' | ',')) +} + +fn editor_agent_tool_call_reply_text(tool_call: &EditorAgentToolCallPlan) -> String { + match tool_call.tool_name { + EditorAgentToolName::GenerateImage => "我来生成图片。".to_string(), + EditorAgentToolName::EditImage => "我来修改图片。".to_string(), + EditorAgentToolName::GenerateCharacter => "我来生成角色形象。".to_string(), + EditorAgentToolName::GenerateIconSpritesheet => "我来生成图标素材。".to_string(), + EditorAgentToolName::GenerateUiDesign => "我来生成 UI 设计图。".to_string(), + } +} + #[cfg(test)] fn heuristic_editor_agent_turn_plan( text: &str, @@ -1260,7 +1595,6 @@ fn heuristic_editor_agent_turn_plan( } } -#[cfg(test)] fn contains_any(text: &str, needles: &[&str]) -> bool { needles.iter().any(|needle| text.contains(needle)) } @@ -1308,6 +1642,7 @@ async fn execute_editor_agent_tool_call( conversation: &EditorAgentConversationRecord, user_message: &EditorAgentMessage, tool_call_id: &str, + latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, tool_call: EditorAgentToolCallPlan, ) -> Result { let project = state @@ -1332,21 +1667,26 @@ async fn execute_editor_agent_tool_call( }; let tool_request_context = editor_agent_tool_request_context(request_context, tool_call_id); let attachment_sources = editor_agent_attachment_sources(user_message.attachments.as_slice()); + let previous_generation_source = (tool_call.tool_name == EditorAgentToolName::EditImage + && user_message.attachments.is_empty()) + .then(|| latest_generation_reference.and_then(editor_agent_generated_reference_source)) + .flatten(); + let tool_reference_sources = previous_generation_source + .clone() + .map(|source| vec![source]) + .unwrap_or_else(|| attachment_sources.clone()); + let generation_reference_context = editor_agent_tool_generation_reference_context( + user_message.attachments.as_slice(), + latest_generation_reference, + previous_generation_source.as_deref(), + ); let generation_inputs = Some(json!({ "source": "editor-agent", "conversationId": conversation.conversation_id, "messageId": user_message.id, "toolCallId": tool_call_id, "fields": [{ "title": "用户指令", "value": tool_call.prompt }], - "references": user_message.attachments.iter().map(|attachment| json!({ - "title": attachment.label.clone().unwrap_or_else(|| "对话附件".to_string()), - "label": attachment.label, - "refType": match attachment.source { - EditorAgentAttachmentSource::CanvasResource => "project-resource", - EditorAgentAttachmentSource::LibraryAsset => "asset", - }, - "refId": attachment.reference_id, - })).collect::>(), + "references": generation_reference_context, })); let tool_name = tool_call.tool_name; @@ -1371,32 +1711,31 @@ async fn execute_editor_agent_tool_call( model: tool_call.model.clone(), aspect_ratio: tool_call.aspect_ratio.clone(), image_size: tool_call.image_size.clone(), - reference_image_srcs: Some(attachment_sources), + reference_image_srcs: Some(tool_reference_sources.clone()), project_id: Some(conversation.project_id.clone()), asset_kind: Some(editor_agent_tool_asset_kind(tool_name).to_string()), generation_inputs, asset_folder_id: Some(editor_agent_default_asset_folder_id()), asset_label: tool_call.summary.clone(), - source_resource_id: user_message - .attachments - .first() - .map(|attachment| attachment.reference_id.clone()), + source_resource_id: editor_agent_tool_source_resource_id( + user_message.attachments.as_slice(), + latest_generation_reference, + previous_generation_source.as_deref(), + ), canvas_completion: Some(completion), }, ) .await } EditorAgentToolName::EditImage => { - let (source_image_src, reference_image_srcs) = - split_editor_agent_edit_sources(attachment_sources).ok_or_else(|| { - EditorAgentToolExecutionError { - tool_name, - model: tool_model.clone(), - error: editor_agent_bad_request( - "修改图片需要先选择一张画布或素材库图片附件", - ), - } - })?; + let (source_image_src, reference_image_srcs) = split_editor_agent_edit_sources( + tool_reference_sources.clone(), + ) + .ok_or_else(|| EditorAgentToolExecutionError { + tool_name, + model: tool_model.clone(), + error: editor_agent_bad_request("修改图片需要先选择一张画布或素材库图片附件"), + })?; edit_editor_image_for_owner( state, &tool_request_context, @@ -1412,10 +1751,11 @@ async fn execute_editor_agent_tool_call( generation_inputs, asset_folder_id: Some(editor_agent_default_asset_folder_id()), asset_label: tool_call.summary.clone(), - source_resource_id: user_message - .attachments - .first() - .map(|attachment| attachment.reference_id.clone()), + source_resource_id: editor_agent_tool_source_resource_id( + user_message.attachments.as_slice(), + latest_generation_reference, + previous_generation_source.as_deref(), + ), target_layer_id: None, canvas_completion: Some(completion), }, @@ -1508,18 +1848,20 @@ fn build_editor_agent_tool_execution( .get("model") .and_then(Value::as_str) .map(str::to_string); + let resolved_summary = summary.unwrap_or_else(|| editor_agent_tool_default_summary(tool_name)); EditorAgentToolExecution { record: EditorAgentGenerationRecord { tool_call_id: tool_call_id.to_string(), tool_name, + summary: Some(resolved_summary.clone()), task_id, status: EditorAgentGenerationStatus::Completed, model, images, error: None, }, - summary: summary.unwrap_or_else(|| editor_agent_tool_default_summary(tool_name)), + summary: resolved_summary, } } @@ -1642,6 +1984,67 @@ fn editor_agent_attachment_sources(attachments: &[EditorAgentAttachmentRef]) -> .collect() } +fn editor_agent_tool_generation_reference_context( + attachments: &[EditorAgentAttachmentRef], + latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, + previous_generation_source: Option<&str>, +) -> Vec { + if previous_generation_source.is_some() { + return latest_generation_reference + .map(|reference| { + vec![json!({ + "title": reference + .summary + .clone() + .unwrap_or_else(|| "上一张生成图".to_string()), + "label": reference.summary, + "refType": "previous-generation", + "refId": reference.resource_id, + "resourceId": reference.resource_id, + "objectKey": reference.object_key, + "assetObjectId": reference.asset_object_id, + "toolCallId": reference.tool_call_id, + "toolName": reference.tool_name, + "sourceMessageId": reference.source_message_id, + "source": previous_generation_source, + "implicit": true, + })] + }) + .unwrap_or_default(); + } + + attachments + .iter() + .map(|attachment| { + json!({ + "title": attachment.label.clone().unwrap_or_else(|| "对话附件".to_string()), + "label": attachment.label, + "refType": match attachment.source { + EditorAgentAttachmentSource::CanvasResource => "project-resource", + EditorAgentAttachmentSource::LibraryAsset => "asset", + }, + "refId": attachment.reference_id, + "resourceId": (attachment.source == EditorAgentAttachmentSource::CanvasResource) + .then(|| attachment.reference_id.clone()), + "objectKey": attachment.object_key, + }) + }) + .collect() +} + +fn editor_agent_tool_source_resource_id( + attachments: &[EditorAgentAttachmentRef], + latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, + previous_generation_source: Option<&str>, +) -> Option { + if previous_generation_source.is_some() { + return latest_generation_reference.and_then(|reference| reference.resource_id.clone()); + } + attachments + .first() + .map(|attachment| attachment.reference_id.clone()) +} + fn split_editor_agent_edit_sources(sources: Vec) -> Option<(String, Vec)> { let mut iter = sources.into_iter(); let first = iter.next()?; @@ -1795,6 +2198,8 @@ where #[cfg(test)] mod tests { use super::*; + use crate::AppConfig; + use platform_llm::LlmTextProtocol; #[test] fn echo_assistant_text_uses_attachment_fallback() { @@ -1994,6 +2399,195 @@ mod tests { ); } + #[test] + fn editor_agent_turn_plan_replaces_structural_reply_fragment() { + let plan = parse_editor_agent_turn_plan( + r#"{"replyText":"{","toolCall":{"toolName":"generate_image","prompt":"生成一张角色规范图","summary":"生成角色规范图"}}"#, + ) + .expect("valid tool call should parse"); + + assert_eq!(plan.reply_text, "我来生成图片。"); + assert_eq!( + plan.tool_call.as_ref().map(|tool_call| tool_call.tool_name), + Some(EditorAgentToolName::GenerateImage) + ); + } + + #[test] + fn editor_agent_llm_system_prompt_describes_each_tool() { + let prompt = editor_agent_llm_system_prompt(); + + for expected in [ + "generate_image：从文字生成一张全新图片", + "edit_image：修改已有图片", + "generate_character：生成新的角色形象", + "generate_icon_spritesheet：生成一组图标素材或图标图集", + "generate_ui_design：生成一张完整 UI 设计图或界面稿", + "规范图/视觉规范图/风格规范图/素材规范展板", + "当前使用 generate_image", + "角色规范图/角色美术视觉规范设定图", + "图标规范图/图标视觉规范展板", + "统一视角、线条粗细", + "色卡/色号", + "工具选择优先级", + "明确修改/指代已有图 => edit_image", + "规范图/视觉规范图/风格规范图/素材规范展板 => generate_image", + ] { + assert!( + prompt.contains(expected), + "system prompt should contain {expected}" + ); + } + } + + #[test] + fn latest_generation_reference_enters_llm_prompt() { + let conversation = test_conversation_record(); + let user_message = test_user_message("把刚才那个衣服换成蓝色"); + let mut document = empty_messages_document(conversation.conversation_id.as_str()); + document.messages.push(EditorAgentMessage { + id: "editor-agent-message-assistant-1".to_string(), + role: EditorAgentMessageRole::Assistant, + kind: EditorAgentMessageKind::Chat, + text: "已生成角色图。".to_string(), + attachments: Vec::new(), + generations: vec![test_completed_generation_record()], + status: EditorAgentMessageStatus::Completed, + created_at: "2026-07-05T00:00:00Z".to_string(), + }); + document.messages.push(user_message.clone()); + + let reference = latest_editor_agent_generated_image_reference(&document) + .expect("latest generated image should be available"); + let prompt = build_editor_agent_llm_user_prompt( + &conversation, + &document, + &user_message, + Some(&reference), + ); + let prompt_json: Value = serde_json::from_str(prompt.as_str()).expect("prompt is JSON"); + + assert_eq!( + prompt_json["latestGeneratedImage"]["toolName"], + json!("generate_image") + ); + assert_eq!( + prompt_json["latestGeneratedImage"]["summary"], + json!("生成红衣角色") + ); + assert_eq!( + prompt_json["latestGeneratedImage"]["resourceId"], + json!("resource-generated-1") + ); + assert_eq!( + prompt_json["latestGeneratedImage"]["objectKey"], + json!("generated-editor-assets/result.png") + ); + assert_eq!( + prompt_json["recentMessages"][0]["generations"][0]["images"][0]["objectKey"], + json!("generated-editor-assets/result.png") + ); + } + + #[test] + fn previous_generation_edit_default_coerces_generate_image_to_edit_image() { + let user_message = test_user_message("把衣服换成蓝色"); + let reference = test_generated_image_reference(); + let plan = EditorAgentTurnPlan { + reply_text: "我来生成图片。".to_string(), + tool_call: Some(EditorAgentToolCallPlan { + tool_name: EditorAgentToolName::GenerateImage, + prompt: "把衣服换成蓝色".to_string(), + summary: Some("生成蓝色衣服".to_string()), + model: None, + aspect_ratio: Some("1:1".to_string()), + image_size: Some("1K".to_string()), + icon_descriptions: Vec::new(), + }), + }; + + let adjusted = + apply_previous_generation_edit_default(plan, &user_message, Some(&reference)); + let tool_call = adjusted.tool_call.expect("tool should be forced"); + + assert_eq!(tool_call.tool_name, EditorAgentToolName::EditImage); + assert_eq!(tool_call.prompt, "把衣服换成蓝色"); + assert_eq!(adjusted.reply_text, "我会基于上一张生成图按你的要求修改。"); + } + + #[test] + fn previous_generation_reference_becomes_implicit_edit_source() { + let reference = test_generated_image_reference(); + + let sources = editor_agent_generated_reference_source(&reference); + let context = editor_agent_tool_generation_reference_context( + &[], + Some(&reference), + sources.as_deref(), + ); + + assert_eq!( + sources.as_deref(), + Some("generated-editor-assets/result.png") + ); + assert_eq!(context[0]["refType"], json!("previous-generation")); + assert_eq!(context[0]["resourceId"], json!("resource-generated-1")); + assert_eq!( + context[0]["objectKey"], + json!("generated-editor-assets/result.png") + ); + assert_eq!(context[0]["implicit"], json!(true)); + } + + #[test] + fn editor_agent_tool_execution_persists_summary() { + let execution = build_editor_agent_tool_execution( + "editor-agent-tool-1", + EditorAgentToolName::EditImage, + Some("换蓝色衣服".to_string()), + json!({ + "ok": true, + "data": { + "model": "gpt-image-2", + "resource": { + "resourceId": "resource-generated-2", + "imageSrc": "/generated-editor-assets/blue.png", + "objectKey": "generated-editor-assets/blue.png", + "assetObjectId": "asset-object-blue", + "width": 1024, + "height": 1024 + } + } + }), + ); + + assert_eq!(execution.summary, "换蓝色衣服"); + assert_eq!(execution.record.summary.as_deref(), Some("换蓝色衣服")); + } + + #[tokio::test] + async fn editor_agent_llm_request_uses_vector_engine_chat_model() { + let conversation = test_conversation_record(); + let document = empty_messages_document(conversation.conversation_id.as_str()); + let user_message = test_user_message("帮我看看这张图还能怎么改"); + let state = AppState::new(AppConfig::default()).expect("state should build"); + + let request = + build_editor_agent_llm_request(&state, &conversation, &document, &user_message, None) + .await; + + assert_eq!(request.protocol, LlmTextProtocol::ChatCompletions); + assert_eq!( + request.model.as_deref(), + Some(platform_agent::CREATIVE_AGENT_GPT5_MODEL) + ); + assert_eq!(request.max_tokens, Some(EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS)); + assert_eq!( + request.request_timeout_ms, + Some(EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS) + ); + } + #[test] fn editor_agent_heuristic_routes_tools_from_user_text() { let character_plan = @@ -2124,4 +2718,54 @@ mod tests { updated_at: "2026-07-03T00:00:00Z".to_string(), } } + + fn test_user_message(text: &str) -> EditorAgentMessage { + EditorAgentMessage { + id: "editor-agent-message-user-1".to_string(), + role: EditorAgentMessageRole::User, + kind: EditorAgentMessageKind::Chat, + text: text.to_string(), + attachments: Vec::new(), + generations: Vec::new(), + status: EditorAgentMessageStatus::Completed, + created_at: "2026-07-05T00:00:00Z".to_string(), + } + } + + fn test_completed_generation_record() -> EditorAgentGenerationRecord { + EditorAgentGenerationRecord { + tool_call_id: "editor-agent-tool-generated-1".to_string(), + tool_name: EditorAgentToolName::GenerateImage, + summary: Some("生成红衣角色".to_string()), + task_id: Some("task-generated-1".to_string()), + status: EditorAgentGenerationStatus::Completed, + model: Some("gpt-image-2".to_string()), + images: vec![EditorAgentGeneratedImage { + resource_id: Some("resource-generated-1".to_string()), + object_key: Some("generated-editor-assets/result.png".to_string()), + asset_object_id: Some("asset-object-result".to_string()), + image_src: "/generated-editor-assets/result.png".to_string(), + thumbnail_src: Some("/generated-editor-assets/result-thumb.png".to_string()), + width: Some(1024), + height: Some(1024), + }], + error: None, + } + } + + fn test_generated_image_reference() -> EditorAgentGeneratedImageReference { + let mut document = empty_messages_document("editor-agent-conv-1"); + document.messages.push(EditorAgentMessage { + id: "editor-agent-message-assistant-1".to_string(), + role: EditorAgentMessageRole::Assistant, + kind: EditorAgentMessageKind::Chat, + text: "已生成角色图。".to_string(), + attachments: Vec::new(), + generations: vec![test_completed_generation_record()], + status: EditorAgentMessageStatus::Completed, + created_at: "2026-07-05T00:00:00Z".to_string(), + }); + latest_editor_agent_generated_image_reference(&document) + .expect("latest generated image should be available") + } } diff --git a/server-rs/crates/api-server/src/llm.rs b/server-rs/crates/api-server/src/llm.rs index b312b77a3..44cabb10b 100644 --- a/server-rs/crates/api-server/src/llm.rs +++ b/server-rs/crates/api-server/src/llm.rs @@ -191,13 +191,14 @@ mod tests { body: r#"{"id":"resp_api_server_01","model":"ark-router-test","choices":[{"message":{"content":"代理成功"},"finish_reason":"stop"}]}"#.to_string(), extra_headers: Vec::new(), }]); - let state = seed_authenticated_state(AppConfig { + let (state, user_id) = seed_authenticated_state(AppConfig { llm_base_url: server_url, llm_api_key: Some("test-key".to_string()), + llm_model: "test-model".to_string(), ..AppConfig::default() }) .await; - let token = issue_access_token(&state); + let token = issue_access_token(&state, &user_id); let app = build_router(state); let response = app @@ -266,13 +267,14 @@ mod tests { .to_string(), extra_headers: vec![("x-request-id", "req_llm_stream_01")], }]); - let state = seed_authenticated_state(AppConfig { + let (state, user_id) = seed_authenticated_state(AppConfig { llm_base_url: server_url, llm_api_key: Some("test-key".to_string()), + llm_model: "test-model".to_string(), ..AppConfig::default() }) .await; - let token = issue_access_token(&state); + let token = issue_access_token(&state, &user_id); let app = build_router(state); let response = app @@ -322,21 +324,20 @@ mod tests { assert!(body_text.contains("data: [DONE]")); } - async fn seed_authenticated_state(config: AppConfig) -> AppState { + async fn seed_authenticated_state(config: AppConfig) -> (AppState, String) { let state = AppState::new(config).expect("state should build"); - state + let user_id = state .seed_test_phone_user_with_password("13800138101", "secret123") .await .id; - state + (state, user_id) } - fn issue_access_token(state: &AppState) -> String { + fn issue_access_token(state: &AppState, user_id: &str) -> String { let claims = AccessTokenClaims::from_input( AccessTokenClaimsInput { - user_id: "user_00000001".to_string(), - session_id: state - .seed_test_refresh_session_for_user_id("user_00000001", "sess_llm_proxy"), + user_id: user_id.to_string(), + session_id: state.seed_test_refresh_session_for_user_id(user_id, "sess_llm_proxy"), provider: AuthProvider::Password, roles: vec!["user".to_string()], token_version: 2, diff --git a/server-rs/crates/api-server/src/state.rs b/server-rs/crates/api-server/src/state.rs index e7c6b098c..923e7906b 100644 --- a/server-rs/crates/api-server/src/state.rs +++ b/server-rs/crates/api-server/src/state.rs @@ -1507,8 +1507,7 @@ fn build_creative_agent_gpt5_client( config.llm_request_timeout_ms, 0, config.llm_retry_backoff_ms, - )? - .with_official_fallback(true); + )?; Ok(Some(LlmClient::new(llm_config)?)) } @@ -1641,10 +1640,10 @@ mod tests { platform_agent::CREATIVE_AGENT_GPT5_MODEL ); assert_eq!( - client.config().responses_url(), - "https://api.vectorengine.test/v1/responses" + client.config().chat_completions_url(), + "https://api.vectorengine.test/v1/chat/completions" ); - assert!(client.config().official_fallback()); + assert!(!client.config().official_fallback()); } #[test] diff --git a/server-rs/crates/platform-agent/src/apimart_gpt5_adapter.rs b/server-rs/crates/platform-agent/src/apimart_gpt5_adapter.rs index 4a1d7389c..9be721636 100644 --- a/server-rs/crates/platform-agent/src/apimart_gpt5_adapter.rs +++ b/server-rs/crates/platform-agent/src/apimart_gpt5_adapter.rs @@ -5,7 +5,7 @@ use platform_llm::{ use crate::error::PlatformAgentError; -pub const CREATIVE_AGENT_GPT5_MODEL: &str = "gpt-5"; +pub const CREATIVE_AGENT_GPT5_MODEL: &str = "gpt-5.4-mini"; #[derive(Clone)] pub struct Gpt5ResponsesAgentClient { @@ -54,7 +54,7 @@ pub fn build_gpt5_multimodal_request( max_tokens: None, request_timeout_ms: None, enable_web_search: false, - protocol: LlmTextProtocol::Responses, + protocol: LlmTextProtocol::ChatCompletions, response_reasoning_effort: None, response_text_verbosity: None, } diff --git a/server-rs/crates/shared-contracts/src/editor_agent.rs b/server-rs/crates/shared-contracts/src/editor_agent.rs index de93f755c..c3790a276 100644 --- a/server-rs/crates/shared-contracts/src/editor_agent.rs +++ b/server-rs/crates/shared-contracts/src/editor_agent.rs @@ -111,6 +111,8 @@ pub struct EditorAgentGenerationRecord { pub tool_call_id: String, pub tool_name: EditorAgentToolName, #[serde(default)] + pub summary: Option, + #[serde(default)] pub task_id: Option, pub status: EditorAgentGenerationStatus, #[serde(default)] diff --git a/src/components/image-editor/useEditorAgentConversation.ts b/src/components/image-editor/useEditorAgentConversation.ts index c779ded02..07e598207 100644 --- a/src/components/image-editor/useEditorAgentConversation.ts +++ b/src/components/image-editor/useEditorAgentConversation.ts @@ -150,6 +150,7 @@ function upsertGenerationRecord( ? { ...record, ...nextRecord, + summary: nextRecord.summary ?? record.summary, taskId: nextRecord.taskId ?? record.taskId, model: nextRecord.model ?? record.model, images: nextRecord.images.length ? nextRecord.images : record.images, @@ -416,6 +417,7 @@ export function useEditorAgentConversation({ const nextRecord: EditorAgentGenerationRecord = { toolCallId: event.data.toolCallId, toolName: event.data.toolName, + summary: event.data.summary ?? null, taskId: event.data.taskId ?? null, status, model: event.data.model ?? null, @@ -450,6 +452,7 @@ export function useEditorAgentConversation({ const nextRecord: EditorAgentGenerationRecord = { toolCallId: event.data.toolCallId, toolName: event.data.toolName, + summary: null, taskId: null, status: 'completed', model: event.data.model, diff --git a/src/services/llmClient.test.ts b/src/services/llmClient.test.ts index a46df0496..c4f6e3d7c 100644 --- a/src/services/llmClient.test.ts +++ b/src/services/llmClient.test.ts @@ -1,6 +1,9 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { streamPlainTextCompletion } from './llmClient'; +import { + requestPlainTextCompletion, + streamPlainTextCompletion, +} from './llmClient'; function createSseResponse(body: string) { const encoder = new TextEncoder(); @@ -48,4 +51,61 @@ describe('llmClient streamPlainTextCompletion', () => { expect(onUpdate).toHaveBeenNthCalledWith(2, '溪上春风'); expect(onUpdate).toHaveBeenCalledTimes(2); }); + + it('reads api-server SSE delta events', async () => { + const onUpdate = vi.fn(); + const fetchMock = vi.fn().mockResolvedValue( + createSseResponse( + [ + 'event: delta\r\n', + 'data: {"delta":"你","content":"你","finishReason":null}\r\n\r\n', + 'event: delta\r\n', + 'data: {"delta":"好","content":"你好","finishReason":null}\r\n\r\n', + 'event: complete\r\n', + 'data: {"id":"resp_01","model":"gpt-5.4-mini","content":"你好","finishReason":"stop"}\r\n\r\n', + 'data: [DONE]\r\n\r\n', + ].join(''), + ), + ); + vi.stubGlobal('fetch', fetchMock); + + const result = await streamPlainTextCompletion('system', 'user', { + onUpdate, + }); + + expect(result).toBe('你好'); + expect(onUpdate).toHaveBeenNthCalledWith(1, '你'); + expect(onUpdate).toHaveBeenNthCalledWith(2, '你好'); + expect(onUpdate).toHaveBeenCalledTimes(2); + }); +}); + +describe('llmClient requestPlainTextCompletion', () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it('reads api-server response envelope content', async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + ok: true, + data: { + id: 'resp_01', + model: 'gpt-5.4-mini', + content: '代理成功', + finishReason: 'stop', + }, + error: null, + meta: {requestId: 'req_01'}, + }), + ), + ); + vi.stubGlobal('fetch', fetchMock); + + const result = await requestPlainTextCompletion('system', 'user'); + + expect(result).toBe('代理成功'); + }); }); diff --git a/src/services/llmClient.ts b/src/services/llmClient.ts index 5182cc353..9af0aefc5 100644 --- a/src/services/llmClient.ts +++ b/src/services/llmClient.ts @@ -1,5 +1,5 @@ import type {TextStreamOptions} from './aiTypes'; -import { fetchWithApiAuth } from './apiClient'; +import { fetchWithApiAuth, type ApiRequestOptions } from './apiClient'; import { parseSseJsonObject, readSseStream } from './sseStream'; const ENV: Partial = import.meta.env ?? {}; @@ -65,6 +65,67 @@ function readLlmStreamDeltaContent(parsed: Record) { return typeof content === 'string' && content.length > 0 ? content : null; } +function readProjectLlmDeltaContent(parsed: Record) { + const delta = parsed.delta; + return typeof delta === 'string' && delta.length > 0 ? delta : null; +} + +function readProjectLlmCompleteContent(parsed: Record) { + const content = parsed.content; + return typeof content === 'string' && content.length > 0 ? content : null; +} + +function readOpenAiMessageContent(parsed: Record) { + const choices = parsed.choices; + if (!Array.isArray(choices)) { + return null; + } + + const [firstChoice] = choices; + if (typeof firstChoice !== 'object' || firstChoice === null) { + return null; + } + + const message = (firstChoice as {message?: unknown}).message; + if (typeof message !== 'object' || message === null) { + return null; + } + + const content = (message as {content?: unknown}).content; + return typeof content === 'string' && content.length > 0 ? content : null; +} + +function readLlmResponseContent(parsed: unknown) { + if (typeof parsed !== 'object' || parsed === null) { + return null; + } + + const record = parsed as Record; + const directContent = readProjectLlmCompleteContent(record); + if (directContent) { + return directContent; + } + + const openAiContent = readOpenAiMessageContent(record); + if (openAiContent) { + return openAiContent; + } + + const data = record.data; + if (typeof data === 'object' && data !== null) { + return readLlmResponseContent(data); + } + + return null; +} + +function readLlmStreamErrorMessage(parsed: Record) { + const message = parsed.message; + return typeof message === 'string' && message.trim() + ? message.trim() + : 'LLM stream returned an error event.'; +} + const NODE_ENV = getNodeEnv(); const IS_SERVER_RUNTIME = typeof window === 'undefined'; const SERVER_API_KEY = @@ -145,7 +206,11 @@ function normalizeLlmError(error: unknown): never { throw error; } -function requestLlmEndpoint(input: string, init: RequestInit = {}) { +function requestLlmEndpoint( + input: string, + init: RequestInit = {}, + options: ApiRequestOptions = {}, +) { const headers = resolveHeaders(init.headers); if (IS_SERVER_RUNTIME && SERVER_API_KEY.trim()) { headers.Authorization = `Bearer ${SERVER_API_KEY.trim()}`; @@ -158,7 +223,7 @@ function requestLlmEndpoint(input: string, init: RequestInit = {}) { return IS_SERVER_RUNTIME ? fetch(input, nextInit) - : fetchWithApiAuth(input, nextInit); + : fetchWithApiAuth(input, nextInit, options); } export function isLlmConnectivityError(error: unknown): error is LlmConnectivityError { @@ -221,8 +286,8 @@ async function requestMessageContent( } const data = JSON.parse(rawResponseText); - const content = data?.choices?.[0]?.message?.content; - if (!content || typeof content !== 'string') { + const content = readLlmResponseContent(data); + if (!content) { throw new Error('LLM response did not include message content.'); } @@ -279,19 +344,23 @@ export async function streamPlainTextCompletion( const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); try { - const response = await requestLlmEndpoint(`${API_BASE_URL}/chat/completions`, { - method: 'POST', - headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({ - model: MODEL, - stream: true, - messages: [ - {role: 'system' as const, content: systemPrompt}, - {role: 'user' as const, content: userPrompt}, - ], - }), - signal: controller.signal, - }); + const response = await requestLlmEndpoint( + `${API_BASE_URL}/chat/completions`, + { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ + model: MODEL, + stream: true, + messages: [ + {role: 'system' as const, content: systemPrompt}, + {role: 'user' as const, content: userPrompt}, + ], + }), + signal: controller.signal, + }, + {omitEnvelopeHeader: true}, + ); if (!response.ok) { const rawResponseText = await response.text(); @@ -314,13 +383,28 @@ export async function streamPlainTextCompletion( let accumulatedText = ''; - await readSseStream(response, ({ data }) => { + await readSseStream(response, ({ data, eventName }) => { if (data === '[DONE]') { return false; } const parsed = parseSseJsonObject(data); - const delta = parsed ? readLlmStreamDeltaContent(parsed) : null; + if (parsed && eventName === 'error') { + throw new Error(readLlmStreamErrorMessage(parsed)); + } + + if (parsed && eventName === 'complete') { + const content = readLlmResponseContent(parsed); + if (content && content !== accumulatedText) { + accumulatedText = content; + options.onUpdate?.(accumulatedText); + } + return; + } + + const delta = parsed + ? (readLlmStreamDeltaContent(parsed) ?? readProjectLlmDeltaContent(parsed)) + : null; if (delta) { accumulatedText += delta; options.onUpdate?.(accumulatedText);