Fix/修复ui-editor调用llm与实际配置不一致 (#235)
Project CI / Repository checks (push) Successful in 2m54s
Project CI / Frontend tests (push) Successful in 4m56s
Project CI / Backend tests (push) Successful in 6m39s
Project CI / Native shell tests (push) Successful in 14m28s

复用统一的LLM调用, 原来的方法内部会默认不stream传输,
一些上游要求流式传输会出现问题

---------

Co-authored-by: 段舒康 <kdletters@qq.com>
Reviewed-on: http://192.168.35.82/git/GenarrativeAI/Genarrative/pulls/235
Co-authored-by: 王德宇 <kvtodev@outlook.com>
Co-committed-by: 王德宇 <kvtodev@outlook.com>
This commit was merged in pull request #235.
This commit is contained in:
2026-09-01 14:35:45 +08:00
committed by 段舒康
parent f0142f7019
commit ae79d50b14
7 changed files with 124 additions and 54 deletions
@@ -73,6 +73,7 @@ pub(crate) async fn request_game_creator_ui_editor_llm_at(
.with_api_kind(api_kind)
.with_model(config.llm.model.clone())
.with_request_timeout_ms(config.llm.request_timeout_ms);
let request = apply_game_creator_llm_reasoning_effort(request, &config.llm)?;
let snapshot = {
let _lock = acquire_game_creator_agent_runtime_project_write_lock_with_wait(
root,
@@ -1,6 +1,8 @@
use crate::config::build_game_creator_llm_client_from_config;
use crate::config::build_game_creator_llm_client_from_llm_config;
use crate::config::load_game_creator_app_config;
use crate::ui_editor::commands::utils::{
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, strict_json_schema,
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm,
strict_json_schema,
};
use crate::ui_editor::component::text::FontSource;
use crate::ui_editor::component::Component;
@@ -343,10 +345,21 @@ pub(crate) async fn bind_components_impl_with_provider(
});
parts.push(LlmMessageContentPart::InputImage { image_url });
}
let client = if provider_identity.is_none() {
Some(build_game_creator_llm_client_from_config()?)
let (llm, client) = if provider_identity.is_none() {
let llm = load_game_creator_app_config()
.map_err(|error| {
eprintln!("ui_binding.error stage=build_client error={error}");
error
})?
.llm;
let client =
build_game_creator_llm_client_from_llm_config(&llm, "llm").map_err(|error| {
eprintln!("ui_binding.error stage=build_client error={error}");
error
})?;
(Some(llm), Some(client))
} else {
None
(None, None)
};
let tool = LlmFunctionTool::new(
"bind_ui_components",
@@ -371,11 +384,15 @@ pub(crate) async fn bind_components_impl_with_provider(
.await
.map_err(platform_llm::LlmError::InvalidRequest)
} else {
client
.as_ref()
.expect("provider client exists without runtime identity")
.run(request)
.await
request_ui_editor_llm(
client
.as_ref()
.expect("provider client exists without runtime identity"),
llm.as_ref()
.expect("LLM config exists without runtime identity"),
request,
)
.await
}
.map_err(|error| format!("组件绑定失败:{error}"))?;
let call = response
@@ -1,5 +1,8 @@
use crate::config::build_game_creator_llm_client_from_config;
use crate::ui_editor::commands::utils::{parse_limited_llm_tool_arguments, strict_json_schema};
use crate::config::build_game_creator_llm_client_from_llm_config;
use crate::config::load_game_creator_app_config;
use crate::ui_editor::commands::utils::{
parse_limited_llm_tool_arguments, request_ui_editor_llm, strict_json_schema,
};
use crate::ui_editor::state::{State, UITree};
use platform_llm::{LlmFunctionTool, LlmMessage, LlmRunRequest, LlmToolChoice};
use serde::{Deserialize, Serialize};
@@ -426,15 +429,21 @@ pub(crate) async fn merge_ui_impl_with_provider(
);
return Err(format!("UI 合并输入超过 {MAX_MERGE_INPUT_BYTES} 字节上限"));
}
let client = if provider_identity.is_none() {
Some(
build_game_creator_llm_client_from_config().map_err(|error| {
let (llm, client) = if provider_identity.is_none() {
let llm = load_game_creator_app_config()
.map_err(|error| {
eprintln!("ui_merge.error stage=build_client error={error}");
error
})?,
)
})?
.llm;
let client =
build_game_creator_llm_client_from_llm_config(&llm, "llm").map_err(|error| {
eprintln!("ui_merge.error stage=build_client error={error}");
error
})?;
(Some(llm), Some(client))
} else {
None
(None, None)
};
let schema = llm_contract::schema().map_err(|error| {
eprintln!("ui_merge.error stage=build_schema error={error}");
@@ -463,11 +472,15 @@ pub(crate) async fn merge_ui_impl_with_provider(
.await
.map_err(platform_llm::LlmError::InvalidRequest)
} else {
client
.as_ref()
.expect("provider client exists without runtime identity")
.run(request)
.await
request_ui_editor_llm(
client
.as_ref()
.expect("provider client exists without runtime identity"),
llm.as_ref()
.expect("LLM config exists without runtime identity"),
request,
)
.await
}
.map_err(|error| {
eprintln!("ui_merge.error stage=llm_request error={error}");
@@ -1,6 +1,8 @@
use crate::config::build_game_creator_llm_client_from_config;
use crate::config::build_game_creator_llm_client_from_llm_config;
use crate::config::load_game_creator_app_config;
use crate::ui_editor::commands::utils::{
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, strict_json_schema,
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm,
strict_json_schema,
};
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
use crate::ui_editor::layout::control_layout::ControlLayout;
@@ -608,15 +610,21 @@ pub(crate) async fn recognize_ui_impl_with_provider(
eprintln!("ui_recognition.error stage=validate reason=no_root_image");
return Err("至少需要一张可作为识别上下文根的界面图".to_string());
}
let client = if provider_identity.is_none() {
Some(
build_game_creator_llm_client_from_config().map_err(|error| {
let (llm, client) = if provider_identity.is_none() {
let llm = load_game_creator_app_config()
.map_err(|error| {
eprintln!("ui_recognition.error stage=build_client error={error}");
error
})?,
)
})?
.llm;
let client =
build_game_creator_llm_client_from_llm_config(&llm, "llm").map_err(|error| {
eprintln!("ui_recognition.error stage=build_client error={error}");
error
})?;
(Some(llm), Some(client))
} else {
None
(None, None)
};
let schema = recognition_json_schema().map_err(|error| {
eprintln!("ui_recognition.error stage=build_schema error={error}");
@@ -687,11 +695,15 @@ pub(crate) async fn recognize_ui_impl_with_provider(
.await
.map_err(platform_llm::LlmError::InvalidRequest)
} else {
client
.as_ref()
.expect("provider client exists without runtime identity")
.run(request)
.await
request_ui_editor_llm(
client
.as_ref()
.expect("provider client exists without runtime identity"),
llm.as_ref()
.expect("LLM config exists without runtime identity"),
request,
)
.await
}
.map_err(|error| {
eprintln!(
@@ -1,6 +1,8 @@
use crate::config::build_game_creator_llm_client_from_config;
use crate::config::build_game_creator_llm_client_from_llm_config;
use crate::config::load_game_creator_app_config;
use crate::ui_editor::commands::utils::{
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, strict_json_schema,
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm,
strict_json_schema,
};
use crate::ui_editor::resource::ui_design_image::UIDesignImageRole;
use crate::ui_editor::state::State;
@@ -172,7 +174,13 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
});
parts.push(LlmMessageContentPart::InputImage { image_url });
}
let client = build_game_creator_llm_client_from_config().map_err(|error| {
let llm = load_game_creator_app_config()
.map_err(|error| {
eprintln!("ui_design_suggestion.error stage=build_client error={error}");
error
})?
.llm;
let client = build_game_creator_llm_client_from_llm_config(&llm, "llm").map_err(|error| {
eprintln!("ui_design_suggestion.error stage=build_client error={error}");
error
})?;
@@ -186,20 +194,21 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
schema,
)
.with_strict(true);
let response = client
.run(
LlmRunRequest::new(vec![
LlmMessage::system(SYSTEM_PROMPT),
LlmMessage::user_multimodal(parts),
])
.with_function_tools(vec![tool])
.with_tool_choice(LlmToolChoice::Required),
)
.await
.map_err(|error| {
eprintln!("ui_design_suggestion.error stage=llm_request error={error}");
format!("UI 参考图语义识别失败:{error}")
})?;
let response = request_ui_editor_llm(
&client,
&llm,
LlmRunRequest::new(vec![
LlmMessage::system(SYSTEM_PROMPT),
LlmMessage::user_multimodal(parts),
])
.with_function_tools(vec![tool])
.with_tool_choice(LlmToolChoice::Required),
)
.await
.map_err(|error| {
eprintln!("ui_design_suggestion.error stage=llm_request error={error}");
format!("UI 参考图语义识别失败:{error}")
})?;
eprintln!(
"ui_design_suggestion.llm_output text_present={} tool_call_count={}",
!response.text.trim().is_empty(),
@@ -1,4 +1,7 @@
use crate::agent::request_game_creator_llm_text;
use crate::config::{apply_game_creator_llm_reasoning_effort, parse_game_creator_llm_api_kind};
use base64::Engine as _;
use platform_llm::{LlmClient, LlmError, LlmRunRequest, LlmRunResponse};
use schemars::JsonSchema;
use std::fs::File;
use std::io::Read;
@@ -7,6 +10,21 @@ use std::path::{Path, PathBuf};
pub(crate) const LLM_TOOL_ARGUMENT_MAX_BYTES: usize = 1024 * 1024;
pub(crate) const UI_REFERENCE_IMAGE_MAX_BYTES: usize = 5 * 1024 * 1024;
/// Prepare a UI Editor request and reuse the caller-owned client. The transport
/// selection remains centralized in `request_game_creator_llm_text`.
pub(crate) async fn request_ui_editor_llm(
client: &LlmClient,
llm: &crate::GameCreatorLlmConfig,
request: LlmRunRequest,
) -> Result<LlmRunResponse, LlmError> {
let request = request.with_api_kind(
parse_game_creator_llm_api_kind(&llm.api_kind).map_err(LlmError::InvalidConfig)?,
);
let request =
apply_game_creator_llm_reasoning_effort(request, llm).map_err(LlmError::InvalidConfig)?;
request_game_creator_llm_text(client, llm, request).await
}
pub(crate) fn parse_limited_llm_tool_arguments(
arguments: &str,
) -> Result<serde_json::Value, String> {
@@ -1255,7 +1255,7 @@ game-project/
DirectProject 使用 `approvalPolicy=never`,避免每次原生调用再经过泛化 ToolHost 包装;原生命令网络保持关闭,联网资料继续走受控 `agc_web_search`。多 Agent、Apps、插件、hooks、Goals、Workspace Dependencies、Tool Suggestion 和原生浏览器/电脑控制仍关闭,避免绕过 AGC durable delegation、浏览器证据和副作用审计;图片生成通过客户端审核的 `agc_tools.agc_generate_image` 暴露普通单图、角色图、视觉规范图和 UI 设计图,完整游戏美术包继续使用 `agc_tools.taonier_prepare_game_art`,两者都复用同一客户端登录态、幂等账本、下载校验和 manifest/revision 投影,不开放 Codex 原生 image tool。app-server 使用隔离 `CODEX_HOME`,明确清空外部 MCP 后只注入 `agc_tools`;配置了 AGC LLM Key 或可解析的 `OPENAI_API_KEY` 登录态时,真实 provider 凭据只由 AGC 本地 provider proxy 持有,Codex 仅使用连接级随机代理令牌;无法安全代理的 OAuth `auth.json` 继续关闭 native shell/unified exec。`agc_tools` 的平台授权由 AGC 客户端当前登录会话和受控后端完成,普通客户端不得把 DirectProject 请求改成外部 API Key 请求;401/403 只投影为客户端登录或权限异常,不向用户索要凭据或暴露内部 URL。shell 子进程采用 `shell_environment_policy` core 继承及 secret/proxy/bridge 排除,provider key 和桥接凭据不得进入命令环境。系统提示词不再预注入项目源码快照或 Skill 正文,Codex 按需读取当前 cwd 文件。
## 2026-08-24 AGC UI 原型桥接与自主 UI workflow
- 2026-08-24 起,`ui-prototype` 与 UI 编辑器的 `UI` JSON 资源明确分离。设计图生成后必须由白名单 `ui.workflow.run` 按页面执行 `prepare → recognize → status → finalize`:为每个功能页面创建并关联 `UI` JSON,载入页面设计图和已登记图片/图标/字体,调用 UI Editor 的 provider-backed 结构识别、多树合并与分批组件绑定,持久化 State/revision,写入 `game/` 应用标记,并把 `reference-ready → structure-ready → merge-ready → binding-ready → application-ready → completed` 各阶段的 `generationKind` 和 manifest revision 投影给客户端。Provider 未配置、请求失败、工具调用缺失、结果不匹配、未知字体引用、未产出可渲染组件或仍有待审节点时保留最近真实阶段并返回 blocker,不得使用 deterministic seed 冒充完成。工作台点击 `ui-prototype` 时通过 `ensure_ui_design_resource_for_prototype` 幂等补齐关联资源;工作流完成后自动打开首个页面的 UI 编辑器 `visual-binding` 最终阶段,交给用户检查和手动调整。只生成图片、登记空 JSON 或进入普通图片画布均不构成 UI 工作流完成,详见 [`【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md`](../【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md)。
- 2026-08-24 起,`ui-prototype` 与 UI 编辑器的 `UI` JSON 资源明确分离。设计图生成后必须由白名单 `ui.workflow.run` 按页面执行 `prepare → recognize → status → finalize`:为每个功能页面创建并关联 `UI` JSON,载入页面设计图和已登记图片/图标/字体,调用 UI Editor 的 provider-backed 结构识别、多树合并与分批组件绑定,持久化 State/revision,写入 `game/` 应用标记,并把 `reference-ready → structure-ready → merge-ready → binding-ready → application-ready → completed` 各阶段的 `generationKind` 和 manifest revision 投影给客户端。Provider 未配置、请求失败、工具调用缺失、结果不匹配、未知字体引用、未产出可渲染组件或仍有待审节点时保留最近真实阶段并返回 blocker,不得使用 deterministic seed 冒充完成。工作台点击 `ui-prototype` 时通过 `ensure_ui_design_resource_for_prototype` 幂等补齐关联资源;工作流完成后自动打开首个页面的 UI 编辑器 `visual-binding` 最终阶段,交给用户检查和手动调整。UI 编辑器独立的语义建议请求也必须复用统一 LLM 传输选择,`llm.stream=true` 时发送 `stream=true` 并聚合完整工具调用后再校验结果。只生成图片、登记空 JSON 或进入普通图片画布均不构成 UI 工作流完成,详见 [`【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md`](../【技术方案】UI工作流资源桥接与Runtime执行-2026-08-24.md)。
## 2026-08-28 AGC 自主构建 relaxed 编排覆盖