GenarrativeAI/Genarrative
11
0
Fork 0
Code Issues 18 Pull Requests 4 Actions Packages Projects 3 Releases Wiki Activity

接入平台美术生成接口

Browse Source 
在 AI 游戏创作 Agent loop 中配置 editorApi.apiKey 时调用 External Editor 图片生成 API

将生成结果换签下载到 assets/canvas-generated/ 并登记为 canvas 来源资产

补充平台生图 mock 测试和相关方案决策文档
This commit is contained in:
AIGameCreator App
2026-06-30 15:47:10 +08:00
parent ff95dfa62f
commit e72e689f6a
3 changed files with 338 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/ai-game-creator-shell/src-tauri/src/main.rs
+334 -1
View File
@@ -197,6 +197,15 @@ struct SyncCanvasProjectAssetsResult {
assets: Vec<UploadLocalAssetResult>,
}
#[derive(Debug, Eq, PartialEq)]
struct GeneratedPlatformArtAsset {
asset: UploadLocalAssetResult,
resource_id: Option<String>,
asset_object_id: Option<String>,
task_id: Option<String>,
model: Option<String>,
}
#[derive(Debug, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
struct OpenCanvasProjectResult {
@@ -1728,8 +1737,17 @@ async fn run_game_creator_agent_loop_at(
"agent.role_briefs",
&format!("6 组角色 brief 已完成，第 {pass} 轮交给 Generator"),
);
let platform_art_step =
maybe_generate_platform_art_asset_step(root, prompt, &group_briefs, pass, progress)
.await;
append_group_brief_steps(root, pass, &agenda.relative_path, &group_briefs, &mut steps);
let group_briefs_markdown = render_agent_group_briefs_context(&group_briefs);
if let Some(step) = platform_art_step {
steps.push(step);
}
let group_briefs_markdown = append_prompt_context(
&render_agent_group_briefs_context(&group_briefs),
&render_local_asset_prompt_context(root)?,
);
emit_agent_progress(
progress,
"llm.generator",
@@ -2677,6 +2695,76 @@ fn suggested_canvas_tool_call(
})
}
async fn maybe_generate_platform_art_asset_step(
root: &Path,
prompt: &str,
briefs: &[AgentGroupBrief],
pass: u8,
progress: Option<&AgentProgressEmitter<'_>>,
) -> Option<GameCreationAgentRunStep> {
if !needs_platform_art_asset_generation(root, briefs) || !editor_api_key_is_configured() {
return None;
}
emit_agent_progress(
progress,
"editor.image_generate",
"美术组正在通过平台 External Editor API 生成首版素材",
);
let result = generate_platform_art_asset_at(root, prompt, briefs).await;
let (status, output_paths, summary) = match result {
Ok(generated) => (
"completed",
vec![generated.asset.local_path.clone()],
format!(
"已通过平台 External Editor API 生成首版美术素材：{}",
generated.asset.local_path
),
),
Err(error) => (
"failed",
Vec::new(),
format!("平台 External Editor API 生成首版美术素材失败：{error}"),
),
};
Some(with_task_context(
agent_trace_step_owned(
pass,
"美术组 / PlatformImageGenerate",
status,
vec![
".agent/spec.md".to_string(),
".agent/manifest.json".to_string(),
],
output_paths,
&summary,
"agent.tool.platform.editor_image_generate",
),
"art",
"Asset",
Some("art-asset-plan"),
"asset-generation",
))
}
fn needs_platform_art_asset_generation(root: &Path, briefs: &[AgentGroupBrief]) -> bool {
let canvas_asset_media_types = project_canvas_asset_media_types(root);
briefs.iter().any(|brief| {
brief.role_briefs.iter().any(|role_brief| {
role_brief.status == "completed"
&& role_brief.group_definition.id == "art"
&& role_brief.role_definition.id == "asset"
&& !role_has_canvas_assets(role_brief, &canvas_asset_media_types)
})
})
}
fn editor_api_key_is_configured() -> bool {
load_game_creator_app_config()
.ok()
.and_then(|config| trim_config_string(&config.editor_api.api_key))
.is_some()
}
fn role_has_canvas_assets(role_brief: &AgentRoleBrief, media_types: &[String]) -> bool {
match (
role_brief.group_definition.id,
@@ -2694,6 +2782,141 @@ fn role_has_canvas_assets(role_brief: &AgentRoleBrief, media_types: &[String]) -
}
}
async fn generate_platform_art_asset_at(
root: &Path,
prompt: &str,
briefs: &[AgentGroupBrief],
) -> Result<GeneratedPlatformArtAsset, String> {
init_local_game_project_at(root, "local-project-draft", "未命名游戏原型")?;
let api_base_url = resolve_canvas_sync_api_base_url(None)?;
let api_key = resolve_canvas_sync_api_key(None)?;
let client = reqwest::Client::new();
let generation_prompt = build_platform_art_asset_prompt(prompt, briefs);
let response = client
.post(format!(
"{}/api/external/v1/editor/images/generations",
api_base_url
))
.bearer_auth(&api_key)
.json(&serde_json::json!({
"prompt": generation_prompt,
"aspectRatio": "1:1",
"imageSize": "1K",
"assetKind": "game-art",
"assetLabel": "AI 游戏首版美术素材",
}))
.send()
.await
.map_err(|error| format!("请求平台图片生成失败：{error}"))?;
let status = response.status();
if !status.is_success() {
return Err(format!("请求平台图片生成失败：HTTP {}", status.as_u16()));
}
let payload = response
.json::<serde_json::Value>()
.await
.map_err(|error| format!("解析平台图片生成响应失败：{error}"))?;
let generated = payload.get("data").unwrap_or(&payload);
let download = resolve_canvas_resource_download(&client, &api_base_url, &api_key, generated)
.await?
.ok_or_else(|| "平台图片生成响应缺少可下载图片".to_string())?;
let null = serde_json::Value::Null;
let resource = generated.get("resource").unwrap_or(&null);
let asset = generated.get("asset").unwrap_or(&null);
let resource_id = json_string_field(resource, "resourceId");
let task_id =
json_string_field(generated, "taskId").or_else(|| json_string_field(resource, "taskId"));
let asset_object_id = json_string_field(generated, "assetObjectId")
.or_else(|| json_string_field(resource, "assetObjectId"))
.or_else(|| json_string_field(asset, "assetObjectId"));
let generated_prompt = json_string_field(generated, "actualPrompt")
.or_else(|| json_string_field(generated, "prompt"))
.or_else(|| json_string_field(resource, "actualPrompt"))
.or_else(|| json_string_field(resource, "prompt"));
let model =
json_string_field(generated, "model").or_else(|| json_string_field(resource, "model"));
let provider = json_string_field(generated, "provider")
.or_else(|| json_string_field(resource, "provider"));
let asset_kind = json_string_field(resource, "assetKind")
.or_else(|| json_string_field(asset, "assetKind"))
.unwrap_or_else(|| "game-art".to_string());
let source_hint = json_string_field(generated, "objectKey")
.or_else(|| json_string_field(generated, "imageSrc"));
let extension = infer_file_extension(source_hint.as_deref(), &download.media_type);
let file_stem = resource_id
.as_deref()
.or(task_id.as_deref())
.unwrap_or("platform-art");
let local_path = format!(
"assets/canvas-generated/{}-{}.{}",
unix_millis(),
sanitize_file_name(file_stem),
extension
);
let absolute_path = root.join(&local_path);
if let Some(parent) = absolute_path.parent() {
fs::create_dir_all(parent)
.map_err(|error| format!("创建平台生成素材目录失败：{}: {error}", parent.display()))?;
}
fs::write(&absolute_path, &download.bytes)
.map_err(|error| format!("写入平台生成素材失败：{}: {error}", absolute_path.display()))?;
let canvas_project_id = json_string_field(resource, "projectId")
.or_else(|| json_string_field(generated, "projectId"));
let registered = register_local_asset_entry(
root,
&local_path,
&asset_kind,
&download.media_type,
"platform-art",
GameCreationAppAssetSource {
kind: GameCreationAppAssetSourceKind::Canvas,
canvas_project_id,
resource_id: resource_id.clone(),
asset_object_id: asset_object_id.clone(),
task_id: task_id.clone(),
prompt: generated_prompt.clone(),
model: model.clone(),
},
)?;
append_agent_db_record(
root,
serde_json::json!({
"recordType": "canvas.asset_generate",
"assetId": registered.id.clone(),
"localPath": registered.local_path.clone(),
"resourceId": resource_id.clone(),
"assetObjectId": asset_object_id.clone(),
"taskId": task_id.clone(),
"model": model.clone(),
"provider": provider.clone(),
}),
)?;
Ok(GeneratedPlatformArtAsset {
asset: registered,
resource_id,
asset_object_id,
task_id,
model,
})
}
fn build_platform_art_asset_prompt(prompt: &str, briefs: &[AgentGroupBrief]) -> String {
let art_asset_brief = briefs
.iter()
.flat_map(|brief| brief.role_briefs.iter())
.find(|role_brief| {
role_brief.group_definition.id == "art" && role_brief.role_definition.id == "asset"
})
.map(|role_brief| role_brief.markdown.trim())
.filter(|markdown| !markdown.is_empty())
.unwrap_or("需要一张可直接用于 Web 小游戏首版原型的核心美术素材。");
format!(
"为 Web 小游戏首版原型生成一张核心美术素材，适合放入本地 assets 并被 canvas 游戏直接引用。\n用户需求：{}\n美术资产 brief：{}",
truncate_inline(prompt, 240),
truncate_prompt_context(art_asset_brief)
)
}
fn render_agent_group_brief_markdown(role_briefs: &[AgentRoleBrief]) -> String {
let mut output = String::new();
for role_brief in role_briefs {
@@ -7015,6 +7238,41 @@ mod tests {
}
})
.to_string();
let generation_body = serde_json::json!({
"imageSrc": "/generated/canvas/hero.png",
"objectKey": "generated/canvas/hero.png",
"assetObjectId": "asset-object-1",
"width": 64,
"height": 64,
"sourceType": "generated",
"prompt": "像素月光主角",
"actualPrompt": "透明 PNG 像素月光主角",
"model": "gpt-image-2",
"provider": "VectorEngine",
"taskId": "task-1",
"resource": {
"resourceId": "resource-1",
"projectId": "canvas-project-1",
"imageSrc": "/generated/canvas/hero.png",
"objectKey": "generated/canvas/hero.png",
"assetObjectId": "asset-object-1",
"width": 64,
"height": 64,
"sourceType": "generated",
"prompt": "像素月光主角",
"actualPrompt": "透明 PNG 像素月光主角",
"model": "gpt-image-2",
"provider": "VectorEngine",
"taskId": "task-1",
"assetKind": "character"
},
"asset": {
"assetId": "asset-1",
"assetObjectId": "asset-object-1",
"assetKind": "character"
}
})
.to_string();
let read_body = serde_json::json!({
"read": {
"provider": "aliyun-oss",
@@ -7033,13 +7291,19 @@ mod tests {
let mut request_buffer = [0_u8; 8192];
let read_len = stream.read(&mut request_buffer).unwrap_or(0);
let request = String::from_utf8_lossy(&request_buffer[..read_len]);
let normalized_request = request.to_ascii_lowercase();
let (content_type, body) = if request
.starts_with("GET /api/external/v1/editor/projects/canvas-project-1 ")
{
assert!(normalized_request.contains("authorization: bearer "));
("application/json", project_body.as_bytes().to_vec())
} else if request.starts_with("POST /api/external/v1/editor/images/generations ") {
assert!(normalized_request.contains("authorization: bearer "));
("application/json", generation_body.as_bytes().to_vec())
} else if request.starts_with(
"GET /api/external/v1/assets/read-url?objectKey=generated%2Fcanvas%2Fhero.png ",
) {
assert!(normalized_request.contains("authorization: bearer "));
("application/json", read_body.as_bytes().to_vec())
} else if request.starts_with("GET /signed/hero.png ") {
("image/png", b"fake-png".to_vec())
@@ -8475,6 +8739,75 @@ mod tests {
fs::remove_dir_all(root).ok();
}
#[tokio::test]
async fn generate_platform_art_asset_downloads_and_registers_external_image() {
let root = unique_project_path();
let config_dir = unique_project_path();
let base_url = spawn_mock_external_canvas_api_server();
fs::create_dir_all(&config_dir).expect("runtime config dir");
fs::write(
config_dir.join(GAME_CREATOR_CONFIG_FILE_NAME),
serde_json::json!({
"editorApi": {
"baseUrl": base_url,
"apiKey": "editor-unit-key"
}
})
.to_string(),
)
.expect("write runtime config");
let _guard = use_test_runtime_config_dir(config_dir.clone());
let art_role = AgentRoleBrief {
group_definition: GAME_CREATOR_AGENT_GROUP_DEFINITIONS[2],
role_definition: ART_AGENT_ROLES[1],
markdown: "需要像素月光主角和厨房场景素材。".to_string(),
relative_path: ".agent/passes/pass-1/groups/art/asset.md".to_string(),
status: "completed".to_string(),
tool_id: "agent.role.brief.art.asset".to_string(),
summary: "规划首版美术资产".to_string(),
};
let briefs = vec![AgentGroupBrief {
definition: GAME_CREATOR_AGENT_GROUP_DEFINITIONS[2],
markdown: "美术组汇总。".to_string(),
relative_path: ".agent/passes/pass-1/groups/art.md".to_string(),
role_briefs: vec![art_role],
}];
let result = generate_platform_art_asset_at(&root, "月光弹幕厨房", &briefs)
.await
.expect("platform art generation");
assert!(result
.asset
.local_path
.starts_with("assets/canvas-generated/"));
assert_eq!(fs::read(&result.asset.absolute_path).unwrap(), b"fake-png");
assert_eq!(result.resource_id.as_deref(), Some("resource-1"));
assert_eq!(result.asset_object_id.as_deref(), Some("asset-object-1"));
assert_eq!(result.task_id.as_deref(), Some("task-1"));
assert_eq!(result.model.as_deref(), Some("gpt-image-2"));
let manifest: Value =
serde_json::from_str(&fs::read_to_string(root.join(".agent/manifest.json")).unwrap())
.expect("manifest json");
let asset = &manifest["assets"][0];
assert_eq!(asset["kind"], "character");
assert_eq!(asset["mediaType"], "image/png");
assert_eq!(asset["source"]["kind"], "canvas");
assert_eq!(asset["source"]["canvasProjectId"], "canvas-project-1");
assert_eq!(asset["source"]["resourceId"], "resource-1");
assert_eq!(asset["source"]["assetObjectId"], "asset-object-1");
assert_eq!(asset["source"]["taskId"], "task-1");
assert_eq!(asset["source"]["model"], "gpt-image-2");
let agent_db = fs::read_to_string(root.join(".agent/agent.db")).expect("agent db");
assert!(agent_db.contains("\"recordType\":\"canvas.asset_generate\""));
assert!(!agent_db.contains("editor-unit-key"));
fs::remove_dir_all(root).ok();
fs::remove_dir_all(config_dir).ok();
}
#[test]
fn canvas_project_url_defaults_to_local_editor_route() {
let url = build_canvas_project_url(None, Some("canvas-project-1")).expect("canvas url");
docs/project-memory/shared-memory/decision-log.md
+2 -1
View File
@@ -3847,7 +3847,8 @@
- 2026-06-25 调整：普通用户通过聊天输入 `/trace` 触发只读 `agent.trace_read`，读取 `.agent/run.latest.json` 并在聊天里摘要 loop 轮次、stopReason、nextStep、active / carry-over 任务、repairRoutes、agent 建议命令和最近 step。trace 面板仍只在开发窗口展示，普通用户窗口不新增面板。
- 2026-06-25 调整：普通用户通过聊天输入 `/import-canvas-export /绝对/画板素材.zip 画板项目ID` 触发待确认 `canvas.export_import`，读取现有 `/editor/canvas` 素材导出 ZIP。导入命令只读取用户指定 ZIP，写入当前本地项目 `assets/canvas-imports/`，基础护栏限制路径逃逸、文件数量和解压体积；导出包没有真实 resourceId 时，用 `canvas-export:<file>` 作为可追踪 assetObjectId，不伪造后端画板资源行。
- 2026-06-25 调整，2026-06-30 更新：普通用户通过聊天输入 `/sync-canvas-project 画板项目ID` 触发待确认 `canvas.project_sync`，复用现有 `/api/external/v1/editor/projects/{projectId}` 读取画板项目快照，再用 `/api/external/v1/assets/read-url` 对 objectKey 或 legacy path 换签，下载资源到本地项目 `assets/canvas-sync/` 并登记为 `canvas` 来源资产；该命令从客户端配置项 `editorApi.apiKey` 读取平台 API Key，默认 base URL 为 `http://127.0.0.1:8082`，可用 `editorApi.baseUrl` 覆盖。API Key 不写入 manifest、agent.db、trace 或日志。该路径不伪装浏览器登录态，也不绕过画板生成、钱包扣费或外部生成 worker；它只同步用户 API Key 已有权限读取的画板资源。
- 2026-06-25 调整：美术组 `Asset` 和音乐组 `SFX` 角色在 loop 中读取 `.agent/manifest.json`；当本地项目还没有对应类型的 `canvas` 来源资产时，角色 step 会追加 `agent.tool.suggest.canvas.project_sync` toolCall：美术组需要 `image/*` 或 `application/vnd.genarrative.image-sequence`，音乐组需要 `audio/*`。该 toolCall 只作为 trace 中的建议，不自动调用 External Editor API，也不绕过用户确认。
- 2026-06-30 调整：`game.generate_draft` 在 `editorApi.apiKey` 已配置且美术组缺少 `canvas` 来源图片资产时，直接调用 External Editor API 的 `/api/external/v1/editor/images/generations` 生成首版美术素材，再通过 `/api/external/v1/assets/read-url` 换签下载到本地 `assets/canvas-generated/`，登记为 `canvas` 来源资产并追加 `canvas.asset_generate` 本地索引记录。API Key 不写入 manifest、agent.db、trace 或日志；生成失败不阻断本地原型生成，会在 trace 中记录失败并保留同步建议。
- 2026-06-25 调整，2026-06-30 更新：美术组 `Asset` 和音乐组 `SFX` 角色在 loop 中读取 `.agent/manifest.json`；当本地项目还没有对应类型的 `canvas` 来源资产时，角色 step 会追加 `agent.tool.suggest.canvas.project_sync` toolCall：美术组需要 `image/*` 或 `application/vnd.genarrative.image-sequence`，音乐组需要 `audio/*`。美术组在未配置 `editorApi.apiKey` 或平台生图失败时仍只给出同步建议；音乐组不调用图片生成接口，只建议同步已有音频资源。
- 2026-06-24 调整：同一本地项目多次 `game.generate_draft` 必须追加 `memory/session.md` 与 `memory/project.md`，不得覆盖历史对话和创作目标记录。
- 2026-06-24 调整：普通用户通过聊天输入 `/help` 发现可用内置命令；命令发现必须留在聊天消息里，不得因此暴露开发面板。
- 2026-06-24 调整：聊天区待确认命令的日志语义必须区分 `permission.pending`、`permission.confirm` 和 `permission.cancel`；待确认卡片必须展示本地写入目标路径，避免用户在不知道落盘位置时确认。
docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md
+2 -2
View File
@@ -67,7 +67,7 @@ game-project/
- `canvas.project_open` 只打开本机 Genarrative 编辑器的 `/editor/canvas?projectid=...`，默认地址为 `http://127.0.0.1:3000`，开发者可在开发窗口改成本机端口；不允许打开远程站点或任意 URL。
- 画板资源回流到本地项目 `assets/`，并在 manifest 中记录画板项目、资源 ID、assetObjectId、prompt、model、taskId 和 assetKind；当前最小落地提供 `asset.register` 登记项目内已有资产，并提供 `canvas.export_import` 读取现有画板素材导出 ZIP。
- `canvas.project_sync` 复用 Genarrative External Editor API，读取用户平台 API Key 可访问的画板项目快照，通过 `/api/external/v1/assets/read-url` 换签并把资源下载到本地项目 `assets/canvas-sync/`；默认 API base URL 为 `http://127.0.0.1:8082`，可用 Tauri 应用配置目录中的 `game-creator.config.json` 的 `editorApi.baseUrl` 覆盖，API Key 从同一配置的 `editorApi.apiKey` 读取，不写入项目文件、trace、manifest 或日志。
- Agent loop 中美术组 `Asset` 和音乐组 `SFX` 会读取 `.agent/manifest.json`；当本地项目尚无对应类型的 `canvas` 来源资产时，会在 `.agent/run.latest.json` 的角色 step 中追加 `agent.tool.suggest.canvas.project_sync` 建议：美术组需要 `image/*` 或 `application/vnd.genarrative.image-sequence`，音乐组需要 `audio/*`。该建议提示用户通过聊天确认 `/sync-canvas-project <画板项目ID>` 回流画板资源，只进入 trace，不自动调用外部 API，也不绕过用户确认。
- Agent loop 中美术组 `Asset` 和音乐组 `SFX` 会读取 `.agent/manifest.json`；当美术组缺少 `image/*` 或 `application/vnd.genarrative.image-sequence` 类型的 `canvas` 来源资产，且 `editorApi.apiKey` 已配置时，App 直接调用 `/api/external/v1/editor/images/generations` 生成首版美术素材，再通过 `/api/external/v1/assets/read-url` 换签下载到本地项目 `assets/canvas-generated/`，登记为 `canvas` 来源资产并追加 `canvas.asset_generate` 本地索引记录。API Key 不写入项目文件、agent.db、trace、manifest 或日志；未配置 Key 或生成失败时，仍保留 `agent.tool.suggest.canvas.project_sync` 建议让用户用 `/sync-canvas-project <画板项目ID>` 回流已有资源。音乐组仍只建议同步已有音频资源，不调用图片生成接口。
- `canvas.asset_import` 当前作为最小真实链路：导入项目目录内已有文件为 `canvas` 来源资产，并要求记录画板项目 ID 以及 resourceId 或 assetObjectId。
- `canvas.export_import` 复用 `/editor/canvas` 已有素材导出 ZIP 格式，读取根 `metadata.json`、复制 `images/` / `media/` / `sequences/` 到本地项目 `assets/canvas-imports/`，再按导出层登记为 `canvas` 来源资产；导出包不保存真实 resourceId 时，使用 `canvas-export:<file>` 作为可追踪 assetObjectId，不伪造后端资源行。
@@ -153,7 +153,7 @@ game-project/
- 开发模式可把项目内已有文件登记为资产；`asset.register` 支持 `uploaded`、`generated`、`canvas` 三种来源，并可记录画板 project/resource/object 元数据。
- 开发模式可执行 `canvas.project_open`，打开本机 Genarrative 编辑器里的指定画板项目。
- 开发模式可执行 `canvas.project_sync`，同步平台画板项目中的已有资源到本地项目资产目录；该命令要求 External API Key 具备 `editor:project` 与 `editor:asset` 相关 scope。
- 开发模式可执行 `canvas.asset_import`，将项目内已有文件按画板来源导入 manifest；也可执行 `canvas.export_import`，把现有画板素材导出 ZIP 回流为本地项目资产。后续若要从独立 App 发起新的画板生成，再复用 External Editor API 的生成接口，不改本地产物与 manifest 落点。
- 开发模式可执行 `canvas.asset_import`，将项目内已有文件按画板来源导入 manifest；也可执行 `canvas.export_import`，把现有画板素材导出 ZIP 回流为本地项目资产。`game.generate_draft` 在配置 `editorApi.apiKey` 时会自动复用 External Editor API 的图片生成接口生成首版美术素材，不新增平行资产模型。
- 普通模式只渲染聊天区；上传文件入口在聊天输入区内，Tauri 主窗口保持聊天窗口尺寸，不承载工具台布局，也不承载游戏预览画面。
- 聊天生成草案后会尝试启动只读 `127.0.0.1:<port>` 静态 HTTP server，调用系统外部浏览器打开预览，并把预览地址回到聊天消息；外部浏览器打开失败时保留本地 URL 供用户手动复制。
- 开发模式仅在 Vite dev 环境响应 `?dev` 或 `#dev`，Tauri dev 会自动额外打开 `developer` 窗口显示专业组、本地项目、预览 iframe 和内置命令日志；正式构建忽略 dev 参数，release 配置只登记 `main` 聊天窗口。