内联 DirectProject 附件与图片内容
Project CI / AI game creator shell Rust shard 3/4 (pull_request) Successful in 5m30s
Project CI / AI game creator shell Rust shard 1/4 (pull_request) Successful in 5m35s
Project CI / AI game creator shell Rust shard 2/4 (pull_request) Successful in 5m54s
Project CI / AI game creator shell Rust shard 4/4 (pull_request) Successful in 6m1s
Project CI / Backend tests (pull_request) Failing after 9s
Project CI / AI game creator shell Rust smoke (pull_request) Successful in 1m52s
Project CI / Repository checks (pull_request) Failing after 10s
Project CI / AI game creator shell Rust crates (pull_request) Successful in 2m41s
Project CI / Frontend tests (pull_request) Failing after 3m54s
Project CI / AI game creator shell web tests (pull_request) Failing after 3m41s
Project CI / Native shell tests (pull_request) Successful in 6m9s

新增附件与图片 canonical content part 及 ts-rs 绑定

移除 DirectProject Tauri 附件 sidecar 追加路径

修正前端引用字段与生成类型适配

补充附件图片顺序投影测试并同步里程碑文档
This commit is contained in:
2026-09-16 13:03:11 +08:00
parent 5fb3db662b
commit 46e3cd9744
17 changed files with 167 additions and 64 deletions
@@ -5,8 +5,9 @@ mod validation;
mod wire;
pub(crate) use model::{
DirectCodexUserContentPart, DirectCodexUserItem, DirectCodexUserMessageEnvelope,
DirectCodexUserMessageItem, DirectCodexUserRole, DirectCodexUserRuntimeRegionPart,
DirectCodexUserAttachmentReferencePart, DirectCodexUserContentPart, DirectCodexUserItem,
DirectCodexUserMessageEnvelope, DirectCodexUserMessageItem, DirectCodexUserRole,
DirectCodexUserRuntimeRegionPart,
};
pub(crate) use validation::validate_direct_codex_user_item;
pub(crate) use wire::{
@@ -36,6 +36,23 @@ pub(crate) enum DirectCodexUserContentPart {
AgcResourceReference { resource_id: String },
#[serde(rename = "agc_runtime_region_reference")]
AgcRuntimeRegionReference(DirectCodexUserRuntimeRegionPart),
/// Uploaded project attachment kept inline in canonical content.
#[serde(rename = "agc_attachment_reference")]
AgcAttachmentReference(DirectCodexUserAttachmentReferencePart),
#[serde(rename = "agc_image_reference")]
AgcImageReference(DirectCodexUserAttachmentReferencePart),
}
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/project-workspace/generated/"))]
pub(crate) struct DirectCodexUserAttachmentReferencePart {
pub(crate) name: String,
pub(crate) media_type: String,
#[ts(type = "number")]
pub(crate) size: u64,
pub(crate) local_path: String,
pub(crate) status: String,
}
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
@@ -40,6 +40,19 @@ pub(crate) fn validate_direct_codex_user_item(
reference_count = reference_count.saturating_add(1);
validate_runtime_region_reference(&manifest, reference)?;
}
DirectCodexUserContentPart::AgcAttachmentReference(reference)
| DirectCodexUserContentPart::AgcImageReference(reference) => {
if reference.name.trim().is_empty() {
return Err("附件缺少文件名".to_string());
}
if !reference.local_path.trim().is_empty() {
sanitize_attachment_local_path(&reference.local_path)
.ok_or_else(|| "附件项目路径无效".to_string())?;
}
if !matches!(reference.status.trim(), "imported" | "failed") {
return Err("附件状态无效".to_string());
}
}
}
}
if reference_count > MAX_DIRECT_CODEX_REFERENCES {
@@ -100,6 +100,21 @@ pub(crate) fn direct_codex_user_item_to_wire_input(
summary.push(']');
summary
}
DirectCodexUserContentPart::AgcAttachmentReference(reference)
| DirectCodexUserContentPart::AgcImageReference(reference) => {
let mut summary = format!(
"[附件:名称={};类型={};大小={} 字节",
reference.name.trim(),
reference.media_type.trim(),
reference.size
);
if !reference.local_path.trim().is_empty() {
summary.push_str(&format!(";项目路径={}", reference.local_path.trim()));
}
summary.push_str(&format!(";状态={}", reference.status.trim()));
summary.push(']');
summary
}
};
input.push(serde_json::json!({ "type": "text", "text": text }));
}
@@ -174,4 +189,28 @@ mod tests {
.expect_err("history item without type must fail");
assert!(error.contains("缺少 type"), "{error}");
}
#[test]
fn attachment_and_image_parts_remain_in_canonical_order_when_projected() {
let root = tempfile::tempdir().expect("temp project");
crate::init_local_game_project_at(root.path(), "wire-test", "wire 投影测试")
.expect("init project");
let item = json!({
"type": "message",
"role": "user",
"id": "turn-1:user",
"content": [
{"type": "input_text", "text": "先看"},
{"type": "agc_image_reference", "name": "hero.png", "mediaType": "image/png", "size": 12, "localPath": "assets/hero.png", "status": "imported"},
{"type": "agc_attachment_reference", "name": "notes.txt", "mediaType": "text/plain", "size": 4, "localPath": "assets/notes.txt", "status": "imported"}
]
});
let projected = direct_codex_user_item_to_response_item(root.path(), &item)
.expect("user response item should project");
let content = projected["content"].as_array().expect("content array");
assert_eq!(content.len(), 3);
assert!(content[0]["text"].as_str().unwrap().contains("先看"));
assert!(content[1]["text"].as_str().unwrap().contains("hero.png"));
assert!(content[2]["text"].as_str().unwrap().contains("notes.txt"));
}
}
@@ -31,10 +31,9 @@ pub(crate) fn normalize_direct_client_turn_id(
pub(crate) async fn chat_with_game_creator_direct_codex(
project_path: String,
prompt: String,
mut user_item: DirectCodexUserItem,
user_item: DirectCodexUserItem,
creation_type: Option<String>,
client_turn_id: Option<String>,
attachments: Option<Vec<DirectCodexTurnAttachment>>,
) -> Result<String, String> {
let root = Path::new(project_path.trim());
let turn_id = normalize_direct_client_turn_id(client_turn_id.as_deref())?;
@@ -43,24 +42,7 @@ pub(crate) async fn chat_with_game_creator_direct_codex(
redact_agent_runtime_error(root, &format!("恢复上一轮陶泥儿整包事务失败:{error}"), 500)
})?;
let turn_emitter = DirectGameCreatorTurnUpdateEmitter::new(root, turn_id.clone());
let mut audit = DirectCodexTurnAudit::start(
root,
&turn_id,
&prompt,
attachments.as_deref().unwrap_or_default(),
);
let attachments = attachments.unwrap_or_default();
if !attachments.is_empty() {
let attachment_context =
render_direct_codex_user_prompt("", &attachments).map_err(|error| {
audit.finish(false);
error
})?;
let DirectCodexUserItem::Message(message) = &mut user_item;
message.content.push(DirectCodexUserContentPart::InputText {
text: attachment_context,
});
}
let mut audit = DirectCodexTurnAudit::start(root, &turn_id, &prompt, &[]);
validate_direct_codex_user_item(root, &user_item).map_err(|error| {
audit.finish(false);
error
+40 -9
View File
@@ -893,8 +893,7 @@ export function App({
/**
* direct-codex chip
* `chat_with_game_creator_direct_codex` `attachments`
*
* canonical user item `content[]`
*/
const [chatAttachments, setChatAttachments] = useState<
DirectCodexTurnAttachment[]
@@ -7035,12 +7034,32 @@ export function App({
if (directProjectPath && directProjectId && directInvoke) {
const clientTurnId =
directConversationTurnId ?? createDirectCodexConversationTurnId();
const effectiveUserItem =
const baseUserItem =
userItem ??
chatComposerDraftToDirectCodexUserItem(
{ text: prompt, references: references ?? [], content: [] },
directCodexConversationMessageId(clientTurnId, 'user'),
);
const effectiveUserItem = attachments?.length
? {
...baseUserItem,
content: [
...baseUserItem.content,
...attachments.map((attachment) => ({
type: attachment.mediaType.toLowerCase().startsWith('image/')
? ('agc_image_reference' as const)
: ('agc_attachment_reference' as const),
name: attachment.name,
mediaType: attachment.mediaType,
size: attachment.size ?? 0,
localPath: attachment.localPath ?? '',
status:
attachment.status ??
(attachment.localPath ? 'imported' : 'failed'),
})),
],
}
: baseUserItem;
if (
!directPolicyChecked &&
projectConversationWriteConfirmedRef.current !== directProjectPath
@@ -7181,7 +7200,6 @@ export function App({
prompt: string;
clientTurnId: string;
creationType?: HomeCreationType;
attachments?: DirectCodexTurnAttachment[];
userItem: ReturnType<typeof chatComposerDraftToDirectCodexUserItem>;
} = {
projectPath: directProjectPath,
@@ -7192,9 +7210,6 @@ export function App({
if (creationType) {
directTurnInput.creationType = creationType;
}
if (attachments?.length) {
directTurnInput.attachments = attachments;
}
directTurnInput.userItem = effectiveUserItem;
const reply = await withDirectCodexSessionRefresh(() => {
// 每次调用都会新建 Rust 事件流;续期重试需重新接收同一回合的进度。
@@ -12595,6 +12610,20 @@ export function App({
content?: DirectCodexUserContentPart[];
}) {
const clientTurnId = createDirectCodexConversationTurnId();
const attachmentContent: DirectCodexUserContentPart[] = (
input.attachments ?? []
).map((attachment) => ({
type: attachment.mediaType.toLowerCase().startsWith('image/')
? ('agc_image_reference' as const)
: ('agc_attachment_reference' as const),
name: attachment.name,
mediaType: attachment.mediaType,
size: attachment.size ?? 0,
localPath: attachment.localPath ?? '',
status:
attachment.status ?? (attachment.localPath ? 'imported' : 'failed'),
}));
const content = [...(input.content ?? []), ...attachmentContent];
supervisorChatShouldFollowLatestRef.current = true;
setMessages((current) => [
...current,
@@ -12609,16 +12638,18 @@ export function App({
void executeChatAgentReply({
prompt: input.prompt,
clientTurnId,
attachments: input.attachments?.length ? input.attachments : undefined,
references: input.references,
userItem: chatComposerDraftToDirectCodexUserItem(
{
text: input.prompt,
references: input.references ?? [],
content: input.content ?? [],
content,
},
directCodexConversationMessageId(clientTurnId, 'user'),
),
// DirectProject 的附件已经是 canonical content part;不能再作为 sidecar
// 传给 Rust,否则会重复追加。
attachments: undefined,
});
}
@@ -0,0 +1,9 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type DirectCodexUserAttachmentReferencePart = {
name: string;
mediaType: string;
size: number;
localPath: string;
status: string;
};
@@ -1,5 +1,5 @@
// This file is generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DirectCodexUserAttachmentReferencePart } from './DirectCodexUserAttachmentReferencePart';
import type { DirectCodexUserRuntimeRegionPart } from './DirectCodexUserRuntimeRegionPart';
export type DirectCodexUserContentPart =
@@ -7,4 +7,8 @@ export type DirectCodexUserContentPart =
| { type: 'agc_resource_reference'; resourceId: string }
| ({
type: 'agc_runtime_region_reference';
} & DirectCodexUserRuntimeRegionPart);
} & DirectCodexUserRuntimeRegionPart)
| ({
type: 'agc_attachment_reference';
} & DirectCodexUserAttachmentReferencePart)
| ({ type: 'agc_image_reference' } & DirectCodexUserAttachmentReferencePart);
@@ -1,5 +1,9 @@
// This file is generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DirectCodexUserMessageItem } from './DirectCodexUserMessageItem';
export type DirectCodexUserItem = DirectCodexUserMessageItem;
/**
* DirectProject 本轮 user input 的唯一结构化入口。
*/
export type DirectCodexUserItem = {
type: 'message';
} & DirectCodexUserMessageItem;
@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DirectCodexUserItem } from './DirectCodexUserItem';
export type DirectCodexUserMessageEnvelope = { item: DirectCodexUserItem };
@@ -1,11 +1,9 @@
// This file is generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DirectCodexUserContentPart } from './DirectCodexUserContentPart';
import type { DirectCodexUserRole } from './DirectCodexUserRole';
export type DirectCodexUserMessageItem = {
type: 'message';
role: DirectCodexUserRole;
content: DirectCodexUserContentPart[];
content: Array<DirectCodexUserContentPart>;
id: string;
};
@@ -1,3 +1,3 @@
// This file is generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type DirectCodexUserRole = 'user';
@@ -1,13 +1,13 @@
// This file is generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type DirectCodexUserRuntimeRegionPart = {
label: string;
runId?: string;
versionId?: string;
elementTag?: string;
elementRole?: string;
text?: string;
width?: number;
height?: number;
resourceIds: string[];
runId: string | null;
versionId: string | null;
elementTag: string | null;
elementRole: string | null;
text: string | null;
width: number | null;
height: number | null;
resourceIds: Array<string>;
};
@@ -1,5 +1,7 @@
export type { DirectCodexUserContentPart } from './DirectCodexUserContentPart';
export type { DirectCodexUserAttachmentReferencePart } from './DirectCodexUserAttachmentReferencePart';
export type { DirectCodexUserItem } from './DirectCodexUserItem';
export type { DirectCodexUserMessageItem } from './DirectCodexUserMessageItem';
export type { DirectCodexUserRole } from './DirectCodexUserRole';
export type { DirectCodexUserRuntimeRegionPart } from './DirectCodexUserRuntimeRegionPart';
export type { DirectCodexUserMessageEnvelope } from './DirectCodexUserMessageEnvelope';
@@ -13,7 +13,6 @@ import {
import type {
DirectCodexUserContentPart,
DirectCodexUserItem,
DirectCodexUserMessageItem,
} from './generated';
export type ResourceReferenceSource =
@@ -103,13 +102,13 @@ export function chatReferenceToContentPart(
return {
type: 'agc_runtime_region_reference',
label: reference.label,
runId: reference.runId,
versionId: reference.versionId,
elementTag: reference.elementTag,
elementRole: reference.elementRole,
text: reference.text,
width: reference.width,
height: reference.height,
runId: reference.runId ?? null,
versionId: reference.versionId ?? null,
elementTag: reference.elementTag ?? null,
elementRole: reference.elementRole ?? null,
text: reference.text ?? null,
width: reference.width ?? null,
height: reference.height ?? null,
resourceIds: reference.resourceIds,
};
}
@@ -126,7 +125,7 @@ export function chatComposerDraftToDirectCodexUserItem(
role: 'user',
content,
id,
} satisfies DirectCodexUserMessageItem;
} satisfies DirectCodexUserItem;
}
export function resourceDisplayName(asset: GameCreationAppAssetManifestEntry) {
@@ -9,7 +9,7 @@
## 修改边界
- 允许修改:AGC 壳 Rust agent 输入合同、DirectProject 历史适配、前端聊天引用模型、ts-rs 生成配置、当前聊天素材文档。
- 明确不修改:assistant 返回协议、工具 activity、附件/图片 DTO 协议、SpacetimeDB、HTTP API仅补齐既有 sidecar 在 DirectProject prompt 的消费
- 明确不修改:assistant 返回协议、工具 activity、附件/图片上传 DTO、SpacetimeDB、HTTP API上传 DTO 仅在入口转换为 canonical `content[]`,不再以 sidecar 文本追加
## 实现顺序
@@ -19,13 +19,13 @@
- `agc_runtime_region_reference` 保留运行区域语义摘要。
- Rust 在持久化前完成白名单、manifest 与路径校验。
- 现有标准 `response_item` 原样兼容;legacy conversation 行不提供 fallback。
- 保持 assistant 返回、工具 activity、附件/图片协议不变;沿用 sidecar DTO 将附件与图片路径映射纳入 DirectProject prompt
- 保持 assistant 返回、工具 activity 与上传 DTO 不变;但 DirectProject canonical user item 不再保存附件 sidecar,附件和图片必须作为 `content[]` 中按顺序排列的 `agc_attachment_reference` / `agc_image_reference`
## 不在范围内
- assistant item 前端投影或 Tauri 返回值改造。
- 工具 item、reasoning、file change、MCP item 的 UI 模型化。
- 附件/图片 content part(不新增 content-part,继续消费现有 sidecar DTO
- 附件/图片 content part:本里程碑明确新增并生成绑定;上传 DTO 只作为输入适配,不作为历史事实源
- SpacetimeDB schema 或 HTTP API 变更。
## 依赖与前置条件
@@ -42,7 +42,7 @@
- [x] 未知 part、失效资源或非法路径在持久化前失败关闭。
- [x] canonical item 以 `response_item` 写入历史,标准旧 item 原样可读。
- [x] Codex wire input 不含 AGC 私有 part,且顺序与 canonical content 一致。
- [x] assistant、附件和工具链路行为无变化;附件-only 输入也能进入 DirectProject prompt。
- [x] assistant、附件和工具链路行为无变化;附件与图片按 content 顺序内联,附件-only 输入也能进入 DirectProject prompt。
## 证据要求