diff --git a/apps/ai-game-creator-shell/src/App.tsx b/apps/ai-game-creator-shell/src/App.tsx index 58526a3db..8f567adce 100644 --- a/apps/ai-game-creator-shell/src/App.tsx +++ b/apps/ai-game-creator-shell/src/App.tsx @@ -221,6 +221,7 @@ import { } from './features/project-workspace/agentRunTrace'; import { DeveloperProjectPanels } from './features/project-workspace/DeveloperProjectPanels'; import { DeveloperRuntimePanels } from './features/project-workspace/DeveloperRuntimePanels'; +import type { DirectCodexUserContentPart } from './features/project-workspace/generated'; import { appendMemoryContent, memoryScopeLabel, @@ -258,6 +259,7 @@ import type { ChatReference, } from './features/project-workspace/resourceReferences'; import { + chatComposerDraftToDirectCodexUserItem, RESOURCE_REFERENCE_INSERT_EVENT, type ResourceReferenceInsertEventDetail, } from './features/project-workspace/resourceReferences'; @@ -549,6 +551,7 @@ type ExecuteChatAgentReplyInput = { attachments?: DirectCodexTurnAttachment[]; directPolicyChecked?: boolean; references?: ChatReference[]; + userItem?: ReturnType; }; /** @@ -737,6 +740,9 @@ export function App({ : '', ); const [chatReferences, setChatReferences] = useState([]); + const [chatContent, setChatContent] = useState( + [], + ); const chatComposerRef = useRef(null); const [chatAgentBusy, setChatAgentBusy] = useState(false); const [directCodexProgress, setDirectCodexProgress] = useState(''); @@ -3685,6 +3691,7 @@ export function App({ setChatInput(readSupervisorChatDraft(openedProject.projectPath)); } setChatReferences([]); + setChatContent([]); setManifest(openedProject.manifest); setProjectFiles([]); setProjectCheckpoints([]); @@ -3925,12 +3932,14 @@ export function App({ function prepareChatCommandDraft(commandDraft: string) { setChatInput(commandDraft); setChatReferences([]); + setChatContent([]); window.setTimeout(() => chatInputRef.current?.focus(), 0); } function handleChatComposerChange(draft: ChatComposerDraft) { setChatInput(draft.text); setChatReferences(draft.references); + setChatContent(draft.content ?? []); } useEffect(() => { @@ -4970,6 +4979,7 @@ export function App({ setChatInput(''); setChatReferences([]); + setChatContent([]); setMessages((current) => [ ...current, { @@ -6057,7 +6067,12 @@ export function App({ return; } - void executeChatAgentReply({ prompt, references }); + const clientTurnId = createDirectCodexConversationTurnId(); + const userItem = chatComposerDraftToDirectCodexUserItem( + { text: prompt, references, content: chatContent }, + directCodexConversationMessageId(clientTurnId, 'user'), + ); + void executeChatAgentReply({ prompt, references, userItem, clientTurnId }); } async function executeLlmConfigStatus() { @@ -6286,6 +6301,7 @@ export function App({ attachments, directPolicyChecked = false, references, + userItem, }: ExecuteChatAgentReplyInput) { if (planningV2ActiveRef.current || planningStartMode) { const nextProjectPath = resolveChatProjectPath(localProject); @@ -6316,6 +6332,12 @@ export function App({ if (directProjectPath && directInvoke) { const clientTurnId = directConversationTurnId ?? createDirectCodexConversationTurnId(); + const effectiveUserItem = + userItem ?? + chatComposerDraftToDirectCodexUserItem( + { text: prompt, references: references ?? [], content: [] }, + directCodexConversationMessageId(clientTurnId, 'user'), + ); if ( !directPolicyChecked && projectConversationWriteConfirmedRef.current !== directProjectPath @@ -6327,6 +6349,7 @@ export function App({ creationType, attachments, references, + userItem: effectiveUserItem, }); try { const policyPaused = await queueProjectPolicyConfirmationIfNeeded( @@ -6451,11 +6474,12 @@ export function App({ clientTurnId: string; creationType?: HomeCreationType; attachments?: DirectCodexTurnAttachment[]; - references?: ChatReference[]; + userItem: ReturnType; } = { projectPath: directProjectPath, prompt, clientTurnId, + userItem: effectiveUserItem, }; if (creationType) { directTurnInput.creationType = creationType; @@ -6463,9 +6487,7 @@ export function App({ if (attachments?.length) { directTurnInput.attachments = attachments; } - if (references?.length) { - directTurnInput.references = references; - } + directTurnInput.userItem = effectiveUserItem; const reply = await withDirectCodexSessionRefresh(() => { // 每次调用都会新建 Rust 事件流;续期重试需重新接收同一回合的进度。 activeDirectCodexTurnRef.current = { @@ -11796,6 +11818,7 @@ export function App({ } setChatInput(''); setChatReferences([]); + setChatContent([]); void loadProjectConversation(nextProjectPath, false, 'replace'); return; } @@ -11825,8 +11848,15 @@ export function App({ const directConversationTurnId = directCodexProductRuntime ? createDirectCodexConversationTurnId() : undefined; + const directUserItem = directConversationTurnId + ? chatComposerDraftToDirectCodexUserItem( + { text: prompt, references, content: chatContent }, + directCodexConversationMessageId(directConversationTurnId, 'user'), + ) + : undefined; setChatInput(''); setChatReferences([]); + setChatContent([]); setMessages((current) => [ ...current, { @@ -11848,6 +11878,7 @@ export function App({ prompt, clientTurnId: directConversationTurnId, references, + userItem: directUserItem, }); } diff --git a/apps/ai-game-creator-shell/src/features/project-workspace/ResourceReferenceInput.tsx b/apps/ai-game-creator-shell/src/features/project-workspace/ResourceReferenceInput.tsx index 1971660b5..14d2febcc 100644 --- a/apps/ai-game-creator-shell/src/features/project-workspace/ResourceReferenceInput.tsx +++ b/apps/ai-game-creator-shell/src/features/project-workspace/ResourceReferenceInput.tsx @@ -70,6 +70,7 @@ import { writeChatPromptPolishReminderDisabled, } from './chatPromptPolish'; import { ChatPromptPolishReminder } from './ChatPromptPolishReminder'; +import type { DirectCodexUserContentPart } from './generated'; import { $createResourceReferenceNode, $isResourceReferenceNode, @@ -176,7 +177,8 @@ function referenceListKey(references: ChatReference[]) { function sameDraft(left: ChatComposerDraft, right: ChatComposerDraft) { return ( left.text === right.text && - referenceListKey(left.references) === referenceListKey(right.references) + referenceListKey(left.references) === referenceListKey(right.references) && + JSON.stringify(left.content ?? []) === JSON.stringify(right.content ?? []) ); } @@ -184,26 +186,50 @@ function collectDraftParts( node: LexicalNode, textParts: string[], references: ChatReference[], + content: DirectCodexUserContentPart[], ) { if ($isTextNode(node)) { - textParts.push(node.getTextContent()); + const text = node.getTextContent(); + textParts.push(text); + if (text) content.push({ type: 'input_text', text }); return; } if ($isLineBreakNode(node)) { textParts.push('\n'); + content.push({ type: 'input_text', text: '\n' }); return; } if ($isResourceReferenceNode(node)) { textParts.push(`@${node.__reference.label}`); references.push(node.__reference); + content.push( + node.__reference.type === 'resource' + ? { + type: 'agc_resource_reference', + resourceId: node.__reference.resourceId, + } + : { + type: 'agc_runtime_region_reference', + label: node.__reference.label, + runId: node.__reference.runId, + versionId: node.__reference.versionId, + elementTag: node.__reference.elementTag, + elementRole: node.__reference.elementRole, + text: node.__reference.text, + width: node.__reference.width, + height: node.__reference.height, + resourceIds: node.__reference.resourceIds, + }, + ); return; } if ($isElementNode(node)) { node.getChildren().forEach((child, index) => { if (index > 0 && node.getType() === 'root') { textParts.push('\n'); + content.push({ type: 'input_text', text: '\n' }); } - collectDraftParts(child, textParts, references); + collectDraftParts(child, textParts, references, content); }); } } @@ -212,10 +238,12 @@ function collectDraftParts( function readDraftFromNodes(): ChatComposerDraft { const textParts: string[] = []; const references: ChatReference[] = []; - collectDraftParts($getRoot(), textParts, references); + const content: DirectCodexUserContentPart[] = []; + collectDraftParts($getRoot(), textParts, references, content); return { text: textParts.join('').trim(), references: dedupeChatReferences(references), + content, }; } diff --git a/apps/ai-game-creator-shell/src/features/project-workspace/resourceReferences.ts b/apps/ai-game-creator-shell/src/features/project-workspace/resourceReferences.ts index 320fd9a15..c08fba49a 100644 --- a/apps/ai-game-creator-shell/src/features/project-workspace/resourceReferences.ts +++ b/apps/ai-game-creator-shell/src/features/project-workspace/resourceReferences.ts @@ -10,6 +10,11 @@ import { buildGameCreationAppAssetTagLibrary, type GameCreationAppAssetTagLibraryEntry, } from '../../../../../packages/shared/src/contracts/gameCreationAppAssetTagLibrary'; +import type { + DirectCodexUserContentPart, + DirectCodexUserItem, + DirectCodexUserMessageItem, +} from './generated'; export type ResourceReferenceSource = | 'asset-picker' @@ -47,6 +52,8 @@ export type ChatReference = ResourceReference | RuntimeRegionReference; export type ChatComposerDraft = { text: string; references: ChatReference[]; + /** Lexical 顺序对应的 canonical user content;仅由编辑器读回时提供。 */ + content?: DirectCodexUserContentPart[]; }; export const RESOURCE_REFERENCE_INSERT_EVENT = 'agc-resource-reference-insert'; @@ -84,8 +91,53 @@ export function isResourceReferenceOverlayTarget(target: EventTarget | null) { export const EMPTY_CHAT_COMPOSER_DRAFT: ChatComposerDraft = { text: '', references: [], + content: [], }; +export function chatReferenceToContentPart( + reference: ChatReference, +): DirectCodexUserContentPart { + if (reference.type === 'resource') { + return { type: 'agc_resource_reference', resourceId: reference.resourceId }; + } + return { + type: 'agc_runtime_region_reference', + label: reference.label, + runId: reference.runId, + versionId: reference.versionId, + elementTag: reference.elementTag, + elementRole: reference.elementRole, + text: reference.text, + width: reference.width, + height: reference.height, + resourceIds: reference.resourceIds, + }; +} + +export function chatComposerDraftToDirectCodexUserItem( + draft: ChatComposerDraft, + id: string, +): DirectCodexUserItem { + const content = draft.content?.length + ? draft.content + : draft.references.length > 0 + ? [ + ...(draft.text + ? [{ type: 'input_text' as const, text: draft.text }] + : []), + ...draft.references.map(chatReferenceToContentPart), + ] + : draft.text + ? [{ type: 'input_text' as const, text: draft.text }] + : []; + return { + type: 'message', + role: 'user', + content, + id, + } satisfies DirectCodexUserMessageItem; +} + export function resourceDisplayName(asset: GameCreationAppAssetManifestEntry) { const fileName = asset.localPath.split(/[\\/]/u).pop() ?? asset.id; return fileName.replace(/\.[^.]+$/u, '').trim() || asset.id;