f5368c825f
重构了editor agent. 使得它能进行多轮工具调用, 把工具调用的结果嵌入到上下文里。 对于上下文的图片, 使用哈希后的id用来引用,不暴露细节信息to llm。 当前实现, 状态直接维护在json doc里, 需要对整个doc加锁,任务目前只能串行。 把SSE改成一个简单请求( 因为生图工具耗时需要二次确认, 不需要再实时展示给用户进度)。 增加确认/取消生图操作。 把tool的参数和错误情况做了反馈。 TODO: 工具调用的规范/prompt还可以改进。 改用external job来做素材生成, 针对来自editor agent 的生成会把生成资产的引用加入到结果json里, 客户端轮询 external job 确定生成状态, 发现结束了或者失败了就 重新get 会话,后端重新提供会话的时候把 结果插入回会话历史里,用来让 ai 引用 以及显示 --------- Co-authored-by: 段舒康 <kdletters@qq.com> Reviewed-on: https://git.genarrative.world/git/GenarrativeAI/Genarrative/pulls/76 Reviewed-by: 段舒康 <kdletters@qq.com> Co-authored-by: 王德宇 <kvtodev@outlook.com> Co-committed-by: 王德宇 <kvtodev@outlook.com>
179 lines
6.4 KiB
TypeScript
179 lines
6.4 KiB
TypeScript
import { Check, Image as ImageIcon, Loader2, Volume2, X } from 'lucide-react';
|
|
import { useEffect, useRef, useState } from 'react';
|
|
|
|
import type { EditorAgentToolCall } from '@/packages/shared/src/contracts';
|
|
import { editorAgentToolLabel } from '@/src/components/image-editor/EditorAgentConversation/toolCallPresentation.ts';
|
|
import { ResolvedAssetAudio } from '@/src/components/ResolvedAssetAudio.tsx';
|
|
import { ResolvedAssetImage } from '@/src/components/ResolvedAssetImage.tsx';
|
|
import { ResolvedAssetVideo } from '@/src/components/ResolvedAssetVideo.tsx';
|
|
import { getExternalGenerationJobStatus } from '@/src/services/external-generation';
|
|
|
|
function ToolCallView({
|
|
toolCall,
|
|
onJobCompleted,
|
|
}: {
|
|
toolCall: EditorAgentToolCall;
|
|
onJobCompleted?: () => void;
|
|
}) {
|
|
const videos = toolCall.videos ?? [];
|
|
const audios = toolCall.audios ?? [];
|
|
const jobId = toolCall.externalJobId?.trim() || null;
|
|
const displaySourceKey = jobId ?? toolCall.status;
|
|
const initialDisplayStatus =
|
|
toolCall.status === 'completed'
|
|
? 'completed'
|
|
: toolCall.status === 'failed'
|
|
? 'failed'
|
|
: toolCall.status === 'cancelled'
|
|
? 'cancelled'
|
|
: 'pending';
|
|
const [displayStatus, setDisplayStatus] = useState(initialDisplayStatus);
|
|
const [displayError, setDisplayError] = useState<string | null>(
|
|
toolCall.error ?? null,
|
|
);
|
|
const terminalNotifiedRef = useRef(false);
|
|
const onJobCompletedRef = useRef(onJobCompleted);
|
|
useEffect(() => {
|
|
onJobCompletedRef.current = onJobCompleted;
|
|
}, [onJobCompleted]);
|
|
useEffect(() => {
|
|
if (toolCall.status !== 'not_completed' || !jobId) {
|
|
terminalNotifiedRef.current = false;
|
|
return;
|
|
}
|
|
// Only a different job may reinitialize the local display state.
|
|
setDisplayStatus(initialDisplayStatus);
|
|
setDisplayError(toolCall.error ?? null);
|
|
terminalNotifiedRef.current = false;
|
|
let disposed = false;
|
|
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
const poll = async () => {
|
|
try {
|
|
const response = await getExternalGenerationJobStatus(jobId);
|
|
if (disposed) return;
|
|
if (
|
|
response.job.status === 'completed' ||
|
|
response.job.status === 'failed'
|
|
) {
|
|
setDisplayStatus(response.job.status);
|
|
setDisplayError(response.job.error ?? null);
|
|
if (!terminalNotifiedRef.current) {
|
|
terminalNotifiedRef.current = true;
|
|
onJobCompletedRef.current?.();
|
|
}
|
|
return;
|
|
}
|
|
timeoutId = setTimeout(poll, 1500);
|
|
} catch {
|
|
if (!disposed) timeoutId = setTimeout(poll, 3000);
|
|
}
|
|
};
|
|
void poll();
|
|
return () => {
|
|
disposed = true;
|
|
if (timeoutId) clearTimeout(timeoutId);
|
|
};
|
|
}, [displaySourceKey]);
|
|
const isCancelled = displayStatus === 'cancelled';
|
|
const isCompleted = displayStatus === 'completed';
|
|
const isFailed = displayStatus === 'failed';
|
|
const isExecuting = displayStatus === 'pending' && Boolean(jobId);
|
|
const statusLabel = isCompleted
|
|
? '已完成'
|
|
: isFailed
|
|
? '失败'
|
|
: isCancelled
|
|
? '已取消'
|
|
: isExecuting
|
|
? '执行中'
|
|
: '待确认';
|
|
return (
|
|
<div className="rounded-lg border border-slate-200 bg-white/80 p-2 text-xs text-slate-600">
|
|
<div className="flex items-center gap-2">
|
|
{isExecuting ? (
|
|
<Loader2 className="h-3.5 w-3.5 animate-spin" aria-hidden="true" />
|
|
) : isCompleted ? (
|
|
<Check className="h-3.5 w-3.5" aria-hidden="true" />
|
|
) : isCancelled || isFailed ? (
|
|
<X className="h-3.5 w-3.5" aria-hidden="true" />
|
|
) : (
|
|
<ImageIcon className="h-3.5 w-3.5" aria-hidden="true" />
|
|
)}
|
|
<span>{editorAgentToolLabel(toolCall.toolName)}</span>
|
|
<span className="ml-auto text-slate-400">{statusLabel}</span>
|
|
</div>
|
|
{displayError ? (
|
|
<div className="mt-1 text-red-600">{displayError}</div>
|
|
) : null}
|
|
{toolCall.images.length ? (
|
|
<div className="mt-2 grid grid-cols-3 gap-2">
|
|
{toolCall.images.map((image, index) => (
|
|
<div
|
|
key={`${toolCall.toolName}-${image.resourceId ?? index}`}
|
|
className="overflow-hidden rounded-xl border border-slate-200 bg-slate-100"
|
|
>
|
|
<ResolvedAssetImage
|
|
src={image.thumbnailSrc ?? image.imageSrc}
|
|
objectKey={image.objectKey}
|
|
refreshKey={image.resourceId ?? toolCall.toolName}
|
|
alt=""
|
|
className="h-20 w-full object-cover"
|
|
/>
|
|
</div>
|
|
))}
|
|
</div>
|
|
) : null}
|
|
{videos.length ? (
|
|
<div className="mt-2 grid grid-cols-1 gap-2">
|
|
{videos.map((video, index) => (
|
|
<div
|
|
key={`${toolCall.toolName}-${video.resourceId ?? video.objectKey ?? index}`}
|
|
className="overflow-hidden rounded-xl border border-slate-200 bg-slate-100"
|
|
>
|
|
<ResolvedAssetVideo
|
|
src={video.videoSrc}
|
|
objectKey={video.objectKey}
|
|
refreshKey={
|
|
video.resourceId ?? video.objectKey ?? toolCall.toolName
|
|
}
|
|
poster={video.thumbnailSrc ?? undefined}
|
|
controls
|
|
playsInline
|
|
preload="metadata"
|
|
className="max-h-56 w-full bg-black object-contain"
|
|
/>
|
|
</div>
|
|
))}
|
|
</div>
|
|
) : null}
|
|
{audios.length ? (
|
|
<div className="mt-2 space-y-2">
|
|
{audios.map((audio, index) => (
|
|
<div
|
|
key={`${toolCall.toolName}-${audio.resourceId ?? audio.objectKey ?? index}`}
|
|
className="flex items-center gap-2 rounded-xl border border-slate-200 bg-slate-50 p-2"
|
|
>
|
|
<Volume2
|
|
className="h-4 w-4 shrink-0 text-slate-500"
|
|
aria-hidden="true"
|
|
/>
|
|
<ResolvedAssetAudio
|
|
src={audio.audioSrc}
|
|
objectKey={audio.objectKey}
|
|
refreshKey={
|
|
audio.resourceId ?? audio.objectKey ?? toolCall.toolName
|
|
}
|
|
controls
|
|
preload="metadata"
|
|
className="min-w-0 flex-1"
|
|
/>
|
|
</div>
|
|
))}
|
|
</div>
|
|
) : null}
|
|
</div>
|
|
);
|
|
}
|
|
|
|
export default ToolCallView;
|