Files
Genarrative/src/components/image-editor/useImageCanvasBackgroundMusicPromptAssist.ts
T
lhk229 281c84b7bf
Project CI / Repository checks (push) Successful in 1m15s
Project CI / Frontend tests (push) Failing after 2m11s
Project CI / Backend tests (push) Successful in 3m33s
Project CI / Native shell tests (push) Successful in 13m57s
BGM生成路径优化 (#142)
Reviewed-on: http://192.168.35.82/git/GenarrativeAI/Genarrative/pulls/142
Co-authored-by: Linghong <ink29535@proton.me>
Co-committed-by: Linghong <ink29535@proton.me>
2026-08-06 17:54:40 +08:00

714 lines
22 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import {
useCallback,
useEffect,
useLayoutEffect,
useMemo,
useRef,
useState,
} from 'react';
import {
completeEditorBackgroundMusicPrompt,
simplifyEditorBackgroundMusicPrompt,
} from '../../services/image-editor/editorProjectClient';
import {
type BackgroundMusicPromptDialogState,
type BackgroundMusicPromptOperation,
type BackgroundMusicPromptOperationStatus,
canonicalizeBackgroundMusicPrompt,
countPromptCodePoints,
createBackgroundMusicPromptStateModel,
} from './ImageCanvasBackgroundMusicPromptModel';
import type { CanvasGenerationDialogState } from './ImageCanvasEditorTypes';
type CanvasGenerationDialogUpdater = (
dialog: CanvasGenerationDialogState,
) => CanvasGenerationDialogState | null;
type ActivePromptOperation = {
operation: BackgroundMusicPromptOperation;
abortController: AbortController | null;
scopeKey: string | null | undefined;
};
export type BackgroundMusicPromptAssistDialogState =
BackgroundMusicPromptDialogState & {
errorMessage: string | null;
};
export type BackgroundMusicPromptAssistActionResult = {
started: boolean;
applied: boolean;
reason:
| 'applied'
| 'duplicate'
| 'ineligible'
| 'locked'
| 'stale'
| 'failed';
errorMessage: string | null;
};
export type BackgroundMusicPromptSubmissionClaim = {
operation: BackgroundMusicPromptOperation;
prompt: string;
};
export type BackgroundMusicPromptAssistController = ReturnType<
typeof useImageCanvasBackgroundMusicPromptAssist
>;
/**
* 共享音频 composer 的 BGM 分支只需要读状态和触发四个用户动作。`beginSubmission` / `finishSubmission`
* 留给 T5 的正式提交链路,不进入视图 props。
*/
export type BackgroundMusicPromptAssistComposerController = Pick<
BackgroundMusicPromptAssistController,
| 'getDialogState'
| 'completePrompt'
| 'simplifyPrompt'
| 'preparePreset'
| 'undoPrompt'
>;
function isBackgroundMusicDialog(
dialog: CanvasGenerationDialogState | undefined,
): dialog is CanvasGenerationDialogState & {
mode: 'audio-background-music';
} {
return dialog?.mode === 'audio-background-music';
}
function isAbortError(error: unknown) {
return (
(typeof DOMException !== 'undefined' &&
error instanceof DOMException &&
error.name === 'AbortError') ||
(error instanceof Error && error.name === 'AbortError')
);
}
function resolvePromptAssistErrorMessage(error: unknown) {
if (isAbortError(error)) {
return null;
}
if (error instanceof Error && error.message.trim()) {
return error.message.trim();
}
return 'AI 暂时无法处理背景音乐描述,请稍后重试';
}
function invalidPromptAssistResponseError() {
return new Error('AI 返回的背景音乐描述无效,请稍后重试');
}
/**
* Prompt 助手临时状态同时属于当前账号和当前项目,任一侧切换都必须让旧 operation
* 失效。这里把两者压成一个稳定字符串,调用方按引用比较对象时不会误判 scope 变化。
*/
function resolveBackgroundMusicPromptScopeKey(
currentUserId: string | null | undefined,
projectId: string | null | undefined,
) {
return JSON.stringify([currentUserId ?? null, projectId ?? null]);
}
export function useImageCanvasBackgroundMusicPromptAssist({
canvasGenerationDialogs,
getCanvasGenerationDialogById,
updateCanvasGenerationDialogById,
currentUserId,
projectId,
}: {
canvasGenerationDialogs: CanvasGenerationDialogState[];
getCanvasGenerationDialogById: (
dialogId: string,
) => CanvasGenerationDialogState | undefined;
updateCanvasGenerationDialogById: (
dialogId: string,
updater: CanvasGenerationDialogUpdater,
) => void;
currentUserId?: string | null;
projectId?: string | null;
}) {
const modelRef = useRef(createBackgroundMusicPromptStateModel());
const activeOperationsRef = useRef(new Map<string, ActivePromptOperation>());
const errorMessagesRef = useRef(new Map<string, string>());
const knownDialogIdsRef = useRef(new Set<string>());
const scopeKey = useMemo(
() => resolveBackgroundMusicPromptScopeKey(currentUserId, projectId),
[currentUserId, projectId],
);
// 只在已提交的生命周期里推进;render 阶段改写会让被放弃的并发 render 污染
// 已提交 operation 的迟到响应判断。
const committedScopeKeyRef = useRef(scopeKey);
const [dialogStates, setDialogStates] = useState<
Record<string, BackgroundMusicPromptAssistDialogState>
>({});
const publishDialogState = useCallback((dialogId: string) => {
const state = modelRef.current.getDialogState(dialogId);
const nextState: BackgroundMusicPromptAssistDialogState = {
...state,
errorMessage: errorMessagesRef.current.get(dialogId) ?? null,
};
setDialogStates((currentStates) => ({
...currentStates,
[dialogId]: nextState,
}));
return nextState;
}, []);
const isSubmissionLocked = useCallback(
(dialogId: string) =>
modelRef.current.getDialogState(dialogId).status === 'submitting' ||
activeOperationsRef.current.get(dialogId)?.operation.status ===
'submitting',
[],
);
const updateDialogPrompt = useCallback(
(dialogId: string, prompt: string) => {
updateCanvasGenerationDialogById(dialogId, (dialog) =>
dialog.mode === 'audio-background-music'
? {
...dialog,
prompt,
status: dialog.status === 'failed' ? 'idle' : dialog.status,
errorMessage:
dialog.status === 'failed' ? undefined : dialog.errorMessage,
}
: dialog,
);
},
[updateCanvasGenerationDialogById],
);
const clearActiveOperation = useCallback(
(
dialogId: string,
options: {
abort: boolean;
removeDialogState: boolean;
},
) => {
const activeOperation = activeOperationsRef.current.get(dialogId);
if (activeOperation) {
if (options.abort) {
activeOperation.abortController?.abort();
}
const resolution = modelRef.current.rejectOperation(
activeOperation.operation,
);
activeOperationsRef.current.delete(dialogId);
if (
!options.removeDialogState &&
resolution.applied &&
resolution.prompt
) {
updateDialogPrompt(dialogId, resolution.prompt);
}
}
errorMessagesRef.current.delete(dialogId);
if (options.removeDialogState) {
modelRef.current.closeDialog(dialogId);
setDialogStates((currentStates) => {
if (!(dialogId in currentStates)) {
return currentStates;
}
const nextStates = { ...currentStates };
delete nextStates[dialogId];
return nextStates;
});
return;
}
if (activeOperation) {
publishDialogState(dialogId);
}
},
[publishDialogState, updateDialogPrompt],
);
/**
* 迟到响应被判定为 stale 时,状态模型可能仍把这次 operation 当作当前 operation。
* 只删 Map 记录会让面板永久停在 `completing` / `simplifying`,所以这里先让模型
* 完成 reject 再放弃它;输入框保留用户当前文本,不写回任何候选。
*/
const discardStaleOperation = useCallback(
(operation: BackgroundMusicPromptOperation) => {
if (
modelRef.current.getDialogState(operation.dialogId).operationId !==
operation.operationId
) {
return;
}
modelRef.current.rejectOperation(operation);
activeOperationsRef.current.delete(operation.dialogId);
publishDialogState(operation.dialogId);
},
[publishDialogState],
);
useEffect(() => {
const currentDialogIds = new Set(
canvasGenerationDialogs
.filter(isBackgroundMusicDialog)
.map((dialog) => dialog.id),
);
for (const dialogId of knownDialogIdsRef.current) {
if (!currentDialogIds.has(dialogId)) {
clearActiveOperation(dialogId, {
abort: true,
removeDialogState: true,
});
}
}
for (const [dialogId, activeOperation] of activeOperationsRef.current) {
if (activeOperation.operation.status === 'submitting') {
continue;
}
const dialog = getCanvasGenerationDialogById(dialogId);
if (!isBackgroundMusicDialog(dialog) || dialog.composerOpen === false) {
clearActiveOperation(dialogId, {
abort: true,
removeDialogState: !isBackgroundMusicDialog(dialog),
});
}
}
knownDialogIdsRef.current = currentDialogIds;
}, [
canvasGenerationDialogs,
clearActiveOperation,
getCanvasGenerationDialogById,
]);
// 必须是 layout effectpassive effect 排在 commit 之后的独立任务里,新 scope 已经
// commit 而清理尚未执行的窗口中,迟到的旧账号 / 旧项目响应会通过 operation 与 scope
// 检查,并把候选写进同 ID 的当前 dialog;随后的清理只会 reset 助手状态,不会撤销
// 已经写入的 Prompt。layout effect 在 commit 内同步执行,promise continuation 无法
// 插进这段同步代码,因此不存在该窗口;它同样只对已提交的 render 执行,被放弃的并发
// render 依旧不会推进 scope。
useLayoutEffect(() => {
if (Object.is(committedScopeKeyRef.current, scopeKey)) {
return;
}
// 账号或项目切换必须原子完成:中止旧请求、清空 operation、reset 状态模型、
// 清空错误与公开 dialog 状态,最后按新 scope 重建已知 dialog 集合。
committedScopeKeyRef.current = scopeKey;
for (const activeOperation of activeOperationsRef.current.values()) {
activeOperation.abortController?.abort();
}
activeOperationsRef.current.clear();
errorMessagesRef.current.clear();
modelRef.current.reset();
knownDialogIdsRef.current = new Set(
canvasGenerationDialogs
.filter(isBackgroundMusicDialog)
.map((dialog) => dialog.id),
);
setDialogStates({});
}, [canvasGenerationDialogs, scopeKey]);
useEffect(
() => () => {
for (const activeOperation of activeOperationsRef.current.values()) {
activeOperation.abortController?.abort();
}
activeOperationsRef.current.clear();
},
[],
);
const runAiOperation = useCallback(
async (
dialogId: string,
status: Extract<
BackgroundMusicPromptOperationStatus,
'completing' | 'simplifying'
>,
): Promise<BackgroundMusicPromptAssistActionResult> => {
const dialog = getCanvasGenerationDialogById(dialogId);
if (!isBackgroundMusicDialog(dialog) || dialog.composerOpen === false) {
return {
started: false,
applied: false,
reason: 'stale',
errorMessage: null,
};
}
if (isSubmissionLocked(dialogId)) {
return {
started: false,
applied: false,
reason: 'locked',
errorMessage: null,
};
}
const canonicalPrompt = canonicalizeBackgroundMusicPrompt(dialog.prompt);
updateDialogPrompt(dialogId, canonicalPrompt);
const start = modelRef.current.beginOperation({
dialogId,
status,
prompt: canonicalPrompt,
});
if (!start.started || !start.operation) {
publishDialogState(dialogId);
return {
started: false,
applied: false,
reason: start.reason === 'started' ? 'ineligible' : start.reason,
errorMessage: null,
};
}
const previousOperation = activeOperationsRef.current.get(dialogId);
previousOperation?.abortController?.abort();
const abortController = new AbortController();
const operation = start.operation;
const operationScopeKey = committedScopeKeyRef.current;
activeOperationsRef.current.set(dialogId, {
operation,
abortController,
scopeKey: operationScopeKey,
});
errorMessagesRef.current.delete(dialogId);
publishDialogState(dialogId);
try {
const response =
status === 'completing'
? await completeEditorBackgroundMusicPrompt(
{ currentPrompt: operation.canonicalPrompt },
{ signal: abortController.signal },
)
: await simplifyEditorBackgroundMusicPrompt(
{ currentPrompt: operation.canonicalPrompt },
{ signal: abortController.signal },
);
const activeOperation = activeOperationsRef.current.get(dialogId);
const latestDialog = getCanvasGenerationDialogById(dialogId);
if (
activeOperation?.operation.operationId !== operation.operationId ||
!Object.is(activeOperation.scopeKey, operationScopeKey) ||
!Object.is(operationScopeKey, committedScopeKeyRef.current)
) {
discardStaleOperation(operation);
return {
started: true,
applied: false,
reason: 'stale',
errorMessage: null,
};
}
if (
!isBackgroundMusicDialog(latestDialog) ||
latestDialog.composerOpen === false
) {
clearActiveOperation(dialogId, {
abort: false,
removeDialogState: !isBackgroundMusicDialog(latestDialog),
});
return {
started: true,
applied: false,
reason: 'stale',
errorMessage: null,
};
}
const canonicalPrompt = canonicalizeBackgroundMusicPrompt(
response.prompt,
);
if (
!Number.isInteger(response.charCount) ||
response.charCount !== countPromptCodePoints(canonicalPrompt)
) {
throw invalidPromptAssistResponseError();
}
const resolution = modelRef.current.resolveAiOperation(
operation,
canonicalPrompt,
);
activeOperationsRef.current.delete(dialogId);
if (!resolution.applied || !resolution.prompt) {
errorMessagesRef.current.set(
dialogId,
invalidPromptAssistResponseError().message,
);
publishDialogState(dialogId);
return {
started: true,
applied: false,
reason: 'failed',
errorMessage: invalidPromptAssistResponseError().message,
};
}
errorMessagesRef.current.delete(dialogId);
updateDialogPrompt(dialogId, resolution.prompt);
publishDialogState(dialogId);
return {
started: true,
applied: true,
reason: 'applied',
errorMessage: null,
};
} catch (error) {
const activeOperation = activeOperationsRef.current.get(dialogId);
if (
activeOperation?.operation.operationId !== operation.operationId ||
!Object.is(activeOperation.scopeKey, operationScopeKey) ||
!Object.is(operationScopeKey, committedScopeKeyRef.current)
) {
discardStaleOperation(operation);
return {
started: true,
applied: false,
reason: 'stale',
errorMessage: null,
};
}
const latestDialog = getCanvasGenerationDialogById(dialogId);
if (
!isBackgroundMusicDialog(latestDialog) ||
latestDialog.composerOpen === false
) {
clearActiveOperation(dialogId, {
abort: false,
removeDialogState: !isBackgroundMusicDialog(latestDialog),
});
return {
started: true,
applied: false,
reason: 'stale',
errorMessage: null,
};
}
const resolution = modelRef.current.rejectOperation(operation);
activeOperationsRef.current.delete(dialogId);
if (resolution.applied && resolution.prompt) {
updateDialogPrompt(dialogId, resolution.prompt);
}
const errorMessage = resolvePromptAssistErrorMessage(error);
if (errorMessage) {
errorMessagesRef.current.set(dialogId, errorMessage);
} else {
errorMessagesRef.current.delete(dialogId);
}
publishDialogState(dialogId);
return {
started: true,
applied: false,
reason: errorMessage ? 'failed' : 'stale',
errorMessage,
};
} finally {
const activeOperation = activeOperationsRef.current.get(dialogId);
if (
activeOperation?.operation.operationId === operation.operationId &&
Object.is(activeOperation.scopeKey, operationScopeKey)
) {
activeOperationsRef.current.delete(dialogId);
}
}
},
[
clearActiveOperation,
discardStaleOperation,
getCanvasGenerationDialogById,
isSubmissionLocked,
publishDialogState,
updateDialogPrompt,
],
);
const completePrompt = useCallback(
(dialogId: string) => runAiOperation(dialogId, 'completing'),
[runAiOperation],
);
const simplifyPrompt = useCallback(
(dialogId: string) => runAiOperation(dialogId, 'simplifying'),
[runAiOperation],
);
const preparePreset = useCallback(
(dialogId: string) => {
const dialog = getCanvasGenerationDialogById(dialogId);
if (!isBackgroundMusicDialog(dialog) || dialog.composerOpen === false) {
return null;
}
if (isSubmissionLocked(dialogId)) {
return null;
}
const previousOperation = activeOperationsRef.current.get(dialogId);
previousOperation?.abortController?.abort();
activeOperationsRef.current.delete(dialogId);
const canonicalPrompt = canonicalizeBackgroundMusicPrompt(dialog.prompt);
updateDialogPrompt(dialogId, canonicalPrompt);
const prepared = modelRef.current.preparePreset(
dialogId,
canonicalPrompt,
);
errorMessagesRef.current.delete(dialogId);
publishDialogState(dialogId);
return prepared.prompt;
},
[
getCanvasGenerationDialogById,
isSubmissionLocked,
publishDialogState,
updateDialogPrompt,
],
);
const undoPrompt = useCallback(
(dialogId: string) => {
const dialog = getCanvasGenerationDialogById(dialogId);
if (!isBackgroundMusicDialog(dialog) || dialog.composerOpen === false) {
return false;
}
const currentState = modelRef.current.getDialogState(dialogId);
if (
currentState.status !== 'idle' ||
currentState.undoPromptSnapshot === null
) {
return false;
}
const canonicalPrompt = canonicalizeBackgroundMusicPrompt(dialog.prompt);
updateDialogPrompt(dialogId, canonicalPrompt);
const swapped = modelRef.current.swapUndoSnapshot(
dialogId,
canonicalPrompt,
);
if (!swapped.applied) {
return false;
}
errorMessagesRef.current.delete(dialogId);
updateDialogPrompt(dialogId, swapped.prompt);
publishDialogState(dialogId);
return true;
},
[getCanvasGenerationDialogById, publishDialogState, updateDialogPrompt],
);
const beginSubmission = useCallback(
(dialogId: string): BackgroundMusicPromptSubmissionClaim | null => {
const dialog = getCanvasGenerationDialogById(dialogId);
if (!isBackgroundMusicDialog(dialog) || dialog.composerOpen === false) {
return null;
}
const start = modelRef.current.beginOperation({
dialogId,
status: 'submitting',
prompt: dialog.prompt,
});
if (start.reason !== 'duplicate') {
updateDialogPrompt(dialogId, start.canonicalPrompt);
}
if (!start.started || !start.operation) {
publishDialogState(dialogId);
return null;
}
const previousOperation = activeOperationsRef.current.get(dialogId);
previousOperation?.abortController?.abort();
activeOperationsRef.current.set(dialogId, {
operation: start.operation,
abortController: null,
scopeKey: committedScopeKeyRef.current,
});
errorMessagesRef.current.delete(dialogId);
publishDialogState(dialogId);
return {
operation: start.operation,
prompt: start.canonicalPrompt,
};
},
[getCanvasGenerationDialogById, publishDialogState, updateDialogPrompt],
);
const finishSubmission = useCallback(
({
operation,
accepted,
}: {
operation: BackgroundMusicPromptOperation;
accepted: boolean;
}) => {
const activeOperation = activeOperationsRef.current.get(
operation.dialogId,
);
if (activeOperation?.operation.operationId !== operation.operationId) {
return false;
}
const latestDialog = getCanvasGenerationDialogById(operation.dialogId);
if (
!Object.is(activeOperation.scopeKey, committedScopeKeyRef.current) ||
!isBackgroundMusicDialog(latestDialog)
) {
clearActiveOperation(operation.dialogId, {
abort: false,
removeDialogState: !isBackgroundMusicDialog(latestDialog),
});
return false;
}
const resolution = accepted
? modelRef.current.completeSubmittingOperation(operation)
: modelRef.current.rejectOperation(operation);
activeOperationsRef.current.delete(operation.dialogId);
if (resolution.applied && resolution.prompt) {
updateDialogPrompt(operation.dialogId, resolution.prompt);
}
publishDialogState(operation.dialogId);
return resolution.applied;
},
[
clearActiveOperation,
getCanvasGenerationDialogById,
publishDialogState,
updateDialogPrompt,
],
);
const getDialogState = useCallback(
(dialogId: string): BackgroundMusicPromptAssistDialogState =>
dialogStates[dialogId] ?? {
...modelRef.current.getDialogState(dialogId),
errorMessage: errorMessagesRef.current.get(dialogId) ?? null,
},
[dialogStates],
);
return useMemo(
() => ({
dialogStates,
getDialogState,
completePrompt,
simplifyPrompt,
preparePreset,
undoPrompt,
beginSubmission,
finishSubmission,
}),
[
beginSubmission,
completePrompt,
dialogStates,
finishSubmission,
getDialogState,
preparePreset,
simplifyPrompt,
undoPrompt,
],
);
}