GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Simplify custom world result editing controls

Browse Source
This commit is contained in:
高物
2026-04-08 19:07:46 +08:00
parent bd9fdcbe31
commit a02f7b6414
125 changed files with 8804 additions and 1462 deletions
Download Patch File Download Diff File Expand all files Collapse all files

242
scripts/dev-server/characterAssetStudioPlugins.ts
View File

@@ -9,6 +9,12 @@ import path from 'node:path';
import { loadEnv, type Plugin } from 'vite';
import {
buildMasterPrompt,
buildVideoActionPrompt,
getActionTemplateById,
} from '../../src/tools/qwenSpriteSheetToolModel';
const CHARACTER_VISUAL_GENERATE_PATH = '/api/character-visual/generate';
const CHARACTER_VISUAL_JOBS_PATH = '/api/character-visual/jobs/';
const CHARACTER_ANIMATION_GENERATE_PATH = '/api/animation/generate';
@@ -17,7 +23,7 @@ const CHARACTER_ANIMATION_IMPORT_VIDEO_PATH = '/api/animation/import-video';
const CHARACTER_ANIMATION_TEMPLATES_PATH = '/api/animation/templates';
const DEFAULT_DASHSCOPE_BASE_URL = 'https://dashscope.aliyuncs.com/api/v1';
const DEFAULT_CHARACTER_VISUAL_MODEL = 'wan2.7-image-pro';
const DEFAULT_CHARACTER_VIDEO_MODEL = 'wan2.7-i2v';
const DEFAULT_CHARACTER_VIDEO_MODEL = 'wan2.2-kf2v-flash';
const DEFAULT_CHARACTER_REFERENCE_VIDEO_MODEL = 'wan2.7-r2v';
const DEFAULT_CHARACTER_MOTION_TRANSFER_MODEL = 'wan2.2-animate-move';
const DASHSCOPE_IMAGE_TASK_POLL_INTERVAL_MS = 2500;
@@ -302,6 +308,18 @@ async function resolveMediaSourcePayload(
};
}
async function resolveMediaSourceAsDataUrl(
rootDir: string,
source: string,
) {
if (/^data:/u.test(source)) {
return source;
}
const payload = await resolveMediaSourcePayload(rootDir, source);
return `data:${payload.mimeType};base64,${payload.buffer.toString('base64')}`;
}
function requestResponse(
urlString: string,
options: {
@@ -684,14 +702,15 @@ function extractImageUrls(payload: Record<string, unknown>) {
return [...new Set(urls)];
}
function buildNpcVisualPrompt(promptText: string) {
const trimmed = promptText.trim();
return [
'单人 NPC 角色形象，全身，侧身朝右，站姿稳定，武器与手完整可见。',
'画面简洁，背景干净，角色轮廓清楚，适合后续做动作与裁切。',
'不要多人，不要复杂场景，不要夸张透视，不要截断脚底。',
trimmed || '江湖风格角色，服装完整，姿态自然。',
].join(' ');
function buildNpcVisualPrompt(
promptText: string,
characterBriefText = '',
) {
const mergedBrief = [characterBriefText.trim(), promptText.trim()]
.filter(Boolean)
.join('\n');
return buildMasterPrompt(mergedBrief || '江湖风格角色，服装完整，姿态自然。');
}
function buildImageSequencePrompt(
@@ -713,19 +732,36 @@ function buildImageSequencePrompt(
.join(' ');
}
function buildNpcAnimationPrompt(
animation: string,
promptText: string,
useChromaKey: boolean,
) {
function buildNpcAnimationPrompt(options: {
animation: string;
promptText: string;
useChromaKey: boolean;
characterBriefText?: string;
actionTemplateId?: string;
}) {
if (options.actionTemplateId) {
return buildVideoActionPrompt({
actionTemplate: getActionTemplateById(
options.actionTemplateId as Parameters<typeof getActionTemplateById>[0],
),
actionDetailText: options.promptText,
useChromaKey: options.useChromaKey,
characterBrief:
options.characterBriefText?.trim() || `${options.animation} 动作角色`,
});
}
return [
`单人 NPC 全身动作视频，动作主题是 ${animation}。`,
`单人 NPC 全身动作视频，动作主题是 ${options.animation}。`,
'角色固定为同一人，侧身朝右，镜头稳定，轮廓清晰，武器不可丢失。',
'动作连贯，避免服装、发型、面部、武器随机漂移。',
useChromaKey
options.useChromaKey
? '背景为纯绿色绿幕，无其他人物和场景元素，方便后期抠像。'
: '背景简洁纯净，无复杂场景。',
promptText.trim(),
options.characterBriefText?.trim()
? `角色设定：${options.characterBriefText.trim()}`
: '',
options.promptText.trim(),
]
.filter(Boolean)
.join(' ');
@@ -787,13 +823,17 @@ async function handleGenerateCharacterVisuals(
typeof body.characterId === 'string'
? body.characterId.trim()
: 'character';
const sourceMode =
typeof body.sourceMode === 'string' ? body.sourceMode.trim() : '';
const promptText =
typeof body.promptText === 'string' ? body.promptText.trim() : '';
const referenceImageDataUrls = isStringArray(body.referenceImageDataUrls)
? body.referenceImageDataUrls.slice(0, 4)
: [];
const sourceMode =
typeof body.sourceMode === 'string' ? body.sourceMode.trim() : '';
const promptText =
typeof body.promptText === 'string' ? body.promptText.trim() : '';
const characterBriefText =
typeof body.characterBriefText === 'string'
? body.characterBriefText.trim()
: '';
const referenceImageDataUrls = isStringArray(body.referenceImageDataUrls)
? body.referenceImageDataUrls.slice(0, 4)
: [];
const candidateCountRaw =
typeof body.candidateCount === 'number' ? body.candidateCount : 3;
const candidateCount = Math.max(
@@ -818,20 +858,20 @@ async function handleGenerateCharacterVisuals(
return;
}
if (!promptText && sourceMode === 'text-to-image') {
sendJson(res, 400, {
error: { message: '文生主形象需要填写角色设定。' },
});
if (!promptText && !characterBriefText && sourceMode === 'text-to-image') {
sendJson(res, 400, {
error: { message: '文生主形象需要填写角色设定。' },
});
return;
}
let activeTaskId = '';
let activePrompt = '';
try {
const finalPrompt = buildNpcVisualPrompt(promptText);
activePrompt = finalPrompt;
const content = [
{ text: finalPrompt },
let activeTaskId = '';
let activePrompt = '';
try {
const finalPrompt = buildNpcVisualPrompt(promptText, characterBriefText);
activePrompt = finalPrompt;
const content = [
{ text: finalPrompt },
...referenceImageDataUrls.map((image) => ({ image })),
];
const createTaskResponse = await proxyJsonRequest(
@@ -1059,6 +1099,14 @@ async function handleGenerateCharacterAnimation(
typeof body.animation === 'string' ? body.animation.trim() : 'idle';
const promptText =
typeof body.promptText === 'string' ? body.promptText.trim() : '';
const characterBriefText =
typeof body.characterBriefText === 'string'
? body.characterBriefText.trim()
: '';
const actionTemplateId =
typeof body.actionTemplateId === 'string'
? body.actionTemplateId.trim()
: '';
const visualSource =
typeof body.visualSource === 'string' ? body.visualSource.trim() : '';
const referenceImageDataUrls = isStringArray(body.referenceImageDataUrls)
@@ -1076,7 +1124,7 @@ async function handleGenerateCharacterAnimation(
typeof body.frameCount === 'number' && Number.isFinite(body.frameCount)
? Math.max(2, Math.min(16, Math.round(body.frameCount)))
: 8;
const durationSeconds =
const requestedDurationSeconds =
typeof body.durationSeconds === 'number' &&
Number.isFinite(body.durationSeconds)
? Math.max(1, Math.min(8, Math.round(body.durationSeconds)))
@@ -1098,6 +1146,10 @@ async function handleGenerateCharacterAnimation(
? body.videoModel.trim()
: runtimeEnv.DASHSCOPE_CHARACTER_VIDEO_MODEL ||
DEFAULT_CHARACTER_VIDEO_MODEL;
const durationSeconds =
videoModel === 'wan2.2-kf2v-flash' ? 5 : requestedDurationSeconds;
const normalizedResolution =
videoModel === 'wan2.2-kf2v-flash' ? '480P' : resolution;
const referenceVideoModel =
typeof body.referenceVideoModel === 'string' &&
body.referenceVideoModel.trim()
@@ -1295,62 +1347,70 @@ async function handleGenerateCharacterAnimation(
return;
}
const modelForVisualUpload =
strategy === 'reference-to-video'
? referenceVideoModel
: strategy === 'motion-transfer'
? motionTransferModel
: videoModel;
const visualUrl = await uploadFileToDashScope(
baseUrl,
apiKey,
modelForVisualUpload,
`${characterId}-${animation}-visual`,
await resolveMediaSourcePayload(rootDir, visualSource),
);
if (strategy === 'image-to-video') {
const finalPrompt = buildNpcAnimationPrompt(
const finalPrompt = buildNpcAnimationPrompt({
animation,
promptText,
useChromaKey,
);
characterBriefText,
actionTemplateId,
});
activePrompt = finalPrompt;
activeModel = videoModel;
const media = [
{ type: 'image', url: visualUrl, role: 'first_frame' },
...(lastFrameImageDataUrl
? [
{
type: 'image',
url: await uploadFileToDashScope(
baseUrl,
apiKey,
videoModel,
`${characterId}-${animation}-last-frame`,
await resolveMediaSourcePayload(
rootDir,
lastFrameImageDataUrl,
),
),
role: 'last_frame',
},
]
: []),
];
const isKf2vFlash = videoModel === 'wan2.2-kf2v-flash';
const visualInputRef = isKf2vFlash
? await resolveMediaSourceAsDataUrl(rootDir, visualSource)
: await uploadFileToDashScope(
baseUrl,
apiKey,
videoModel,
`${characterId}-${animation}-visual`,
await resolveMediaSourcePayload(rootDir, visualSource),
);
const lastFrameRef = lastFrameImageDataUrl
? isKf2vFlash
? await resolveMediaSourceAsDataUrl(rootDir, lastFrameImageDataUrl)
: await uploadFileToDashScope(
baseUrl,
apiKey,
videoModel,
`${characterId}-${animation}-last-frame`,
await resolveMediaSourcePayload(
rootDir,
lastFrameImageDataUrl,
),
)
: '';
const inputPayload =
isKf2vFlash
? {
prompt: finalPrompt,
first_frame_url: visualInputRef,
...(lastFrameRef ? { last_frame_url: lastFrameRef } : {}),
}
: {
prompt: finalPrompt,
media: [
{ type: 'first_frame', url: visualInputRef },
...(lastFrameRef
? [{ type: 'last_frame', url: lastFrameRef }]
: []),
],
};
const videoSynthesisEndpoint = isKf2vFlash
? `${baseUrl}/services/aigc/image2video/video-synthesis`
: `${baseUrl}/services/aigc/video-generation/video-synthesis`;
const createTaskResponse = await proxyJsonRequest(
`${baseUrl}/services/aigc/video-generation/video-synthesis`,
videoSynthesisEndpoint,
apiKey,
{
model: videoModel,
input: {
prompt: finalPrompt,
media,
},
input: inputPayload,
parameters: {
duration: durationSeconds,
resolution,
resolution: normalizedResolution,
...(isKf2vFlash ? { prompt_extend: true, watermark: false } : {}),
},
},
{
@@ -1482,6 +1542,20 @@ async function handleGenerateCharacterAnimation(
return;
}
const modelForVisualUpload =
strategy === 'reference-to-video'
? referenceVideoModel
: strategy === 'motion-transfer'
? motionTransferModel
: videoModel;
const visualUrl = await uploadFileToDashScope(
baseUrl,
apiKey,
modelForVisualUpload,
`${characterId}-${animation}-visual`,
await resolveMediaSourcePayload(rootDir, visualSource),
);
if (strategy === 'motion-transfer') {
if (referenceVideoDataUrls.length === 0) {
sendJson(res, 400, {
@@ -1490,11 +1564,12 @@ async function handleGenerateCharacterAnimation(
return;
}
const finalPrompt = buildNpcAnimationPrompt(
const finalPrompt = buildNpcAnimationPrompt({
animation,
promptText,
useChromaKey,
);
characterBriefText,
});
activePrompt = finalPrompt;
activeModel = motionTransferModel;
const referenceVideoUrl = await uploadFileToDashScope(
@@ -1679,11 +1754,12 @@ async function handleGenerateCharacterAnimation(
return;
}
const finalPrompt = buildNpcAnimationPrompt(
const finalPrompt = buildNpcAnimationPrompt({
animation,
promptText,
useChromaKey,
);
characterBriefText,
});
activePrompt = finalPrompt;
activeModel = referenceVideoModel;
const createTaskResponse = await proxyJsonRequest(

195
scripts/dev-server/localApiPlugins.ts
View File

@@ -27,7 +27,7 @@ const CHARACTER_VISUAL_PUBLISH_PATH = '/api/character-visual/publish';
const CHARACTER_ANIMATION_PUBLISH_PATH = '/api/animation/publish';
const CUSTOM_WORLD_SCENE_IMAGE_PATH = '/api/custom-world/scene-image';
const DEFAULT_DASHSCOPE_BASE_URL = 'https://dashscope.aliyuncs.com/api/v1';
const DEFAULT_DASHSCOPE_SCENE_IMAGE_MODEL = 'wan2.2-t2i-flash';
const DEFAULT_DASHSCOPE_SCENE_IMAGE_MODEL = 'wan2.7-image';
const DASHSCOPE_TASK_POLL_INTERVAL_MS = 2000;
const DASHSCOPE_TASK_TIMEOUT_MS = 150000;
@@ -541,6 +541,43 @@ async function resolveAssetSourcePayload(
};
}
async function resolveAssetSourceAsDataUrl(
rootDir: string,
source: string,
fallbackMessage: string,
) {
if (/^data:image\/[^;]+;base64,/u.test(source)) {
return source;
}
const payload = await resolveAssetSourcePayload(
rootDir,
source,
fallbackMessage,
);
const mimeType = (() => {
switch (payload.extension) {
case 'jpg':
case 'jpeg':
return 'image/jpeg';
case 'webp':
return 'image/webp';
default:
return 'image/png';
}
})();
return `data:${mimeType};base64,${payload.buffer.toString('base64')}`;
}
function resolveDashScopeSceneImageModel(model: string) {
if (/^wan2\.7-image(?:-pro)?$/u.test(model)) {
return model;
}
return DEFAULT_DASHSCOPE_SCENE_IMAGE_MODEL;
}
function resolveImageExtension(
contentTypeHeader: string | string[] | undefined,
sourceUrl: string,
@@ -657,6 +694,44 @@ function getDashScopeImageUrl(taskResponse: Record<string, unknown>) {
}
}
const choices = output && Array.isArray(output.choices) ? output.choices : [];
for (const choice of choices) {
if (!isRecordValue(choice)) {
continue;
}
const message = isRecordValue(choice.message) ? choice.message : null;
const content =
message && Array.isArray(message.content) ? message.content : [];
for (const entry of content) {
if (!isRecordValue(entry)) {
continue;
}
const imageUrl =
typeof entry.image === 'string' && entry.image.trim()
? entry.image.trim()
: typeof entry.url === 'string' && entry.url.trim()
? entry.url.trim()
: '';
if (imageUrl) {
return {
url: imageUrl,
actualPrompt:
typeof entry.actual_prompt === 'string' &&
entry.actual_prompt.trim()
? entry.actual_prompt.trim()
: typeof entry.revised_prompt === 'string' &&
entry.revised_prompt.trim()
? entry.revised_prompt.trim()
: undefined,
};
}
}
}
throw new Error('场景图片生成成功，但没有返回可下载的图片地址。');
}
@@ -886,10 +961,11 @@ function createCustomWorldSceneImagePlugin(
typeof body.size === 'string' && body.size.trim()
? body.size.trim()
: '1280*720';
const model =
const requestedModel =
typeof body.model === 'string' && body.model.trim()
? body.model.trim()
: defaultModel;
const model = resolveDashScopeSceneImageModel(requestedModel);
const worldName =
typeof body.worldName === 'string' ? body.worldName.trim() : '';
const profileId =
@@ -898,6 +974,10 @@ function createCustomWorldSceneImagePlugin(
typeof body.landmarkName === 'string' ? body.landmarkName.trim() : '';
const landmarkId =
typeof body.landmarkId === 'string' ? body.landmarkId.trim() : '';
const referenceImageSrc =
typeof body.referenceImageSrc === 'string'
? body.referenceImageSrc.trim()
: '';
if (!prompt) {
sendJson(res, 400, { error: { message: 'prompt is required.' } });
@@ -912,20 +992,37 @@ function createCustomWorldSceneImagePlugin(
}
try {
const messageContent: Array<{ image: string } | { text: string }> = [];
if (referenceImageSrc) {
messageContent.push({
image: await resolveAssetSourceAsDataUrl(
rootDir,
referenceImageSrc,
'参考图必须来自 public 目录或使用 Data URL。',
),
});
}
messageContent.push({ text: prompt });
const createTaskResponse = await proxyJsonRequest(
`${baseUrl}/services/aigc/text2image/image-synthesis`,
`${baseUrl}/services/aigc/image-generation/generation`,
apiKey,
{
model,
input: {
prompt,
...(negativePrompt ? { negative_prompt: negativePrompt } : {}),
messages: [
{
role: 'user',
content: messageContent,
},
],
},
parameters: {
n: 1,
size,
prompt_extend: true,
watermark: false,
...(negativePrompt ? { negative_prompt: negativePrompt } : {}),
},
},
{
@@ -1023,6 +1120,7 @@ function createCustomWorldSceneImagePlugin(
size,
prompt,
negativePrompt,
referenceImageSrc: referenceImageSrc || undefined,
actualPrompt: imageResult.actualPrompt,
remoteUrl: imageResult.url,
imageSrc,
@@ -1189,6 +1287,7 @@ function createCharacterVisualPublishPlugin(rootDir: string): Plugin {
typeof body.height === 'number' && Number.isFinite(body.height)
? body.height
: 1536;
const updateCharacterOverride = body.updateCharacterOverride !== false;
if (!characterId) {
sendJson(res, 400, { error: { message: 'characterId is required.' } });
@@ -1259,26 +1358,30 @@ function createCharacterVisualPublishPlugin(rootDir: string): Plugin {
'utf8',
);
const overrideMap = await readJsonObjectFile(characterOverridesFilePath);
const existingOverride = overrideMap[characterId];
const nextOverride =
existingOverride &&
typeof existingOverride === 'object' &&
!Array.isArray(existingOverride)
? { ...(existingOverride as Record<string, unknown>) }
: {};
nextOverride.generatedVisualAssetId = assetId;
nextOverride.portrait = masterImagePath;
overrideMap[characterId] = nextOverride;
await writeJsonObjectFile(characterOverridesFilePath, overrideMap);
let overrideMap: Record<string, unknown> = {};
if (updateCharacterOverride) {
overrideMap = await readJsonObjectFile(characterOverridesFilePath);
const existingOverride = overrideMap[characterId];
const nextOverride =
existingOverride &&
typeof existingOverride === 'object' &&
!Array.isArray(existingOverride)
? { ...(existingOverride as Record<string, unknown>) }
: {};
nextOverride.generatedVisualAssetId = assetId;
nextOverride.portrait = masterImagePath;
overrideMap[characterId] = nextOverride;
await writeJsonObjectFile(characterOverridesFilePath, overrideMap);
}
sendJson(res, 200, {
ok: true,
assetId,
portraitPath: masterImagePath,
overrideMap,
saveMessage:
'主形象已发布到 public/generated-characters，并更新角色覆盖。',
saveMessage: updateCharacterOverride
? '主形象已发布到 public/generated-characters，并更新角色覆盖。'
: '主形象已保存到 public/generated-characters，可直接写回当前自定义世界角色。',
});
} catch (error) {
sendJson(res, 500, {
@@ -1333,6 +1436,7 @@ function createCharacterAnimationPublishPlugin(rootDir: string): Plugin {
!Array.isArray(body.animations)
? (body.animations as Record<string, unknown>)
: null;
const updateCharacterOverride = body.updateCharacterOverride !== false;
if (!characterId) {
sendJson(res, 400, { error: { message: 'characterId is required.' } });
@@ -1476,35 +1580,40 @@ function createCharacterAnimationPublishPlugin(rootDir: string): Plugin {
'utf8',
);
const overrideMap = await readJsonObjectFile(characterOverridesFilePath);
const existingOverride = overrideMap[characterId];
const nextOverride =
existingOverride &&
typeof existingOverride === 'object' &&
!Array.isArray(existingOverride)
? { ...(existingOverride as Record<string, unknown>) }
: {};
const existingAnimationMap =
nextOverride.animationMap &&
typeof nextOverride.animationMap === 'object' &&
!Array.isArray(nextOverride.animationMap)
? (nextOverride.animationMap as Record<string, unknown>)
: {};
nextOverride.generatedAnimationSetId = animationSetId;
nextOverride.generatedVisualAssetId = visualAssetId;
nextOverride.animationMap = {
...existingAnimationMap,
...nextAnimationMap,
};
overrideMap[characterId] = nextOverride;
await writeJsonObjectFile(characterOverridesFilePath, overrideMap);
let overrideMap: Record<string, unknown> = {};
if (updateCharacterOverride) {
overrideMap = await readJsonObjectFile(characterOverridesFilePath);
const existingOverride = overrideMap[characterId];
const nextOverride =
existingOverride &&
typeof existingOverride === 'object' &&
!Array.isArray(existingOverride)
? { ...(existingOverride as Record<string, unknown>) }
: {};
const existingAnimationMap =
nextOverride.animationMap &&
typeof nextOverride.animationMap === 'object' &&
!Array.isArray(nextOverride.animationMap)
? (nextOverride.animationMap as Record<string, unknown>)
: {};
nextOverride.generatedAnimationSetId = animationSetId;
nextOverride.generatedVisualAssetId = visualAssetId;
nextOverride.animationMap = {
...existingAnimationMap,
...nextAnimationMap,
};
overrideMap[characterId] = nextOverride;
await writeJsonObjectFile(characterOverridesFilePath, overrideMap);
}
sendJson(res, 200, {
ok: true,
animationSetId,
overrideMap,
saveMessage:
'基础动作资源已发布到 public/generated-animations，并更新角色覆盖。',
animationMap: nextAnimationMap,
saveMessage: updateCharacterOverride
? '基础动作资源已发布到 public/generated-animations，并更新角色覆盖。'
: '基础动作资源已保存到 public/generated-animations，可直接写回当前自定义世界角色。',
});
} catch (error) {
sendJson(res, 500, {