This commit is contained in:
2026-04-18 13:05:29 +08:00
parent 09d4c0c31b
commit 5032701c38
77 changed files with 8538 additions and 2413 deletions

View File

@@ -6,6 +6,7 @@ import path from 'node:path';
import test from 'node:test';
import express from 'express';
import { PNG } from 'pngjs';
import type { AppConfig } from '../../config.js';
import { createCharacterAssetRoutes } from './characterAssetRoutes.js';
@@ -16,6 +17,31 @@ const PNG_BUFFER = Buffer.from(
);
const MP4_BUFFER = Buffer.from('mock-video');
function createGreenScreenFixturePngBuffer() {
const png = new PNG({ width: 2, height: 1 });
png.data[0] = 0;
png.data[1] = 255;
png.data[2] = 0;
png.data[3] = 255;
png.data[4] = 220;
png.data[5] = 48;
png.data[6] = 72;
png.data[7] = 255;
return PNG.sync.write(png);
}
function readPngAlphaValues(buffer: Buffer) {
const png = PNG.sync.read(buffer);
return Array.from({ length: png.width * png.height }, (_, index) => {
return png.data[index * 4 + 3] ?? 0;
});
}
const GREEN_SCREEN_PNG_BUFFER = createGreenScreenFixturePngBuffer();
function createTestConfig(
projectRoot: string,
dashScopeBaseUrl: string,
@@ -165,7 +191,7 @@ test('character visual generation converts public reference images into data url
if (req.method === 'GET' && url.pathname === '/downloads/visual.png') {
res.statusCode = 200;
res.setHeader('Content-Type', 'image/png');
res.end(PNG_BUFFER);
res.end(GREEN_SCREEN_PNG_BUFFER);
return;
}
@@ -219,6 +245,7 @@ test('character visual generation converts public reference images into data url
const savedDraftPath = path.join(tempRoot, 'public', payload.drafts[0]!.imageSrc.slice(1));
assert.equal(fs.existsSync(savedDraftPath), true);
assert.deepEqual(readPngAlphaValues(fs.readFileSync(savedDraftPath)), [0, 255]);
});
},
);
@@ -404,6 +431,110 @@ test('character workflow cache skips rewriting unchanged payloads', async () =>
);
});
test('character animation publish returns frame dimensions in animation map', async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-animation-publish-'));
await withAssetRouteServer(
createTestConfig(tempRoot, 'http://127.0.0.1:9/api/v1'),
async (assetBaseUrl) => {
const response = await fetch(`${assetBaseUrl}/api/assets/character-animation/publish`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
characterId: 'harbor-guide',
visualAssetId: 'visual-1',
updateCharacterOverride: false,
animations: {
run: {
framesDataUrls: [`data:image/png;base64,${PNG_BUFFER.toString('base64')}`],
fps: 12,
loop: true,
frameWidth: 144,
frameHeight: 192,
previewVideoPath: '/generated-character-drafts/harbor-guide/animation/run/preview.mp4',
},
},
}),
});
assert.equal(response.status, 200);
const payload = (await response.json()) as {
animationMap: Record<
string,
{
frameWidth?: number;
frameHeight?: number;
fps?: number;
loop?: boolean;
previewVideoPath?: string;
}
>;
};
assert.equal(payload.animationMap.run?.frameWidth, 144);
assert.equal(payload.animationMap.run?.frameHeight, 192);
assert.equal(payload.animationMap.run?.fps, 12);
assert.equal(payload.animationMap.run?.loop, true);
assert.equal(
payload.animationMap.run?.previewVideoPath,
'/generated-character-drafts/harbor-guide/animation/run/preview.mp4',
);
},
);
});
test('character visual publish removes green screen before saving master and previews', async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-visual-publish-'));
const publicDir = path.join(tempRoot, 'public');
fs.mkdirSync(publicDir, { recursive: true });
fs.writeFileSync(path.join(publicDir, 'draft.png'), GREEN_SCREEN_PNG_BUFFER);
await withAssetRouteServer(
createTestConfig(tempRoot, 'http://127.0.0.1:9/api/v1'),
async (assetBaseUrl) => {
const response = await fetch(`${assetBaseUrl}/api/assets/character-visual/publish`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
characterId: 'harbor-guide',
sourceMode: 'image-to-image',
promptText: '潮雾港向导',
selectedPreviewSource: '/draft.png',
previewSources: ['/draft.png'],
width: 1024,
height: 1024,
updateCharacterOverride: false,
}),
});
assert.equal(response.status, 200);
const payload = (await response.json()) as {
portraitPath: string;
};
const savedMasterPath = path.join(tempRoot, 'public', payload.portraitPath.slice(1));
const savedPreviewPath = path.join(
tempRoot,
'public',
'generated-characters',
'harbor-guide',
'visual',
path.basename(path.dirname(savedMasterPath)),
'preview-1.png',
);
assert.equal(fs.existsSync(savedMasterPath), true);
assert.equal(fs.existsSync(savedPreviewPath), true);
assert.deepEqual(readPngAlphaValues(fs.readFileSync(savedMasterPath)), [0, 255]);
assert.deepEqual(readPngAlphaValues(fs.readFileSync(savedPreviewPath)), [0, 255]);
},
);
});
test('character animation image-to-video flow uploads a public visual source and submits the resolved oss url', async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-video-'));
const publicDir = path.join(tempRoot, 'public');
@@ -524,3 +655,318 @@ test('character animation image-to-video flow uploads a public visual source and
},
);
});
test('character animation non-loop image-to-video uses first and last master frames', async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-kf2v-'));
const publicDir = path.join(tempRoot, 'public');
fs.mkdirSync(publicDir, { recursive: true });
fs.writeFileSync(path.join(publicDir, 'visual.png'), PNG_BUFFER);
let videoSynthesisPayloadText = '';
await withHttpServer(
(dashScopeBaseUrl) => async (req, res) => {
const url = new URL(req.url || '/', dashScopeBaseUrl);
if (
req.method === 'POST' &&
url.pathname === '/api/v1/services/aigc/image2video/video-synthesis'
) {
videoSynthesisPayloadText = (await readRequestBody(req)).toString('utf8');
sendJson(res, {
output: {
task_id: 'video-task-kf2v-1',
},
});
return;
}
if (req.method === 'GET' && url.pathname === '/api/v1/tasks/video-task-kf2v-1') {
sendJson(res, {
output: {
task_status: 'SUCCEEDED',
video_url: `${dashScopeBaseUrl}/downloads/preview.mp4`,
},
});
return;
}
if (req.method === 'GET' && url.pathname === '/downloads/preview.mp4') {
res.statusCode = 200;
res.setHeader('Content-Type', 'video/mp4');
res.end(MP4_BUFFER);
return;
}
res.statusCode = 404;
res.end('not found');
},
async (dashScopeBaseUrl) => {
const config = createTestConfig(tempRoot, `${dashScopeBaseUrl}/api/v1`);
await withAssetRouteServer(config, async (assetBaseUrl) => {
const response = await fetch(
`${assetBaseUrl}/api/assets/character-animation/generate`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
characterId: 'harbor-guide',
strategy: 'image-to-video',
animation: 'attack',
promptText: '短促挥击后收招',
characterBriefText: '旧港守望者',
visualSource: '/visual.png',
referenceImageDataUrls: [],
referenceVideoDataUrls: [],
frameCount: 8,
fps: 8,
durationSeconds: 4,
loop: false,
useChromaKey: true,
resolution: '720P',
imageSequenceModel: 'wan2.7-image-pro',
videoModel: 'wan2.7-i2v',
referenceVideoModel: 'wan2.7-r2v',
motionTransferModel: 'wan2.2-animate-move',
}),
},
);
assert.equal(response.status, 200);
const videoPayload = JSON.parse(videoSynthesisPayloadText) as {
model: string;
input: {
first_frame_url?: string;
last_frame_url?: string;
};
parameters: {
resolution?: string;
};
};
assert.equal(videoPayload.model, 'wan2.2-kf2v-flash');
assert.match(videoPayload.input.first_frame_url ?? '', /^data:image\/png;base64,/u);
assert.match(videoPayload.input.last_frame_url ?? '', /^data:image\/png;base64,/u);
assert.equal(videoPayload.parameters.resolution, '480P');
});
},
);
});
test('character animation loop image-to-video uses wan2.6-i2v-flash with img_url only', async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-i2v-loop-'));
const publicDir = path.join(tempRoot, 'public');
fs.mkdirSync(publicDir, { recursive: true });
fs.writeFileSync(path.join(publicDir, 'visual.png'), PNG_BUFFER);
let videoSynthesisPayloadText = '';
await withHttpServer(
(dashScopeBaseUrl) => async (req, res) => {
const url = new URL(req.url || '/', dashScopeBaseUrl);
if (
req.method === 'POST' &&
url.pathname === '/api/v1/services/aigc/video-generation/video-synthesis'
) {
videoSynthesisPayloadText = (await readRequestBody(req)).toString('utf8');
sendJson(res, {
output: {
task_id: 'video-task-i2v-loop-1',
},
});
return;
}
if (req.method === 'GET' && url.pathname === '/api/v1/tasks/video-task-i2v-loop-1') {
sendJson(res, {
output: {
task_status: 'SUCCEEDED',
video_url: `${dashScopeBaseUrl}/downloads/preview.mp4`,
},
});
return;
}
if (req.method === 'GET' && url.pathname === '/downloads/preview.mp4') {
res.statusCode = 200;
res.setHeader('Content-Type', 'video/mp4');
res.end(MP4_BUFFER);
return;
}
res.statusCode = 404;
res.end('not found');
},
async (dashScopeBaseUrl) => {
const config = createTestConfig(tempRoot, `${dashScopeBaseUrl}/api/v1`);
await withAssetRouteServer(config, async (assetBaseUrl) => {
const response = await fetch(
`${assetBaseUrl}/api/assets/character-animation/generate`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
characterId: 'harbor-guide',
strategy: 'image-to-video',
animation: 'run',
promptText: '稳定循环奔跑',
characterBriefText: '旧港守望者',
visualSource: '/visual.png',
referenceImageDataUrls: [],
referenceVideoDataUrls: [],
frameCount: 8,
fps: 8,
durationSeconds: 4,
loop: true,
useChromaKey: true,
resolution: '720P',
imageSequenceModel: 'wan2.7-image-pro',
videoModel: 'wan2.6-i2v-flash',
referenceVideoModel: 'wan2.7-r2v',
motionTransferModel: 'wan2.2-animate-move',
}),
},
);
assert.equal(response.status, 200);
const videoPayload = JSON.parse(videoSynthesisPayloadText) as {
model: string;
input: {
img_url?: string;
first_frame_url?: string;
last_frame_url?: string;
};
parameters: {
audio?: boolean;
resolution?: string;
};
};
assert.equal(videoPayload.model, 'wan2.6-i2v-flash');
assert.match(videoPayload.input.img_url ?? '', /^data:image\/png;base64,/u);
assert.equal(videoPayload.input.first_frame_url, undefined);
assert.equal(videoPayload.input.last_frame_url, undefined);
assert.equal(videoPayload.parameters.audio, false);
assert.equal(videoPayload.parameters.resolution, '720P');
});
},
);
});
test('character animation reference-to-video can use only reference image media', async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-r2v-'));
const publicDir = path.join(tempRoot, 'public');
fs.mkdirSync(publicDir, { recursive: true });
fs.writeFileSync(path.join(publicDir, 'visual.png'), PNG_BUFFER);
let videoSynthesisPayloadText = '';
await withHttpServer(
(dashScopeBaseUrl) => async (req, res) => {
const url = new URL(req.url || '/', dashScopeBaseUrl);
if (req.method === 'GET' && url.pathname === '/api/v1/uploads') {
sendJson(res, {
data: {
upload_host: `${dashScopeBaseUrl}/upload`,
upload_dir: 'uploads/test-dir',
policy: 'policy',
signature: 'signature',
oss_access_key_id: 'oss-key',
},
});
return;
}
if (req.method === 'POST' && url.pathname === '/upload') {
await readRequestBody(req);
res.statusCode = 200;
res.end('ok');
return;
}
if (
req.method === 'POST' &&
url.pathname === '/api/v1/services/aigc/video-generation/video-synthesis'
) {
videoSynthesisPayloadText = (await readRequestBody(req)).toString('utf8');
sendJson(res, {
output: {
task_id: 'video-task-r2v-1',
},
});
return;
}
if (req.method === 'GET' && url.pathname === '/api/v1/tasks/video-task-r2v-1') {
sendJson(res, {
output: {
task_status: 'SUCCEEDED',
video_url: `${dashScopeBaseUrl}/downloads/preview.mp4`,
},
});
return;
}
if (req.method === 'GET' && url.pathname === '/downloads/preview.mp4') {
res.statusCode = 200;
res.setHeader('Content-Type', 'video/mp4');
res.end(MP4_BUFFER);
return;
}
res.statusCode = 404;
res.end('not found');
},
async (dashScopeBaseUrl) => {
const config = createTestConfig(tempRoot, `${dashScopeBaseUrl}/api/v1`);
await withAssetRouteServer(config, async (assetBaseUrl) => {
const response = await fetch(
`${assetBaseUrl}/api/assets/character-animation/generate`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
characterId: 'harbor-guide',
strategy: 'reference-to-video',
animation: 'run',
promptText: '稳定循环奔跑',
characterBriefText: '旧港守望者',
visualSource: '/visual.png',
referenceImageDataUrls: [],
referenceVideoDataUrls: [],
frameCount: 8,
fps: 8,
durationSeconds: 4,
loop: true,
useChromaKey: true,
resolution: '720P',
imageSequenceModel: 'wan2.7-image-pro',
videoModel: 'wan2.7-i2v',
referenceVideoModel: 'wan2.7-r2v',
motionTransferModel: 'wan2.2-animate-move',
}),
},
);
assert.equal(response.status, 200);
const videoPayload = JSON.parse(videoSynthesisPayloadText) as {
input: {
media: Array<{ type: string; url: string }>;
};
};
assert.equal(videoPayload.input.media[0]?.type, 'reference_image');
assert.match(videoPayload.input.media[0]?.url ?? '', /^oss:\/\/uploads\/test-dir\//u);
assert.equal(videoPayload.input.media.length, 1);
});
},
);
});

View File

@@ -7,7 +7,8 @@ import http, {
import https from 'node:https';
import path from 'node:path';
import { type NextFunction, type Request, type Response,Router } from 'express';
import { type NextFunction, type Request, type Response, Router } from 'express';
import { PNG } from 'pngjs';
import {
buildMasterPrompt,
@@ -32,6 +33,7 @@ const CHARACTER_ANIMATION_TEMPLATES_PATH = '/api/assets/character-animation/temp
const DEFAULT_DASHSCOPE_BASE_URL = 'https://dashscope.aliyuncs.com/api/v1';
const DEFAULT_CHARACTER_VISUAL_MODEL = 'wan2.7-image-pro';
const DEFAULT_CHARACTER_VIDEO_MODEL = 'wan2.2-kf2v-flash';
const DEFAULT_CHARACTER_LOOP_VIDEO_MODEL = 'wan2.6-i2v-flash';
const DEFAULT_CHARACTER_REFERENCE_VIDEO_MODEL = 'wan2.7-r2v';
const DEFAULT_CHARACTER_MOTION_TRANSFER_MODEL = 'wan2.2-animate-move';
const DASHSCOPE_IMAGE_TASK_POLL_INTERVAL_MS = 2500;
@@ -107,6 +109,67 @@ type DecodedMediaPayload = {
extension: string;
};
function applyGreenScreenAlphaToPngBuffer(buffer: Buffer) {
try {
const png = PNG.sync.read(buffer);
const pixels = png.data;
let changed = false;
for (let index = 0; index < pixels.length; index += 4) {
const red = pixels[index] ?? 0;
const green = pixels[index + 1] ?? 0;
const blue = pixels[index + 2] ?? 0;
const alpha = pixels[index + 3] ?? 0;
const greenRatio = green / Math.max(1, red + blue);
if (alpha === 0) {
continue;
}
const greenLead = green - Math.max(red, blue);
if (green <= 72 || greenLead <= 20 || greenRatio <= 0.72) {
continue;
}
let nextAlpha = Math.min(alpha, Math.max(0, 255 - greenLead * 6));
if (green > 120 && greenLead > 48 && greenRatio > 1.12) {
nextAlpha = 0;
}
if (nextAlpha === alpha) {
continue;
}
pixels[index + 3] = nextAlpha;
if (nextAlpha > 0) {
pixels[index + 1] = Math.min(
green,
Math.max(red, blue) + Math.max(6, Math.round(greenLead * 0.18)),
);
}
changed = true;
}
return changed ? PNG.sync.write(png) : buffer;
} catch {
return buffer;
}
}
function applyChromaKeyToMediaPayload(payload: DecodedMediaPayload) {
if (payload.mimeType !== 'image/png' && payload.extension !== 'png') {
return payload;
}
return {
...payload,
buffer: applyGreenScreenAlphaToPngBuffer(payload.buffer),
mimeType: 'image/png',
extension: 'png',
} satisfies DecodedMediaPayload;
}
type CharacterPromptBundle = {
visualPromptText: string;
animationPromptText: string;
@@ -355,6 +418,92 @@ function sanitizeCharacterPromptBundle(
};
}
function sanitizeAnimationPromptText(value: string, maxLength: number) {
return value
.replace(/\s+/gu, ' ')
.replace(/||||||||/gu, '')
.replace(/||/gu, '')
.replace(/|/gu, '')
.replace(/|/gu, '')
.trim()
.slice(0, maxLength);
}
function buildCompactAnimationCharacterBrief(value: string) {
const normalized = sanitizeAnimationPromptText(value, 160);
if (!normalized) {
return '';
}
return normalized
.split(/[\/|\n,;]+/u)
.map((item) => item.trim())
.filter(Boolean)
.slice(0, 4)
.join('');
}
function isInappropriateContentMessage(value: string) {
return /finappropriate-content|inappropriate content||/iu.test(
value,
);
}
async function proxyJsonRequestWithPromptFallback(params: {
urlString: string;
apiKey: string;
buildBody: (prompt: string) => Record<string, unknown>;
primaryPrompt: string;
fallbackPrompt?: string;
extraHeaders?: Record<string, string>;
}) {
const firstResponse = await proxyJsonRequest(
params.urlString,
params.apiKey,
params.buildBody(params.primaryPrompt),
params.extraHeaders,
);
if (firstResponse.statusCode >= 200 && firstResponse.statusCode < 300) {
return {
response: firstResponse,
prompt: params.primaryPrompt,
moderationFallbackApplied: false,
};
}
const fallbackPrompt = params.fallbackPrompt?.trim() ?? '';
const errorMessage = extractApiErrorMessage(
firstResponse.bodyText,
'视频生成请求失败。',
);
if (
!fallbackPrompt ||
fallbackPrompt === params.primaryPrompt ||
!isInappropriateContentMessage(errorMessage)
) {
return {
response: firstResponse,
prompt: params.primaryPrompt,
moderationFallbackApplied: false,
};
}
const secondResponse = await proxyJsonRequest(
params.urlString,
params.apiKey,
params.buildBody(fallbackPrompt),
params.extraHeaders,
);
return {
response: secondResponse,
prompt: fallbackPrompt,
moderationFallbackApplied: true,
};
}
function buildCharacterPromptBundleUserPrompt(params: {
roleKind: string;
characterBriefText: string;
@@ -463,13 +612,24 @@ async function writeJsonObjectFile(
}
function decodeMediaDataUrl(dataUrl: string): DecodedMediaPayload {
const matched = /^data:([^;]+);base64,(.+)$/u.exec(dataUrl);
const matched = /^data:([^,]+),(.+)$/u.exec(dataUrl);
if (!matched) {
throw new Error('不支持的媒体数据,要求使用 Base64 Data URL。');
}
const mimeType = matched[1];
const metadata = matched[1];
const base64Payload = matched[2];
const metadataParts = metadata
.split(';')
.map((item) => item.trim())
.filter(Boolean);
const mimeType = metadataParts[0] ?? 'application/octet-stream';
const isBase64 = metadataParts.some((item) => item.toLowerCase() === 'base64');
if (!isBase64) {
throw new Error('不支持的媒体数据,要求使用 Base64 Data URL。');
}
const extension = (() => {
switch (mimeType) {
case 'image/jpeg':
@@ -484,6 +644,8 @@ function decodeMediaDataUrl(dataUrl: string): DecodedMediaPayload {
return 'mov';
case 'video/x-msvideo':
return 'avi';
case 'video/webm':
return 'webm';
default:
return mimeType.split('/')[1] ?? 'bin';
}
@@ -552,6 +714,15 @@ async function resolveMediaSourcePayload(
};
}
async function resolveCharacterVisualPayload(
rootDir: string,
source: string,
): Promise<DecodedMediaPayload> {
return applyChromaKeyToMediaPayload(
await resolveMediaSourcePayload(rootDir, source),
);
}
async function resolveMediaSourceAsDataUrl(
rootDir: string,
source: string,
@@ -982,19 +1153,32 @@ function buildNpcAnimationPrompt(options: {
animation: string;
promptText: string;
useChromaKey: boolean;
loop: boolean;
characterBriefText?: string;
actionTemplateId?: string;
}) {
const characterBrief = buildCompactAnimationCharacterBrief(
options.characterBriefText ?? '',
);
const actionDetailText = sanitizeAnimationPromptText(options.promptText, 140);
const loopRule = options.loop
? '这是循环动作,直接进入动作循环中段,不要开场静止站桩,不要把主参考图原样作为第一帧。'
: '这是非循环动作,首帧和尾帧都要回到参考主图角色形象,中段完成动作变化。';
if (options.actionTemplateId) {
return buildVideoActionPrompt({
actionTemplate: getActionTemplateById(
options.actionTemplateId as Parameters<typeof getActionTemplateById>[0],
),
actionDetailText: options.promptText,
useChromaKey: options.useChromaKey,
characterBrief:
options.characterBriefText?.trim() || `${options.animation} 动作角色`,
});
return [
buildVideoActionPrompt({
actionTemplate: getActionTemplateById(
options.actionTemplateId as Parameters<typeof getActionTemplateById>[0],
),
actionDetailText,
useChromaKey: options.useChromaKey,
characterBrief: characterBrief || `${options.animation} 动作角色`,
}),
loopRule,
]
.filter(Boolean)
.join(' ');
}
return [
@@ -1004,15 +1188,50 @@ function buildNpcAnimationPrompt(options: {
options.useChromaKey
? '背景为纯绿色绿幕,无其他人物和场景元素,方便后期抠像。'
: '背景简洁纯净,无复杂场景。',
options.characterBriefText?.trim()
? `角色设定:${options.characterBriefText.trim()}`
characterBrief
? `角色设定:${characterBrief}`
: '',
options.promptText.trim(),
actionDetailText,
loopRule,
]
.filter(Boolean)
.join(' ');
}
function buildFallbackModerationSafeAnimationPrompt(options: {
animation: string;
loop: boolean;
useChromaKey: boolean;
}) {
return [
`单人全身角色动作视频,动作主题是 ${options.animation}`,
'角色固定为同一人,右向斜侧身,镜头稳定,轮廓清楚。',
options.loop
? '循环动作直接进入稳定循环,不要静止开场,不要定格首帧。'
: '非循环动作首尾回到角色标准站姿,中段完成动作变化。',
options.useChromaKey
? '背景为纯绿色绿幕,无其他人物和场景元素。'
: '背景简洁纯净。',
]
.filter(Boolean)
.join(' ');
}
function getLowestSupportedVideoResolution(model: string, fallback: string) {
switch (model) {
case 'wan2.6-i2v-flash':
case 'wan2.6-i2v':
case 'wan2.6-i2v-us':
return '720P';
case 'wan2.2-kf2v-flash':
case 'wan2.2-i2v-flash':
case 'wan2.5-i2v-preview':
return '480P';
default:
return fallback;
}
}
async function handleGenerateCharacterPromptBundle(
config: AppConfig,
req: IncomingMessage & { body?: unknown },
@@ -1318,7 +1537,7 @@ async function handleGenerateCharacterVisuals(
const imageSrc = await writeDraftBinaryFile(
rootDir,
path.posix.join(draftRelativeDir, fileName),
imageResponse.body,
applyGreenScreenAlphaToPngBuffer(imageResponse.body),
);
return {
@@ -1475,6 +1694,7 @@ async function handleGenerateCharacterAnimation(
Number.isFinite(body.durationSeconds)
? Math.max(1, Math.min(8, Math.round(body.durationSeconds)))
: 4;
const loop = body.loop === true;
const useChromaKey = body.useChromaKey !== false;
const resolution =
typeof body.resolution === 'string' && body.resolution.trim()
@@ -1487,15 +1707,28 @@ async function handleGenerateCharacterAnimation(
: runtimeEnv.DASHSCOPE_CHARACTER_IMAGE_SEQUENCE_MODEL ||
runtimeEnv.DASHSCOPE_CHARACTER_VISUAL_MODEL ||
DEFAULT_CHARACTER_VISUAL_MODEL;
const videoModel =
const requestedVideoModel =
typeof body.videoModel === 'string' && body.videoModel.trim()
? body.videoModel.trim()
: runtimeEnv.DASHSCOPE_CHARACTER_VIDEO_MODEL ||
DEFAULT_CHARACTER_VIDEO_MODEL;
const loopVideoModel =
runtimeEnv.DASHSCOPE_CHARACTER_LOOP_VIDEO_MODEL ||
(requestedVideoModel === 'wan2.2-kf2v-flash'
? DEFAULT_CHARACTER_LOOP_VIDEO_MODEL
: requestedVideoModel) ||
DEFAULT_CHARACTER_LOOP_VIDEO_MODEL;
const keyframeVideoModel =
runtimeEnv.DASHSCOPE_CHARACTER_KEYFRAME_VIDEO_MODEL ||
DEFAULT_CHARACTER_VIDEO_MODEL;
const videoModel =
strategy === 'image-to-video' ? (loop ? loopVideoModel : keyframeVideoModel) : requestedVideoModel;
const durationSeconds =
videoModel === 'wan2.2-kf2v-flash' ? 5 : requestedDurationSeconds;
const normalizedResolution =
videoModel === 'wan2.2-kf2v-flash' ? '480P' : resolution;
const normalizedResolution = getLowestSupportedVideoResolution(
videoModel,
videoModel === 'wan2.2-kf2v-flash' ? '480P' : resolution,
);
const referenceVideoModel =
typeof body.referenceVideoModel === 'string' &&
body.referenceVideoModel.trim()
@@ -1707,13 +1940,21 @@ async function handleGenerateCharacterAnimation(
animation,
promptText,
useChromaKey,
loop,
characterBriefText,
actionTemplateId,
});
const fallbackPrompt = buildFallbackModerationSafeAnimationPrompt({
animation,
loop,
useChromaKey,
});
activePrompt = finalPrompt;
activeModel = videoModel;
const isKf2vFlash = videoModel === 'wan2.2-kf2v-flash';
const visualInputRef = isKf2vFlash
const isWan26I2vFlash = videoModel === 'wan2.6-i2v-flash';
const visualInputRef =
isKf2vFlash || isWan26I2vFlash
? await resolveMediaSourceAsDataUrl(rootDir, visualSource)
: await uploadFileToDashScope(
baseUrl,
@@ -1722,9 +1963,12 @@ async function handleGenerateCharacterAnimation(
`${characterId}-${animation}-visual`,
await resolveMediaSourcePayload(rootDir, visualSource),
);
const lastFrameRef = lastFrameImageDataUrl
const resolvedLastFrameSource = !loop
? lastFrameImageDataUrl || visualSource
: '';
const lastFrameRef = resolvedLastFrameSource
? isKf2vFlash
? await resolveMediaSourceAsDataUrl(rootDir, lastFrameImageDataUrl)
? await resolveMediaSourceAsDataUrl(rootDir, resolvedLastFrameSource)
: await uploadFileToDashScope(
baseUrl,
apiKey,
@@ -1732,47 +1976,59 @@ async function handleGenerateCharacterAnimation(
`${characterId}-${animation}-last-frame`,
await resolveMediaSourcePayload(
rootDir,
lastFrameImageDataUrl,
resolvedLastFrameSource,
),
)
: '';
const inputPayload =
isKf2vFlash
const createVideoRequestBody = (prompt: string) => ({
model: videoModel,
input: isKf2vFlash
? {
prompt: finalPrompt,
prompt,
first_frame_url: visualInputRef,
...(lastFrameRef ? { last_frame_url: lastFrameRef } : {}),
}
: {
prompt: finalPrompt,
media: [
{ type: 'first_frame', url: visualInputRef },
...(lastFrameRef
? [{ type: 'last_frame', url: lastFrameRef }]
: []),
],
};
: isWan26I2vFlash
? {
prompt,
img_url: visualInputRef,
}
: {
prompt,
media: [
{ type: 'first_frame', url: visualInputRef },
...(lastFrameRef
? [{ type: 'last_frame', url: lastFrameRef }]
: []),
],
},
parameters: {
duration: durationSeconds,
resolution: normalizedResolution,
...(isKf2vFlash
? { prompt_extend: true, watermark: false }
: {}),
...(isWan26I2vFlash ? { audio: false } : {}),
},
});
const videoSynthesisEndpoint = isKf2vFlash
? `${baseUrl}/services/aigc/image2video/video-synthesis`
: `${baseUrl}/services/aigc/video-generation/video-synthesis`;
const createTaskResponse = await proxyJsonRequest(
videoSynthesisEndpoint,
apiKey,
{
model: videoModel,
input: inputPayload,
parameters: {
duration: durationSeconds,
resolution: normalizedResolution,
...(isKf2vFlash ? { prompt_extend: true, watermark: false } : {}),
const { response: createTaskResponse, prompt: submittedPrompt } =
await proxyJsonRequestWithPromptFallback({
urlString: videoSynthesisEndpoint,
apiKey,
buildBody: createVideoRequestBody,
primaryPrompt: finalPrompt,
fallbackPrompt,
extraHeaders: {
'X-DashScope-Async': 'enable',
'X-DashScope-OssResourceResolve': 'enable',
},
},
{
'X-DashScope-Async': 'enable',
'X-DashScope-OssResourceResolve': 'enable',
},
);
});
activePrompt = submittedPrompt;
if (
createTaskResponse.statusCode < 200 ||
@@ -1809,7 +2065,7 @@ async function handleGenerateCharacterAnimation(
animation,
strategy,
model: videoModel,
prompt: finalPrompt,
prompt: submittedPrompt,
createdAt,
updatedAt: createdAt,
});
@@ -1859,7 +2115,7 @@ async function handleGenerateCharacterAnimation(
model: videoModel,
strategy,
animation,
prompt: finalPrompt,
prompt: submittedPrompt,
createdAt: new Date().toISOString(),
videoUrl,
},
@@ -1877,7 +2133,7 @@ async function handleGenerateCharacterAnimation(
animation,
strategy,
model: videoModel,
prompt: finalPrompt,
prompt: submittedPrompt,
createdAt,
updatedAt: new Date().toISOString(),
result: {
@@ -1891,7 +2147,7 @@ async function handleGenerateCharacterAnimation(
taskId,
strategy: 'image-to-video',
model: videoModel,
prompt: finalPrompt,
prompt: submittedPrompt,
previewVideoPath,
});
return;
@@ -1923,6 +2179,7 @@ async function handleGenerateCharacterAnimation(
animation,
promptText,
useChromaKey,
loop,
characterBriefText,
});
activePrompt = finalPrompt;
@@ -2081,8 +2338,8 @@ async function handleGenerateCharacterAnimation(
}
if (strategy === 'reference-to-video') {
const uploadedReferenceUrls = await Promise.all([
...referenceImageDataUrls.map(async (source, index) =>
const uploadedReferenceImages = await Promise.all(
referenceImageDataUrls.map(async (source, index) =>
uploadFileToDashScope(
baseUrl,
apiKey,
@@ -2091,7 +2348,9 @@ async function handleGenerateCharacterAnimation(
await resolveMediaSourcePayload(rootDir, source),
),
),
...referenceVideoDataUrls.map(async (source, index) =>
);
const uploadedReferenceVideos = await Promise.all(
referenceVideoDataUrls.map(async (source, index) =>
uploadFileToDashScope(
baseUrl,
apiKey,
@@ -2100,9 +2359,13 @@ async function handleGenerateCharacterAnimation(
await resolveMediaSourcePayload(rootDir, source),
),
),
]);
);
if (uploadedReferenceUrls.length === 0) {
if (
!visualUrl &&
uploadedReferenceImages.length === 0 &&
uploadedReferenceVideos.length === 0
) {
sendJson(res, 400, {
error: { message: '参考生视频至少需要一张参考图或一段参考视频。' },
});
@@ -2113,6 +2376,7 @@ async function handleGenerateCharacterAnimation(
animation,
promptText,
useChromaKey,
loop,
characterBriefText,
});
activePrompt = finalPrompt;
@@ -2124,11 +2388,24 @@ async function handleGenerateCharacterAnimation(
model: referenceVideoModel,
input: {
prompt: finalPrompt,
reference_urls: [visualUrl, ...uploadedReferenceUrls],
media: [
{ type: 'reference_image', url: visualUrl },
...uploadedReferenceImages.map((url) => ({
type: 'reference_image' as const,
url,
})),
...uploadedReferenceVideos.map((url) => ({
type: 'reference_video' as const,
url,
})),
],
},
parameters: {
duration: durationSeconds,
resolution,
resolution: getLowestSupportedVideoResolution(
referenceVideoModel,
resolution,
),
prompt_optimizer: true,
},
},
@@ -2688,7 +2965,7 @@ async function handlePublishCharacterVisual(
);
await mkdir(visualDir, { recursive: true });
const masterPayload = await resolveMediaSourcePayload(
const masterPayload = await resolveCharacterVisualPayload(
rootDir,
selectedPreviewSource,
);
@@ -2697,7 +2974,7 @@ async function handlePublishCharacterVisual(
const previewImagePaths: string[] = [];
for (let index = 0; index < previewSources.length; index += 1) {
const previewPayload = await resolveMediaSourcePayload(
const previewPayload = await resolveCharacterVisualPayload(
rootDir,
previewSources[index] ?? '',
);
@@ -2904,6 +3181,11 @@ async function handlePublishCharacterAnimation(
startFrame: 1,
extension: frameExtension,
basePath,
frameWidth,
frameHeight,
fps,
loop,
...(previewVideoPath ? { previewVideoPath } : {}),
};
}