1
This commit is contained in:
@@ -6,6 +6,7 @@ import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
|
||||
import express from 'express';
|
||||
import { PNG } from 'pngjs';
|
||||
|
||||
import type { AppConfig } from '../../config.js';
|
||||
import { createCharacterAssetRoutes } from './characterAssetRoutes.js';
|
||||
@@ -16,6 +17,31 @@ const PNG_BUFFER = Buffer.from(
|
||||
);
|
||||
const MP4_BUFFER = Buffer.from('mock-video');
|
||||
|
||||
function createGreenScreenFixturePngBuffer() {
|
||||
const png = new PNG({ width: 2, height: 1 });
|
||||
|
||||
png.data[0] = 0;
|
||||
png.data[1] = 255;
|
||||
png.data[2] = 0;
|
||||
png.data[3] = 255;
|
||||
|
||||
png.data[4] = 220;
|
||||
png.data[5] = 48;
|
||||
png.data[6] = 72;
|
||||
png.data[7] = 255;
|
||||
|
||||
return PNG.sync.write(png);
|
||||
}
|
||||
|
||||
function readPngAlphaValues(buffer: Buffer) {
|
||||
const png = PNG.sync.read(buffer);
|
||||
return Array.from({ length: png.width * png.height }, (_, index) => {
|
||||
return png.data[index * 4 + 3] ?? 0;
|
||||
});
|
||||
}
|
||||
|
||||
const GREEN_SCREEN_PNG_BUFFER = createGreenScreenFixturePngBuffer();
|
||||
|
||||
function createTestConfig(
|
||||
projectRoot: string,
|
||||
dashScopeBaseUrl: string,
|
||||
@@ -165,7 +191,7 @@ test('character visual generation converts public reference images into data url
|
||||
if (req.method === 'GET' && url.pathname === '/downloads/visual.png') {
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', 'image/png');
|
||||
res.end(PNG_BUFFER);
|
||||
res.end(GREEN_SCREEN_PNG_BUFFER);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -219,6 +245,7 @@ test('character visual generation converts public reference images into data url
|
||||
|
||||
const savedDraftPath = path.join(tempRoot, 'public', payload.drafts[0]!.imageSrc.slice(1));
|
||||
assert.equal(fs.existsSync(savedDraftPath), true);
|
||||
assert.deepEqual(readPngAlphaValues(fs.readFileSync(savedDraftPath)), [0, 255]);
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -404,6 +431,110 @@ test('character workflow cache skips rewriting unchanged payloads', async () =>
|
||||
);
|
||||
});
|
||||
|
||||
test('character animation publish returns frame dimensions in animation map', async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-animation-publish-'));
|
||||
|
||||
await withAssetRouteServer(
|
||||
createTestConfig(tempRoot, 'http://127.0.0.1:9/api/v1'),
|
||||
async (assetBaseUrl) => {
|
||||
const response = await fetch(`${assetBaseUrl}/api/assets/character-animation/publish`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
characterId: 'harbor-guide',
|
||||
visualAssetId: 'visual-1',
|
||||
updateCharacterOverride: false,
|
||||
animations: {
|
||||
run: {
|
||||
framesDataUrls: [`data:image/png;base64,${PNG_BUFFER.toString('base64')}`],
|
||||
fps: 12,
|
||||
loop: true,
|
||||
frameWidth: 144,
|
||||
frameHeight: 192,
|
||||
previewVideoPath: '/generated-character-drafts/harbor-guide/animation/run/preview.mp4',
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
const payload = (await response.json()) as {
|
||||
animationMap: Record<
|
||||
string,
|
||||
{
|
||||
frameWidth?: number;
|
||||
frameHeight?: number;
|
||||
fps?: number;
|
||||
loop?: boolean;
|
||||
previewVideoPath?: string;
|
||||
}
|
||||
>;
|
||||
};
|
||||
|
||||
assert.equal(payload.animationMap.run?.frameWidth, 144);
|
||||
assert.equal(payload.animationMap.run?.frameHeight, 192);
|
||||
assert.equal(payload.animationMap.run?.fps, 12);
|
||||
assert.equal(payload.animationMap.run?.loop, true);
|
||||
assert.equal(
|
||||
payload.animationMap.run?.previewVideoPath,
|
||||
'/generated-character-drafts/harbor-guide/animation/run/preview.mp4',
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('character visual publish removes green screen before saving master and previews', async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-visual-publish-'));
|
||||
const publicDir = path.join(tempRoot, 'public');
|
||||
fs.mkdirSync(publicDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(publicDir, 'draft.png'), GREEN_SCREEN_PNG_BUFFER);
|
||||
|
||||
await withAssetRouteServer(
|
||||
createTestConfig(tempRoot, 'http://127.0.0.1:9/api/v1'),
|
||||
async (assetBaseUrl) => {
|
||||
const response = await fetch(`${assetBaseUrl}/api/assets/character-visual/publish`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
characterId: 'harbor-guide',
|
||||
sourceMode: 'image-to-image',
|
||||
promptText: '潮雾港向导',
|
||||
selectedPreviewSource: '/draft.png',
|
||||
previewSources: ['/draft.png'],
|
||||
width: 1024,
|
||||
height: 1024,
|
||||
updateCharacterOverride: false,
|
||||
}),
|
||||
});
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
const payload = (await response.json()) as {
|
||||
portraitPath: string;
|
||||
};
|
||||
|
||||
const savedMasterPath = path.join(tempRoot, 'public', payload.portraitPath.slice(1));
|
||||
const savedPreviewPath = path.join(
|
||||
tempRoot,
|
||||
'public',
|
||||
'generated-characters',
|
||||
'harbor-guide',
|
||||
'visual',
|
||||
path.basename(path.dirname(savedMasterPath)),
|
||||
'preview-1.png',
|
||||
);
|
||||
|
||||
assert.equal(fs.existsSync(savedMasterPath), true);
|
||||
assert.equal(fs.existsSync(savedPreviewPath), true);
|
||||
assert.deepEqual(readPngAlphaValues(fs.readFileSync(savedMasterPath)), [0, 255]);
|
||||
assert.deepEqual(readPngAlphaValues(fs.readFileSync(savedPreviewPath)), [0, 255]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('character animation image-to-video flow uploads a public visual source and submits the resolved oss url', async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-video-'));
|
||||
const publicDir = path.join(tempRoot, 'public');
|
||||
@@ -524,3 +655,318 @@ test('character animation image-to-video flow uploads a public visual source and
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('character animation non-loop image-to-video uses first and last master frames', async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-kf2v-'));
|
||||
const publicDir = path.join(tempRoot, 'public');
|
||||
fs.mkdirSync(publicDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(publicDir, 'visual.png'), PNG_BUFFER);
|
||||
|
||||
let videoSynthesisPayloadText = '';
|
||||
|
||||
await withHttpServer(
|
||||
(dashScopeBaseUrl) => async (req, res) => {
|
||||
const url = new URL(req.url || '/', dashScopeBaseUrl);
|
||||
|
||||
if (
|
||||
req.method === 'POST' &&
|
||||
url.pathname === '/api/v1/services/aigc/image2video/video-synthesis'
|
||||
) {
|
||||
videoSynthesisPayloadText = (await readRequestBody(req)).toString('utf8');
|
||||
sendJson(res, {
|
||||
output: {
|
||||
task_id: 'video-task-kf2v-1',
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && url.pathname === '/api/v1/tasks/video-task-kf2v-1') {
|
||||
sendJson(res, {
|
||||
output: {
|
||||
task_status: 'SUCCEEDED',
|
||||
video_url: `${dashScopeBaseUrl}/downloads/preview.mp4`,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && url.pathname === '/downloads/preview.mp4') {
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', 'video/mp4');
|
||||
res.end(MP4_BUFFER);
|
||||
return;
|
||||
}
|
||||
|
||||
res.statusCode = 404;
|
||||
res.end('not found');
|
||||
},
|
||||
async (dashScopeBaseUrl) => {
|
||||
const config = createTestConfig(tempRoot, `${dashScopeBaseUrl}/api/v1`);
|
||||
await withAssetRouteServer(config, async (assetBaseUrl) => {
|
||||
const response = await fetch(
|
||||
`${assetBaseUrl}/api/assets/character-animation/generate`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
characterId: 'harbor-guide',
|
||||
strategy: 'image-to-video',
|
||||
animation: 'attack',
|
||||
promptText: '短促挥击后收招',
|
||||
characterBriefText: '旧港守望者',
|
||||
visualSource: '/visual.png',
|
||||
referenceImageDataUrls: [],
|
||||
referenceVideoDataUrls: [],
|
||||
frameCount: 8,
|
||||
fps: 8,
|
||||
durationSeconds: 4,
|
||||
loop: false,
|
||||
useChromaKey: true,
|
||||
resolution: '720P',
|
||||
imageSequenceModel: 'wan2.7-image-pro',
|
||||
videoModel: 'wan2.7-i2v',
|
||||
referenceVideoModel: 'wan2.7-r2v',
|
||||
motionTransferModel: 'wan2.2-animate-move',
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
|
||||
const videoPayload = JSON.parse(videoSynthesisPayloadText) as {
|
||||
model: string;
|
||||
input: {
|
||||
first_frame_url?: string;
|
||||
last_frame_url?: string;
|
||||
};
|
||||
parameters: {
|
||||
resolution?: string;
|
||||
};
|
||||
};
|
||||
assert.equal(videoPayload.model, 'wan2.2-kf2v-flash');
|
||||
assert.match(videoPayload.input.first_frame_url ?? '', /^data:image\/png;base64,/u);
|
||||
assert.match(videoPayload.input.last_frame_url ?? '', /^data:image\/png;base64,/u);
|
||||
assert.equal(videoPayload.parameters.resolution, '480P');
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('character animation loop image-to-video uses wan2.6-i2v-flash with img_url only', async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-i2v-loop-'));
|
||||
const publicDir = path.join(tempRoot, 'public');
|
||||
fs.mkdirSync(publicDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(publicDir, 'visual.png'), PNG_BUFFER);
|
||||
|
||||
let videoSynthesisPayloadText = '';
|
||||
|
||||
await withHttpServer(
|
||||
(dashScopeBaseUrl) => async (req, res) => {
|
||||
const url = new URL(req.url || '/', dashScopeBaseUrl);
|
||||
|
||||
if (
|
||||
req.method === 'POST' &&
|
||||
url.pathname === '/api/v1/services/aigc/video-generation/video-synthesis'
|
||||
) {
|
||||
videoSynthesisPayloadText = (await readRequestBody(req)).toString('utf8');
|
||||
sendJson(res, {
|
||||
output: {
|
||||
task_id: 'video-task-i2v-loop-1',
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && url.pathname === '/api/v1/tasks/video-task-i2v-loop-1') {
|
||||
sendJson(res, {
|
||||
output: {
|
||||
task_status: 'SUCCEEDED',
|
||||
video_url: `${dashScopeBaseUrl}/downloads/preview.mp4`,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && url.pathname === '/downloads/preview.mp4') {
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', 'video/mp4');
|
||||
res.end(MP4_BUFFER);
|
||||
return;
|
||||
}
|
||||
|
||||
res.statusCode = 404;
|
||||
res.end('not found');
|
||||
},
|
||||
async (dashScopeBaseUrl) => {
|
||||
const config = createTestConfig(tempRoot, `${dashScopeBaseUrl}/api/v1`);
|
||||
await withAssetRouteServer(config, async (assetBaseUrl) => {
|
||||
const response = await fetch(
|
||||
`${assetBaseUrl}/api/assets/character-animation/generate`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
characterId: 'harbor-guide',
|
||||
strategy: 'image-to-video',
|
||||
animation: 'run',
|
||||
promptText: '稳定循环奔跑',
|
||||
characterBriefText: '旧港守望者',
|
||||
visualSource: '/visual.png',
|
||||
referenceImageDataUrls: [],
|
||||
referenceVideoDataUrls: [],
|
||||
frameCount: 8,
|
||||
fps: 8,
|
||||
durationSeconds: 4,
|
||||
loop: true,
|
||||
useChromaKey: true,
|
||||
resolution: '720P',
|
||||
imageSequenceModel: 'wan2.7-image-pro',
|
||||
videoModel: 'wan2.6-i2v-flash',
|
||||
referenceVideoModel: 'wan2.7-r2v',
|
||||
motionTransferModel: 'wan2.2-animate-move',
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
|
||||
const videoPayload = JSON.parse(videoSynthesisPayloadText) as {
|
||||
model: string;
|
||||
input: {
|
||||
img_url?: string;
|
||||
first_frame_url?: string;
|
||||
last_frame_url?: string;
|
||||
};
|
||||
parameters: {
|
||||
audio?: boolean;
|
||||
resolution?: string;
|
||||
};
|
||||
};
|
||||
assert.equal(videoPayload.model, 'wan2.6-i2v-flash');
|
||||
assert.match(videoPayload.input.img_url ?? '', /^data:image\/png;base64,/u);
|
||||
assert.equal(videoPayload.input.first_frame_url, undefined);
|
||||
assert.equal(videoPayload.input.last_frame_url, undefined);
|
||||
assert.equal(videoPayload.parameters.audio, false);
|
||||
assert.equal(videoPayload.parameters.resolution, '720P');
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('character animation reference-to-video can use only reference image media', async () => {
|
||||
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'genarrative-character-r2v-'));
|
||||
const publicDir = path.join(tempRoot, 'public');
|
||||
fs.mkdirSync(publicDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(publicDir, 'visual.png'), PNG_BUFFER);
|
||||
|
||||
let videoSynthesisPayloadText = '';
|
||||
|
||||
await withHttpServer(
|
||||
(dashScopeBaseUrl) => async (req, res) => {
|
||||
const url = new URL(req.url || '/', dashScopeBaseUrl);
|
||||
|
||||
if (req.method === 'GET' && url.pathname === '/api/v1/uploads') {
|
||||
sendJson(res, {
|
||||
data: {
|
||||
upload_host: `${dashScopeBaseUrl}/upload`,
|
||||
upload_dir: 'uploads/test-dir',
|
||||
policy: 'policy',
|
||||
signature: 'signature',
|
||||
oss_access_key_id: 'oss-key',
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'POST' && url.pathname === '/upload') {
|
||||
await readRequestBody(req);
|
||||
res.statusCode = 200;
|
||||
res.end('ok');
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
req.method === 'POST' &&
|
||||
url.pathname === '/api/v1/services/aigc/video-generation/video-synthesis'
|
||||
) {
|
||||
videoSynthesisPayloadText = (await readRequestBody(req)).toString('utf8');
|
||||
sendJson(res, {
|
||||
output: {
|
||||
task_id: 'video-task-r2v-1',
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && url.pathname === '/api/v1/tasks/video-task-r2v-1') {
|
||||
sendJson(res, {
|
||||
output: {
|
||||
task_status: 'SUCCEEDED',
|
||||
video_url: `${dashScopeBaseUrl}/downloads/preview.mp4`,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && url.pathname === '/downloads/preview.mp4') {
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', 'video/mp4');
|
||||
res.end(MP4_BUFFER);
|
||||
return;
|
||||
}
|
||||
|
||||
res.statusCode = 404;
|
||||
res.end('not found');
|
||||
},
|
||||
async (dashScopeBaseUrl) => {
|
||||
const config = createTestConfig(tempRoot, `${dashScopeBaseUrl}/api/v1`);
|
||||
await withAssetRouteServer(config, async (assetBaseUrl) => {
|
||||
const response = await fetch(
|
||||
`${assetBaseUrl}/api/assets/character-animation/generate`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
characterId: 'harbor-guide',
|
||||
strategy: 'reference-to-video',
|
||||
animation: 'run',
|
||||
promptText: '稳定循环奔跑',
|
||||
characterBriefText: '旧港守望者',
|
||||
visualSource: '/visual.png',
|
||||
referenceImageDataUrls: [],
|
||||
referenceVideoDataUrls: [],
|
||||
frameCount: 8,
|
||||
fps: 8,
|
||||
durationSeconds: 4,
|
||||
loop: true,
|
||||
useChromaKey: true,
|
||||
resolution: '720P',
|
||||
imageSequenceModel: 'wan2.7-image-pro',
|
||||
videoModel: 'wan2.7-i2v',
|
||||
referenceVideoModel: 'wan2.7-r2v',
|
||||
motionTransferModel: 'wan2.2-animate-move',
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
|
||||
const videoPayload = JSON.parse(videoSynthesisPayloadText) as {
|
||||
input: {
|
||||
media: Array<{ type: string; url: string }>;
|
||||
};
|
||||
};
|
||||
assert.equal(videoPayload.input.media[0]?.type, 'reference_image');
|
||||
assert.match(videoPayload.input.media[0]?.url ?? '', /^oss:\/\/uploads\/test-dir\//u);
|
||||
assert.equal(videoPayload.input.media.length, 1);
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -7,7 +7,8 @@ import http, {
|
||||
import https from 'node:https';
|
||||
import path from 'node:path';
|
||||
|
||||
import { type NextFunction, type Request, type Response,Router } from 'express';
|
||||
import { type NextFunction, type Request, type Response, Router } from 'express';
|
||||
import { PNG } from 'pngjs';
|
||||
|
||||
import {
|
||||
buildMasterPrompt,
|
||||
@@ -32,6 +33,7 @@ const CHARACTER_ANIMATION_TEMPLATES_PATH = '/api/assets/character-animation/temp
|
||||
const DEFAULT_DASHSCOPE_BASE_URL = 'https://dashscope.aliyuncs.com/api/v1';
|
||||
const DEFAULT_CHARACTER_VISUAL_MODEL = 'wan2.7-image-pro';
|
||||
const DEFAULT_CHARACTER_VIDEO_MODEL = 'wan2.2-kf2v-flash';
|
||||
const DEFAULT_CHARACTER_LOOP_VIDEO_MODEL = 'wan2.6-i2v-flash';
|
||||
const DEFAULT_CHARACTER_REFERENCE_VIDEO_MODEL = 'wan2.7-r2v';
|
||||
const DEFAULT_CHARACTER_MOTION_TRANSFER_MODEL = 'wan2.2-animate-move';
|
||||
const DASHSCOPE_IMAGE_TASK_POLL_INTERVAL_MS = 2500;
|
||||
@@ -107,6 +109,67 @@ type DecodedMediaPayload = {
|
||||
extension: string;
|
||||
};
|
||||
|
||||
function applyGreenScreenAlphaToPngBuffer(buffer: Buffer) {
|
||||
try {
|
||||
const png = PNG.sync.read(buffer);
|
||||
const pixels = png.data;
|
||||
let changed = false;
|
||||
|
||||
for (let index = 0; index < pixels.length; index += 4) {
|
||||
const red = pixels[index] ?? 0;
|
||||
const green = pixels[index + 1] ?? 0;
|
||||
const blue = pixels[index + 2] ?? 0;
|
||||
const alpha = pixels[index + 3] ?? 0;
|
||||
const greenRatio = green / Math.max(1, red + blue);
|
||||
|
||||
if (alpha === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const greenLead = green - Math.max(red, blue);
|
||||
if (green <= 72 || greenLead <= 20 || greenRatio <= 0.72) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let nextAlpha = Math.min(alpha, Math.max(0, 255 - greenLead * 6));
|
||||
|
||||
if (green > 120 && greenLead > 48 && greenRatio > 1.12) {
|
||||
nextAlpha = 0;
|
||||
}
|
||||
|
||||
if (nextAlpha === alpha) {
|
||||
continue;
|
||||
}
|
||||
|
||||
pixels[index + 3] = nextAlpha;
|
||||
if (nextAlpha > 0) {
|
||||
pixels[index + 1] = Math.min(
|
||||
green,
|
||||
Math.max(red, blue) + Math.max(6, Math.round(greenLead * 0.18)),
|
||||
);
|
||||
}
|
||||
changed = true;
|
||||
}
|
||||
|
||||
return changed ? PNG.sync.write(png) : buffer;
|
||||
} catch {
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
function applyChromaKeyToMediaPayload(payload: DecodedMediaPayload) {
|
||||
if (payload.mimeType !== 'image/png' && payload.extension !== 'png') {
|
||||
return payload;
|
||||
}
|
||||
|
||||
return {
|
||||
...payload,
|
||||
buffer: applyGreenScreenAlphaToPngBuffer(payload.buffer),
|
||||
mimeType: 'image/png',
|
||||
extension: 'png',
|
||||
} satisfies DecodedMediaPayload;
|
||||
}
|
||||
|
||||
type CharacterPromptBundle = {
|
||||
visualPromptText: string;
|
||||
animationPromptText: string;
|
||||
@@ -355,6 +418,92 @@ function sanitizeCharacterPromptBundle(
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeAnimationPromptText(value: string, maxLength: number) {
|
||||
return value
|
||||
.replace(/\s+/gu, ' ')
|
||||
.replace(/血浆|喷血|鲜血|断肢|斩首|裸体|裸露|色情|性交/gu, '')
|
||||
.replace(/死亡|死去|击杀/gu, '倒地结束')
|
||||
.replace(/受击|受伤/gu, '失衡')
|
||||
.replace(/砍杀|斩击/gu, '挥击')
|
||||
.trim()
|
||||
.slice(0, maxLength);
|
||||
}
|
||||
|
||||
function buildCompactAnimationCharacterBrief(value: string) {
|
||||
const normalized = sanitizeAnimationPromptText(value, 160);
|
||||
if (!normalized) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return normalized
|
||||
.split(/[\/|\n,,。;;]+/u)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean)
|
||||
.slice(0, 4)
|
||||
.join(',');
|
||||
}
|
||||
|
||||
function isInappropriateContentMessage(value: string) {
|
||||
return /finappropriate-content|inappropriate content|不适当内容|违规内容/iu.test(
|
||||
value,
|
||||
);
|
||||
}
|
||||
|
||||
async function proxyJsonRequestWithPromptFallback(params: {
|
||||
urlString: string;
|
||||
apiKey: string;
|
||||
buildBody: (prompt: string) => Record<string, unknown>;
|
||||
primaryPrompt: string;
|
||||
fallbackPrompt?: string;
|
||||
extraHeaders?: Record<string, string>;
|
||||
}) {
|
||||
const firstResponse = await proxyJsonRequest(
|
||||
params.urlString,
|
||||
params.apiKey,
|
||||
params.buildBody(params.primaryPrompt),
|
||||
params.extraHeaders,
|
||||
);
|
||||
|
||||
if (firstResponse.statusCode >= 200 && firstResponse.statusCode < 300) {
|
||||
return {
|
||||
response: firstResponse,
|
||||
prompt: params.primaryPrompt,
|
||||
moderationFallbackApplied: false,
|
||||
};
|
||||
}
|
||||
|
||||
const fallbackPrompt = params.fallbackPrompt?.trim() ?? '';
|
||||
const errorMessage = extractApiErrorMessage(
|
||||
firstResponse.bodyText,
|
||||
'视频生成请求失败。',
|
||||
);
|
||||
|
||||
if (
|
||||
!fallbackPrompt ||
|
||||
fallbackPrompt === params.primaryPrompt ||
|
||||
!isInappropriateContentMessage(errorMessage)
|
||||
) {
|
||||
return {
|
||||
response: firstResponse,
|
||||
prompt: params.primaryPrompt,
|
||||
moderationFallbackApplied: false,
|
||||
};
|
||||
}
|
||||
|
||||
const secondResponse = await proxyJsonRequest(
|
||||
params.urlString,
|
||||
params.apiKey,
|
||||
params.buildBody(fallbackPrompt),
|
||||
params.extraHeaders,
|
||||
);
|
||||
|
||||
return {
|
||||
response: secondResponse,
|
||||
prompt: fallbackPrompt,
|
||||
moderationFallbackApplied: true,
|
||||
};
|
||||
}
|
||||
|
||||
function buildCharacterPromptBundleUserPrompt(params: {
|
||||
roleKind: string;
|
||||
characterBriefText: string;
|
||||
@@ -463,13 +612,24 @@ async function writeJsonObjectFile(
|
||||
}
|
||||
|
||||
function decodeMediaDataUrl(dataUrl: string): DecodedMediaPayload {
|
||||
const matched = /^data:([^;]+);base64,(.+)$/u.exec(dataUrl);
|
||||
const matched = /^data:([^,]+),(.+)$/u.exec(dataUrl);
|
||||
if (!matched) {
|
||||
throw new Error('不支持的媒体数据,要求使用 Base64 Data URL。');
|
||||
}
|
||||
|
||||
const mimeType = matched[1];
|
||||
const metadata = matched[1];
|
||||
const base64Payload = matched[2];
|
||||
const metadataParts = metadata
|
||||
.split(';')
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
const mimeType = metadataParts[0] ?? 'application/octet-stream';
|
||||
const isBase64 = metadataParts.some((item) => item.toLowerCase() === 'base64');
|
||||
|
||||
if (!isBase64) {
|
||||
throw new Error('不支持的媒体数据,要求使用 Base64 Data URL。');
|
||||
}
|
||||
|
||||
const extension = (() => {
|
||||
switch (mimeType) {
|
||||
case 'image/jpeg':
|
||||
@@ -484,6 +644,8 @@ function decodeMediaDataUrl(dataUrl: string): DecodedMediaPayload {
|
||||
return 'mov';
|
||||
case 'video/x-msvideo':
|
||||
return 'avi';
|
||||
case 'video/webm':
|
||||
return 'webm';
|
||||
default:
|
||||
return mimeType.split('/')[1] ?? 'bin';
|
||||
}
|
||||
@@ -552,6 +714,15 @@ async function resolveMediaSourcePayload(
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveCharacterVisualPayload(
|
||||
rootDir: string,
|
||||
source: string,
|
||||
): Promise<DecodedMediaPayload> {
|
||||
return applyChromaKeyToMediaPayload(
|
||||
await resolveMediaSourcePayload(rootDir, source),
|
||||
);
|
||||
}
|
||||
|
||||
async function resolveMediaSourceAsDataUrl(
|
||||
rootDir: string,
|
||||
source: string,
|
||||
@@ -982,19 +1153,32 @@ function buildNpcAnimationPrompt(options: {
|
||||
animation: string;
|
||||
promptText: string;
|
||||
useChromaKey: boolean;
|
||||
loop: boolean;
|
||||
characterBriefText?: string;
|
||||
actionTemplateId?: string;
|
||||
}) {
|
||||
const characterBrief = buildCompactAnimationCharacterBrief(
|
||||
options.characterBriefText ?? '',
|
||||
);
|
||||
const actionDetailText = sanitizeAnimationPromptText(options.promptText, 140);
|
||||
const loopRule = options.loop
|
||||
? '这是循环动作,直接进入动作循环中段,不要开场静止站桩,不要把主参考图原样作为第一帧。'
|
||||
: '这是非循环动作,首帧和尾帧都要回到参考主图角色形象,中段完成动作变化。';
|
||||
|
||||
if (options.actionTemplateId) {
|
||||
return buildVideoActionPrompt({
|
||||
actionTemplate: getActionTemplateById(
|
||||
options.actionTemplateId as Parameters<typeof getActionTemplateById>[0],
|
||||
),
|
||||
actionDetailText: options.promptText,
|
||||
useChromaKey: options.useChromaKey,
|
||||
characterBrief:
|
||||
options.characterBriefText?.trim() || `${options.animation} 动作角色`,
|
||||
});
|
||||
return [
|
||||
buildVideoActionPrompt({
|
||||
actionTemplate: getActionTemplateById(
|
||||
options.actionTemplateId as Parameters<typeof getActionTemplateById>[0],
|
||||
),
|
||||
actionDetailText,
|
||||
useChromaKey: options.useChromaKey,
|
||||
characterBrief: characterBrief || `${options.animation} 动作角色`,
|
||||
}),
|
||||
loopRule,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
return [
|
||||
@@ -1004,15 +1188,50 @@ function buildNpcAnimationPrompt(options: {
|
||||
options.useChromaKey
|
||||
? '背景为纯绿色绿幕,无其他人物和场景元素,方便后期抠像。'
|
||||
: '背景简洁纯净,无复杂场景。',
|
||||
options.characterBriefText?.trim()
|
||||
? `角色设定:${options.characterBriefText.trim()}`
|
||||
characterBrief
|
||||
? `角色设定:${characterBrief}`
|
||||
: '',
|
||||
options.promptText.trim(),
|
||||
actionDetailText,
|
||||
loopRule,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
function buildFallbackModerationSafeAnimationPrompt(options: {
|
||||
animation: string;
|
||||
loop: boolean;
|
||||
useChromaKey: boolean;
|
||||
}) {
|
||||
return [
|
||||
`单人全身角色动作视频,动作主题是 ${options.animation}。`,
|
||||
'角色固定为同一人,右向斜侧身,镜头稳定,轮廓清楚。',
|
||||
options.loop
|
||||
? '循环动作直接进入稳定循环,不要静止开场,不要定格首帧。'
|
||||
: '非循环动作首尾回到角色标准站姿,中段完成动作变化。',
|
||||
options.useChromaKey
|
||||
? '背景为纯绿色绿幕,无其他人物和场景元素。'
|
||||
: '背景简洁纯净。',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
function getLowestSupportedVideoResolution(model: string, fallback: string) {
|
||||
switch (model) {
|
||||
case 'wan2.6-i2v-flash':
|
||||
case 'wan2.6-i2v':
|
||||
case 'wan2.6-i2v-us':
|
||||
return '720P';
|
||||
case 'wan2.2-kf2v-flash':
|
||||
case 'wan2.2-i2v-flash':
|
||||
case 'wan2.5-i2v-preview':
|
||||
return '480P';
|
||||
default:
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
async function handleGenerateCharacterPromptBundle(
|
||||
config: AppConfig,
|
||||
req: IncomingMessage & { body?: unknown },
|
||||
@@ -1318,7 +1537,7 @@ async function handleGenerateCharacterVisuals(
|
||||
const imageSrc = await writeDraftBinaryFile(
|
||||
rootDir,
|
||||
path.posix.join(draftRelativeDir, fileName),
|
||||
imageResponse.body,
|
||||
applyGreenScreenAlphaToPngBuffer(imageResponse.body),
|
||||
);
|
||||
|
||||
return {
|
||||
@@ -1475,6 +1694,7 @@ async function handleGenerateCharacterAnimation(
|
||||
Number.isFinite(body.durationSeconds)
|
||||
? Math.max(1, Math.min(8, Math.round(body.durationSeconds)))
|
||||
: 4;
|
||||
const loop = body.loop === true;
|
||||
const useChromaKey = body.useChromaKey !== false;
|
||||
const resolution =
|
||||
typeof body.resolution === 'string' && body.resolution.trim()
|
||||
@@ -1487,15 +1707,28 @@ async function handleGenerateCharacterAnimation(
|
||||
: runtimeEnv.DASHSCOPE_CHARACTER_IMAGE_SEQUENCE_MODEL ||
|
||||
runtimeEnv.DASHSCOPE_CHARACTER_VISUAL_MODEL ||
|
||||
DEFAULT_CHARACTER_VISUAL_MODEL;
|
||||
const videoModel =
|
||||
const requestedVideoModel =
|
||||
typeof body.videoModel === 'string' && body.videoModel.trim()
|
||||
? body.videoModel.trim()
|
||||
: runtimeEnv.DASHSCOPE_CHARACTER_VIDEO_MODEL ||
|
||||
DEFAULT_CHARACTER_VIDEO_MODEL;
|
||||
const loopVideoModel =
|
||||
runtimeEnv.DASHSCOPE_CHARACTER_LOOP_VIDEO_MODEL ||
|
||||
(requestedVideoModel === 'wan2.2-kf2v-flash'
|
||||
? DEFAULT_CHARACTER_LOOP_VIDEO_MODEL
|
||||
: requestedVideoModel) ||
|
||||
DEFAULT_CHARACTER_LOOP_VIDEO_MODEL;
|
||||
const keyframeVideoModel =
|
||||
runtimeEnv.DASHSCOPE_CHARACTER_KEYFRAME_VIDEO_MODEL ||
|
||||
DEFAULT_CHARACTER_VIDEO_MODEL;
|
||||
const videoModel =
|
||||
strategy === 'image-to-video' ? (loop ? loopVideoModel : keyframeVideoModel) : requestedVideoModel;
|
||||
const durationSeconds =
|
||||
videoModel === 'wan2.2-kf2v-flash' ? 5 : requestedDurationSeconds;
|
||||
const normalizedResolution =
|
||||
videoModel === 'wan2.2-kf2v-flash' ? '480P' : resolution;
|
||||
const normalizedResolution = getLowestSupportedVideoResolution(
|
||||
videoModel,
|
||||
videoModel === 'wan2.2-kf2v-flash' ? '480P' : resolution,
|
||||
);
|
||||
const referenceVideoModel =
|
||||
typeof body.referenceVideoModel === 'string' &&
|
||||
body.referenceVideoModel.trim()
|
||||
@@ -1707,13 +1940,21 @@ async function handleGenerateCharacterAnimation(
|
||||
animation,
|
||||
promptText,
|
||||
useChromaKey,
|
||||
loop,
|
||||
characterBriefText,
|
||||
actionTemplateId,
|
||||
});
|
||||
const fallbackPrompt = buildFallbackModerationSafeAnimationPrompt({
|
||||
animation,
|
||||
loop,
|
||||
useChromaKey,
|
||||
});
|
||||
activePrompt = finalPrompt;
|
||||
activeModel = videoModel;
|
||||
const isKf2vFlash = videoModel === 'wan2.2-kf2v-flash';
|
||||
const visualInputRef = isKf2vFlash
|
||||
const isWan26I2vFlash = videoModel === 'wan2.6-i2v-flash';
|
||||
const visualInputRef =
|
||||
isKf2vFlash || isWan26I2vFlash
|
||||
? await resolveMediaSourceAsDataUrl(rootDir, visualSource)
|
||||
: await uploadFileToDashScope(
|
||||
baseUrl,
|
||||
@@ -1722,9 +1963,12 @@ async function handleGenerateCharacterAnimation(
|
||||
`${characterId}-${animation}-visual`,
|
||||
await resolveMediaSourcePayload(rootDir, visualSource),
|
||||
);
|
||||
const lastFrameRef = lastFrameImageDataUrl
|
||||
const resolvedLastFrameSource = !loop
|
||||
? lastFrameImageDataUrl || visualSource
|
||||
: '';
|
||||
const lastFrameRef = resolvedLastFrameSource
|
||||
? isKf2vFlash
|
||||
? await resolveMediaSourceAsDataUrl(rootDir, lastFrameImageDataUrl)
|
||||
? await resolveMediaSourceAsDataUrl(rootDir, resolvedLastFrameSource)
|
||||
: await uploadFileToDashScope(
|
||||
baseUrl,
|
||||
apiKey,
|
||||
@@ -1732,47 +1976,59 @@ async function handleGenerateCharacterAnimation(
|
||||
`${characterId}-${animation}-last-frame`,
|
||||
await resolveMediaSourcePayload(
|
||||
rootDir,
|
||||
lastFrameImageDataUrl,
|
||||
resolvedLastFrameSource,
|
||||
),
|
||||
)
|
||||
: '';
|
||||
const inputPayload =
|
||||
isKf2vFlash
|
||||
const createVideoRequestBody = (prompt: string) => ({
|
||||
model: videoModel,
|
||||
input: isKf2vFlash
|
||||
? {
|
||||
prompt: finalPrompt,
|
||||
prompt,
|
||||
first_frame_url: visualInputRef,
|
||||
...(lastFrameRef ? { last_frame_url: lastFrameRef } : {}),
|
||||
}
|
||||
: {
|
||||
prompt: finalPrompt,
|
||||
media: [
|
||||
{ type: 'first_frame', url: visualInputRef },
|
||||
...(lastFrameRef
|
||||
? [{ type: 'last_frame', url: lastFrameRef }]
|
||||
: []),
|
||||
],
|
||||
};
|
||||
: isWan26I2vFlash
|
||||
? {
|
||||
prompt,
|
||||
img_url: visualInputRef,
|
||||
}
|
||||
: {
|
||||
prompt,
|
||||
media: [
|
||||
{ type: 'first_frame', url: visualInputRef },
|
||||
...(lastFrameRef
|
||||
? [{ type: 'last_frame', url: lastFrameRef }]
|
||||
: []),
|
||||
],
|
||||
},
|
||||
parameters: {
|
||||
duration: durationSeconds,
|
||||
resolution: normalizedResolution,
|
||||
...(isKf2vFlash
|
||||
? { prompt_extend: true, watermark: false }
|
||||
: {}),
|
||||
...(isWan26I2vFlash ? { audio: false } : {}),
|
||||
},
|
||||
});
|
||||
const videoSynthesisEndpoint = isKf2vFlash
|
||||
? `${baseUrl}/services/aigc/image2video/video-synthesis`
|
||||
: `${baseUrl}/services/aigc/video-generation/video-synthesis`;
|
||||
|
||||
const createTaskResponse = await proxyJsonRequest(
|
||||
videoSynthesisEndpoint,
|
||||
apiKey,
|
||||
{
|
||||
model: videoModel,
|
||||
input: inputPayload,
|
||||
parameters: {
|
||||
duration: durationSeconds,
|
||||
resolution: normalizedResolution,
|
||||
...(isKf2vFlash ? { prompt_extend: true, watermark: false } : {}),
|
||||
const { response: createTaskResponse, prompt: submittedPrompt } =
|
||||
await proxyJsonRequestWithPromptFallback({
|
||||
urlString: videoSynthesisEndpoint,
|
||||
apiKey,
|
||||
buildBody: createVideoRequestBody,
|
||||
primaryPrompt: finalPrompt,
|
||||
fallbackPrompt,
|
||||
extraHeaders: {
|
||||
'X-DashScope-Async': 'enable',
|
||||
'X-DashScope-OssResourceResolve': 'enable',
|
||||
},
|
||||
},
|
||||
{
|
||||
'X-DashScope-Async': 'enable',
|
||||
'X-DashScope-OssResourceResolve': 'enable',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
activePrompt = submittedPrompt;
|
||||
|
||||
if (
|
||||
createTaskResponse.statusCode < 200 ||
|
||||
@@ -1809,7 +2065,7 @@ async function handleGenerateCharacterAnimation(
|
||||
animation,
|
||||
strategy,
|
||||
model: videoModel,
|
||||
prompt: finalPrompt,
|
||||
prompt: submittedPrompt,
|
||||
createdAt,
|
||||
updatedAt: createdAt,
|
||||
});
|
||||
@@ -1859,7 +2115,7 @@ async function handleGenerateCharacterAnimation(
|
||||
model: videoModel,
|
||||
strategy,
|
||||
animation,
|
||||
prompt: finalPrompt,
|
||||
prompt: submittedPrompt,
|
||||
createdAt: new Date().toISOString(),
|
||||
videoUrl,
|
||||
},
|
||||
@@ -1877,7 +2133,7 @@ async function handleGenerateCharacterAnimation(
|
||||
animation,
|
||||
strategy,
|
||||
model: videoModel,
|
||||
prompt: finalPrompt,
|
||||
prompt: submittedPrompt,
|
||||
createdAt,
|
||||
updatedAt: new Date().toISOString(),
|
||||
result: {
|
||||
@@ -1891,7 +2147,7 @@ async function handleGenerateCharacterAnimation(
|
||||
taskId,
|
||||
strategy: 'image-to-video',
|
||||
model: videoModel,
|
||||
prompt: finalPrompt,
|
||||
prompt: submittedPrompt,
|
||||
previewVideoPath,
|
||||
});
|
||||
return;
|
||||
@@ -1923,6 +2179,7 @@ async function handleGenerateCharacterAnimation(
|
||||
animation,
|
||||
promptText,
|
||||
useChromaKey,
|
||||
loop,
|
||||
characterBriefText,
|
||||
});
|
||||
activePrompt = finalPrompt;
|
||||
@@ -2081,8 +2338,8 @@ async function handleGenerateCharacterAnimation(
|
||||
}
|
||||
|
||||
if (strategy === 'reference-to-video') {
|
||||
const uploadedReferenceUrls = await Promise.all([
|
||||
...referenceImageDataUrls.map(async (source, index) =>
|
||||
const uploadedReferenceImages = await Promise.all(
|
||||
referenceImageDataUrls.map(async (source, index) =>
|
||||
uploadFileToDashScope(
|
||||
baseUrl,
|
||||
apiKey,
|
||||
@@ -2091,7 +2348,9 @@ async function handleGenerateCharacterAnimation(
|
||||
await resolveMediaSourcePayload(rootDir, source),
|
||||
),
|
||||
),
|
||||
...referenceVideoDataUrls.map(async (source, index) =>
|
||||
);
|
||||
const uploadedReferenceVideos = await Promise.all(
|
||||
referenceVideoDataUrls.map(async (source, index) =>
|
||||
uploadFileToDashScope(
|
||||
baseUrl,
|
||||
apiKey,
|
||||
@@ -2100,9 +2359,13 @@ async function handleGenerateCharacterAnimation(
|
||||
await resolveMediaSourcePayload(rootDir, source),
|
||||
),
|
||||
),
|
||||
]);
|
||||
);
|
||||
|
||||
if (uploadedReferenceUrls.length === 0) {
|
||||
if (
|
||||
!visualUrl &&
|
||||
uploadedReferenceImages.length === 0 &&
|
||||
uploadedReferenceVideos.length === 0
|
||||
) {
|
||||
sendJson(res, 400, {
|
||||
error: { message: '参考生视频至少需要一张参考图或一段参考视频。' },
|
||||
});
|
||||
@@ -2113,6 +2376,7 @@ async function handleGenerateCharacterAnimation(
|
||||
animation,
|
||||
promptText,
|
||||
useChromaKey,
|
||||
loop,
|
||||
characterBriefText,
|
||||
});
|
||||
activePrompt = finalPrompt;
|
||||
@@ -2124,11 +2388,24 @@ async function handleGenerateCharacterAnimation(
|
||||
model: referenceVideoModel,
|
||||
input: {
|
||||
prompt: finalPrompt,
|
||||
reference_urls: [visualUrl, ...uploadedReferenceUrls],
|
||||
media: [
|
||||
{ type: 'reference_image', url: visualUrl },
|
||||
...uploadedReferenceImages.map((url) => ({
|
||||
type: 'reference_image' as const,
|
||||
url,
|
||||
})),
|
||||
...uploadedReferenceVideos.map((url) => ({
|
||||
type: 'reference_video' as const,
|
||||
url,
|
||||
})),
|
||||
],
|
||||
},
|
||||
parameters: {
|
||||
duration: durationSeconds,
|
||||
resolution,
|
||||
resolution: getLowestSupportedVideoResolution(
|
||||
referenceVideoModel,
|
||||
resolution,
|
||||
),
|
||||
prompt_optimizer: true,
|
||||
},
|
||||
},
|
||||
@@ -2688,7 +2965,7 @@ async function handlePublishCharacterVisual(
|
||||
);
|
||||
await mkdir(visualDir, { recursive: true });
|
||||
|
||||
const masterPayload = await resolveMediaSourcePayload(
|
||||
const masterPayload = await resolveCharacterVisualPayload(
|
||||
rootDir,
|
||||
selectedPreviewSource,
|
||||
);
|
||||
@@ -2697,7 +2974,7 @@ async function handlePublishCharacterVisual(
|
||||
|
||||
const previewImagePaths: string[] = [];
|
||||
for (let index = 0; index < previewSources.length; index += 1) {
|
||||
const previewPayload = await resolveMediaSourcePayload(
|
||||
const previewPayload = await resolveCharacterVisualPayload(
|
||||
rootDir,
|
||||
previewSources[index] ?? '',
|
||||
);
|
||||
@@ -2904,6 +3181,11 @@ async function handlePublishCharacterAnimation(
|
||||
startFrame: 1,
|
||||
extension: frameExtension,
|
||||
basePath,
|
||||
frameWidth,
|
||||
frameHeight,
|
||||
fps,
|
||||
loop,
|
||||
...(previewVideoPath ? { previewVideoPath } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user