ziyu/AI_spine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve character cutout prompts and offline candidate evaluation

Browse Source
This commit is contained in:
ziyuzhao-zzy
2026-09-16 01:52:43 +08:00
parent b32fae2d48
commit 3753f110f4
52 changed files with 34245 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipeline.mjs
+136 -22
View File
@@ -8,28 +8,39 @@
*/
import {Buffer} from 'node:buffer';
import {spawnSync} from 'node:child_process';
import {cpSync, existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync} from 'node:fs';
import {cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
import path from 'node:path';
const repoRoot = path.dirname(new URL(import.meta.url).pathname);
const defaultGenielabsDir = '/Users/zhaoziyu/Documents/Codex/2026-09-10/diao/work/repos/spine-animation-ai';
const defaultModel = 'gpt-image-2';
// These prompts are deliberately character-agnostic. The reference is the
// only source of truth; the model must not infer a conventional T-pose or fill
// in a hidden/missing counterpart. Keeping several phrasings is useful since
// image models sometimes satisfy one wording while ignoring another.
const positivePrompt = [
'Create a complete 2D game sprite sheet for Spine animation from the exact reference character.',
'Copy the exact single pose from the reference, including camera angle, body lean, limb bends, hand direction and occlusion.',
'Preserve every visible asymmetry, occlusion, limb bend, hand direction and body lean exactly as shown. Do not mirror, straighten, normalize or symmetrize the pose.',
'Output one set of isolated visible parts only. Do not add alternate poses, duplicate parts, extra props, effects or text. Keep the original style and lay parts on a white background.',
'Preserve the exact art style, face, shading and color palette; do not redesign the character.',
'Create a production-ready 2D cutout sheet for a skeletal animation rig from the supplied reference image.',
'Use the reference as the only ground truth. Preserve the exact camera view, proportions, relative scale, handedness, limb bends, body lean and every visible asymmetry.',
'Make individually movable layers at natural articulation or occlusion boundaries: include only visible regions that actually exist in the reference (for example head, hair, face, hat, torso, clothing, sleeves, upper/lower limbs, hands, feet and accessories as applicable).',
'Move cutout pieces apart by translation only. Keep each piece in its original orientation and preserve screen-left/screen-right ownership. Extract visible surfaces; do not invent a fully hidden limb or a mirrored counterpart.',
'Render one copy of every layer only. Do not mirror, straighten, normalize, symmetrize, pose, redesign, or complete the character. Do not move parts into a different pose or make a second version of any part.',
'Output isolated cutout pieces ONLY. Do not include the original assembled figure, a full-body preview, a reference thumbnail or a before/after comparison anywhere in the output. Keep eyes, pupils, eyebrows, nose and mouth painted on the face rather than emitting separate facial fragments.',
'Preserve the original rendering technique, linework, texture, shading and colors. Leave clear white gaps between every pair of pieces. Use a loose exploded arrangement following the original anatomy, with every piece fully inside the canvas. Do not join neighboring pieces just to preserve their original coordinates.',
].join(' ');
const negativePrompt = [
'assembled preview figure, reference thumbnail, duplicate complete character, redesign, new outfit, alternate pose, T-pose, mirrored pose, duplicated limb, invented hidden part, merged pieces, cut-off layers, isolated pupils, isolated facial features, scenery, background shadows, labels, arrows, text, watermark',
].join(' ');
const negativePrompt = '3D, realistic, altered style, redesign, connected limbs, full body pose, scenery, shadows on background, missing limbs, merged layers, text, watermark';
const promptVariants = [
positivePrompt,
`${positivePrompt} Use a production Spine decomposition: separate each visible upper limb, forearm, hand, thigh, shin, foot, head, hair and clothing layer while retaining the exact reference pose.`,
`Technical 2D cutout sheet. Treat the reference as ground truth. Extract only the parts that exist in the reference, with original left/right orientation and occlusion. Never invent a symmetric counterpart or a second version of any part. ${positivePrompt}`,
`${positivePrompt} Registration pass: keep scale and angle fixed across pieces; only their translation changes to open gaps. Retain the exact screen-left/screen-right ownership.`,
`${positivePrompt} Articulation pass: make cuts only at real joints, clothing seams or occlusion boundaries. Keep hands, shoes, hair and accessories as coherent shapes; never shatter a single shape into noise fragments.`,
`${positivePrompt} Occlusion audit: inspect what is actually visible before cutting. Move visible rear surfaces apart for this sheet; never add an assembled view to demonstrate how they fit together.`,
`${positivePrompt} Conservative pass: prefer fewer complete, useful layers over many tiny fragments. Each output layer must be a meaningful movable part with enough surrounding pixels for animation.`,
`${positivePrompt} Pixel-fidelity pass: preserve the reference's exact edge contours, facial features, costume details, colors and non-symmetric construction. Do not redraw or reinterpret the artwork.`,
];
function args(argv) {
const out = {reference: '', outDir: '', genielabsDir: defaultGenielabsDir, python: '', envFile: '', model: process.env.GENARRATIVE_SPINE_IMAGE_MODEL || defaultModel, size: '1024x1024', animations: 'idle,walk,wave', layoutMode: 'matched', minScore: 0.3, maxParts: 18, attempts: 3, resume: false};
const out = {reference: '', outDir: '', genielabsDir: defaultGenielabsDir, python: '', envFile: '', promptFile: '', model: process.env.GENARRATIVE_SPINE_IMAGE_MODEL || defaultModel, size: '1024x1024', animations: 'idle,walk,wave', layoutMode: 'matched', minScore: 0.3, maxParts: 28, attempts: 6, resume: false};
for (let i = 2; i < argv.length; i += 1) {
const key = argv[i];
if (key === '--reference') out.reference = argv[++i] || '';
@@ -37,13 +48,14 @@ function args(argv) {
else if (key === '--genielabs-dir') out.genielabsDir = argv[++i] || '';
else if (key === '--python') out.python = argv[++i] || 'python3';
else if (key === '--env-file') out.envFile = argv[++i] || '';
else if (key === '--prompt-file') out.promptFile = argv[++i] || '';
else if (key === '--image-model') out.model = argv[++i] || defaultModel;
else if (key === '--size') out.size = argv[++i] || '1024x1024';
else if (key === '--animations') out.animations = argv[++i] || 'idle';
else if (key === '--layout-mode') out.layoutMode = argv[++i] || 'matched';
else if (key === '--min-score') out.minScore = Number(argv[++i] || 0.55);
else if (key === '--max-parts') out.maxParts = Number(argv[++i] || 18);
else if (key === '--attempts') out.attempts = Math.max(1, Math.min(3, Number(argv[++i] || 3)));
else if (key === '--attempts') out.attempts = Math.max(1, Math.min(promptVariants.length, Number(argv[++i] || promptVariants.length)));
else if (key === '--resume') out.resume = true;
else if (key === '--help' || key === '-h') { console.log('用法：node pipeline.mjs --reference ref.png --out-dir ./output --genielabs-dir /path/to/spine-animation-ai'); process.exit(0); }
}
@@ -115,7 +127,7 @@ async function requestAtlas(reference, model, size, config, prompt) {
if (!response.ok) {
const message = payload.error?.message || 'unknown error';
if ((response.status === 429 || response.status >= 500) && attempt < 2) { await new Promise((resolve) => setTimeout(resolve, 2 ** attempt * 2000)); continue; }
throw new Error(`VectorEngine 图片编辑失败（${response.status}）：${message}`);
throw Object.assign(new Error(`VectorEngine 图片编辑失败（${response.status}）：${message}`), {status: response.status});
}
const item = payload.data?.[0] || payload.images?.[0] || payload.output?.[0];
if (!item) throw new Error('VectorEngine 未返回图片');
@@ -189,8 +201,6 @@ function filterLayout(layoutPath, partsDir, minScore, maxParts) {
}
for (const name of Object.keys(layout.parts || {})) {
if (!keep.has(name)) {
const file = path.join(partsDir, `${name}.png`);
if (existsSync(file)) unlinkSync(file);
delete layout.parts[name];
}
}
@@ -206,6 +216,98 @@ function pngSize(file) {
throw new Error('参考图目前需要 PNG 格式');
}
// Score a generated sheet without assuming a particular character or an
// expected number of limbs. Count-only selection used to prefer exactly 12
// parts, which rewarded hallucinated duplicates on simple characters and
// dropped valid parts on unusual silhouettes. This score instead penalizes
// obvious failure modes: tiny noise components, one giant merged silhouette,
// implausible foreground coverage and extreme component counts.
function candidateQuality(stats) {
const width = Number(stats.atlas_width || 0);
const height = Number(stats.atlas_height || 0);
const area = Math.max(1, width * height);
const parts = Array.isArray(stats.parts) ? stats.parts : [];
const count = parts.length;
if (!count) return {score: Number.POSITIVE_INFINITY, count, tinyRatio: 1, giantRatio: 1, borderRatio: 1, coverage: 0};
const componentAreas = parts.map((part) => Math.max(0, Number(part.area || 0)));
const minUsefulArea = Math.max(500, area * 0.00045);
const tiny = componentAreas.filter((value) => value < minUsefulArea).length;
const giant = componentAreas.filter((value) => value > area * 0.42).length;
const border = parts.filter((part) => Number(part.x || 0) <= 1 || Number(part.y || 0) <= 1 || Number(part.x || 0) + Number(part.width || 0) >= width - 1 || Number(part.y || 0) + Number(part.height || 0) >= height - 1).length;
const coverage = componentAreas.reduce((sum, value) => sum + value, 0) / area;
const countPenalty = count < 4 ? (4 - count) * 5 : count > 28 ? (count - 28) * 1.5 : 0;
const tinyPenalty = (tiny / count) * 18;
const giantPenalty = giant * 20;
const borderPenalty = (border / count) * 8;
const coveragePenalty = coverage < 0.015 ? 8 : coverage > 0.88 ? 8 : 0;
return {
score: countPenalty + tinyPenalty + giantPenalty + borderPenalty + coveragePenalty,
count,
tinyRatio: tiny / count,
giantRatio: giant / count,
borderRatio: border / count,
coverage,
};
}
function evaluateCandidatePlacement(reference, partsDir, outputPath, python, positionScript) {
const result = spawnSync(python, [positionScript, '--reference', reference, '--parts', partsDir, '--output', outputPath], {encoding: 'utf8'});
if (result.status !== 0 || !existsSync(outputPath)) return {score: 50, matchedParts: 0, meanMatch: 0, overlapPenalty: 0, error: (result.stderr || '').trim().slice(-400)};
try {
const layout = JSON.parse(readFileSync(outputPath, 'utf8').replace(/\b(?:NaN|Infinity|-Infinity)\b/gu, 'null'));
const parts = Object.values(layout.parts || {});
const matchScores = parts.map((part) => Number(part.score || 0)).filter(Number.isFinite);
let overlapPenalty = 0;
for (let i = 0; i < parts.length; i += 1) {
for (let j = i + 1; j < parts.length; j += 1) {
const a = parts[i]; const b = parts[j];
const ix = Math.max(0, Math.min(a.x + a.width, b.x + b.width) - Math.max(a.x, b.x));
const iy = Math.max(0, Math.min(a.y + a.height, b.y + b.height) - Math.max(a.y, b.y));
const intersection = ix * iy;
const union = Math.max(1, a.width * a.height + b.width * b.height - intersection);
const iou = intersection / union;
if (iou > 0.55) overlapPenalty += (iou - 0.55) * 12;
}
}
const meanMatch = matchScores.length ? matchScores.reduce((sum, value) => sum + value, 0) / matchScores.length : 0;
const score = (1 - meanMatch) * 10 + overlapPenalty + Math.max(0, 5 - parts.length) * 2;
return {score, matchedParts: parts.length, meanMatch, overlapPenalty};
} catch (error) {
return {score: 50, matchedParts: 0, meanMatch: 0, overlapPenalty: 0, error: error.message};
}
}
function repairLayerOrder(layoutPath) {
const layout = JSON.parse(readFileSync(layoutPath, 'utf8').replace(/\b(?:NaN|Infinity|-Infinity)\b/gu, 'null'));
const order = [...(layout.z_order || Object.keys(layout.parts || {}))];
const index = new Map(order.map((name, i) => [name, i]));
const parts = layout.parts || {};
// Broad top accessories such as a hat brim should sit behind the face. A
// visual occlusion vote can invert this relation because both layers share
// similar colours. Enforce only the distinctive geometry pattern: a part
// above the canvas midpoint, at least 1.45x wider than a lower overlapping
// part. This remains character-agnostic and leaves hair/torso relations
// untouched in ordinary cases.
for (const [wideName, wide] of Object.entries(parts)) {
const canvasHeight = Number(layout.canvas_height || 0);
if (Number(wide.height) > Number(wide.width) * 0.65 || Number(wide.y) + Number(wide.height) / 2 > canvasHeight * 0.35) continue;
for (const [narrowName, narrow] of Object.entries(parts)) {
if (wideName === narrowName || Number(wide.y) >= Number(narrow.y) || Number(wide.width) < Number(narrow.width) * 1.45) continue;
const ix = Math.max(0, Math.min(wide.x + wide.width, narrow.x + narrow.width) - Math.max(wide.x, narrow.x));
const iy = Math.max(0, Math.min(wide.y + wide.height, narrow.y + narrow.height) - Math.max(wide.y, narrow.y));
const overlap = (ix * iy) / Math.max(1, Number(narrow.width) * Number(narrow.height));
if (overlap < 0.08) continue;
const wi = index.get(wideName); const ni = index.get(narrowName);
if (wi === undefined || ni === undefined || wi < ni) continue;
order.splice(wi, 1);
order.splice(order.indexOf(narrowName), 0, wideName);
order.forEach((name, i) => index.set(name, i));
}
}
layout.z_order = order;
writeFileSync(layoutPath, JSON.stringify(layout, null, 2));
}
function buildAnchoredLayout(reference, partsDir, layoutPath) {
const source = pngSize(reference);
const manifest = JSON.parse(readFileSync(path.join(partsDir, 'parts-manifest.json'), 'utf8'));
@@ -258,7 +360,7 @@ async function main() {
for (const file of [positionScript, buildScript, atlasScript, playerScript]) if (!existsSync(file)) throw new Error(`Genielabs 脚本不存在：${file}`);
const out = path.resolve(options.outDir); const parts = path.join(out, 'parts'); const atlasDir = path.join(out, 'atlas');
mkdirSync(out, {recursive: true}); mkdirSync(parts, {recursive: true}); mkdirSync(atlasDir, {recursive: true});
const model = resolveModel(options.model); const config = envConfig(options.envFile);
const model = resolveModel(options.model);
const atlasPath = path.join(out, 'generated-parts-sheet.png');
writeFileSync(path.join(out, 'source-original.png'), readFileSync(options.reference));
const positionReference = path.join(out, 'reference-position.png');
@@ -267,10 +369,12 @@ async function main() {
writeFileSync(path.join(out, 'manifest.json'), JSON.stringify({status: 'generating', model, requestedModel: options.model, pipelineVersion: 'genarrative-genielabs-spine-v1', source: path.basename(options.reference)}, null, 2));
const candidateRoot = path.join(out, 'candidates'); mkdirSync(candidateRoot, {recursive: true});
let selectedCandidate = null;
let candidateDiagnostics = [];
if (options.resume && existsSync(atlasPath) && existsSync(path.join(parts, 'parts-manifest.json'))) {
selectedCandidate = {atlas: atlasPath, parts};
console.log(`复用已有拆件工作图：${atlasPath}`);
} else {
const config = envConfig(options.envFile);
const candidates = [];
for (let index = 0; index < options.attempts; index += 1) {
const candidateDir = path.join(candidateRoot, `attempt-${index + 1}`); mkdirSync(candidateDir, {recursive: true});
@@ -278,13 +382,22 @@ async function main() {
const candidateParts = path.join(candidateDir, 'parts'); mkdirSync(candidateParts, {recursive: true});
try {
console.log(`尝试第 ${index + 1}/${options.attempts} 个通用拆件提示词`);
writeFileSync(candidateAtlas, await requestAtlas(options.reference, model, options.size, config, promptVariants[index % promptVariants.length]));
const prompt = options.promptFile ? readFileSync(options.promptFile, 'utf8') : promptVariants[index % promptVariants.length];
writeFileSync(path.join(candidateDir, 'prompt.txt'), `${prompt}\n\nAvoid: ${negativePrompt}\n`);
writeFileSync(candidateAtlas, await requestAtlas(options.reference, model, options.size, config, prompt));
run(options.python, [splitScript, '--atlas', candidateAtlas, '--output-dir', candidateParts]);
const stats = JSON.parse(readFileSync(path.join(candidateParts, 'parts-manifest.json'), 'utf8'));
const count = stats.parts.length; const score = Math.abs(count - 12) + (count > 24 ? count - 24 : 0);
candidates.push({atlas: candidateAtlas, parts: candidateParts, count, score});
console.log(`候选 ${index + 1}: ${count} 个部件，质量分 ${score}`);
} catch (error) { console.warn(`候选 ${index + 1} 失败：${error.message}`); }
const quality = candidateQuality(stats);
const placement = evaluateCandidatePlacement(positionReference, candidateParts, path.join(candidateDir, 'layout-eval.json'), options.python, positionScript);
const totalScore = quality.score + placement.score;
candidates.push({atlas: candidateAtlas, parts: candidateParts, count: quality.count, score: totalScore, quality, placement});
candidateDiagnostics.push({attempt: index + 1, count: quality.count, score: totalScore, quality, placement});
console.log(`候选 ${index + 1}: ${quality.count} 个部件，综合分 ${totalScore.toFixed(2)}（拆件 ${quality.score.toFixed(2)}，定位 ${placement.score.toFixed(2)}，匹配 ${(placement.meanMatch * 100).toFixed(0)}%，重叠罚分 ${placement.overlapPenalty.toFixed(2)}）`);
} catch (error) {
candidateDiagnostics.push({attempt: index + 1, status: 'failed', error: error.message});
console.warn(`候选 ${index + 1} 失败：${error.message}`);
if (error.status === 401 || error.status === 403) break;
}
}
if (!candidates.length) throw new Error('所有拆件候选均失败');
selectedCandidate = candidates.sort((a, b) => a.score - b.score)[0];
@@ -295,6 +408,7 @@ async function main() {
if (options.layoutMode === 'matched') {
run(options.python, [positionScript, '--reference', positionReference, '--parts', parts, '--output', layout]);
filterLayout(layout, parts, options.minScore, options.maxParts);
repairLayerOrder(layout);
} else {
buildAnchoredLayout(options.reference, parts, layout);
}
@@ -309,8 +423,8 @@ async function main() {
writeFileSync(skeleton, JSON.stringify(skeletonJson, null, 2));
run(options.python, [atlasScript, '--parts', parts, '--output', atlasDir, '--name', 'skeleton']);
const riggedPreview = path.join(out, 'preview-rigged.html'); run(options.python, [playerScript, '--skeleton', skeleton, '--atlas', path.join(atlasDir, 'skeleton.atlas'), '--atlas-image', path.join(atlasDir, 'skeleton.png'), '--output', riggedPreview, '--animation', options.animations.split(',')[0] || 'idle']);
const preview = path.join(out, 'preview.html'); writeReferencePreview(options.reference, preview);
const manifest = {status: 'completed', pipelineVersion: 'genarrative-genielabs-spine-v1', requestedModel: options.model, model, source: path.basename(options.reference), attempts: options.attempts, selectedCandidate: selectedCandidate.count ?? null, promptVariants: promptVariants.length, outputs: {parts, layout, skeleton, atlas: atlasDir, preview, riggedPreview}, limitations: ['默认 preview.html 显示参考图的真实非对称姿态，保证形体结构正常；preview-rigged.html 显示自动拆件 rig，需人工校正。', '当前自动生成的是 region attachments；mesh/weights/IK 仍需在 Spine 中人工校正。', 'Genielabs 脚本为外部依赖，请按其 PolyForm Noncommercial 许可证处理商业授权。']};
const preview = path.join(out, 'preview.html'); writeReferencePreview(path.join(out, 'assembled.png'), preview);
const manifest = {status: 'completed', pipelineVersion: 'genarrative-genielabs-spine-v1', requestedModel: options.model, model, source: path.basename(options.reference), attempts: options.attempts, selectedCandidate: selectedCandidate.count ?? null, selectedCandidateScore: selectedCandidate.score ?? null, candidateDiagnostics, promptVariants: promptVariants.length, outputs: {parts, layout, skeleton, atlas: atlasDir, preview, riggedPreview}, limitations: ['默认 preview.html 显示参考图的真实非对称姿态，保证形体结构正常；preview-rigged.html 显示自动拆件 rig，需人工校正。', '当前自动生成的是 region attachments；mesh/weights/IK 仍需在 Spine 中人工校正。', 'Genielabs 脚本为外部依赖，请按其 PolyForm Noncommercial 许可证处理商业授权。']};
writeFileSync(path.join(out, 'manifest.json'), JSON.stringify(manifest, null, 2)); console.log(`Spine pipeline 完成：${out}`);
}