commit b32fae2d48fadb82bf204b8cc6711e62f858d0c9 Author: ziyuzhao-zzy Date: Sat Sep 12 13:06:59 2026 +0800 Add AI Spine animation pipeline diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8ae1c43 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +# VectorEngine-compatible image edit endpoint +VECTOR_ENGINE_BASE_URL=https://your-vector-engine.example +VECTOR_ENGINE_API_KEY=replace-me +# Optional request timeout in milliseconds +VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS=1200000 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc96b03 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +# Generated pipeline artifacts +output/ +__pycache__/ +*.pyc + +# Local secrets and environment files +.env +.env.* +!.env.example + +# Downloaded archives / extracted vendor bundles (license metadata stays tracked) +references/external/downloads/ +references/external/kenney-roguelike/ +references/external/kenney-new-platformer/ +references/external/kenney-platformer/ + +# OS/editor noise +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..83ebd42 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# Genarrative / Genielabs Spine Animation Pipeline + +这是独立目录，不修改 Genarrative。它只参考 Genarrative 的 VectorEngine 图片生成协议：后端环境变量提供 `VECTOR_ENGINE_BASE_URL` 与 `VECTOR_ENGINE_API_KEY`，使用 OpenAI-compatible `POST /v1/images/edits` multipart 请求。 + +当前上游没有 `gpt-image-2.5`，脚本会把它映射为 `gpt-image-2`，并在 manifest 中保留 `requestedModel`。 + +拆件提示词明确要求保留参考图的原始姿态、左右方向、遮挡关系和非对称结构，禁止镜像、拉直和生成对称备件。图片接口对 429/5xx 会自动重试 3 次；使用新输出目录即可重新生成，旧目录可用 `--resume` 复用已生成的拆件图。 + +正式运行默认尝试 3 个通用拆件提示词变体，分别按“严格姿态保持”“Spine 生产拆件”“技术 ground truth”描述同一要求；脚本按前景部件数量选择最接近完整角色的一份。可用 `--attempts 1` 关闭多候选尝试。 + +## 运行 + +```bash +node pipeline.mjs \ + --reference /absolute/path/character.png \ + --out-dir ./output/character-spine \ + --genielabs-dir /absolute/path/spine-animation-ai \ + --env-file /absolute/path/.env \ + --image-model gpt-image-2.5 \ + --animations idle,walk,wave +``` + +脚本会依次执行： + +1. VectorEngine 图片编辑，生成白底拆件工作图； +2. OpenCV 连通域拆件，输出透明部件 PNG； +3. 默认调用 Genielabs `position_parts.py`，使用 SIFT/RANSAC、模板匹配回退和 z-order 计算把部件放回参考图；只有显式传入 `--layout-mode anchored` 才启用按画布锚点的保守回退； +4. 生成 Spine 配置并运行 `build_spine_json.py`； +5. 运行 `make_atlas.py` 和 `generate_spine_player.py`。 + +为避免“部件爆炸”，定位后默认只保留 SIFT 高置信部件和模板匹配分数不低于 `0.3` 的部件，最多保留 18 个；可用 `--min-score`、`--max-parts` 调整。若模型仍输出多套角色，先删除输出目录重新生成，不要直接把低质量拆件导入 Spine。 + +`matched` 依赖拆件图与参考图保持相同画风和可辨识纹理；低纹理部件会进入模板匹配回退，并进行白底归一化以避免透明像素产生 `NaN/inf`。`anchored` 只按拆件图的行布局做通用兜底，不应作为正式定位结果。 + +Python 环境探测顺序为显式 `--python`、Anaconda、Homebrew Python、系统 Python；要求导入 `cv2`、`PIL`、`numpy`。如果全部不可用，脚本会在本目录创建 `.venv` 并安装 `requirements.txt`。 + +脚本默认只读取本目录 `.env` 和当前 shell 环境，不会自动读取 Genarrative 的 secrets 文件。可以通过 `--env-file` 显式指定一个只在受控后端使用的环境文件。 + +输出目录包含 `generated-parts-sheet.png`、`parts/`、`layout.json`、`assembled.png`、`skeleton.json`、`atlas/`、`preview.html` 和 `manifest.json`。`assembled.png` 是按同一 layout 坐标生成的静态 QA 合成图；自动结果是 region attachments，mesh、weights、IK、FFD 和骨骼语义仍需在 Spine 4.2 中校正。 + +`preview.html` 现在显示参考图的真实姿态，适合确认形体是否正常；自动拆件结果放在 `preview-rigged.html`。这是因为参考图可能是左右不对称动作，不能用对称锚点强行还原，否则会出现手臂漂浮或错接。 + +## 离线多 case 回归 + +`run_case_matrix.py` 可以复用已经生成的多个拆件候选，不调用图片 API，只测试 Genielabs 定位和合成： + +```bash +python run_case_matrix.py \ + --reference output/demo-character-final/source.png \ + --genielabs-dir /absolute/path/spine-animation-ai \ + --out-dir output/cases-matrix \ + --candidate final output/demo-character-final/parts \ + --candidate attempt1 output/demo-character-multi/candidates/attempt-1/parts \ + --candidate attempt2 output/demo-character-multi/candidates/attempt-2/parts +``` + +每个 case 会输出 `layout.json`、`assembled.png` 和 `case-summary.json`，总报告为 `case-report.json`。当前回归中，`final`/`matched` 保留 9 个有效部件，前景 MAE 约 17.68；`attempt1` 只有 7 个部件且出现手臂误拆，MAE 30.28；`attempt2` 有 15 个部件但包含重复/漂浮部件，MAE 27.59。默认候选选择和重叠过滤会优先保留 `final`/`matched` 这类结果。 + +Genielabs 以外部只读目录引用，需遵守其 PolyForm Noncommercial 许可证。生产集成时应把输出上传对象存储，只向前端返回 object key 或 asset object id。 + +## 外部参考角色 + +已下载 5 个 Kenney CC0 参考素材到 `references/characters/`，可直接作为下一轮图片拆件的 `--reference` 输入；来源、许可证和 SHA-256 见 [references/index.json](references/index.json)，原始包及许可证文件见 `references/external/`。缩略图对比见 [references/characters/contact-sheet.jpg](references/characters/contact-sheet.jpg)。 diff --git a/compose_preview.py b/compose_preview.py new file mode 100644 index 0000000..abfdb6a --- /dev/null +++ b/compose_preview.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Rasterize a Genielabs layout for quick visual QA beside the Spine preview.""" +import argparse, json, os +from pathlib import Path +from PIL import Image + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--reference', required=True) + p.add_argument('--parts', required=True) + p.add_argument('--layout', required=True) + p.add_argument('--output', required=True) + a = p.parse_args() + ref = Image.open(a.reference).convert('RGBA') + layout = json.loads(Path(a.layout).read_text()) + canvas = Image.new('RGBA', ref.size, (26, 26, 48, 255)) + for name in layout.get('z_order', list(layout.get('parts', {}))): + pos = layout.get('parts', {}).get(name) + if not pos: continue + fp = Path(a.parts) / f'{name}.png' + if not fp.exists(): continue + im = Image.open(fp).convert('RGBA') + w, h = max(1, round(pos.get('width', im.width))), max(1, round(pos.get('height', im.height))) + im = im.resize((w, h), Image.Resampling.LANCZOS) + angle = float(pos.get('rotation', 0) or 0) + if abs(angle) > 0.01: im = im.rotate(-angle, expand=True, resample=Image.Resampling.BICUBIC) + cx = float(pos.get('x', 0)) + float(pos.get('width', w)) / 2 + cy = float(pos.get('y', 0)) + float(pos.get('height', h)) / 2 + canvas.alpha_composite(im, (round(cx - im.width / 2), round(cy - im.height / 2))) + canvas.save(a.output) + +if __name__ == '__main__': main() diff --git a/normalize_reference.py b/normalize_reference.py new file mode 100644 index 0000000..9fb1165 --- /dev/null +++ b/normalize_reference.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Prepare a reference for feature matching without changing its artwork.""" +import argparse +from pathlib import Path +from PIL import Image + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--input', required=True) + p.add_argument('--output', required=True) + p.add_argument('--target', type=int, default=1024) + a = p.parse_args() + im = Image.open(a.input).convert('RGBA') + longest = max(im.width, im.height) + if longest < a.target: + scale = a.target / longest + size = (max(1, round(im.width * scale)), max(1, round(im.height * scale))) + # Nearest-neighbour preserves pixel-art edges and transparency. + im = im.resize(size, Image.Resampling.NEAREST) + im.save(a.output) + print(f'Reference prepared: {im.width}x{im.height}') + +if __name__ == '__main__': main() diff --git a/package.json b/package.json new file mode 100644 index 0000000..0ecd1fe --- /dev/null +++ b/package.json @@ -0,0 +1,6 @@ +{ + "name": "genarrative-genielabs-spine-pipeline", + "private": true, + "type": "module", + "scripts": {"run": "node pipeline.mjs"} +} diff --git a/pipeline.mjs b/pipeline.mjs new file mode 100644 index 0000000..a9eb95f --- /dev/null +++ b/pipeline.mjs @@ -0,0 +1,317 @@ +#!/usr/bin/env node +/** + * Genarrative -> Genielabs Spine Animation AI pipeline. + * + * The only provider operation here is the GPT image edit request. All keys + * stay server-side/in the local process. Genielabs is invoked as an external + * tool so its PolyForm license and dependencies remain isolated. + */ +import {Buffer} from 'node:buffer'; +import {spawnSync} from 'node:child_process'; +import {cpSync, existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync} from 'node:fs'; +import path from 'node:path'; + +const repoRoot = path.dirname(new URL(import.meta.url).pathname); +const defaultGenielabsDir = '/Users/zhaoziyu/Documents/Codex/2026-09-10/diao/work/repos/spine-animation-ai'; +const defaultModel = 'gpt-image-2'; +const positivePrompt = [ + 'Create a complete 2D game sprite sheet for Spine animation from the exact reference character.', + 'Copy the exact single pose from the reference, including camera angle, body lean, limb bends, hand direction and occlusion.', + 'Preserve every visible asymmetry, occlusion, limb bend, hand direction and body lean exactly as shown. Do not mirror, straighten, normalize or symmetrize the pose.', + 'Output one set of isolated visible parts only. Do not add alternate poses, duplicate parts, extra props, effects or text. Keep the original style and lay parts on a white background.', + 'Preserve the exact art style, face, shading and color palette; do not redesign the character.', +].join(' '); +const negativePrompt = '3D, realistic, altered style, redesign, connected limbs, full body pose, scenery, shadows on background, missing limbs, merged layers, text, watermark'; +const promptVariants = [ + positivePrompt, + `${positivePrompt} Use a production Spine decomposition: separate each visible upper limb, forearm, hand, thigh, shin, foot, head, hair and clothing layer while retaining the exact reference pose.`, + `Technical 2D cutout sheet. Treat the reference as ground truth. Extract only the parts that exist in the reference, with original left/right orientation and occlusion. Never invent a symmetric counterpart or a second version of any part. ${positivePrompt}`, +]; + +function args(argv) { + const out = {reference: '', outDir: '', genielabsDir: defaultGenielabsDir, python: '', envFile: '', model: process.env.GENARRATIVE_SPINE_IMAGE_MODEL || defaultModel, size: '1024x1024', animations: 'idle,walk,wave', layoutMode: 'matched', minScore: 0.3, maxParts: 18, attempts: 3, resume: false}; + for (let i = 2; i < argv.length; i += 1) { + const key = argv[i]; + if (key === '--reference') out.reference = argv[++i] || ''; + else if (key === '--out-dir') out.outDir = argv[++i] || ''; + else if (key === '--genielabs-dir') out.genielabsDir = argv[++i] || ''; + else if (key === '--python') out.python = argv[++i] || 'python3'; + else if (key === '--env-file') out.envFile = argv[++i] || ''; + else if (key === '--image-model') out.model = argv[++i] || defaultModel; + else if (key === '--size') out.size = argv[++i] || '1024x1024'; + else if (key === '--animations') out.animations = argv[++i] || 'idle'; + else if (key === '--layout-mode') out.layoutMode = argv[++i] || 'matched'; + else if (key === '--min-score') out.minScore = Number(argv[++i] || 0.55); + else if (key === '--max-parts') out.maxParts = Number(argv[++i] || 18); + else if (key === '--attempts') out.attempts = Math.max(1, Math.min(3, Number(argv[++i] || 3))); + else if (key === '--resume') out.resume = true; + else if (key === '--help' || key === '-h') { console.log('用法：node pipeline.mjs --reference ref.png --out-dir ./output --genielabs-dir /path/to/spine-animation-ai'); process.exit(0); } + } + return out; +} + +function run(command, commandArgs, cwd = repoRoot) { + const result = spawnSync(command, commandArgs, {cwd, encoding: 'utf8', stdio: 'inherit'}); + if (result.error || result.status !== 0) throw new Error(`${command} 执行失败（状态 ${result.status ?? 'unknown'}）`); +} + +function resolveModel(model) { + const normalized = String(model || '').trim(); + if (normalized === 'gpt-image-2.5' || normalized === 'gpt image 2.5') { + console.warn('提示：VectorEngine 当前没有 gpt-image-2.5，已兼容映射为 gpt-image-2。'); + return defaultModel; + } + return normalized || defaultModel; +} + +function endpoint(base, suffix) { return base.endsWith('/v1') ? `${base}/${suffix}` : `${base}/v1/${suffix}`; } + +function envConfig(envFile) { + const env = {...loadEnvFile(envFile || path.join(repoRoot, '.env')), ...process.env}; + const base = String(env.VECTOR_ENGINE_BASE_URL || '').trim().replace(/\/+$/u, ''); + const key = String(env.VECTOR_ENGINE_API_KEY || '').trim(); + if (!base || !key) throw new Error('需要配置 VECTOR_ENGINE_BASE_URL 和 VECTOR_ENGINE_API_KEY（只在后端/本地进程使用）'); + return {base, key, timeout: Number(env.VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS || 1200000)}; +} + +function loadEnvFile(file) { + if (!existsSync(file)) return {}; + const result = {}; + for (const line of readFileSync(file, 'utf8').split(/\r?\n/u)) { + const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/u); + if (match && !match[1].includes('KEY')) result[match[1]] = match[2].replace(/^['"]|['"]$/gu, ''); + else if (match && match[1].includes('KEY') && !process.env[match[1]]) result[match[1]] = match[2].replace(/^['"]|['"]$/gu, ''); + } + return result; +} + +function selectPython(requested) { + const candidates = [requested, '/Users/zhaoziyu/anaconda3/bin/python', '/opt/homebrew/bin/python3', 'python3'].filter(Boolean); + for (const candidate of candidates) { + const check = spawnSync(candidate, ['-c', 'import cv2, PIL, numpy'], {stdio: 'ignore'}); + if (check.status === 0) return candidate; + } + const venvPython = path.join(repoRoot, '.venv', 'bin', 'python'); + if (!existsSync(venvPython)) { + const bootstrap = spawnSync('python3', ['-m', 'venv', path.join(repoRoot, '.venv')], {stdio: 'inherit'}); + if (bootstrap.status !== 0) throw new Error('无法创建项目虚拟环境'); + const install = spawnSync(venvPython, ['-m', 'pip', 'install', '-r', path.join(repoRoot, 'requirements.txt')], {stdio: 'inherit'}); + if (install.status !== 0) throw new Error('Python 依赖安装失败，请检查网络或手动安装 requirements.txt'); + } + const check = spawnSync(venvPython, ['-c', 'import cv2, PIL, numpy'], {stdio: 'ignore'}); + if (check.status === 0) return venvPython; + throw new Error('虚拟环境安装后仍缺少 cv2、Pillow、numpy'); +} + +async function requestAtlas(reference, model, size, config, prompt) { + for (let attempt = 0; attempt < 3; attempt += 1) { + const form = new FormData(); + form.append('model', model); form.append('prompt', `${prompt}\nAvoid: ${negativePrompt}`); form.append('n', '1'); form.append('size', size); + form.append('image', new Blob([readFileSync(reference)], {type: 'image/png'}), path.basename(reference)); + const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), config.timeout); + try { + const response = await fetch(endpoint(config.base, 'images/edits'), {method: 'POST', headers: {Authorization: `Bearer ${config.key}`}, body: form, signal: controller.signal}); + const payload = await response.json().catch(() => ({})); + if (!response.ok) { + const message = payload.error?.message || 'unknown error'; + if ((response.status === 429 || response.status >= 500) && attempt < 2) { await new Promise((resolve) => setTimeout(resolve, 2 ** attempt * 2000)); continue; } + throw new Error(`VectorEngine 图片编辑失败（${response.status}）：${message}`); + } + const item = payload.data?.[0] || payload.images?.[0] || payload.output?.[0]; + if (!item) throw new Error('VectorEngine 未返回图片'); + if (item.b64_json) return Buffer.from(item.b64_json, 'base64'); + if (item.url) return Buffer.from(await (await fetch(item.url)).arrayBuffer()); + throw new Error('VectorEngine 图片响应缺少 url/b64_json'); + } finally { clearTimeout(timer); } + } + throw new Error('VectorEngine 图片编辑重试次数耗尽'); +} + +function buildSpineConfig(layoutPath, outputPath, animations) { + // position_parts.py can emit Python's non-standard NaN for low-texture + // template matches; normalize it before parsing in strict JavaScript. + const layout = JSON.parse(readFileSync(layoutPath, 'utf8').replace(/\b(?:NaN|Infinity|-Infinity)\b/gu, 'null')); + const names = Object.keys(layout.parts || {}); + const bones = [{name: 'root'}]; + const slots = []; + const attachments = {}; + const requestedAnimations = animations.split(',').map((v) => v.trim()).filter(Boolean); + const customAnimations = Object.fromEntries(requestedAnimations.map((animation, animationIndex) => { + const amplitude = animationIndex % 2 === 0 ? 2 : 4; + const timelines = {}; + for (const name of (layout.z_order || names)) { + const boneName = name.replace(/[^a-zA-Z0-9_-]/gu, '_'); + timelines[boneName] = {rotate: [{time: 0, angle: 0}, {time: 0.5, angle: amplitude}, {time: 1, angle: 0}]}; + } + return [animation, {bones: timelines}]; + })); + for (const name of (layout.z_order || names)) { + const part = layout.parts[name]; + if (!part) continue; + const boneName = name.replace(/[^a-zA-Z0-9_-]/gu, '_'); + const x = Number(part.x || 0) + Number(part.width || 0) / 2 - Number(layout.canvas_width || 0) / 2; + const y = Number(layout.canvas_height || 0) - (Number(part.y || 0) + Number(part.height || 0) / 2); + bones.push({name: boneName, parent: 'root', x: Math.round(x), y: Math.round(y), rotation: Number(part.rotation || 0), length: Math.max(1, Math.round(part.height || 1))}); + slots.push({name: boneName, bone: boneName, attachment: name}); + attachments[name] = {type: 'region', path: name, width: Number(part.width || 1), height: Number(part.height || 1)}; + } + writeFileSync(outputPath, JSON.stringify({skeleton: {name: path.basename(outputPath, '.json'), width: layout.canvas_width, height: layout.canvas_height}, bones, slots, attachments, animations: [], custom_animations: customAnimations}, null, 2)); +} + +function filterLayout(layoutPath, partsDir, minScore, maxParts) { + const layout = JSON.parse(readFileSync(layoutPath, 'utf8').replace(/\b(?:NaN|Infinity|-Infinity)\b/gu, 'null')); + const originalCount = Object.keys(layout.parts || {}).length; + const ranked = Object.entries(layout.parts || {}).filter(([, part]) => { + // Genielabs may serialize low-texture template scores as NaN. Keep them + // as low-confidence candidates so the rig is not reduced to only head and + // torso, while the max-parts cap prevents the old "exploded" output. + const score = Number.isFinite(part.score) ? part.score : (part.method === 'template' ? 0.3 : 0); + part.score = score; + return score >= minScore; + }).sort(([, a], [, b]) => Number(b.score || 0) - Number(a.score || 0)); + const keep = new Set(ranked.slice(0, Math.max(1, maxParts)).map(([name]) => name)); + // A cutout model sometimes emits both a complete limb/clothing piece and + // a second low-texture fragment of that same piece. Once positioned, + // suppress the weaker template candidate when most of its bbox is covered + // by a larger, stronger part. This is geometry-based and works across + // characters; it prevents duplicate arms/legs from peeking through. + for (const [name, part] of ranked) { + if (!keep.has(name) || part.method !== 'template') continue; + const area = Math.max(1, Number(part.width || 0) * Number(part.height || 0)); + for (const [otherName, other] of ranked) { + if (name === otherName || !keep.has(otherName)) continue; + const otherArea = Number(other.width || 0) * Number(other.height || 0); + if (otherArea <= area || Number(other.score || 0) < Number(part.score || 0)) continue; + const ix = Math.max(0, Math.min(part.x + part.width, other.x + other.width) - Math.max(part.x, other.x)); + const iy = Math.max(0, Math.min(part.y + part.height, other.y + other.height) - Math.max(part.y, other.y)); + if ((ix * iy) / area >= 0.5) { keep.delete(name); break; } + } + } + for (const name of Object.keys(layout.parts || {})) { + if (!keep.has(name)) { + const file = path.join(partsDir, `${name}.png`); + if (existsSync(file)) unlinkSync(file); + delete layout.parts[name]; + } + } + layout.z_order = (layout.z_order || []).filter((name) => keep.has(name)); + layout.quality = {minScore, maxParts, keptParts: layout.z_order.length, droppedParts: originalCount - layout.z_order.length}; + writeFileSync(layoutPath, JSON.stringify(layout, null, 2)); + if (layout.z_order.length < 3) throw new Error(`有效拆件过少（${layout.z_order.length}），请提高拆件图质量或降低 --min-score`); +} + +function pngSize(file) { + const bytes = readFileSync(file); + if (bytes.readUInt32BE(0) === 0x89504e47) return {width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20)}; + throw new Error('参考图目前需要 PNG 格式'); +} + +function buildAnchoredLayout(reference, partsDir, layoutPath) { + const source = pngSize(reference); + const manifest = JSON.parse(readFileSync(path.join(partsDir, 'parts-manifest.json'), 'utf8')); + const partsByY = [...manifest.parts].sort((a, b) => a.y - b.y); + const top = partsByY.filter((part) => (part.y + part.height / 2) / manifest.atlas_height < 0.25).sort((a, b) => b.area - a.area); + const torso = partsByY.filter((part) => { const y = (part.y + part.height / 2) / manifest.atlas_height; return y >= 0.25 && y < 0.43; }).sort((a, b) => b.area - a.area); + const torsoNames = new Set(torso.map((part) => part.name)); + const middle = partsByY.filter((part) => { const y = (part.y + part.height / 2) / manifest.atlas_height; return y >= 0.35 && y < 0.56 && !torsoNames.has(part.name); }).sort((a, b) => a.x - b.x); + const lower = partsByY.filter((part) => (part.y + part.height / 2) / manifest.atlas_height >= 0.56); + const parts = {}; + const add = (part, x, y, rotation = 0) => { parts[part.name] = {x: Math.round(x - part.width / 2), y: Math.round(y - part.height / 2), width: part.width, height: part.height, original_width: part.width, original_height: part.height, scale: 1, rotation, score: 1, method: 'anchored'}; }; + const cx = source.width / 2; + // Head layers: put the hat behind the face so its brim cannot cover the eyes. + if (top[0]) add(top[0], cx, source.height * 0.13); + if (top[2]) add(top[2], cx, source.height * 0.27); + if (top[1]) add(top[1], cx, source.height * 0.30); + const leftPieces = middle.filter((part) => part.x < manifest.atlas_width * 0.5); + const rightPieces = middle.filter((part) => part.x >= manifest.atlas_width * 0.5); + // The rear arm must be drawn behind the shirt; otherwise it appears to grow + // out of the chest instead of sitting behind the torso. + rightPieces.forEach((part) => add(part, source.width * 0.67, source.height * 0.55, -25)); + const torsoPart = torso[0]; if (torsoPart) add(torsoPart, cx, source.height * 0.53); + const sleeves = torso.slice(1).sort((a, b) => a.x - b.x); if (sleeves[0]) add(sleeves[0], source.width * 0.38, source.height * 0.49); if (sleeves[1]) add(sleeves[1], source.width * 0.68, source.height * 0.50); + leftPieces.sort((a, b) => a.x - b.x).forEach((part, i) => add(part, source.width * (i === 0 ? 0.15 : 0.27), source.height * 0.51, i === 0 ? 5 : 0)); + const legRows = [...lower].sort((a, b) => a.y - b.y); + const upperLegs = legRows.filter((part) => (part.y + part.height / 2) / manifest.atlas_height < 0.72).sort((a, b) => a.x - b.x); + const lowerLegs = legRows.filter((part) => { const y = (part.y + part.height / 2) / manifest.atlas_height; return y >= 0.72 && y < 0.88; }).sort((a, b) => a.x - b.x); + const shoes = legRows.filter((part) => (part.y + part.height / 2) / manifest.atlas_height >= 0.88).sort((a, b) => a.x - b.x); + upperLegs.slice(0, 2).forEach((part, i) => add(part, source.width * (i ? 0.60 : 0.43), source.height * 0.68)); + lowerLegs.slice(0, 2).forEach((part, i) => add(part, source.width * (i ? 0.60 : 0.43), source.height * 0.80)); + shoes.slice(0, 2).forEach((part, i) => add(part, source.width * (i ? 0.60 : 0.43), source.height * 0.91)); + const zOrder = Object.keys(parts); + writeFileSync(layoutPath, JSON.stringify({reference_image: path.basename(reference), canvas_width: source.width, canvas_height: source.height, parts, z_order: zOrder, quality: {mode: 'anchored', keptParts: zOrder.length, sourceSheetParts: manifest.parts.length}}, null, 2)); +} + +function writeReferencePreview(reference, outputPath) { + const data = readFileSync(reference).toString('base64'); + writeFileSync(outputPath, `Reference Pose Previewreference pose`); +} + +async function main() { + const options = args(process.argv); + if (!options.reference || !existsSync(options.reference) || !options.outDir || !options.genielabsDir) throw new Error('必须提供 --reference、--out-dir、--genielabs-dir'); + options.python = selectPython(options.python); + const splitScript = path.join(repoRoot, 'segment.py'); + const positionScript = path.join(options.genielabsDir, 'scripts', 'position_parts.py'); + const buildScript = path.join(options.genielabsDir, 'scripts', 'build_spine_json.py'); + const atlasScript = path.join(options.genielabsDir, 'scripts', 'make_atlas.py'); + const playerScript = path.join(options.genielabsDir, 'scripts', 'generate_spine_player.py'); + for (const file of [positionScript, buildScript, atlasScript, playerScript]) if (!existsSync(file)) throw new Error(`Genielabs 脚本不存在：${file}`); + const out = path.resolve(options.outDir); const parts = path.join(out, 'parts'); const atlasDir = path.join(out, 'atlas'); + mkdirSync(out, {recursive: true}); mkdirSync(parts, {recursive: true}); mkdirSync(atlasDir, {recursive: true}); + const model = resolveModel(options.model); const config = envConfig(options.envFile); + const atlasPath = path.join(out, 'generated-parts-sheet.png'); + writeFileSync(path.join(out, 'source-original.png'), readFileSync(options.reference)); + const positionReference = path.join(out, 'reference-position.png'); + run(options.python, [path.join(repoRoot, 'normalize_reference.py'), '--input', options.reference, '--output', positionReference]); + writeFileSync(path.join(out, 'source.png'), readFileSync(positionReference)); + writeFileSync(path.join(out, 'manifest.json'), JSON.stringify({status: 'generating', model, requestedModel: options.model, pipelineVersion: 'genarrative-genielabs-spine-v1', source: path.basename(options.reference)}, null, 2)); + const candidateRoot = path.join(out, 'candidates'); mkdirSync(candidateRoot, {recursive: true}); + let selectedCandidate = null; + if (options.resume && existsSync(atlasPath) && existsSync(path.join(parts, 'parts-manifest.json'))) { + selectedCandidate = {atlas: atlasPath, parts}; + console.log(`复用已有拆件工作图：${atlasPath}`); + } else { + const candidates = []; + for (let index = 0; index < options.attempts; index += 1) { + const candidateDir = path.join(candidateRoot, `attempt-${index + 1}`); mkdirSync(candidateDir, {recursive: true}); + const candidateAtlas = path.join(candidateDir, 'generated-parts-sheet.png'); + const candidateParts = path.join(candidateDir, 'parts'); mkdirSync(candidateParts, {recursive: true}); + try { + console.log(`尝试第 ${index + 1}/${options.attempts} 个通用拆件提示词`); + writeFileSync(candidateAtlas, await requestAtlas(options.reference, model, options.size, config, promptVariants[index % promptVariants.length])); + run(options.python, [splitScript, '--atlas', candidateAtlas, '--output-dir', candidateParts]); + const stats = JSON.parse(readFileSync(path.join(candidateParts, 'parts-manifest.json'), 'utf8')); + const count = stats.parts.length; const score = Math.abs(count - 12) + (count > 24 ? count - 24 : 0); + candidates.push({atlas: candidateAtlas, parts: candidateParts, count, score}); + console.log(`候选 ${index + 1}: ${count} 个部件，质量分 ${score}`); + } catch (error) { console.warn(`候选 ${index + 1} 失败：${error.message}`); } + } + if (!candidates.length) throw new Error('所有拆件候选均失败'); + selectedCandidate = candidates.sort((a, b) => a.score - b.score)[0]; + cpSync(selectedCandidate.atlas, atlasPath); rmSync(parts, {recursive: true, force: true}); cpSync(selectedCandidate.parts, parts, {recursive: true}); + console.log(`选择候选：${selectedCandidate.count} 个部件`); + } + const layout = path.join(out, 'layout.json'); + if (options.layoutMode === 'matched') { + run(options.python, [positionScript, '--reference', positionReference, '--parts', parts, '--output', layout]); + filterLayout(layout, parts, options.minScore, options.maxParts); + } else { + buildAnchoredLayout(options.reference, parts, layout); + } + // Emit a deterministic raster composite so placement can be inspected + // without loading the Web Player. It uses the same layout coordinates, + // scale, rotation and z-order that are written to the Spine skeleton. + run(options.python, [path.join(repoRoot, 'compose_preview.py'), '--reference', positionReference, '--parts', parts, '--layout', layout, '--output', path.join(out, 'assembled.png')]); + const spineConfig = path.join(out, 'spine-config.json'); buildSpineConfig(layout, spineConfig, options.animations); + const skeleton = path.join(out, 'skeleton.json'); run(options.python, [buildScript, '--config', spineConfig, '--output', skeleton]); + const skeletonJson = JSON.parse(readFileSync(skeleton, 'utf8')); + skeletonJson.skeleton = {...skeletonJson.skeleton, images: './atlas/'}; + writeFileSync(skeleton, JSON.stringify(skeletonJson, null, 2)); + run(options.python, [atlasScript, '--parts', parts, '--output', atlasDir, '--name', 'skeleton']); + const riggedPreview = path.join(out, 'preview-rigged.html'); run(options.python, [playerScript, '--skeleton', skeleton, '--atlas', path.join(atlasDir, 'skeleton.atlas'), '--atlas-image', path.join(atlasDir, 'skeleton.png'), '--output', riggedPreview, '--animation', options.animations.split(',')[0] || 'idle']); + const preview = path.join(out, 'preview.html'); writeReferencePreview(options.reference, preview); + const manifest = {status: 'completed', pipelineVersion: 'genarrative-genielabs-spine-v1', requestedModel: options.model, model, source: path.basename(options.reference), attempts: options.attempts, selectedCandidate: selectedCandidate.count ?? null, promptVariants: promptVariants.length, outputs: {parts, layout, skeleton, atlas: atlasDir, preview, riggedPreview}, limitations: ['默认 preview.html 显示参考图的真实非对称姿态，保证形体结构正常；preview-rigged.html 显示自动拆件 rig，需人工校正。', '当前自动生成的是 region attachments；mesh/weights/IK 仍需在 Spine 中人工校正。', 'Genielabs 脚本为外部依赖，请按其 PolyForm Noncommercial 许可证处理商业授权。']}; + writeFileSync(path.join(out, 'manifest.json'), JSON.stringify(manifest, null, 2)); console.log(`Spine pipeline 完成：${out}`); +} + +main().catch((error) => { console.error(`Spine pipeline 失败：${error.message}`); process.exitCode = 1; }); diff --git a/references/README.md b/references/README.md new file mode 100644 index 0000000..da3aba7 --- /dev/null +++ b/references/README.md @@ -0,0 +1,17 @@ +# 外部参考角色 + +这里保存用于 Spine pipeline 回归的外部素材。首批素材来自 Kenney，页面标注 CC0 1.0；每个文件的来源、尺寸和 SHA-256 记录在 `index.json`，原始压缩包和 `License.txt` 保存在 `external/` 下。它们可以直接替换 `--reference`，也可以作为 GPT image edit 的参考图输入。 + +- `characters/kenney-roguelike-sheet.png`：Roguelike Characters 透明角色条 +- `characters/kenney-beige-idle.png`：New Platformer Pack，idle 姿态 +- `characters/kenney-green-jump.png`：New Platformer Pack，jump 姿态 +- `characters/kenney-purple-duck.png`：New Platformer Pack，duck 姿态 +- `characters/kenney-ooli.png`：Platformer Kit 单角色预览 + +![contact sheet](characters/contact-sheet.jpg) + +来源页面： + +- https://kenney.nl/assets/roguelike-characters +- https://kenney.nl/assets/new-platformer-pack +- https://kenney.nl/assets/platformer-kit diff --git a/references/characters/contact-sheet.jpg b/references/characters/contact-sheet.jpg new file mode 100644 index 0000000..6d6edaf Binary files /dev/null and b/references/characters/contact-sheet.jpg differ diff --git a/references/characters/kenney-beige-idle.png b/references/characters/kenney-beige-idle.png new file mode 100644 index 0000000..58c8962 Binary files /dev/null and b/references/characters/kenney-beige-idle.png differ diff --git a/references/characters/kenney-green-jump.png b/references/characters/kenney-green-jump.png new file mode 100644 index 0000000..79f990c Binary files /dev/null and b/references/characters/kenney-green-jump.png differ diff --git a/references/characters/kenney-ooli.png b/references/characters/kenney-ooli.png new file mode 100644 index 0000000..2d416ee Binary files /dev/null and b/references/characters/kenney-ooli.png differ diff --git a/references/characters/kenney-purple-duck.png b/references/characters/kenney-purple-duck.png new file mode 100644 index 0000000..b0f75bf Binary files /dev/null and b/references/characters/kenney-purple-duck.png differ diff --git a/references/characters/kenney-roguelike-sheet.png b/references/characters/kenney-roguelike-sheet.png new file mode 100644 index 0000000..25d17ee Binary files /dev/null and b/references/characters/kenney-roguelike-sheet.png differ diff --git a/references/index.json b/references/index.json new file mode 100644 index 0000000..9595612 --- /dev/null +++ b/references/index.json @@ -0,0 +1,57 @@ +[ + { + "file": "references/characters/kenney-beige-idle.png", + "width": 128, + "height": 128, + "sha256": "157b0994e98acde8dd3a375fddc644ef0430ebe256487ffb0d1ba16808694082", + "source": "Kenney", + "license": "CC0 1.0", + "source_page": "https://kenney.nl/assets/new-platformer-pack", + "download_url": "https://kenney.nl/media/pages/assets/new-platformer-pack/1896103897-1764756702/kenney_new-platformer-pack-1.1.zip", + "license_file": "references/external/kenney-new-platformer/License.txt" + }, + { + "file": "references/characters/kenney-green-jump.png", + "width": 128, + "height": 128, + "sha256": "9b7d04a44ecaaa809855607aefeb3feef2bc5079e1dc4da16d8b5822a233207b", + "source": "Kenney", + "license": "CC0 1.0", + "source_page": "https://kenney.nl/assets/new-platformer-pack", + "download_url": "https://kenney.nl/media/pages/assets/new-platformer-pack/1896103897-1764756702/kenney_new-platformer-pack-1.1.zip", + "license_file": "references/external/kenney-new-platformer/License.txt" + }, + { + "file": "references/characters/kenney-ooli.png", + "width": 64, + "height": 64, + "sha256": "0644095fa476efbdefcdaf1d8ba4873f18ded5e3e0f2ceb62593ff5b427c6bb4", + "source": "Kenney", + "license": "CC0 1.0", + "source_page": "https://kenney.nl/assets/platformer-kit", + "download_url": "https://kenney.nl/media/pages/assets/platformer-kit/1585cf62b4-1775122253/kenney_platformer-kit.zip", + "license_file": "references/external/kenney-platformer/License.txt" + }, + { + "file": "references/characters/kenney-purple-duck.png", + "width": 128, + "height": 128, + "sha256": "226e9a6baaf4301a828091e1241a9fadf813c9bf4ba44ce603ec9a7cfd261f61", + "source": "Kenney", + "license": "CC0 1.0", + "source_page": "https://kenney.nl/assets/new-platformer-pack", + "download_url": "https://kenney.nl/media/pages/assets/new-platformer-pack/1896103897-1764756702/kenney_new-platformer-pack-1.1.zip", + "license_file": "references/external/kenney-new-platformer/License.txt" + }, + { + "file": "references/characters/kenney-roguelike-sheet.png", + "width": 918, + "height": 203, + "sha256": "771d51ca84b2c5b412027ba80c6a6b8a98bb5a983b2c58ac4f0ccf08ae4d10f2", + "source": "Kenney", + "license": "CC0 1.0", + "source_page": "https://kenney.nl/assets/roguelike-characters", + "download_url": "https://kenney.nl/media/pages/assets/roguelike-characters/53ffff4133-1729196490/kenney_roguelike-characters.zip", + "license_file": "references/external/kenney-roguelike/License.txt" + } +] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1655c70 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +Pillow>=10.0 +numpy>=1.26 +opencv-python>=4.9 diff --git a/run_case_matrix.py b/run_case_matrix.py new file mode 100644 index 0000000..d3ee97f --- /dev/null +++ b/run_case_matrix.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Run the placement stage against several existing split candidates. + +This is an offline regression matrix: it does not call the image API. Point +the input map at generated candidate folders, then inspect each assembled.png +and case-summary.json under --out-dir. +""" +import argparse, json, shutil, subprocess +from pathlib import Path +from PIL import Image +import numpy as np + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--reference', required=True) + ap.add_argument('--genielabs-dir', required=True) + ap.add_argument('--out-dir', required=True) + ap.add_argument('--candidate', action='append', nargs=2, metavar=('NAME','PARTS_DIR'), required=True) + args = ap.parse_args() + root = Path(args.out_dir); root.mkdir(parents=True, exist_ok=True) + py = '/Users/zhaoziyu/anaconda3/bin/python' + ref = Image.open(args.reference).convert('RGBA') + ref_rgb = np.asarray(ref)[:, :, :3].astype(np.float32) + ref_fg = np.asarray(ref)[:, :, 3] > 128 + rows = [] + for name, source in args.candidate: + case = root / name; parts = case / 'parts'; parts.mkdir(parents=True, exist_ok=True) + for f in Path(source).glob('*.png'): shutil.copy2(f, parts / f.name) + layout = case / 'layout.json' + subprocess.run([py, str(Path(args.genielabs_dir)/'scripts/position_parts.py'), '--reference', args.reference, '--parts', str(parts), '--output', str(layout)], check=True) + assembled = case / 'assembled.png' + subprocess.run([py, str(Path(__file__).with_name('compose_preview.py')), '--reference', args.reference, '--parts', str(parts), '--layout', str(layout), '--output', str(assembled)], check=True) + out = np.asarray(Image.open(assembled).convert('RGBA'))[:, :, :3].astype(np.float32) + err = float(np.mean(np.abs(out[ref_fg] - ref_rgb[ref_fg]))) if ref_fg.any() else None + data = json.loads(layout.read_text()) + summary = {'name': name, 'parts': len(data.get('parts', {})), 'sift': sum(v.get('method') == 'sift' for v in data.get('parts', {}).values()), 'template': sum(v.get('method') == 'template' for v in data.get('parts', {}).values()), 'foreground_mae': round(err, 2) if err is not None else None, 'z_order': data.get('z_order', [])} + (case/'case-summary.json').write_text(json.dumps(summary, ensure_ascii=False, indent=2)) + rows.append(summary) + (root/'case-report.json').write_text(json.dumps(rows, ensure_ascii=False, indent=2)) + print(json.dumps(rows, ensure_ascii=False, indent=2)) + +if __name__ == '__main__': main() diff --git a/segment.py b/segment.py new file mode 100644 index 0000000..db2e5dc --- /dev/null +++ b/segment.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Deterministically split a GPT generated white-background parts sheet. + +The model call is intentionally kept in Genarrative's Node runner. This file +only performs the open-source, deterministic connected-components step used by +the Genielabs Spine Animation AI workflow. +""" +import argparse +import json +import os +import sys + +try: + import cv2 +except ImportError as error: + raise SystemExit("缺少 OpenCV，请先执行：python3 -m pip install opencv-python") from error + + +def segment(atlas_path: str, output_dir: str, min_area: int, padding: int, bg_threshold: int): + image = cv2.imread(atlas_path, cv2.IMREAD_UNCHANGED) + if image is None: + raise ValueError(f"无法读取拆件图集：{atlas_path}") + if image.ndim == 2: + image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGRA) + elif image.shape[2] == 3: + image = cv2.cvtColor(image, cv2.COLOR_BGR2BGRA) + + gray = cv2.cvtColor(image[:, :, :3], cv2.COLOR_BGR2GRAY) + _, mask = cv2.threshold(gray, bg_threshold, 255, cv2.THRESH_BINARY_INV) + count, labels, stats, _ = cv2.connectedComponentsWithStats(mask, 8) + os.makedirs(output_dir, exist_ok=True) + height, width = image.shape[:2] + saved = [] + components = [] + for label in range(1, count): + area = int(stats[label, cv2.CC_STAT_AREA]) + if area < min_area: + continue + x, y = int(stats[label, cv2.CC_STAT_LEFT]), int(stats[label, cv2.CC_STAT_TOP]) + w, h = int(stats[label, cv2.CC_STAT_WIDTH]), int(stats[label, cv2.CC_STAT_HEIGHT]) + x1, y1 = max(0, x - padding), max(0, y - padding) + x2, y2 = min(width, x + w + padding), min(height, y + h + padding) + crop = image[y1:y2, x1:x2].copy() + crop[labels[y1:y2, x1:x2] != label] = (0, 0, 0, 0) + name = f"part_{len(saved):02d}.png" + path = os.path.join(output_dir, name) + if not cv2.imwrite(path, crop): + raise ValueError(f"无法写入拆件：{path}") + saved.append(path) + components.append({"name": name[:-4], "x": x, "y": y, "width": w, "height": h, "area": area}) + if not saved: + raise ValueError("拆件图集没有检测到前景，请检查白底和 bg-threshold") + with open(os.path.join(output_dir, "parts-manifest.json"), "w", encoding="utf-8") as handle: + json.dump({"atlas_width": width, "atlas_height": height, "parts": components}, handle, ensure_ascii=False, indent=2) + return saved + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--atlas", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--min-area", type=int, default=500) + parser.add_argument("--padding", type=int, default=12) + parser.add_argument("--bg-threshold", type=int, default=240) + args = parser.parse_args() + try: + paths = segment(args.atlas, args.output_dir, args.min_area, args.padding, args.bg_threshold) + except Exception as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print(f"拆出 {len(paths)} 个部件") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())