From 99d65377ab714f171829acbd04d49d709cf83d5a Mon Sep 17 00:00:00 2001 From: Ukawka <140130066+Ukawka@users.noreply.github.com> Date: Sun, 20 Sep 2026 22:21:50 +0800 Subject: [PATCH] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=AD=A5=E8=A1=A5=E5=85=A8?= =?UTF-8?q?=E6=8E=A5=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 7 + README.md | 38 ++++- completion_geometry.py | 247 ++++++++++++++++++++++++++++++++ configs/tiantoken.json | 45 ++++++ docs/2-segmentation-stage.md | 2 + docs/3-completion-stage.md | 127 ++++++++++++++++ docs/model-stages.md | 4 +- prompts/complete-v1.txt | 8 +- providers/image-edits.mjs | 6 +- stages/completion-stage.mjs | 152 ++++++++++++++++++++ tests/completion-stage.test.mjs | 101 +++++++++++++ tests/model-stages.test.mjs | 14 ++ tests/test_completion.py | 143 ++++++++++++++++++ tools/model-check.mjs | 32 ++++- 14 files changed, 912 insertions(+), 14 deletions(-) create mode 100644 completion_geometry.py create mode 100644 configs/tiantoken.json create mode 100644 docs/3-completion-stage.md create mode 100644 stages/completion-stage.mjs create mode 100644 tests/completion-stage.test.mjs create mode 100644 tests/test_completion.py diff --git a/.env.example b/.env.example index b99649e..9172ee1 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,13 @@ VECTOR_ENGINE_API_KEY=replace-me # Optional request timeout in milliseconds VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS=1200000 +# Optional independent relay: select with --config configs/tiantoken.json +TIANTOKEN_BASE_URL=https://api.tiantoken.com +TIANTOKEN_API_KEY=replace-me +# Defaults in tiantoken.json; independent from SPINE_* model overrides. +# TIANTOKEN_UNDERSTANDING_MODEL=gpt-6-astra +# TIANTOKEN_COMPLETION_MODEL=gpt-image-2 + # Independent model IDs for the annotated-parts pipeline. Use EXACT relay IDs. # Confirm with: node tools/model-check.mjs --list-models SPINE_UNDERSTANDING_MODEL=gpt-6-astra diff --git a/README.md b/README.md index 06ada04..50508be 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ 2. 使用 SAM 根据语义、粗框和程序候选生成 mask，经语义/几何检查后从原图提取可见像素。 3. 使用图片编辑模型补全被遮挡的部件，再进行几何验收和 Spine 导出。 -目前第一阶段和第二阶段均可运行。第二阶段在 `ai-spine` Conda 环境的临时子进程中加载本地 SAM 3.1 权重，使用 GPU 分割并生成半透明标注图、白底拆解图和透明部件。分割结果仍需语义复核；补全结果合并及新管线的 Spine 导出尚未完成。 +目前第一、二阶段可运行，第三阶段已接入部件画布准备、图片编辑请求、候选几何检查及可见像素恢复。第二阶段在 `ai-spine` Conda 环境的临时子进程中加载本地 SAM 3.1 权重，使用 GPU 分割并生成半透明标注图、白底拆解图和透明部件。分割与补全结果仍需语义复核；新管线的 Spine 导出尚未完成。第三阶段已通过离线及 mock 接口验证，真实图片编辑质量尚待验证。 仓库仍保留旧版“AI 生成白底拆件图 → OpenCV 拆分 → Genielabs 定位 → Spine 导出”流程，用于对照和回归测试。 @@ -17,6 +17,7 @@ - [项目修改约定](AGENTS.md)：每次修改都必须遵守，Python 固定使用 `ai-spine` Conda 环境。 - [第一阶段：视觉 LLM 角色结构理解](docs/1-understanding-stage.md)：执行流程、代码对应关系、数据格式和输出文件。 - [第二阶段：本地 SAM 分割](docs/2-segmentation-stage.md)：环境、运行命令、权重适配、输出和质量边界。 +- [第三阶段：部件补全](docs/3-completion-stage.md)：局部画布、遮挡/关节延展区域、生成调用、可见像素恢复及几何检查。 - [模型阶段配置与实验说明](docs/model-stages.md)：模型切换、协议适配、图片编辑探针、缓存和错误处理。 ## 环境准备 @@ -55,6 +56,15 @@ SAM_CODE_DIR=F:/Study/SAM3/sam3 模型 ID 会原样发送给中转站，不进行隐式别名转换。不要把 API Key 写进 JSON 配置或提交到版本库。 +另一个中转站 Tiantoken 使用独立配置 [configs/tiantoken.json](configs/tiantoken.json)。在本地 `.env` 设置 `TIANTOKEN_BASE_URL=https://api.tiantoken.com` 和 `TIANTOKEN_API_KEY`，运行时增加 `--config .\configs\tiantoken.json`；原中转站配置保持独立。 + +```powershell +node .\tools\model-check.mjs --config .\configs\tiantoken.json --list-models +node .\tools\model-check.mjs --config .\configs\tiantoken.json --dry-run +``` + +该配置默认第一阶段使用 `gpt-6-astra`（Responses），第三阶段使用 `gpt-image-2`（images/edits），可通过 `TIANTOKEN_UNDERSTANDING_MODEL`、`TIANTOKEN_COMPLETION_MODEL` 或对应 CLI 参数覆盖。2026-09-20 已验证认证与模型列表访问，返回 113 个模型，含上述两个 ID；尚未验证实际理解、图片编辑或 Responses 图片生成工具能力。此配置不等于已接入 Astra 主导的第三阶段图片生成。 + ### Python 与外部项目 本项目固定使用 **Conda 环境 `ai-spine`**，不使用或创建 `.venv`。第一阶段的 Alpha 校验需要 Python 3、`Pillow`、`numpy`、`opencv-python`。 @@ -188,7 +198,31 @@ conda run --no-capture-output -n ai-spine node .\tools\model-check.mjs ` 加入 `--dry-run` 可检查环境与本地路径，不加载权重推理。输入必须含 `source.png`、v2 `semantic-parts.json`、`segmentation-prompts.json`。完整参数与质量检查见[第二阶段文档](docs/2-segmentation-stage.md)。 -### 测试图片补全接口 +### 第三阶段：按部件补全 + +先从已有 SAM 输出准备画布、允许新增区域和提示词，不请求模型： + +```powershell +node .\tools\model-check.mjs --stage completion ` + --input-dir .\output\sombrero-sam31-v3 ` + --out-dir .\output\sombrero-completion-prepare ` + --completion-model qwen-image-3.0 ` + --prepare-only +``` + +打开 `report.html` 检查部件和允许新增区域。以下命令对 `head` 发起一次部件编辑任务，再检查候选并恢复原图可见像素；会调用中转站图片模型： + +```powershell +node .\tools\model-check.mjs --stage completion ` + --input-dir .\output\sombrero-sam31-v3 ` + --out-dir .\output\sombrero-completion-head ` + --completion-model qwen-image-3.0 ` + --part head +``` + +`--part` 可重复；省略则处理所有具备可见 mask、补全需求和允许区域的部件。当前 SAM 示例仍有错分，应先复核。补全不会自动纠正上游语义；通过几何检查的 PNG 保存到 `parts/`，仍需人工复核。完整参数、离线候选导入与限制见[第三阶段文档](docs/3-completion-stage.md)。 + +### 单独测试图片补全接口 此命令只验证图片编辑适配器并保存原始候选，不代表补全部件已经验收通过： diff --git a/completion_geometry.py b/completion_geometry.py new file mode 100644 index 0000000..3d9f337 --- /dev/null +++ b/completion_geometry.py @@ -0,0 +1,247 @@ +"""Prepare bounded completion canvases and merge generated pixels conservatively.""" +import argparse +import html +import json +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + +from normalize_reference import prepare_rgba + + +def write_json(file, value): + Path(file).write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding='utf-8') + + +def read_json(file): + return json.loads(Path(file).read_text(encoding='utf-8-sig')) + + +def localize(array, origin, size): + """Copy source coordinates to a padded local canvas, without scaling.""" + out = np.zeros((size, size) + array.shape[2:], dtype=array.dtype) + ox, oy = origin + h, w = array.shape[:2] + x0, y0, x1, y1 = max(0, ox), max(0, oy), min(w, ox + size), min(h, oy + size) + if x1 > x0 and y1 > y0: + out[y0-oy:y1-oy, x0-ox:x1-ox] = array[y0:y1, x0:x1] + return out + + +def allowed_regions(part, plan, masks, visible, radius): + h, w = visible.shape + distance = cv2.distanceTransform((~visible).astype(np.uint8), cv2.DIST_L2, 5) + band = (distance <= radius) & ~visible + regions, warnings = {}, [] + flags = part['completion'] + if flags['occlusion_fill']: + occluders = np.zeros_like(visible) + for name in part['occluded_by']: + if name in masks: + occluders |= masks[name] + regions['occlusion_fill'] = band & occluders + if not regions['occlusion_fill'].any(): + warnings.append('NO_OCCLUSION_REGION') + anchors = {} + if flags['joint_extension_parent']: + anchors['joint_extension_parent'] = [part['pivot_norm']] + if flags['joint_extension_child']: + anchors['joint_extension_child'] = [p['pivot_norm'] for p in plan['parts'] + if p['parent'] == part['id'] and p['rig_role'] == 'bone_part'] + for kind, points in anchors.items(): + disks = np.zeros((h, w), np.uint8) + for x, y in points: + cv2.circle(disks, (min(w-1, round(x*w)), min(h-1, round(y*h))), radius, 1, -1) + regions[kind] = band & (disks > 0) + if not regions[kind].any(): + warnings.append(f'NO_{kind.upper()}_REGION') + allowed = np.zeros_like(visible) + for region in regions.values(): + allowed |= region + return allowed, regions, warnings + + +def prepare(directory, selected_ids, size=1024, radius=32): + out = Path(directory) + plan, segmentation = read_json(out/'semantic-parts.json'), read_json(out/'segmentation.json') + source = np.array(Image.open(out/'source.png').convert('RGBA')) + h, w = source.shape[:2] + known = {p['id']: p for p in plan['parts']} + if set(selected_ids) - set(known): + raise ValueError('Unknown requested part ID') + records = {p['id']: p for p in segmentation['parts']} + if len(records) != len(segmentation['parts']) or set(records) - set(known): + raise ValueError('Segmentation IDs do not match semantic plan') + masks = {} + for name in records: + mask = np.array(Image.open(out/'masks'/f'{name}.png').convert('L')) + if mask.shape != (h, w) or not np.isin(mask, [0, 255]).all(): + raise ValueError(f'{name}: mask must be binary and match the source canvas') + mask = mask > 0 + if not mask.any() or (mask & (source[:, :, 3] == 0)).any(): + raise ValueError(f'{name}: empty mask or mask outside original alpha') + masks[name] = mask + tasks, skipped = [], [] + for part in plan['parts']: + name = part['id'] + if selected_ids and name not in selected_ids: + continue + if not part['needs_completion']: + skipped.append({'id': name, 'reason': 'NO_COMPLETION_NEEDED'}) + continue + if name not in masks or part['visibility'] == 'hidden': + skipped.append({'id': name, 'reason': 'NO_VISIBLE_SAM_MASK'}) + continue + visible = masks[name] + allowed, regions, warnings = allowed_regions(part, plan, masks, visible, radius) + if not allowed.any(): + skipped.append({'id': name, 'reason': 'NO_ALLOWED_EXTENSION', 'warnings': warnings}) + continue + ys, xs = np.nonzero(visible | allowed) + bw, bh = int(xs.max()-xs.min()+1), int(ys.max()-ys.min()+1) + if max(bw, bh) + 32 > size: + raise ValueError(f'{name}: completion canvas too small; increase --completion-size (no resizing is performed)') + origin = [int(xs.min()) - (size-bw)//2, int(ys.min()) - (size-bh)//2] + folder = out/'tasks'/name + folder.mkdir(parents=True, exist_ok=True) + cut = source.copy() + cut[~visible] = 0 + original = localize(cut, origin, size) + editable = localize(allowed, origin, size) + Image.fromarray(original).save(folder/'visible.png') + white = Image.new('RGBA', (size, size), 'white') + Image.alpha_composite(white, Image.fromarray(original)).convert('RGB').save(folder/'input.png') + context = Image.fromarray(localize(source, origin, size)) + Image.alpha_composite(white, context).convert('RGB').save(folder/'context.png') + edit_mask = np.full((size, size, 4), 255, np.uint8) + edit_mask[editable, 3] = 0 + Image.fromarray(edit_mask).save(folder/'edit-mask.png') + Image.fromarray(editable.astype(np.uint8)*255).save(folder/'allowed-mask.png') + local_regions, region_bounds = {}, {} + for kind, region in regions.items(): + local = localize(region, origin, size) + Image.fromarray(local.astype(np.uint8)*255).save(folder/f'{kind}.png') + local_regions[kind] = int(local.sum()) + ry, rx = np.nonzero(local) + region_bounds[kind] = [int(rx.min()), int(ry.min()), int(rx.max())+1, int(ry.max())+1] if len(rx) else None + task = {'id': name, 'semantic': part['semantic'], 'completion': part['completion'], + 'origin_xy': origin, 'canvas_size': [size, size], 'source_size': [w, h], + 'allowed_pixels': int(editable.sum()), 'regions': local_regions, + 'region_bounds_xyxy': region_bounds, 'occluded_by': part['occluded_by'], + 'warnings': records[name].get('warnings', []) + warnings, + 'semantic_verified': False, 'status': 'prepared'} + write_json(folder/'task.json', task) + tasks.append(task) + result = {'schema_version': 1, 'tasks': tasks, 'skipped': skipped, + 'quality_status': 'needs_review', 'canvas_size': size, 'extension_radius': radius} + write_json(out/'completion-plan.json', result) + return result + + +def merge_candidate(folder, candidate_file, output_file): + folder = Path(folder) + task = read_json(folder/'task.json') + original = np.array(Image.open(folder/'visible.png').convert('RGBA')) + allowed = np.array(Image.open(folder/'allowed-mask.png').convert('L')) > 0 + candidate = Image.open(candidate_file).convert('RGBA') + qa = {'accepted': False, 'semantic_verified': False, 'reasons': []} + if list(candidate.size) != task['canvas_size']: + qa['reasons'].append('CANVAS_SIZE_MISMATCH') + return qa + try: + candidate, preprocessing = prepare_rgba(candidate) + except ValueError as error: + qa['reasons'].append('CANNOT_EXTRACT_FOREGROUND') + qa['detail'] = str(error) + return qa + generated = np.array(candidate) + visible = original[:, :, 3] > 0 + opaque = original[:, :, 3] >= 240 + if not opaque.any(): + qa['reasons'].append('NO_OPAQUE_ALIGNMENT_ANCHOR') + return qa + # Detect gross movement/redesign before restoring original pixels, otherwise + # restoration could hide a shifted generation attached at the wrong place. + white = Image.new('RGBA', candidate.size, 'white') + generated_rgb = np.array(Image.alpha_composite(white, candidate))[:, :, :3].astype(float) + mae = float(np.abs(generated_rgb[opaque] - original[:, :, :3][opaque]).mean()) + retained = float((generated[:, :, 3][opaque] >= 240).mean()) + proposed = (generated[:, :, 3] > 0) & ~visible + added = proposed & allowed + _, labels = cv2.connectedComponents((visible | added).astype(np.uint8), connectivity=8) + touching = np.unique(labels[visible]) + connected = added & np.isin(labels, touching[touching != 0]) + region_additions = {kind: int((connected & (np.array(Image.open(folder/f'{kind}.png')) > 0)).sum()) + for kind in task['regions']} + missing_regions = [kind for kind, pixels in task['regions'].items() if not pixels or not region_additions[kind]] + qa.update(alignment_mae=mae, visible_retention=retained, + new_pixels=int(connected.sum()), discarded_outside_allowed=int((proposed & ~allowed).sum()), + disconnected_pixels=int((added & ~connected).sum()), region_additions=region_additions, + unfulfilled_regions=missing_regions, preprocessing=preprocessing) + if mae > 30 or retained < .95: + qa['reasons'].append('VISIBLE_ALIGNMENT_MISMATCH') + if not connected.any(): + qa['reasons'].append('NO_CONNECTED_NEW_PIXELS') + if qa['reasons']: + return qa + merged = np.zeros_like(original) + merged[connected] = generated[connected] + merged[visible] = original[visible] + qa['visible_pixels_exact'] = bool(np.array_equal(merged[visible], original[visible])) + qa['accepted'] = True + qa['quality_status'] = 'needs_review' + Image.fromarray(merged).save(output_file) + return qa + + +def finalize(directory): + out = Path(directory) + result = read_json(out/'completion-result.json') + rows = [] + for task in result['parts']: + name = task['id'] + qa = task.get('qa', {}) + if task.get('selected_file'): + im = Image.open(out/task['selected_file']).convert('RGBA') + bbox = im.getbbox() + crop = im.crop(bbox) + (out/'parts').mkdir(exist_ok=True) + crop.save(out/'parts'/f'{name}.png') + meta = read_json(out/'tasks'/name/'task.json') + ox, oy = meta['origin_xy'] + task.update(image=f'parts/{name}.png', crop_xyxy=[bbox[0]+ox, bbox[1]+oy, bbox[2]+ox, bbox[3]+oy]) + view = f'' if task.get('image') else '' + warnings = html.escape(', '.join(task.get('upstream_warnings', []))) + rows.append(f'{html.escape(name)}

{warnings}

任务参数 · 提示词{html.escape(task["status"])}

原图局部参考

{view}
{html.escape(json.dumps(qa, ensure_ascii=False, indent=2))}
') + write_json(out/'completion-result.json', result) + (out/'report.html').write_text('''部件补全 + +

第三阶段：部件补全

新增区域受允许掩码限制，原图可见像素逐像素恢复。几何通过仅表示候选可供复核，不代表遮挡和关节纹理已正确补全。上游 SAM 错分仍需修正。

+'''+''.join(rows)+'
部件 / 上游告警状态原可见区域允许新增区域（白色）补全候选几何检查

跳过

'+html.escape(json.dumps(result['skipped'], ensure_ascii=False, indent=2))+'
', encoding='utf-8') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('action', choices=['prepare', 'merge', 'finalize']) + parser.add_argument('--out', required=True) + parser.add_argument('--part', action='append', default=[]) + parser.add_argument('--size', type=int, default=1024) + parser.add_argument('--radius', type=int, default=32) + parser.add_argument('--candidate') + parser.add_argument('--merged') + parser.add_argument('--qa') + args = parser.parse_args() + if args.action == 'prepare': + if not 128 <= args.size <= 4096 or not 1 <= args.radius <= 256: + parser.error('size must be 128..4096 and radius 1..256') + prepare(args.out, args.part, args.size, args.radius) + elif args.action == 'merge': + write_json(args.qa, merge_candidate(args.out, args.candidate, args.merged)) + else: + finalize(args.out) + + +if __name__ == '__main__': + main() diff --git a/configs/tiantoken.json b/configs/tiantoken.json new file mode 100644 index 0000000..3b736ba --- /dev/null +++ b/configs/tiantoken.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "providers": { + "tiantoken-vision": { + "protocol": "responses", + "base_url_env": "TIANTOKEN_BASE_URL", + "api_key_env": "TIANTOKEN_API_KEY", + "timeout_ms": 180000, + "max_retries": 2 + }, + "tiantoken-image-edit": { + "protocol": "images-edits", + "base_url_env": "TIANTOKEN_BASE_URL", + "api_key_env": "TIANTOKEN_API_KEY", + "timeout_ms": 1200000, + "max_retries": 2, + "capabilities": {"multiple_images": false, "mask": false}, + "image_field": "image", + "mask_editable": "transparent" + } + }, + "stages": { + "understanding": { + "provider": "tiantoken-vision", + "model_env": "TIANTOKEN_UNDERSTANDING_MODEL", + "default_model": "gpt-6-astra", + "prompt": "../prompts/understand-v1.txt", + "output_mode": "text-json", + "stream": true, + "params": {"store": false} + }, + "completion": { + "provider": "tiantoken-image-edit", + "model_env": "TIANTOKEN_COMPLETION_MODEL", + "default_model": "gpt-image-2", + "prompt": "../prompts/complete-v1.txt", + "params": {"n": 1} + }, + "segmentation": { + "provider": "sam-local", + "model_dir_env": "SAM_MODEL_DIR", + "default_model_dir": "F:/Study/SAM3" + } + } +} diff --git a/docs/2-segmentation-stage.md b/docs/2-segmentation-stage.md index 32e508f..310d2d0 100644 --- a/docs/2-segmentation-stage.md +++ b/docs/2-segmentation-stage.md @@ -106,3 +106,5 @@ Node 先启动环境探测子进程，再启动一次推理子进程。模型只 这组结果覆盖原图前景约 67.18%，仍有 64286 像素未覆盖、30569 像素重叠。`shin_left` 和 `foot_left` 的 mask IoU 约 0.843，实际都分到了鞋子；躯干及其他肢体也存在碎片或错分。文本候选对照未解决这些问题。这说明推理和输出链路已跑通，但当前旧语义计划与提示策略尚不能产出合格的最终拆件；需要修正部件粗框并继续验证提示及 mask 语义。 同日 `conda run --no-capture-output -n ai-spine npm test` 的 17 项测试全部通过（包括 Python 分割回归用例）；真实产物 `--resume` 哈希校验复用通过。测试通过仅验证程序行为，不表示上述分割质量合格。 + +后续已接入[第三阶段部件补全](3-completion-stage.md)。该阶段直接读取本目录的 `source.png`、`semantic-parts.json`、`segmentation.json` 和全画布 `masks/`；保留本阶段的告警，不能通过生成隐藏纹理来修复错误的部件归属。 diff --git a/docs/3-completion-stage.md b/docs/3-completion-stage.md new file mode 100644 index 0000000..831dd98 --- /dev/null +++ b/docs/3-completion-stage.md @@ -0,0 +1,127 @@ +# 第三阶段：部件补全 + +本阶段处理第一阶段声明的 `occlusion_fill`、`joint_extension_parent`、`joint_extension_child`，读取第二阶段的可见 mask，为部件生成局部画布，调用图片编辑模型并合并受限的新增像素。原图已可见区域的 RGBA 像素始终从原图恢复，不使用模型重绘的版本。 + +使用 [AGENTS.md](../AGENTS.md) 规定的 Conda `ai-spine` 环境，不需要额外安装依赖。第三阶段几何处理使用本地 Python；图片生成通过现有中转站 `images/edits` 接口完成。 + +切换 Tiantoken 中转站时，在命令中增加 `--config .\configs\tiantoken.json`，并删除示例中的 `--completion-model qwen-image-3.0`，或改为 `--completion-model gpt-image-2`。该配置读取独立的 `TIANTOKEN_BASE_URL` / `TIANTOKEN_API_KEY`，第三阶段默认 `gpt-image-2`。目前仅验证模型列表可访问，实际图片编辑能力尚未实测；`gpt-6-astra` 的 Responses 图片生成工具仍需另行适配。 + +## 命令 + +在 PowerShell 中进入项目并激活环境： + +```powershell +Set-Location F:\Study\taonier\AI_spine +conda activate ai-spine +$runId = Get-Date -Format "yyyyMMdd-HHmmss" +``` + +只准备任务（离线、无需 API Key）： + +```powershell +node .\tools\model-check.mjs --stage completion ` + --input-dir .\output\sombrero-sam31-v3 ` + --out-dir ".\output\$runId-prepare" ` + --completion-model qwen-image-3.0 ` + --prepare-only +``` + +真正调用模型补全一个部件，需要 `.env` 中的中转站地址与密钥，会产生图片请求： + +```powershell +node .\tools\model-check.mjs --stage completion ` + --input-dir .\output\sombrero-sam31-v3 ` + --out-dir ".\output\$runId-complete-head" ` + --completion-model qwen-image-3.0 ` + --part head +``` + +`--part head --part neck` 指定多个部件。省略 `--part` 时处理所有可用且需要补全的部件，每个部件单独发出请求；当前模型配置 `n=1`。程序串行调用，沿用提供方的超时及显式 429/5xx 重试策略，超时/网络中断不自动重复生成。发生请求错误保留候选和诊断，停止后续部件。 + +离线导入已经生成的候选（无需 API Key）： + +```powershell +node .\tools\model-check.mjs --stage completion ` + --input-dir .\output\sombrero-sam31-v3 ` + --out-dir ".\output\$runId-merge" ` + --part head ` + --candidates-dir .\completion-candidates +``` + +候选文件名为 `completion-candidates/head.png`，必须与对应任务的局部画布尺寸、坐标完全一致。准备与合并必须使用相同输入、`--completion-size` 和 `--extension-radius`。缺失候选记为 `missing_candidate`，不会联网补发。可将先前任务保存的原始候选复制为 `ID.png`，离线检查它，避免重复计费。 + +每个输出目录必须是新目录；`--resume` 仅复用已完整结束且输入、参数、源码和产物哈希一致的任务。准备模式与生成模式使用不同目录。失败任务不自动续发请求。 + +| 参数 | 作用 | +| --- | --- | +| `--input-dir` | SAM 输出目录，必须含 `source.png`、v2 计划、`segmentation.json`、`masks/.png` | +| `--part ID` | 指定部件，可重复；省略时处理全部有补全需求的部件 | +| `--prepare-only` | 只准备画布、区域与提示词，不请求图片模型 | +| `--candidates-dir` | 离线读取 `ID.png` 候选，与 prepare-only 互斥 | +| `--completion-size` | 方形局部画布边长，默认 1024，128～4096；不缩放原像素 | +| `--extension-radius` | 新增区域距可见 mask 的最大距离，默认 32 像素，1～256 | +| `--completion-model` | 图片模型原始 ID，当前示例使用 qwen-image-3.0 | +| `--dry-run` | 检查输入文件、计划、参数和 Python 环境，不生成画布或请求模型 | + +旧的 `--stage completion --image ... --prompt-file ...` 保留为独立接口探针，不能与 `--input-dir` 混用。`--stop-after` 目前仍只支持 understand/segment；第三阶段独立调用。 + +## 允许新增区域 + +候选不可以随意扩大部件。程序在原图坐标中构造以下区域，并始终排除原可见 mask： + +- 遮挡补全：可见轮廓外 `extension-radius` 范围，与 `occluded_by` 指向的 SAM mask 的交集。 +- 向父关节延展：同一距离范围，与本部件 `pivot_norm` 附近半径为 `extension-radius` 的圆盘的交集。 +- 向子关节延展：同一距离范围，与直接子骨骼部件的 pivot 圆盘的交集。 + +这些是保守的几何候选区域，不是推断出的完整隐藏形状。缺少遮挡 mask、关节过远或没有子骨骼时，相应区域为空并记录告警。所有区域都为空时跳过该部件。完全隐藏或没有可见 SAM mask 的部件也跳过，不凭空生成完整肢体。 + +局部画布包含可见区域与允许新增区域，至少留 16 像素边距。`origin_xy` 记录局部左上角在原图中的坐标，可为负数；只有平移和补透明边，不进行缩放。所需范围放不下时失败，需增大画布。新增区域目前限定在原图画布范围内。 + +输入给模型的是白底可见部件。只有配置明确声明 `capabilities.mask=true` 时才发送 RGBA 编辑 mask（透明处可编辑）；只有 `multiple_images=true` 时才发送第二张局部原图参考。默认配置二者均为 false，不假定中转站支持这些能力。无论是否发送编辑 mask，返回候选都经过本地允许区域约束。 + +## 返回候选的检查和合并 + +1. 尺寸必须与局部画布完全一致。拒绝自动缩放、重定位或裁切来迎合返回图。 +2. 已有透明背景保留；不透明白底使用边界连通近白区域去底。无法提取前景则拒绝。 +3. 使用原部件 Alpha ≥240 的像素检查对齐：候选保留率至少 95%，RGB 平均绝对误差不大于 30。无不透明锚点则拒绝。这只能发现明显偏移或重绘，不能保证语义正确。 +4. 只保留允许区域中的新增像素，并去掉与原可见部分不连通的新增块。越界和不连通像素数量写入 QA。没有新增像素则拒绝。 +5. 原可见 mask 内 RGBA 逐像素恢复；候选仅提供新增像素。记录各补全子区域新增数量及 `unfulfilled_regions`，存在空缺的要求不会冒充已完成。 +6. 每个返回候选分别检查，选择第一个通过几何条件的候选。输出始终 `semantic_verified=false`、`quality_status=needs_review`。 + +上游 SAM mask 有碎片、重复或语义错分时会原样影响补全，告警保存在 JSON 和报告中。颜色/位置检查也不能验证隐藏纹理是否合理。白底去除可能影响贴边白色细节，仍需人工检查。 + +## 产物 + +| 路径 | 内容 | +| --- | --- | +| `completion-plan.json` | 所有已准备任务、区域及跳过原因 | +| `tasks//input.png` | 发给模型的白底局部部件 | +| `tasks//visible.png` | 从原图保存的 RGBA 可见部件，合并时恢复像素 | +| `tasks//context.png` | 同坐标局部原图参考 | +| `tasks//edit-mask.png` | RGBA 编辑 mask，透明处允许编辑 | +| `tasks//allowed-mask.png` | 白色表示允许新增像素的区域 | +| `tasks//<补全字段>.png` | 三种补全要求各自的允许区域 | +| `tasks//task.json` / `prompt.txt` | 坐标、补全需求、上游告警 / 实际提示词 | +| `tasks//candidate-N.*` | 模型或离线导入的原始候选，拒绝时也保留 | +| `tasks//qa-N.json` | 单候选检查结果与拒绝原因 | +| `tasks//merged-N.png` | 通过几何检查后的局部 RGBA，已恢复可见像素 | +| `parts/.png` | 被选中的补全候选，紧裁剪透明 PNG | +| `completion-result.json` | 每部件状态、候选 QA、原图裁剪坐标和输出路径 | +| `report.html` | 原可见部件、允许新增区域、补全候选和告警 | +| `manifest.json` | 运行模式、网络请求标记、输入/代码/产物哈希及计数 | + +`parts/` 只包含本次被选中且通过几何检查的补全部件。无需补全、未选中和被拒绝的部件不会被伪装成新补全结果；最终组装需结合原 SAM 部件及这里的替换文件。Spine 导出尚未接入。 + +## 实现和验证 + +[stages/completion-stage.mjs](../stages/completion-stage.mjs) 调度准备、请求、候选检查及缓存；[completion_geometry.py](../completion_geometry.py) 实现区域构造、Alpha 提取、原像素恢复和报告；[providers/image-edits.mjs](../providers/image-edits.mjs) 复用现有图片接口。 + +测试覆盖原尺寸坐标映射、Alpha 保真、三个补全字段、越界裁剪、尺寸/位移/空白候选拒绝、离线模式、显式多图与 mask 能力、错误诊断和缓存失效。使用 `conda run --no-capture-output -n ai-spine npm test` 执行，不调用真实付费模型。 + +最初接入时，sombrero-sam31-v3 输出可离线准备 17 个补全任务，11 个部件因无需补全而跳过。当时验证了真实输入的画布准备与 mock 编辑响应的完整合并链路，尚未执行真实 Qwen 图片生成。 + +本轮验证结果：22 项 Node 测试通过（其中包含 7 个 Python 补全用例），真实输入的 `--prepare-only` 和 `--resume` 均成功。准备产物位于 `output/sombrero-completion-prepared-v2/`，打开其中 `report.html` 可检查每个部件的画布与允许新增区域。 + +2026-09-20 用户运行 `output/20260918-183532/sam` 的补全任务时，head 的真实响应同时含有有效 `url` 和空字符串 `b64_json`。原解码器将空字符串误当图片，报“不支持的 PNG/JPEG/WebP 格式”。已修复为空值时使用 URL 下载，并覆盖空串、空白字符串及 null 的回归测试；22 项测试通过。 + +本次已从保存的 `complete/tasks/head/response.raw.json` 下载原生成结果，没有再次调用生成接口。图片保存在同级 `complete-recovered-candidates/head.png`。真实输出为 2048×2048，输入画布为 1024×1024，因此离线合并被 `CANVAS_SIZE_MISMATCH` 拒绝，报告位于 `complete-recovered-head-check/report.html`。这次格式解析问题已解决，但尺寸约束尚未通过；不能把已有 1024 输入任务的候选直接改用 `--completion-size 2048` 合并，因为新画布的像素坐标也会变化。 diff --git a/docs/model-stages.md b/docs/model-stages.md index bb27bb8..10fd4e8 100644 --- a/docs/model-stages.md +++ b/docs/model-stages.md @@ -2,7 +2,7 @@ 本轮实现结构理解、图片编辑适配器、结构 JSON 校验、Alpha 几何校验、候选种子、诊断预览和实验缓存。几何层需要 Python 与 requirements.txt，不运行旧的“拆件图 → SIFT 定位”流程。 -结构理解支持停在 `understand`，也可通过 `--stop-after segment` 接续本地 SAM 分割，或使用 `--stage segmentation --input-dir` 独立执行第二阶段。掩码语义修正、隐藏部件合并、Spine 骨骼导出尚未完成。图片编辑测试保存的是原始候选，不是验收通过的最终资产。 +结构理解支持停在 `understand`，也可通过 `--stop-after segment` 接续本地 SAM 分割，或使用 `--stage segmentation --input-dir` 独立执行第二阶段。第三阶段使用 `--stage completion --input-dir SAM目录` 完成部件画布准备、候选生成、限制新增区域和原像素恢复，详见[第三阶段文档](3-completion-stage.md)。掩码语义修正、补全的语义验收和 Spine 骨骼导出尚未完成。旧的 `--image --prompt-file` 图片编辑探针仍只保存原始候选。 ## 环境与模型 @@ -153,7 +153,7 @@ node .\tools\model-check.mjs --stage completion --image .\local-part.png --promp 同一命令增加 `--resume`，仅当上次完整结束，且模型、协议、参数、提示词、Schema、源图、适配器代码和所有产物校验一致时复用。模型变化时要求新的输出目录，防止覆盖对照组。失败任务也保留原目录，使用新目录重跑。 -理解、SAM 和补全探针各自记录独立缓存。SAM 额外校验输入计划、候选提示、本地源码和权重哈希。对比补全模型时，固定相同局部画布、任务文本与掩码。 +理解、SAM、部件补全和补全探针各自记录独立缓存。SAM 额外校验输入计划、候选提示、本地源码和权重哈希；部件补全校验语义计划、所有输入 mask、画布参数、源码及离线候选哈希。对比补全模型时，固定相同局部画布、任务文本与掩码。 显式 429/5xx 最多按配置重试，超时或网络断开不自动重试，因为服务端可能已经执行任务。完整请求不保证模型内容可复现；seed 只在提供方确认支持时放入 params，usage 不存在时记录 null，不估造价格或 token 数。 diff --git a/prompts/complete-v1.txt b/prompts/complete-v1.txt index 4f1a382..bde5b43 100644 --- a/prompts/complete-v1.txt +++ b/prompts/complete-v1.txt @@ -5,6 +5,8 @@ line thickness and shading. Do not mirror, rotate, resize, recenter or redesign. Keep the canvas dimensions and the position of every already-visible feature. Extend the hidden joint area naturally behind its occluder, without adding other body parts, props, labels, annotations or extra versions of the same part. -If an edit mask is supplied, transparent mask pixels are editable. If no edit -mask is supplied, this request is only an adapter experiment, not a guaranteed -pixel-preserving final asset. A later stage will restore source visible pixels. +If an edit mask is supplied, transparent mask pixels are editable. Otherwise, +use the task's bounded extension instructions and keep the remaining background +pure white or transparent. Return the same canvas size without recentering. +Downstream checks will reject misalignment, restrict additions to allowed regions, +and restore the source visible pixels. Do not rely on restoration to hide changes. diff --git a/providers/image-edits.mjs b/providers/image-edits.mjs index 19eafd1..ed798e7 100644 --- a/providers/image-edits.mjs +++ b/providers/image-edits.mjs @@ -36,8 +36,10 @@ export async function decodeEditedImages(payload, stage, {fetchImpl = fetch} = { const decoded = []; for (const item of items) { let bytes; - if (typeof item.b64_json === 'string') bytes = Buffer.from(item.b64_json, 'base64'); - else if (typeof item.url === 'string') bytes = await downloadImage(item.url, stage, fetchImpl); + // Some relays return both fields, with an empty b64_json for URL output. + // Only a nonempty inline value takes precedence over the download URL. + if (typeof item.b64_json === 'string' && item.b64_json.trim()) bytes = Buffer.from(item.b64_json.trim(), 'base64'); + else if (typeof item.url === 'string' && item.url.trim()) bytes = await downloadImage(item.url.trim(), stage, fetchImpl); else throw new Error('图片编辑响应缺少 b64_json/url'); decoded.push({bytes, ...imageFormat(bytes)}); } diff --git a/stages/completion-stage.mjs b/stages/completion-stage.mjs new file mode 100644 index 0000000..d4d3394 --- /dev/null +++ b/stages/completion-stage.mjs @@ -0,0 +1,152 @@ +import {execFileSync} from 'node:child_process'; +import {copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync} from 'node:fs'; +import path from 'node:path'; +import {projectRoot, publicStage} from '../lib/config.mjs'; +import {selectPython} from '../lib/python-runtime.mjs'; +import {readImage, sha256} from '../lib/images.mjs'; +import {validateParts} from '../lib/validate-parts.mjs'; +import {prepareRun, jsonFile, finishRun} from '../lib/run-artifacts.mjs'; +import {editImage, decodeEditedImages} from '../providers/image-edits.mjs'; +import {redact} from '../providers/http.mjs'; + +const script = path.join(projectRoot, 'completion_geometry.py'); +const readJson = (file) => JSON.parse(readFileSync(file, 'utf8').replace(/^\uFEFF/u, '')); +function pythonCall(python, args) { + try { + execFileSync(python, [script, ...args], {cwd: projectRoot, windowsHide: true, + encoding: 'utf8', timeout: 120000, env: {...process.env, PYTHONUTF8: '1'}, stdio: ['ignore', 'pipe', 'pipe']}); + } catch (error) { throw new Error(`补全几何处理失败：${error.stderr || error.message}`); } +} +function filesIn(dir, prefix = '') { + return readdirSync(dir, {withFileTypes: true}).flatMap((entry) => entry.isDirectory() + ? filesIn(path.join(dir, entry.name), `${prefix}${entry.name}/`) : [`${prefix}${entry.name}`]); +} + +export async function runCompletion({stage, inputDir, outDir, partIds = [], prepareOnly = false, + candidatesDir, completionSize = 1024, extensionRadius = 32, python: requestedPython, + resume = false, dryRun = false}, transport) { + if (!inputDir) throw new Error('部件补全需要 --input-dir 指向 SAM 输出目录'); + if (!outDir && !dryRun) throw new Error('部件补全需要独立的 --out-dir'); + if (prepareOnly && candidatesDir) throw new Error('--prepare-only 与 --candidates-dir 不能同时使用'); + const size = Number(completionSize), radius = Number(extensionRadius); + if (!Number.isInteger(size) || size < 128 || size > 4096 || !Number.isInteger(radius) || radius < 1 || radius > 256) { + throw new Error('--completion-size 必须为 128～4096 的整数；--extension-radius 必须为 1～256 的整数'); + } + const input = path.resolve(inputDir); + const plan = readJson(path.join(input, 'semantic-parts.json')); + const validation = validateParts(plan); + if (plan.schema_version !== 2 || !validation.valid) throw new Error(`补全需要有效 v2 计划：${validation.errors.join(', ')}`); + const names = new Set(plan.parts.map((p) => p.id)); + if (partIds.some((id) => !names.has(id)) || new Set(partIds).size !== partIds.length) throw new Error('--part 包含未知或重复部件 ID'); + const segmentation = readJson(path.join(input, 'segmentation.json')); + if (!Array.isArray(segmentation.parts) || new Set(segmentation.parts.map((p) => p.id)).size !== segmentation.parts.length || segmentation.parts.some((p) => !names.has(p.id))) { + throw new Error('SAM 部件 ID 必须唯一且存在于语义计划中'); + } + const source = readImage(path.join(input, 'source.png'), {pngOnly: true}); + const inputFiles = ['source.png', 'semantic-parts.json', 'segmentation.json', ...segmentation.parts.map((p) => `masks/${p.id}.png`)]; + const inputs = Object.fromEntries(inputFiles.map((file) => [file, sha256(readFileSync(path.join(input, file)))])); + const imported = {}; + if (candidatesDir) { + for (const p of plan.parts.filter((p) => p.needs_completion && (!partIds.length || partIds.includes(p.id)))) { + const file = path.resolve(candidatesDir, `${p.id}.png`); + if (existsSync(file)) imported[p.id] = {file, sha256: sha256(readFileSync(file))}; + } + } + const python = selectPython(requestedPython); + if (dryRun) { + console.log(JSON.stringify({stage: 'completion', input, width: source.width, height: source.height, + selected_parts: partIds, completion_size: size, extension_radius: radius, + mode: prepareOnly ? 'prepare' : candidatesDir ? 'offline_candidates' : 'image_edit', network_requested: false}, null, 2)); + return {dryRun: true}; + } + const codeFiles = ['stages/completion-stage.mjs', 'completion_geometry.py', 'normalize_reference.py', + 'lib/config.mjs', 'lib/python-runtime.mjs', 'lib/run-artifacts.mjs', 'lib/images.mjs', + 'lib/validate-parts.mjs', 'schemas/semantic-parts.schema.json', 'providers/image-edits.mjs', 'providers/http.mjs']; + const identity = {schema_version: 1, stage: 'completion', output_kind: 'bounded_part_completion', + mode: prepareOnly ? 'prepare' : candidatesDir ? 'offline_candidates' : 'image_edit', + resolved_stage: publicStage(stage), endpoint_sha256: sha256(stage.baseUrl), prompt_sha256: sha256(stage.prompt), + inputs, part_ids: partIds, completion_size: size, extension_radius: radius, + imported_candidates: Object.fromEntries(Object.entries(imported).map(([id, value]) => [id, value.sha256])), + code_sha256: sha256(codeFiles.map((file) => `${file}\n${readFileSync(path.join(projectRoot, file), 'utf8')}`).join('\n'))}; + const fingerprint = sha256(JSON.stringify(identity)); + const run = prepareRun(outDir, fingerprint, resume); + if (run.reused) return {out: run.out, reused: true}; + const manifest = {...identity, fingerprint, status: 'running', quality_status: 'pending', started_at: new Date().toISOString(), network_requested: false}; + jsonFile(run.manifestPath, manifest, stage.apiKey); + const result = {schema_version: 1, quality_status: 'needs_review', parts: [], skipped: []}; + try { + for (const file of inputFiles) { + mkdirSync(path.dirname(path.join(run.out, file)), {recursive: true}); + copyFileSync(path.join(input, file), path.join(run.out, file)); + } + pythonCall(python, ['prepare', '--out', run.out, '--size', String(size), '--radius', String(radius), ...partIds.flatMap((id) => ['--part', id])]); + const prepared = readJson(path.join(run.out, 'completion-plan.json')); + result.skipped = prepared.skipped; + result.parts = prepared.tasks.map((task) => ({id: task.id, status: 'prepared', upstream_warnings: task.warnings, semantic_verified: false})); + for (const [index, task] of prepared.tasks.entries()) { + const folder = path.join(run.out, 'tasks', task.id); + const prompt = `${stage.prompt}\n\nPart: ${task.id}: ${task.semantic}\nCompletion requirements: ${JSON.stringify(task.completion)}\nCanvas: ${size} x ${size} pixels. Keep the part at its current position and native scale.\nExtend only the hidden contour and requested joint connections, by at most ${radius} pixels from the visible silhouette. Keep all other background pure white or transparent. Do not add the occluder or neighboring body parts.\n${stage.capabilities.multiple_images ? 'The second image is local context only; do not copy its other parts into the output.' : 'Only the isolated visible part is provided.'}\nReturn exactly the same canvas dimensions.\n`; + const fullPrompt = `${prompt}\nOccluded by: ${task.occluded_by.join(', ') || 'not specified'}.\nLocal pixel bounds for proposed extensions (xyxy, exclusive upper bounds): ${JSON.stringify(task.region_bounds_xyxy)}. Null means no supported region for that requirement. These boxes bound the allowed regions; do not fill whole rectangles.\n`; + writeFileSync(path.join(folder, 'prompt.txt'), fullPrompt); + if (prepareOnly) continue; + const record = result.parts[index]; + console.log(`部件补全 [${index + 1}/${prepared.tasks.length}] ${task.id}`); + let outputs; + if (candidatesDir) { + if (!imported[task.id]) { record.status = 'missing_candidate'; continue; } + outputs = [{bytes: readFileSync(imported[task.id].file), extension: 'png'}]; + } else { + const images = [readImage(path.join(folder, 'input.png'), {pngOnly: true})]; + if (stage.capabilities.multiple_images) images.push(readImage(path.join(folder, 'context.png'), {pngOnly: true})); + const mask = stage.capabilities.mask ? readImage(path.join(folder, 'edit-mask.png'), {pngOnly: true}) : null; + jsonFile(path.join(folder, 'request.json'), {model: stage.model, images: images.map((im) => im.sha256), + edit_mask_sent: Boolean(mask), allowed_mask_sha256: sha256(readFileSync(path.join(folder, 'allowed-mask.png')))}, stage.apiKey); + manifest.network_requested = true; + jsonFile(run.manifestPath, manifest, stage.apiKey); + try { + const {payload, metadata} = await editImage(stage, {images, mask, prompt: fullPrompt}, transport); + jsonFile(path.join(folder, 'response.raw.json'), payload, stage.apiKey); + jsonFile(path.join(folder, 'request-metadata.json'), metadata, stage.apiKey); + outputs = await decodeEditedImages(payload, stage, transport); + } catch (error) { + record.status = 'request_failed'; + if (error.diagnostics) jsonFile(path.join(folder, 'response.error.json'), error.diagnostics, stage.apiKey); + throw error; + } + } + record.candidates = []; + for (const [i, output] of outputs.entries()) { + const filename = `candidate-${i + 1}.${output.extension}`; + writeFileSync(path.join(folder, filename), output.bytes); + const merged = path.join(folder, `merged-${i + 1}.png`), qaFile = path.join(folder, `qa-${i + 1}.json`); + pythonCall(python, ['merge', '--out', folder, '--candidate', path.join(folder, filename), '--merged', merged, '--qa', qaFile]); + const qa = readJson(qaFile); + record.candidates.push({file: `tasks/${task.id}/${filename}`, qa}); + if (qa.accepted && !record.selected_file) { + record.selected_file = `tasks/${task.id}/merged-${i + 1}.png`; + record.qa = qa; + } + } + record.status = record.selected_file ? 'geometry_passed_needs_review' : 'rejected'; + if (!record.qa) record.qa = {accepted: false, reasons: record.candidates.flatMap((c) => c.qa.reasons)}; + jsonFile(path.join(run.out, 'completion-result.json'), result, stage.apiKey); + } + manifest.quality_status = prepareOnly ? 'prepared_needs_review' : 'needs_review'; + manifest.prepared_count = result.parts.length; + manifest.accepted_geometry_count = result.parts.filter((p) => p.selected_file).length; + manifest.rejected_count = result.parts.filter((p) => p.status === 'rejected').length; + manifest.skipped_count = result.skipped.length; + manifest.completed_at = new Date().toISOString(); + jsonFile(path.join(run.out, 'completion-result.json'), result, stage.apiKey); + pythonCall(python, ['finalize', '--out', run.out]); + finishRun(run, manifest, filesIn(run.out).filter((file) => file !== 'manifest.json'), stage.apiKey); + return {out: run.out, reused: false, prepared: result.parts.length, accepted: manifest.accepted_geometry_count, rejected: manifest.rejected_count}; + } catch (error) { + manifest.status = 'failed'; manifest.error = redact(error.message, stage.apiKey); + manifest.quality_status = 'failed'; + jsonFile(run.manifestPath, manifest, stage.apiKey); + jsonFile(path.join(run.out, 'completion-result.json'), result, stage.apiKey); + try { pythonCall(python, ['finalize', '--out', run.out]); } catch { /* Preserve the original error and artifacts. */ } + throw error; + } +} diff --git a/tests/completion-stage.test.mjs b/tests/completion-stage.test.mjs new file mode 100644 index 0000000..9aa0558 --- /dev/null +++ b/tests/completion-stage.test.mjs @@ -0,0 +1,101 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {execFileSync} from 'node:child_process'; +import {mkdtempSync, readFileSync, writeFileSync, mkdirSync, realpathSync, rmSync} from 'node:fs'; +import path from 'node:path'; +import {tmpdir} from 'node:os'; +import {projectRoot, loadConfig, resolveStage} from '../lib/config.mjs'; +import {selectPython} from '../lib/python-runtime.mjs'; +import {runCompletion} from '../stages/completion-stage.mjs'; + +const python = selectPython(); +function fixture(t) { + const base = realpathSync(tmpdir()); + const root = mkdtempSync(path.join(base, 'spine-completion-')); + t.after(() => { + const resolved = realpathSync(root); + assert.equal(path.dirname(resolved), base); + assert.ok(path.basename(resolved).startsWith('spine-completion-')); + rmSync(resolved, {recursive: true}); + }); + const input = path.join(root, 'input'); + execFileSync(python, ['-c', 'import sys; sys.path.insert(0, "tests"); from test_completion import make_fixture; make_fixture(sys.argv[1])', input], {cwd: projectRoot, windowsHide: true}); + const stage = resolveStage(loadConfig({envFile: '__no_test_env__', env: {VECTOR_ENGINE_BASE_URL: 'https://relay.invalid/v1', VECTOR_ENGINE_API_KEY: 'test-secret'}}), 'completion'); + return {root, input, stage}; +} + +test('completion geometry rejects moved/empty outputs and preserves original alpha and coordinates', () => { + const result = execFileSync(python, ['-m', 'unittest', 'discover', '-s', 'tests', '-p', 'test_completion.py'], {cwd: projectRoot, windowsHide: true, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe']}); + assert.equal(result, ''); +}); + +test('part completion prepares without HTTP, sends declared capabilities, merges, resumes, and detects changed masks', async (t) => { + const {root, input, stage} = fixture(t); + const opts = {stage, inputDir: input, outDir: path.join(root, 'run'), completionSize: 128, extensionRadius: 16, partIds: ['torso']}; + let requests = 0; + const transport = {fetchImpl: async (_url, request) => { + requests++; + assert.equal(request.body.get('model'), stage.model); + assert.equal(request.body.getAll('image').length, 1); + assert.equal(request.body.has('mask'), false); + return new Response(JSON.stringify({data: [{b64_json: readFileSync(path.join(input, 'candidate.png')).toString('base64')}]}), {headers: {'Content-Type': 'application/json'}}); + }}; + await runCompletion({...opts, outDir: path.join(root, 'prepared'), prepareOnly: true}, transport); + assert.equal(requests, 0); + const run = await runCompletion(opts, transport); + assert.equal(run.accepted, 1); + const result = JSON.parse(readFileSync(path.join(opts.outDir, 'completion-result.json'))); + assert.equal(result.parts[0].qa.visible_pixels_exact, true); + assert.equal(result.parts[0].semantic_verified, false); + assert.deepEqual(result.parts[0].crop_xyxy, [20, 20, 50, 40]); + assert.equal((await runCompletion({...opts, resume: true}, transport)).reused, true); + assert.equal(requests, 1); + writeFileSync(path.join(input, 'masks', 'torso.png'), readFileSync(path.join(input, 'masks', 'collar.png'))); + await assert.rejects(runCompletion({...opts, resume: true}, transport), /变化/u); + assert.equal(requests, 1); +}); + +test('offline candidate import needs no HTTP; mismatched candidate is retained and rejected', async (t) => { + const {root, input, stage} = fixture(t); + const candidates = path.join(root, 'candidates'); mkdirSync(candidates); + writeFileSync(path.join(candidates, 'torso.png'), readFileSync(path.join(input, 'source.png'))); + const outDir = path.join(root, 'run'); + const result = await runCompletion({stage, inputDir: input, outDir, candidatesDir: candidates, completionSize: 128}, {fetchImpl: () => { throw new Error('Must remain offline'); }}); + assert.equal(result.rejected, 1); + assert.equal(result.accepted, 0); + const data = JSON.parse(readFileSync(path.join(outDir, 'completion-result.json'))); + assert.deepEqual(data.parts[0].qa.reasons, ['CANVAS_SIZE_MISMATCH']); + assert.ok(readFileSync(path.join(outDir, 'tasks', 'torso', 'candidate-1.png')).length); +}); + +test('provider mask/context capabilities are explicit and failure preserves diagnostic report', async (t) => { + const {root, input, stage} = fixture(t); + stage.capabilities = {mask: true, multiple_images: true}; + stage.maxRetries = 0; + const outDir = path.join(root, 'run'); + await assert.rejects(runCompletion({stage, inputDir: input, outDir, completionSize: 128}, {fetchImpl: async (_url, request) => { + assert.equal(request.body.getAll('image').length, 2); + assert.ok(request.body.get('mask')); + return new Response(JSON.stringify({error: {message: 'test-secret failed'}}), {status: 400}); + }}), /HTTP 400/u); + const manifest = readFileSync(path.join(outDir, 'manifest.json'), 'utf8'); + assert.equal(JSON.parse(manifest).status, 'failed'); + assert.ok(!manifest.includes('test-secret')); + assert.ok(readFileSync(path.join(outDir, 'report.html')).length); + assert.ok(readFileSync(path.join(outDir, 'tasks', 'torso', 'response.error.json')).length); +}); + +test('completion CLI prepares without credentials and rejects mixed probe arguments', (t) => { + const {root, input} = fixture(t); + const envFile = path.join(root, 'empty.env'); writeFileSync(envFile, ''); + const outDir = path.join(root, 'cli'); + const args = ['tools/model-check.mjs', '--stage', 'completion', '--input-dir', input, + '--out-dir', outDir, '--prepare-only', '--part', 'torso', '--completion-size', '128', '--env-file', envFile]; + const options = {cwd: projectRoot, windowsHide: true, encoding: 'utf8', + env: {...process.env, VECTOR_ENGINE_BASE_URL: '', VECTOR_ENGINE_API_KEY: ''}, stdio: ['ignore', 'pipe', 'pipe']}; + assert.match(execFileSync(process.execPath, args, options), /未请求模型/u); + const manifest = JSON.parse(readFileSync(path.join(outDir, 'manifest.json'))); + assert.equal(manifest.network_requested, false); + assert.equal(manifest.prepared_count, 1); + assert.throws(() => execFileSync(process.execPath, [...args, '--image', path.join(input, 'source.png')], options), (error) => /不接受探针输入/u.test(error.stderr)); +}); diff --git a/tests/model-stages.test.mjs b/tests/model-stages.test.mjs index 8602322..cde1042 100644 --- a/tests/model-stages.test.mjs +++ b/tests/model-stages.test.mjs @@ -177,6 +177,20 @@ test('image adapter sends multipart, blocks unconfirmed capabilities, decodes in }}); assert.equal(decoded.length, 2); assert.equal(decoded[0].width, 128); + let downloads = 0; + for (const inline of ['', ' \r\n ', null]) { + const recovered = await decodeEditedImages({data: [{b64_json: inline, url: 'https://cdn.invalid/generated.png'}]}, stage, { + fetchImpl: async (url, request) => { + downloads += 1; + assert.equal(String(url), 'https://cdn.invalid/generated.png'); + assert.equal(request.headers, undefined); + return new Response(image.bytes); + }, + }); + assert.deepEqual(recovered[0].bytes, image.bytes); + } + assert.equal(downloads, 3); + await assert.rejects(decodeEditedImages({data: [{b64_json: ' ', url: ''}]}, stage), /缺少 b64_json\/url/u); await assert.rejects(decodeEditedImages({data: [{b64_json: Buffer.from('error').toString('base64')}]}, stage), /图片数据/u); await assert.rejects(decodeEditedImages({data: [{url: 'https://cdn.invalid/404'}]}, stage, {fetchImpl: async () => new Response('', {status: 404})}), /404/u); }); diff --git a/tests/test_completion.py b/tests/test_completion.py new file mode 100644 index 0000000..afe552f --- /dev/null +++ b/tests/test_completion.py @@ -0,0 +1,143 @@ +import copy +import json +from pathlib import Path +import tempfile +import unittest + +import numpy as np +from PIL import Image + +from completion_geometry import prepare, merge_candidate, localize, read_json, write_json, finalize + + +def make_fixture(directory): + root = Path(directory) + root.mkdir(parents=True, exist_ok=True) + (root/'masks').mkdir(exist_ok=True) + source = np.zeros((80, 80, 4), np.uint8) + source[20:40, 20:40] = [180, 50, 20, 255] + source[20, 20, 3] = 80 + source[20:40, 40:50] = [20, 80, 180, 255] + Image.fromarray(source).save(root/'source.png') + part = {'id': 'torso', 'semantic': 'torso', 'parent': None, 'rig_role': 'bone_part', 'bind_to': None, + 'visibility': 'partial', 'bbox_norm_xyxy': [.25, .25, .5, .5], 'pivot_norm': [.4, .4], + 'positive_points_norm': [], 'negative_points_norm': [], 'occluded_by': ['collar'], + 'needs_completion': True, 'confidence': .8, + 'completion': {'occlusion_fill': True, 'joint_extension_parent': False, 'joint_extension_child': False}} + collar = copy.deepcopy(part) + collar.update(id='collar', semantic='collar', parent='torso', bind_to='torso', rig_role='attachment', + visibility='visible', occluded_by=[], needs_completion=False, + completion={key: False for key in part['completion']}) + write_json(root/'semantic-parts.json', {'schema_version': 2, 'coordinate_space': 'normalized_xy_top_left', 'parts': [part, collar], 'notes': []}) + write_json(root/'segmentation.json', {'schema_version': 1, 'parts': [{'id': 'torso', 'warnings': ['SEMANTIC_REVIEW_REQUIRED']}, {'id': 'collar'}]}) + for name, x0, x1 in [('torso', 20, 40), ('collar', 40, 50)]: + mask = np.zeros((80, 80), np.uint8) + mask[20:40, x0:x1] = 255 + Image.fromarray(mask).save(root/'masks'/f'{name}.png') + # Expected 128 canvas: union bbox (20,20,50,40) -> origin (-29,-34). + # Construct candidate independently from production prepare/merge logic. + candidate = np.zeros((128, 128, 4), np.uint8) + candidate[54:74, 49:79] = [180, 50, 20, 255] + candidate[54, 49, 3] = 80 + Image.fromarray(candidate).save(root/'candidate.png') + return source + + +class CompletionTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.source = make_fixture(self.root) + self.plan = prepare(self.root, ['torso'], size=128, radius=16) + self.folder = self.root/'tasks'/'torso' + + def test_canvas_mapping_and_only_occluder_region_editable(self): + task = self.plan['tasks'][0] + self.assertEqual(task['origin_xy'], [-29, -34]) + self.assertEqual(task['allowed_pixels'], 200) + visible = np.array(Image.open(self.folder/'visible.png')) + self.assertEqual(visible[54, 49, 3], 80) + mask = np.array(Image.open(self.folder/'edit-mask.png')) + self.assertTrue(np.all(mask[54:74, 69:79, 3] == 0)) + self.assertTrue(np.all(mask[54:74, 49:69, 3] == 255)) + self.assertEqual(int((mask[:, :, 3] == 0).sum()), 200) + + def test_merge_restores_rgba_exactly_and_clips_extra_pixels(self): + candidate = np.array(Image.open(self.root/'candidate.png')) + candidate[54:74, 49:69, :3] += 5 + candidate[10:14, 10:14] = [255, 0, 0, 255] + Image.fromarray(candidate).save(self.root/'candidate.png') + output = self.folder/'merged.png' + qa = merge_candidate(self.folder, self.root/'candidate.png', output) + self.assertTrue(qa['accepted']) + self.assertEqual(qa['new_pixels'], 200) + self.assertEqual(qa['discarded_outside_allowed'], 16) + self.assertTrue(qa['visible_pixels_exact']) + merged = np.array(Image.open(output)) + np.testing.assert_array_equal(merged[54:74, 49:69], self.source[20:40, 20:40]) + self.assertEqual(merged[11, 11, 3], 0) + write_json(self.root/'completion-result.json', {'parts': [{'id': 'torso', 'status': 'geometry_passed_needs_review', + 'selected_file': 'tasks/torso/merged.png', 'qa': qa}], 'skipped': []}) + finalize(self.root) + result = read_json(self.root/'completion-result.json') + self.assertEqual(result['parts'][0]['crop_xyxy'], [20, 20, 50, 40]) + + def test_reject_resize_shift_no_additions_and_background_only(self): + candidate = Image.open(self.root/'candidate.png').convert('RGBA') + variants = [(candidate.resize((64, 64)), 'CANVAS_SIZE_MISMATCH'), + (Image.fromarray(np.roll(np.array(candidate), 12, axis=1)), 'VISIBLE_ALIGNMENT_MISMATCH'), + (Image.open(self.folder/'visible.png'), 'NO_CONNECTED_NEW_PIXELS'), + (Image.new('RGB', (128, 128), 'white'), 'CANNOT_EXTRACT_FOREGROUND')] + for number, (image, reason) in enumerate(variants): + file = self.root/f'bad-{number}.png' + image.save(file) + output = self.root/f'rejected-{number}.png' + qa = merge_candidate(self.folder, file, output) + self.assertFalse(qa['accepted']) + self.assertIn(reason, qa['reasons']) + self.assertFalse(output.exists()) + + def test_white_background_extraction_preserves_visible_and_adds_only_allowed(self): + candidate = Image.open(self.root/'candidate.png').convert('RGBA') + rgb = Image.alpha_composite(Image.new('RGBA', candidate.size, 'white'), candidate).convert('RGB') + rgb.save(self.root/'white.png') + qa = merge_candidate(self.folder, self.root/'white.png', self.folder/'merged.png') + self.assertTrue(qa['accepted']) + self.assertEqual(qa['new_pixels'], 200) + + def test_disconnected_additions_are_not_merged(self): + candidate = np.array(Image.open(self.folder/'visible.png')) + candidate[54:57, 76:79] = [180, 50, 20, 255] + Image.fromarray(candidate).save(self.root/'detached.png') + qa = merge_candidate(self.folder, self.root/'detached.png', self.folder/'detached-merged.png') + self.assertFalse(qa['accepted']) + self.assertEqual(qa['disconnected_pixels'], 9) + self.assertIn('NO_CONNECTED_NEW_PIXELS', qa['reasons']) + + def test_block_missing_occluder_and_reject_canvas_or_masks(self): + plan = read_json(self.root/'semantic-parts.json') + plan['parts'][0]['occluded_by'] = [] + write_json(self.root/'semantic-parts.json', plan) + self.assertEqual(prepare(self.root, [], 128, 16)['skipped'][0]['reason'], 'NO_ALLOWED_EXTENSION') + with self.assertRaises(ValueError): + prepare(self.root, ['missing'], 128, 16) + Image.new('L', (12, 12), 255).save(self.root/'masks'/'torso.png') + with self.assertRaisesRegex(ValueError, 'mask must'): + prepare(self.root, [], 128, 16) + + def test_joint_regions_use_parent_and_child_pivots(self): + plan = read_json(self.root/'semantic-parts.json') + plan['parts'][0]['completion'] = {'occlusion_fill': False, 'joint_extension_parent': True, 'joint_extension_child': True} + plan['parts'][0]['pivot_norm'] = [.25, .4] + child = copy.deepcopy(plan['parts'][0]) + child.update(id='arm', parent='torso', pivot_norm=[.5, .4]) + plan['parts'].append(child) + write_json(self.root/'semantic-parts.json', plan) + task = prepare(self.root, ['torso'], 128, 8)['tasks'][0] + self.assertGreater(task['regions']['joint_extension_parent'], 0) + self.assertGreater(task['regions']['joint_extension_child'], 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/model-check.mjs b/tools/model-check.mjs index f6f0976..e0bddbf 100644 --- a/tools/model-check.mjs +++ b/tools/model-check.mjs @@ -6,11 +6,13 @@ import {requestJson} from '../providers/http.mjs'; import {runUnderstanding, runCompletionProbe} from '../stages/model-stage.mjs'; import {readImage} from '../lib/images.mjs'; import {runSegmentation} from '../stages/segmentation-stage.mjs'; +import {runCompletion} from '../stages/completion-stage.mjs'; const help = `模型接入与结构理解（新增模式，不调用旧拆件流程） node pipeline.mjs --mode annotated-parts --reference image.png --out-dir output/run --stop-after understand node tools/model-check.mjs --stage completion --image local.png --prompt-file task.txt --out-dir output/edit node tools/model-check.mjs --stage segmentation --input-dir output/understand --out-dir output/segment --sam-dir F:/Study/SAM3 + node tools/model-check.mjs --stage completion --input-dir output/segment --out-dir output/complete --part head --prepare-only node tools/model-check.mjs --list-models node tools/model-check.mjs --dry-run @@ -25,7 +27,12 @@ const help = `模型接入与结构理解（新增模式，不调用旧拆件流 --resume 输入/配置/代码和产物哈希相同时复用 --parts-file FILE 离线读取已有 v1/v2 计划，做 Alpha 校验与种子生成，不请求模型 --python FILE ai-spine Conda 环境解释器（默认使用已激活环境） - --input-dir DIR 已完成的结构理解目录（SAM 阶段输入） + --input-dir DIR SAM 阶段读理解目录；补全阶段读 SAM 目录 + --part ID 补全部件 ID，可重复；默认处理所有需补全部件 + --prepare-only 仅准备补全画布、掩码和提示词，不请求模型 + --candidates-dir DIR 离线导入 ID.png 补全候选，不请求模型 + --completion-size INT 补全方形画布边长，默认 1024，不缩放原像素 + --extension-radius INT 补全区域距可见轮廓的最大像素距离，默认 32 --sam-dir DIR 本地 SAM 根目录，包含 sam3 与 sam3.1checkpoint --sam-code-dir DIR SAM 源码仓库目录覆盖 --sam-checkpoint FILE 本地权重路径覆盖 @@ -35,10 +42,11 @@ const help = `模型接入与结构理解（新增模式，不调用旧拆件流 --mask FILE 透明处可编辑的 RGBA PNG；需配置 capabilities --dry-run 检查配置与本地输入，不请求模型、不生成产物 ---stop-after 支持 understand / segment。SAM 在临时 GPU 子进程运行，完成后退出；补全合并/导出尚未接入。`; +--stop-after 支持 understand / segment。SAM 在临时 GPU 子进程运行，完成后退出。 +部件补全使用 --stage completion --input-dir；可见像素恢复与几何检查已接入，Spine 导出尚未接入。`; export function parseOptions(argv) { - const options = {stage: 'understanding', imagePaths: []}; + const options = {stage: 'understanding', imagePaths: [], partIds: []}; const valueFlags = new Map([ ['--config', 'configFile'], ['--env-file', 'envFile'], ['--reference', 'reference'], ['--out-dir', 'outDir'], ['--understanding-model', 'understandingModel'], ['--completion-model', 'completionModel'], @@ -47,17 +55,20 @@ export function parseOptions(argv) { ['--mode', 'mode'], ['--python', 'python'], ['--stream', 'streamRaw'], ['--parts-file', 'partsFile'], ['--input-dir', 'inputDir'], ['--sam-dir', 'samDir'], ['--sam-code-dir', 'samCodeDir'], ['--sam-checkpoint', 'samCheckpoint'], ['--sam-confidence', 'samConfidence'], ['--sam-precision', 'samPrecision'], + ['--candidates-dir', 'candidatesDir'], ['--completion-size', 'completionSize'], ['--extension-radius', 'extensionRadius'], ]); for (let i = 0; i < argv.length; i += 1) { const flag = argv[i]; if (['--help', '-h'].includes(flag)) options.help = true; else if (flag === '--resume') options.resume = true; else if (flag === '--dry-run') options.dryRun = true; + else if (flag === '--prepare-only') options.prepareOnly = true; else if (flag === '--list-models') options.listModels = true; - else if (valueFlags.has(flag) || flag === '--image') { + else if (valueFlags.has(flag) || flag === '--image' || flag === '--part') { const value = argv[++i]; if (!value || value.startsWith('--')) throw new Error(`${flag} 缺少参数值`); if (flag === '--image') options.imagePaths.push(value); + else if (flag === '--part') options.partIds.push(value); else options[valueFlags.get(flag)] = value; } else throw new Error(`未知参数：${flag}，请用 --help 查看`); } @@ -76,13 +87,24 @@ export async function main(argv = process.argv.slice(2)) { throw new Error('--stop-after 仅支持 understand / segment，且必须匹配阶段'); } const loaded = loadConfig(options); + const partCompletion = options.stage === 'completion' && options.inputDir; + if (!partCompletion && (options.prepareOnly || options.candidatesDir || options.partIds.length || options.completionSize || options.extensionRadius)) { + throw new Error('部件补全参数需要 --stage completion --input-dir'); + } + if (partCompletion) { + if (options.reference || options.imagePaths.length || options.maskPath || options.promptFile || options.listModels) throw new Error('部件补全从 --input-dir 生成画布与提示，不接受探针输入参数'); + const stage = resolveStage(loaded, 'completion', {model: options.completionModel}, {requireKey: !options.dryRun && !options.prepareOnly && !options.candidatesDir}); + const result = await runCompletion({...options, stage}); + if (!result.dryRun) console.log(`部件补全${result.reused ? '复用完成' : options.prepareOnly ? '准备完成（未请求模型）' : '处理完成，需复核'}：${path.join(result.out, 'report.html')}`); + return result; + } if (options.stage === 'segmentation' || (options.stopAfter === 'segment' && options.inputDir)) { if (options.reference || options.partsFile || options.imagePaths.length || options.maskPath || options.promptFile || options.listModels) throw new Error('独立 SAM 阶段使用 --input-dir，不接受模型请求或编辑参数'); const result = await runSegmentation({...options, loaded}); if (!result.dryRun) console.log(`SAM 分割${result.reused ? '复用完成' : '完成'}：${path.join(result.out, 'report.html')}`); return result; } - if (options.inputDir) throw new Error('--input-dir 仅用于 SAM 分割阶段'); + if (options.inputDir) throw new Error('--input-dir 仅用于 SAM 分割或部件补全阶段'); const overrides = options.stage === 'understanding' ? {model: options.understandingModel, protocol: options.understandingProtocol, outputMode: options.outputMode, stream: options.stream} : {model: options.completionModel};