From 639fdf01b6d13c0a73c5287c55b437ee3fbe926b Mon Sep 17 00:00:00 2001 From: lingh Date: Fri, 3 Jul 2026 23:54:28 +0800 Subject: [PATCH 1/5] Gate chroma suppression by the matte's own confidence suppress_alpha_by_chroma was the one stage where colour evidence could veto the matting model outright, and every observed failure had the same shape: ViTMatte was right (raw ~0.97 on a white shirt over pastel blue, on blush over pink) and the bgc-keyed suppression crushed it, because bgc magnitude cannot tell a background-coloured subject from background residue. The matte's raw alpha can: it rates enclosed background pockets low (~0.28 median) and subject texture high. Suppression is now scaled by (1 - smoothstep(raw_alpha, suppress_raw_lo, suppress_raw_hi)) (defaults 0.85/0.98): colour evidence only vetoes the matte where the matte itself is unsure. The gate applies only when alpha really comes from ViTMatte; chroma-seeded alpha passes raw_alpha=None and keeps old behaviour. Set suppress_raw_lo: 1.0 to disable (verified byte-identical to the old output). Validated on both screen colours: pink #F4D8E8 face restored (skin alpha 0.841 -> 0.993, holes 17.2% -> 0.6%), pastel blue #CFEFFF unchanged-to-better (shirt +0.01, wisps identical, gap clearing 99.8%). Also fixes the long-deferred chroma-only pipeline defects (white shirt 0.900, light skin 0.989). CLI output verified byte-identical to the reviewed prototype on both images. Co-Authored-By: Claude Opus 4.8 --- bgfilter/alpha_post.py | 11 +++++++++++ bgfilter/pipeline.py | 11 ++++++++--- bgfilter/settings.py | 7 +++++++ configs/default.yaml | 5 +++++ 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/bgfilter/alpha_post.py b/bgfilter/alpha_post.py index c0f3634..3b413cf 100644 --- a/bgfilter/alpha_post.py +++ b/bgfilter/alpha_post.py @@ -23,6 +23,7 @@ def suppress_alpha_by_chroma( bg_confidence: np.ndarray, trimap: np.ndarray, settings: AlphaPostSettings, + raw_alpha: np.ndarray | None = None, ) -> np.ndarray: """Pull alpha toward 0 where background-colour confidence is high. @@ -30,6 +31,12 @@ def suppress_alpha_by_chroma( pixels are never thinned. Background colour that survived in hair gaps / hole pockets becomes transparent; this also clears the colour fringe that over-unmixing leaves on those low-alpha pixels, since alpha ~ 0 hides it. + + ``raw_alpha`` is the matting model's own prediction before any constraint. + When given, suppression fades where the model is confident the pixel is + opaque (ramp ``suppress_raw_lo -> suppress_raw_hi``): colour evidence may + only veto the matte where the matte is unsure. Background pockets it rates + low stay suppressible; a background-coloured subject it rates ~1 is kept. """ if not settings.chroma_suppress: return alpha @@ -38,6 +45,10 @@ def suppress_alpha_by_chroma( settings.chroma_suppress_bg_low, settings.chroma_suppress_bg_high, ) + if raw_alpha is not None and settings.suppress_raw_lo < 1.0: + suppress = suppress * ( + 1.0 - _smoothstep(raw_alpha, settings.suppress_raw_lo, settings.suppress_raw_hi) + ) factor = 1.0 - settings.chroma_suppress_strength * suppress out = alpha.astype(np.float32).copy() band = trimap == 128 diff --git a/bgfilter/pipeline.py b/bgfilter/pipeline.py index 4611dad..e6aafd9 100644 --- a/bgfilter/pipeline.py +++ b/bgfilter/pipeline.py @@ -103,10 +103,15 @@ def _run_image( else: seg_mask = None trimap, trimap_stats = generate_trimap(bg_confidence, settings.trimap) - alpha, alpha_source = pipeline._predict_alpha(rgb, trimap, bg_confidence) + raw_alpha, alpha_source = pipeline._predict_alpha(rgb, trimap, bg_confidence) - alpha = enforce_trimap(alpha, trimap) - alpha = suppress_alpha_by_chroma(alpha, bg_confidence, trimap, settings.alpha_post) + alpha = enforce_trimap(raw_alpha, trimap) + alpha = suppress_alpha_by_chroma( + alpha, bg_confidence, trimap, settings.alpha_post, + # The matte-confidence gate only makes sense for a real matting prediction; + # a chroma-seeded alpha is itself colour evidence, so no gate there. + raw_alpha=raw_alpha if alpha_source == "vitmatte" else None, + ) alpha = clean_alpha(alpha, trimap, settings.alpha_post) foreground = estimate_foreground_rgb( rgb, alpha, bg_confidence, model, settings.foreground diff --git a/bgfilter/settings.py b/bgfilter/settings.py index f660130..9d2b431 100644 --- a/bgfilter/settings.py +++ b/bgfilter/settings.py @@ -54,6 +54,13 @@ class AlphaPostSettings: chroma_suppress_bg_low: float = 0.35 chroma_suppress_bg_high: float = 0.80 chroma_suppress_strength: float = 1.0 + # Matte-confidence gate: fade the suppression where ViTMatte itself is confident + # the pixel is opaque (raw alpha ramps lo -> hi), so colour evidence only vetoes + # the matte where the matte is unsure. Protects background-coloured subjects + # (white shirt on pastel blue, blush on pink) that chroma alone cannot tell from + # background residue. Set suppress_raw_lo >= 1.0 to disable the gate. + suppress_raw_lo: float = 0.85 + suppress_raw_hi: float = 0.98 @dataclass(frozen=True) diff --git a/configs/default.yaml b/configs/default.yaml index bda0325..7da6e81 100644 --- a/configs/default.yaml +++ b/configs/default.yaml @@ -40,6 +40,11 @@ alpha_post: chroma_suppress_bg_low: 0.35 chroma_suppress_bg_high: 0.80 chroma_suppress_strength: 1.0 + # Matte-confidence gate: suppression fades where ViTMatte's raw alpha is high + # (ramp lo -> hi), so colour evidence only vetoes the matte where it is unsure. + # Set suppress_raw_lo: 1.0 to disable. + suppress_raw_lo: 0.85 + suppress_raw_hi: 0.98 foreground: enabled: true From adbbb9d62024ed9a30c5a791095a37ea87db8f8a Mon Sep 17 00:00:00 2001 From: lingh Date: Sat, 4 Jul 2026 19:52:24 +0800 Subject: [PATCH 2/5] Add cross-model veto of hair-gap background residue (default on) Colour-drifted background trapped between hair strands defeats every single-signal defence: the chroma key reads it as foreground (bgc ~0.07), the segmenter backs it, ViTMatte rates it opaque, and post-hoc removal is a proven dead end (it shreds the hair volume the same pixels belong to). A second, trimap-free matting model (BiRefNet_HR-matting) is the only tested model that separates this residue from the subject, so its opinion is fused in as a veto: min-fusion that may only LOWER alpha, restricted to the background-hued bright suspect zone (proj >= 3, L >= 45, feathered) and gated by primary-alpha confidence (0.70 -> 0.95 ramp) so soft wisps and dark hair are exempt by construction. - settings/config/CLI: cross_check block, --cross-check/--no-cross-check - alpha_post.cross_check_alpha after clean_alpha; second opinion reuses BiRefNetSegmenter; saved to debug as cross_check_alpha.png - chroma.bg_hue_projection extracted and shared with the trimap - docs: methodology.md (new), hair_gap_artifacts.md (investigation log) Verified: cross-check ON reproduces the visually-reviewed B1gate prototype byte-for-byte on TestImage3; --no-cross-check reproduces the previous baseline byte-for-byte; pink-bg FixImage1 face untouched. Co-Authored-By: Claude Fable 5 --- README.md | 10 ++ bgfilter/alpha_post.py | 43 ++++++++- bgfilter/chroma.py | 15 +++ bgfilter/cli.py | 2 + bgfilter/config.py | 8 +- bgfilter/pipeline.py | 47 ++++++++- bgfilter/settings.py | 19 ++++ bgfilter/trimap.py | 11 +-- configs/default.yaml | 16 ++++ docs/hair_gap_artifacts.md | 128 +++++++++++++++++++++++++ docs/methodology.md | 190 +++++++++++++++++++++++++++++++++++++ 11 files changed, 474 insertions(+), 15 deletions(-) create mode 100644 docs/hair_gap_artifacts.md create mode 100644 docs/methodology.md diff --git a/README.md b/README.md index 0272fb0..661d955 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ RGB input -> chroma bg-confidence (keyed to the auto-detected or given colour) -> [optional] semantic segmentation mask -> trimap -> ViTMatte -> alpha cleanup + -> cross-model veto (second matting opinion on bg-hued residue) -> pymatting foreground -> despill -> RGBA PNG -> QA previews ``` @@ -28,6 +29,15 @@ Both share pymatting foreground estimation and a colour de-spill. There is **no* green-contamination rescue / recolour layer — with clean source images it is unnecessary, so it was removed. +Both also run a **cross-model veto** by default: a second, trimap-free matting +model (`ZhengPeng7/BiRefNet_HR-matting`) may only *lower* alpha, only on +background-hued bright pixels the primary result is confident about — this clears +colour-drifted background residue trapped between hair strands that the chroma +key, the segmenter and ViTMatte all read as foreground. Costs one extra model +download (~0.9 GB) and one inference pass per image; disable with +`--no-cross-check` (see the `cross_check` config section, and +`docs/hair_gap_artifacts.md` for the analysis behind it). + The segmentation trimap defaults to `directional` mode (chroma + seg + a hue-direction split: it keeps a background-coloured garment such as a white shirt while dropping a background-hued residual such as blue trapped between hair strands). Switch with diff --git a/bgfilter/alpha_post.py b/bgfilter/alpha_post.py index 3b413cf..8f40a66 100644 --- a/bgfilter/alpha_post.py +++ b/bgfilter/alpha_post.py @@ -3,7 +3,7 @@ from __future__ import annotations import numpy as np from .deps import require_cv2 -from .settings import AlphaPostSettings +from .settings import AlphaPostSettings, CrossCheckSettings def enforce_trimap(alpha: np.ndarray, trimap: np.ndarray) -> np.ndarray: @@ -56,6 +56,47 @@ def suppress_alpha_by_chroma( return np.clip(out, 0.0, 1.0) +def cross_check_alpha( + alpha: np.ndarray, + second_alpha: np.ndarray, + proj: np.ndarray, + lightness: np.ndarray, + trimap: np.ndarray, + settings: CrossCheckSettings, +) -> np.ndarray: + """Veto falsely-confident alpha with an independent second matting opinion. + + Background residue trapped between hair strands drifts in colour until the + chroma key reads it as foreground, the segmenter backs it, and the primary + matte rates it opaque -- every single-signal defence fails. A second model + that does separate it (see docs/hair_gap_artifacts.md) may pull alpha DOWN + (min-fusion; it can never raise it), and only where + + - the pixel is background-hued and not dark (the suspect zone), and + - the primary alpha is high (``gate_lo -> gate_hi`` ramp): pixels the + pipeline already renders soft (outer wisps) are exempt by construction. + + Inside the zone this deliberately overrides the trimap-FG clamp -- the + residue it exists to clear is mostly trimap-FG. Sure background cannot be + disturbed: min-fusion keeps alpha 0 at 0. + """ + if not settings.enabled: + return alpha + cv2 = require_cv2() + zone = ( + (proj >= settings.proj_min) + & (lightness >= settings.l_min) + & (trimap != 0) + ) + weight = zone.astype(np.float32) + if settings.feather_sigma > 0: + blur = cv2.GaussianBlur(weight, (0, 0), settings.feather_sigma) + weight = np.where(zone, 1.0, np.clip(blur, 0.0, 1.0)).astype(np.float32) + gate = weight * _smoothstep(alpha, settings.gate_lo, settings.gate_hi) + out = alpha * (1.0 - gate) + np.minimum(alpha, second_alpha) * gate + return np.clip(out, 0.0, 1.0).astype(np.float32) + + def _remove_small_components(mask: np.ndarray, min_area: int) -> np.ndarray: cv2 = require_cv2() count, labels, stats, _ = cv2.connectedComponentsWithStats(mask.astype(np.uint8), 8) diff --git a/bgfilter/chroma.py b/bgfilter/chroma.py index 817af75..64d4afb 100644 --- a/bgfilter/chroma.py +++ b/bgfilter/chroma.py @@ -39,6 +39,21 @@ def convert_color_spaces(rgb: np.ndarray) -> tuple[np.ndarray, np.ndarray]: return rgb_f, lab +def bg_hue_projection( + lab: np.ndarray, lab_center: tuple[float, float, float] +) -> np.ndarray: + """Per-pixel projection of Lab chroma (a*, b*) onto the background hue direction. + + Positive = displaced toward the background hue, ~0 for neutral pixels. Returns + zeros when the background is near-neutral (no meaningful hue direction). + """ + center = np.asarray(lab_center, dtype=np.float32) + mag = float(np.hypot(center[1], center[2])) + if mag < 1e-3: + return np.zeros(lab.shape[:2], dtype=np.float32) + return (lab[..., 1] * center[1] + lab[..., 2] * center[2]) / mag + + def parse_hex_color(text: str) -> tuple[float, float, float]: """Parse ``#RRGGBB`` (or ``RRGGBB``) into an RGB triple normalised to 0..1.""" s = text.strip().lstrip("#") diff --git a/bgfilter/cli.py b/bgfilter/cli.py index 08c0928..9d1ac93 100644 --- a/bgfilter/cli.py +++ b/bgfilter/cli.py @@ -29,6 +29,7 @@ def main( unknown_radius_ratio: float | None = typer.Option(None, "--unknown-radius-ratio", min=0.0), fg_safe_radius_ratio: float | None = typer.Option(None, "--fg-safe-radius-ratio", min=0.0), despill: bool | None = typer.Option(None, "--despill/--no-despill"), + cross_check: bool | None = typer.Option(None, "--cross-check/--no-cross-check", help="Second-opinion veto of background-hued residue between hair strands (default: on; costs one extra model inference)"), trimap_mode: str | None = typer.Option(None, "--trimap-mode", help="Trimap mode (segmentation pipeline): directional | seg | directional-hard-bg"), seg_backend: str | None = typer.Option(None, "--seg-backend", help="Segmentation backend: birefnet (default) | anime-seg"), ) -> None: @@ -45,6 +46,7 @@ def main( unknown_radius_ratio=unknown_radius_ratio, fg_safe_radius_ratio=fg_safe_radius_ratio, despill=despill, + cross_check=cross_check, trimap_mode=trimap_mode, seg_backend=seg_backend, ) diff --git a/bgfilter/config.py b/bgfilter/config.py index bd38528..f395c49 100644 --- a/bgfilter/config.py +++ b/bgfilter/config.py @@ -7,6 +7,7 @@ from typing import Any, TypeVar from .settings import ( AlphaPostSettings, ChromaSettings, + CrossCheckSettings, DespillSettings, ForegroundSettings, ModelSettings, @@ -48,7 +49,7 @@ def _update_dataclass(instance: T, values: dict[str, Any] | None) -> T: def settings_from_dict(data: dict[str, Any]) -> PipelineSettings: data = dict(data) screen_color = data.pop("screen_color", None) - allowed_sections = {"chroma", "trimap", "alpha_post", "foreground", "despill", "model", "segmentation"} + allowed_sections = {"chroma", "trimap", "alpha_post", "cross_check", "foreground", "despill", "model", "segmentation"} unknown_sections = sorted(set(data) - allowed_sections) if unknown_sections: raise ValueError(f"Unknown config section(s): {', '.join(unknown_sections)}") @@ -57,6 +58,7 @@ def settings_from_dict(data: dict[str, Any]) -> PipelineSettings: chroma=_update_dataclass(ChromaSettings(), data.get("chroma")), trimap=_update_dataclass(TrimapSettings(), data.get("trimap")), alpha_post=_update_dataclass(AlphaPostSettings(), data.get("alpha_post")), + cross_check=_update_dataclass(CrossCheckSettings(), data.get("cross_check")), foreground=_update_dataclass(ForegroundSettings(), data.get("foreground")), despill=_update_dataclass(DespillSettings(), data.get("despill")), model=_update_dataclass(ModelSettings(), data.get("model")), @@ -100,6 +102,9 @@ def override_settings(settings: PipelineSettings, **overrides: Any) -> PipelineS despill_updates: dict[str, Any] = {} if overrides.get("despill") is not None: despill_updates["enabled"] = overrides["despill"] + cross_check_updates: dict[str, Any] = {} + if overrides.get("cross_check") is not None: + cross_check_updates["enabled"] = overrides["cross_check"] seg_updates: dict[str, Any] = {} if overrides.get("device") is not None: seg_updates["device"] = overrides["device"] @@ -121,6 +126,7 @@ def override_settings(settings: PipelineSettings, **overrides: Any) -> PipelineS chroma=chroma, trimap=_update_dataclass(trimap, trimap_updates), alpha_post=alpha_post, + cross_check=_update_dataclass(settings.cross_check, cross_check_updates), foreground=foreground, despill=_update_dataclass(despill, despill_updates), model=_update_dataclass(model, model_updates), diff --git a/bgfilter/pipeline.py b/bgfilter/pipeline.py index e6aafd9..8a7ee5f 100644 --- a/bgfilter/pipeline.py +++ b/bgfilter/pipeline.py @@ -6,13 +6,23 @@ from pathlib import Path import numpy as np -from .alpha_post import clean_alpha, enforce_trimap, suppress_alpha_by_chroma -from .chroma import compute_bg_confidence, parse_hex_color +from .alpha_post import ( + clean_alpha, + cross_check_alpha, + enforce_trimap, + suppress_alpha_by_chroma, +) +from .chroma import ( + bg_hue_projection, + compute_bg_confidence, + convert_color_spaces, + parse_hex_color, +) from .despill import despill from .foreground import estimate_foreground_rgb from .io import load_rgb, save_gray, save_rgb, save_rgba, write_text from .qa import BACKGROUND_COLORS, composite, make_qa_grid, save_previews -from .settings import PipelineSettings +from .settings import PipelineSettings, SegmentationSettings from .trimap import ( fuse_trimap, fuse_trimap_directional, @@ -27,6 +37,7 @@ class MattingPipeline: self.settings = settings self._runner: ViTMatteRunner | None = None self._segmenter = None + self._cross_checker = None def _segment(self, rgb: np.ndarray) -> np.ndarray: if self._segmenter is None: @@ -35,6 +46,20 @@ class MattingPipeline: self._segmenter = make_segmenter(self.settings.segmentation) return self._segmenter.mask(rgb) + def _second_opinion(self, rgb: np.ndarray) -> np.ndarray: + if self._cross_checker is None: + from .segmentation import BiRefNetSegmenter + + check = self.settings.cross_check + self._cross_checker = BiRefNetSegmenter( + SegmentationSettings( + model_name=check.model_name, + device=self.settings.segmentation.device, + input_size=check.input_size, + ) + ) + return self._cross_checker.mask(rgb) + def _predict_alpha(self, rgb: np.ndarray, trimap: np.ndarray, bg_confidence: np.ndarray) -> tuple[np.ndarray, str]: if self.settings.model.matting_method == "chroma": return trimap_to_alpha_seed(trimap, bg_confidence), "chroma" @@ -89,8 +114,6 @@ def _run_image( if mode == "seg": trimap, trimap_stats = fuse_trimap(seg_mask, bg_confidence, settings.trimap) elif mode in ("directional", "directional-hard-bg"): - from .chroma import convert_color_spaces - lab = convert_color_spaces(rgb)[1] trimap, trimap_stats = fuse_trimap_directional( seg_mask, bg_confidence, lab, model.lab_center, settings.trimap @@ -113,6 +136,18 @@ def _run_image( raw_alpha=raw_alpha if alpha_source == "vitmatte" else None, ) alpha = clean_alpha(alpha, trimap, settings.alpha_post) + second_alpha = None + if settings.cross_check.enabled: + lab = convert_color_spaces(rgb)[1] + second_alpha = pipeline._second_opinion(rgb) + alpha = cross_check_alpha( + alpha, + second_alpha, + bg_hue_projection(lab, model.lab_center), + lab[..., 0], + trimap, + settings.cross_check, + ) foreground = estimate_foreground_rgb( rgb, alpha, bg_confidence, model, settings.foreground ) @@ -137,6 +172,8 @@ def _run_image( save_gray(debug / "trimap.png", trimap) if seg_mask is not None: save_gray(debug / "seg_mask.png", seg_mask) + if second_alpha is not None: + save_gray(debug / "cross_check_alpha.png", second_alpha) save_gray(debug / "alpha.png", alpha) save_rgb(debug / "foreground_rgb.png", corrected_rgb) save_gray(debug / "color_mask.png", color_mask) diff --git a/bgfilter/settings.py b/bgfilter/settings.py index 9d2b431..0274f32 100644 --- a/bgfilter/settings.py +++ b/bgfilter/settings.py @@ -63,6 +63,24 @@ class AlphaPostSettings: suppress_raw_hi: float = 0.98 +@dataclass(frozen=True) +class CrossCheckSettings: + # Cross-model veto: a trimap-free matting model gives a second opinion that + # may only LOWER alpha (min-fusion), only on background-hued, non-dark pixels + # (the suspect zone) where the primary result is confident (gate_lo->gate_hi + # alpha ramp). Clears colour-drifted background residue between hair strands + # that chroma, segmentation and the primary matte all read as foreground; + # already-soft wisps and dark hair are exempt by construction. + enabled: bool = True + model_name: str = "ZhengPeng7/BiRefNet_HR-matting" + input_size: int = 2048 + proj_min: float = 3.0 # bg-hue projection above which a pixel is suspect + l_min: float = 45.0 # Lab lightness below which a pixel is exempt (dark hair) + feather_sigma: float = 2.0 # Gaussian feather of the zone boundary, in px + gate_lo: float = 0.70 # primary alpha below this -> fully exempt + gate_hi: float = 0.95 # primary alpha above this -> fully vetoable + + @dataclass(frozen=True) class ForegroundSettings: enabled: bool = True @@ -117,6 +135,7 @@ class PipelineSettings: chroma: ChromaSettings = ChromaSettings() trimap: TrimapSettings = TrimapSettings() alpha_post: AlphaPostSettings = AlphaPostSettings() + cross_check: CrossCheckSettings = CrossCheckSettings() foreground: ForegroundSettings = ForegroundSettings() despill: DespillSettings = DespillSettings() model: ModelSettings = ModelSettings() diff --git a/bgfilter/trimap.py b/bgfilter/trimap.py index 6c8b168..bd03ff0 100644 --- a/bgfilter/trimap.py +++ b/bgfilter/trimap.py @@ -2,6 +2,7 @@ from __future__ import annotations import numpy as np +from .chroma import bg_hue_projection from .deps import require_cv2 from .settings import TrimapSettings @@ -137,14 +138,8 @@ def fuse_trimap_directional( chroma_unknown = ~chroma_bg & ~chroma_fg core = seg_mask >= settings.seg_core_threshold - # Directional chroma: projection of (a*, b*) onto the background chroma - # direction. Positive = colour displaced toward the background hue. - lab_c = np.asarray(lab_center, dtype=np.float32) - mag = float(np.hypot(lab_c[1], lab_c[2])) - if mag < 1e-3: - proj = np.zeros(shape, dtype=np.float32) - else: - proj = (lab[..., 1] * lab_c[1] + lab[..., 2] * lab_c[2]) / mag + # Directional chroma: positive = colour displaced toward the background hue. + proj = bg_hue_projection(lab, lab_center) bg_hued = proj >= settings.bg_hue_proj_min bg = chroma_bg | ((seg_mask < settings.seg_low) & ~chroma_fg) diff --git a/configs/default.yaml b/configs/default.yaml index 7da6e81..48fbe9a 100644 --- a/configs/default.yaml +++ b/configs/default.yaml @@ -46,6 +46,22 @@ alpha_post: suppress_raw_lo: 0.85 suppress_raw_hi: 0.98 +cross_check: + # Cross-model veto: a second, trimap-free matting model may only LOWER alpha + # (min-fusion), only on background-hued bright pixels the primary result is + # confident about — clears colour-drifted background residue between hair + # strands that chroma, segmentation and ViTMatte all read as foreground. + # Costs one extra model (~0.9 GB download) and one inference pass per image. + # Disable with enabled: false or --no-cross-check. + enabled: true + model_name: ZhengPeng7/BiRefNet_HR-matting + input_size: 2048 + proj_min: 3.0 # bg-hue projection above which a pixel is suspect + l_min: 45.0 # Lab lightness below which a pixel is exempt (dark hair) + feather_sigma: 2.0 # zone-boundary feather, px + gate_lo: 0.70 # primary alpha below this -> fully exempt + gate_hi: 0.95 # primary alpha above this -> fully vetoable + foreground: enabled: true method: ml diff --git a/docs/hair_gap_artifacts.md b/docs/hair_gap_artifacts.md new file mode 100644 index 0000000..50a2dce --- /dev/null +++ b/docs/hair_gap_artifacts.md @@ -0,0 +1,128 @@ +# 发丝间隙残留瑕疵:问题分析与候选优化方向 + +状态:**待验证**(2026-07 记录)。这是当前管线剩下最顽固的质量问题。 + +## 问题描述 + +发丝间隙和个别镂空洞中残留蓝白色瑕疵点。颜色已偏离背景色(如 #CFEFFF), +是背景与发丝的稀释混合色。在深色合成背景上最显眼。 + +## 为什么现有机制全部够不着(失败链,每环均已实测) + +1. **chroma 抓不到**:混合色漂移后 bgc ≈ 0.04,远低于 sure_fg 阈值 0.12, + chroma 直接判为前景。 +2. **seg 抓不到**:瑕疵被头发包在轮廓内部,seg 置信度极高 + (seg_loose 拉到 0.95 仍保留,sweep 已验证)。 +3. **ViTMatte 分不出来**:几像素宽的亮色细缝,与反光亮发丝/头发软体积在 + 局部纹理上几乎不可区分,模型给出中等偏高 alpha。 +4. **chroma_suppress 救不了**:压制由 bgc 驱动(smoothstep 0.35→0.80), + bgc 0.04 不触发;matte-confidence gate(639fdf0)对它们无关—— + 还没到门就被 bgc 挡在外面。 +5. **后处理摘除已证死路**:speck 检测器(L 50–78 / proj≥3.5 / 暗邻域 / + 小连通域)能命中,但同批像素同时是真实头发的软体积,压 alpha 或挖洞 + 都会打碎右腰/胳膊间的发簇。多背景评审判定 baseline 最好。 + +**核心教训:问题不在"检测不到",而在"动作是毁灭性的"。** 只要动作是 +"把 alpha 压向 0",检测误报就直接毁头发,而误报无法消除——瑕疵与发丝 +软体积在颜色、拓扑、alpha 邻域上全部重叠(拓扑检测器 v1–v4 全失败)。 + +## 已证伪、不要再试 + +- 任何"检测 + 压 alpha / 挖洞"的变体(见 5)。 +- trimap 阈值/带宽类调参(seg_loose 0.10–0.95 两轮 sweep 无增益)。 +- 拓扑/连通域/到背景距离类检测(口袋与内部拓扑不可分)。 +- 换 seg 模型(BEN2 与 BiRefNet 打平,已废弃)。 + +## 候选方向(按优先级) + +### A. 局部放大重推理 —— 已验证,证伪(2026-07,不要落地) + +根因假设是:原分辨率下缝隙仅 2–3 像素,模型证据不足;放大重推理应能让 +ViTMatte 把缝隙 alpha 判低。先导验证(TestImage3,4 个瑕疵密集 crop, +2×2 对照:放大 1x/2x/4x × trimap 原样/可疑 FG 降级 unknown/降级+膨胀) +结果: + +- 只放大(trimap 原样):speck 中位 alpha 1.000 → 1.000,零变化(2x、4x 同)。 +- 放大 + 降级:中位仍 0.98–0.99,与不放大只降级打平甚至更差。 +- 放大 + 降级 + 膨胀 11px(给模型最大自由):3/4 crop 中位 0.88–0.96, + 最好的 crop 也只到 0.57——仍远高于视觉消除所需;且 4x 常比 2x 更高 + (更多证据让模型**更确信**是前景)。 +- 视觉:右腰/胳膊间关键区域各变体与 baseline 几乎不可区分。 + +诊断补充:瑕疵点 46.6% 是 trimap-FG(rule 2:bgc 0.07 中位 ≤0.12 且 +seg 0.94 中位),其余 53.4% 在 unknown 带但 ViTMatte raw 中位 0.989。 + +**结论:不是分辨率/证据不足,是画风层面歧义**——anime 发丝间的亮色细条 +在 ViTMatte 眼里就是头发高光/软体积,给它更多像素只会更确信。任何以 +ViTMatte 重判为动作的方案(含分块放大整图版)都不必再做。这同时大幅 +降低方向 B 的期望:歧义在样式而非模型容量,换 matting 模型大概率同判 +(参考 BEN2 教训);若试 B,先用同样的 4 crop 先导验证,不要直接接管线。 + +### B. 换更强的 matting 模型 —— 先导验证部分通过(2026-07,待视觉评审) + +**同族换权重:证伪。** ViTMatte Distinctions-646(唯一零接入候选;AEMatter +不在 HF,FBA/DiffMatte 仅权重无代码)与 Com-1K 判决一致(demote 后 +speck 中位 0.87–0.97),同架构换训练数据无效。 + +**跨族第二意见:通过。** BiRefNet_HR-matting(trimap-free,2048 输入, +trust_remote_code 加载方式与现有 seg 相同)是**第一个把瑕疵和主体拉开 +的模型**:speck med 0.74 / p10 0.36,而暗发核心 0.996、衬衫 1.0、 +软发丝 med 0.18(本就该半透明)。直接替换不可行(软发丝太薄、丢硬保证), +但可作**交叉模型融合**: + + zone = (proj>=3.0) & (L>=45) & (trimap!=0) # 可疑色区,向外羽化 + gate = smoothstep(alpha_base, 0.7, 0.95) # 只否决假自信像素 + B1: alpha = lerp(alpha, min(alpha, hrmat), zone*gate) + B2: alpha = lerp(alpha, alpha*smoothstep(hrmat, 0.5, 0.95), zone*gate) + +置信门是关键:第一版没有它,软发丝被全灭(med 0.477→0.000);加门后 +speck mean 0.769→0.714(B1)/0.673(B2),发核心/衬衫/软发丝逐像素不变。 +视觉:右腰发簇完好,缝隙蓝白明显消退。效果图 Outputs/dirB_hrmat/。 +落地成本:+1 个模型(~0.9GB)+1 次推理(cuda ~2s)。遗留:两模型都判 +~1.0 的瑕疵子集(p90=1.0)不受影响。 + +**视觉评审结论(2026-07-04):B1gate 胜出。** 用户多区域评审后选保守的 +min 融合;B2 的意见锐化(remap 0.5→0.95)被否——数字更低(0.673 vs +0.714)但视觉代价不值。落地形态因此更简:无 remap 参数,第二意见原样 +`min()`,可调参数只剩 zone(proj_min/L_min/羽化)与置信门(lo/hi)。 + +**已落地(2026-07-04,默认开启)。** 新增 `cross_check` 设置块 + +`--cross-check/--no-cross-check`;pipeline 在 clean_alpha 后插入 +`cross_check_alpha`(alpha_post.py),第二意见加载复用 BiRefNetSegmenter。 +验证:开启时 TestImage3 输出与评审原型逐字节一致;`--no-cross-check` 与 +旧 baseline 逐字节一致;FixImage1(粉底)脸部无损、发缘粉残留被清, +唯一副作用是羽化权重外溢到暗豁免区边缘(弓弦零星像素最多 -0.5,中位 +无变化,视觉连续)——若在意可把羽化后的权重对 L 面向平色背景 AI 角色图像的离线抠图算法。交叉模型否决(§7)经视觉评审 +> 后已落地,默认开启。 + +## 1. 问题设定与总体框架 + +给定一幅在**单一平色背景**上生成的角色图像 $I \in \mathbb{R}^{H\times W\times 3}$, +目标是估计逐像素不透明度 $\alpha \in [0,1]^{H\times W}$ 与前景色 +$F$,输出高质量 RGBA。任务难点集中在发丝、半透明边缘、衣物镂空, +以及与背景色相近的主体区域(白衬衫、浅肤色)。 + +算法的总体框架是**三方分权**: + +- **色度先验(chroma)** 掌握"什么颜色是背景"——提供连续的背景置信度场, + 并锐化平背景边界; +- **语义分割(segmentation)** 掌握"哪里是主体"——决定拓扑(把染色发丝留在 + 前景、把透视孔洞留在背景); +- **matting 模型(ViTMatte)** 掌握"边界处的混合比例"——只在双方都不确定的 + 未知带内做精细 alpha 推理。 + +三方通过一张三值 trimap($\{0, 128, 255\}$,即背景/未知/前景)交换决定权; +连续证据(置信度、模型原始输出)则绕过三值接口,在后处理阶段以软方式继续 +参与(§4)。管线存在一个退化形态:关闭分割后成为纯 chroma + ViTMatte 的 +两方管线,本文以三方管线为主线。 + +## 2. 背景颜色先验与置信度场 + +### 2.1 背景色自动探测(带失败闸门) + +无人工输入时,从图像边框条带(宽度约 $4\%$ 边长)探测背景色:对边框像素的 +Lab 值做粗直方图(bin 宽 6),取众数 bin 的均值为峰中心,再以半径 12 聚出 +背景簇。设两道质量闸门: + +1. 簇须占边框像素 $\geq 55\%$; +2. 四角(各取 $3\%$ 短边见方)中位色与簇中心距离 $\leq 20$ 的须 $\geq 3$ 角。 + +任一不满足即抛出异常而非硬算——渐变、双色、主体压角等场景应显式失败, +避免对错误的背景色抠图。探测器与后续算法解耦,仅输出一个颜色。 + +### 2.2 背景颜色模型 + +对样本集合(自动模式取边框簇像素;人工输入 hex 时取 Lab 距目标 $\leq 25$ +的像素,优先边框,不足则回退全图最近 10%)做稳健统计:中位数为中心 +$c_{\mathrm{lab}}, c_{\mathrm{rgb}}$,以绝对偏差 75 分位 $\times 1.4826$ +为尺度 $\sigma$(设下限,防止过窄的方差把轻微压缩噪声推出背景)。 + +### 2.3 背景置信度场 + +对每个像素以双空间高斯核取较大者: + +$$C = \max\!\Big(\exp\big(-\tfrac{1}{2}(d_{\mathrm{lab}}/\sigma_{\mathrm{lab}})^2\big),\; +0.85\,\exp\big(-\tfrac{1}{2}(d_{\mathrm{rgb}}/2.5)^2\big)\Big)$$ + +其中 $d_{\mathrm{rgb}}$ 为按通道 $\sigma$ 归一的欧氏距离。$C$ 是全管线的 +色度主证据:参与 trimap、alpha 压制与前景估计。 + +## 3. 语义–色度三信号三值 Trimap + +### 3.1 三个信号 + +- $C$:背景置信度(§2.3),阈值 $\tau_{bg}=0.92$、$\tau_{fg}=0.12$ 把像素分为 + chroma-背景 / chroma-前景 / chroma-未知三档; +- $s$:分割软掩码(BiRefNet,1024 输入;可切换 anime-seg), + 阈值 $\tau_{low}=0.15$、$\tau_{core}=0.60$; +- $\rho$:**色调方向投影**。设背景 Lab 色度向量为 $(a_c, b_c)$,则 + +$$\rho(p) = \frac{a_p a_c + b_p b_c}{\lVert (a_c, b_c) \rVert}$$ + +$\rho \geq 4.0$ 记为"偏背景色调"(bg-hued)。$\rho$ 与 $C$ 互补:$C$ 度量 +到背景色的**距离**,$\rho$ 度量色偏的**方向**——白衬衫距背景不远但方向中性, +发间蓝残留距背景也不近但方向朝蓝,二者由 $\rho$ 分开。 + +### 3.2 融合规则(directional 模式,默认) + +$$\mathrm{BG}: \quad C \geq \tau_{bg} \;\lor\; (s < \tau_{low} \land C > \tau_{fg})$$ +$$\mathrm{FG}: \quad C \leq \tau_{fg} \land s \geq \tau_{low}$$ +$$\text{Rule 3}: \quad \underbrace{\tau_{fg} < C < \tau_{bg}}_{\text{chroma-未知}} \land\; s \geq \tau_{core} \land \rho < 4.0 \;\Rightarrow\; \mathrm{FG}$$ + +其余像素保持未知。规则体现**证据分级原则**:规则 2 中分割仅需 +$s \geq 0.15$,因为它有 chroma 的独立佐证;规则 3 中分割单独作证 +(chroma 弃权),故门槛提高到 $0.60$,且被 $\rho$ 一票否决——偏背景色调的 +高置信像素(发丝间的背景残留)**留在未知带**,交由 §4 的压制处理,而非 +硬定前景。变体 `seg`(无色调切分)与 `directional-hard-bg`(bg-hued 直接 +判背景,激进,可能误伤冷色阴影下的白布)按参数选用。 + +最后沿分割轮廓(loose 掩码 $s \geq 0.08$ 的膨胀−腐蚀差)保留一圈细未知带 +(不覆盖 chroma-背景),供 matting 模型抗锯齿;带宽刻意小,以保住指缝等 +小孔洞。 + +## 4. Trimap 引导的 Alpha 推理与后验修正 + +### 4.1 推理与硬钳制 + +ViTMatte(base,Composition-1K 权重)以 RGB + trimap 为输入输出原始 +alpha $\alpha_{raw}$。随后 `enforce_trimap` 施加硬保证: + +$$\alpha[\mathrm{BG}] \equiv 0, \qquad \alpha[\mathrm{FG}] \equiv 1$$ + +这使"确定背景恒透明"成为结构性质而非统计性质——模型幻觉无法漏入 +已判定的背景。ViTMatte 不可用时可回退 chroma 种子 +$\alpha = \mathrm{clip}(1 - C)$(按参数启用)。 + +### 4.2 带 matte 置信门的色度压制 + +未知带内残存的背景色(发丝缝隙、孔洞死角)由色度证据压除。记 +$S(x; e_0, e_1)$ 为 smoothstep,压制系数为 + +$$\lambda = S(C;\, 0.35,\, 0.80)\; \cdot\; \big(1 - S(\alpha_{raw};\, 0.85,\, 0.98)\big)$$ +$$\alpha' = \alpha \cdot (1 - k\,\lambda), \quad k=1, \quad \text{仅作用于未知带}$$ + +第二个因子是 **matte 置信门**:ViTMatte 自信不透明($\alpha_{raw} \to 1$)的 +像素豁免压制。设计原则为"**色度证据只在模型不确定处拥有否决权**"—— +背景色残留处模型原始输出低(实测中位 $\approx 0.28$),仍可压除;而背景色 +的主体(浅蓝底上的白衬衫、粉底上的腮红)模型原始输出高(实测 +$\approx 0.97$),被门保护。该门同时修复了纯 chroma 管线长期存在的 +白衬衫/浅肤穿洞问题。 + +### 4.3 形态学清理 + +`clean_alpha` 移除未知带内面积 $< 10^{-5} HW$ 的孤立前景连通域;并将完全 +落在 trimap-前景内部、面积 $\leq 2\times10^{-5} HW$ 且不触及图像边缘的 +非实心区域填实(消除模型在确定前景内部的针孔)。 + +## 5. 前景颜色恢复与定向去溢色 + +### 5.1 前景色估计 + +以 pymatting 的多级前景估计(正则 $10^{-5}$)从 $I = \alpha F + (1-\alpha)B$ +中解出 $F$,使半透明像素的颜色摆脱背景污染;不可用时回退启发式反混合。 + +### 5.2 定向去溢色(despill) + +以背景 Lab 色度方向 $\hat d = (a_c, b_c)/\lVert\cdot\rVert$ 为轴,对前景带 +(含向外扩张 2 px)中沿该轴超出边距 $m = 4.0$ 的色度分量按权回拉: + +$$\mathrm{excess} = \max(\rho - m,\, 0), \qquad +(a, b) \leftarrow (a, b) - w \cdot \mathrm{excess} \cdot \hat d$$ + +权重 $w$ 由 alpha 权(不透明区全力,近透明区按下限 0.35 缓和)与色度权 +(excess 归一)相乘。只动 $a^*b^*$、保留亮度 $L$,故发丝纹理与明暗不受损。 +该式对任意平色背景成立:饱和绿幕退化为经典通道抑制,低饱和 pastel +背景(无主导 RGB 通道)由同一投影统一处理。 + +## 6. 设计原则小结 + +1. **分权与制衡**:拓扑归分割、颜色归色度、混合比例归 matting 模型; + 任何一方不得越界(如色度不得推翻分割的拓扑判断)。 +2. **证据分级**:证据越孤立,启用门槛越高(规则 2 的 $0.15$ 对规则 3 的 + $0.60$)。 +3. **弱者才可被否决**:后验修正(压制)只作用于上游不确定的区域—— + trimap 限定作用域(未知带),matte 置信门限定作用强度。 +4. **硬保证优先**:三值接口 + 硬钳制换来"确定区零泄漏"的结构保证; + 连续证据经旁路(压制、置信门、alpha 种子)参与,不稀释保证。 +5. **显式失败**:背景探测不达标即报错,拒绝在错误先验上静默产出。 + +## 7. 局限与评估中的扩展 + +**已知局限**:发丝间隙中颜色已漂移的背景残留(bgc $\approx 0.07$,偏离 +$\tau_{fg}$ 之下)同时骗过三方——chroma 视其为前景、分割置信度高、 +ViTMatte 判其为发丝高光($\alpha_{raw}$ 中位 $0.989$),四个模型族 +(ViTMatte Com-1K / D646、BiRefNet_HR-matting、SEMat-SAM2)对其中约四成 +像素一致判不透明。详见 docs/hair_gap_artifacts.md。 + +**交叉模型置信否决(已落地,默认开启)**:引入一个 trimap-free matting +模型(BiRefNet_HR-matting, 2048)作第二意见 $\alpha_{2nd}$,仅在色度定义 +的可疑区($\rho \geq 3$ 且 $L \geq 45$ 且非确定背景,边界高斯羽化)内、 +且主结果**假自信**的像素上行使否决,采用保守的 min 融合(视觉评审否决了 +更激进的意见锐化变体): + +$$\alpha' = \mathrm{lerp}\big(\alpha,\; \min(\alpha,\, \alpha_{2nd}),\; g\big), +\qquad g = \mathrm{zone} \times S(\alpha;\, 0.70,\, 0.95)$$ + +该设计延续原则 3:第二意见的发言范围由色度划定,可否决对象由置信门限定, +且只许压低、不许抬高——确定背景的零 alpha 结构上不可能被扰动;软发丝 +(本就半透明,门控豁免)与暗发丝(亮度豁免)不受影响。可疑区内它有意 +凌驾于 trimap-FG 硬钳制之上:待清除的残留多数恰是 trimap-FG。 + +## 附录:关键默认参数 + +| 模块 | 参数 | 默认值 | +|---|---|---| +| 背景探测 | 簇半径 / 边框占比门限 / 角容差 | 12 / 0.55 / 20 | +| 置信度场 | $\sigma_{\mathrm{lab}}$ 下限 / $\sigma_{\mathrm{rgb}}$ 下限 | 10 / 0.08 | +| Trimap | $\tau_{bg}$ / $\tau_{fg}$ / $\tau_{low}$ / $\tau_{core}$ / $\rho$ 门限 | 0.92 / 0.12 / 0.15 / 0.60 / 4.0 | +| 压制 | bgc 斜坡 / matte 门斜坡 / 强度 | 0.35–0.80 / 0.85–0.98 / 1.0 | +| 清理 | 最小连通域 / 填孔面积比 | $10^{-5}$ / $2\times10^{-5}$ | +| 去溢色 | 边距 $m$ / 强度 / alpha 权下限 | 4.0 / 0.92 / 0.35 | +| 交叉否决 | $\rho$ 门限 / $L$ 门限 / 羽化 $\sigma$ / 置信门 lo–hi | 3.0 / 45 / 2.0 / 0.70–0.95 | +| 模型 | 分割 / matting / 第二意见 / 设备 | BiRefNet@1024 / ViTMatte-base / BiRefNet_HR-matting@2048 / cpu | From 11c591e5b5e28f1af1dba5caa58dab03a8b5aa99 Mon Sep 17 00:00:00 2001 From: lingh Date: Sat, 4 Jul 2026 20:57:39 +0800 Subject: [PATCH 3/5] Default the HTTP service to cross-check OFF (opt in via BGFILTER_CROSS_CHECK) The cross-model veto adds one extra large-model inference per request (measured ~35 s/image on CPU: 14.4 s -> 49.1 s), which is too slow as a service default. The service now disables it regardless of the config default; BGFILTER_CROSS_CHECK=1 re-enables it (GPU or quality-first deployments). CLI behaviour is unchanged (still on by default). Startup log and /healthz report the effective state. Co-Authored-By: Claude Fable 5 --- README.md | 5 +++++ app.py | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8dce818..3ca2b22 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,11 @@ Config comes from `BGFILTER_CONFIG` (defaults to `configs/default.yaml` when pre caps input size (default ~4MP → `413`); `BGFILTER_PRELOAD=1` loads the default models at startup. Run a single worker (`--workers 1`) — each worker loads its own copy of the models. +Unlike the CLI, the service defaults the **cross-model veto to off** — it adds one +extra large-model inference (~35 s/image on CPU, 14 s → 49 s total). Opt in with +`BGFILTER_CROSS_CHECK=1` (worth it on GPU or when hair-gap residue matters more than +latency); `/healthz` reports the effective state as `crossCheck`. + ## Quality Check The quality checker measures alpha validity and edge spill on semi-transparent edge diff --git a/app.py b/app.py index 500e8c7..9130d80 100644 --- a/app.py +++ b/app.py @@ -62,6 +62,12 @@ def _build_settings(): if device: # Covers both settings.model.device and settings.segmentation.device. settings = override_settings(settings, device=device) + # The cross-model veto adds one extra large-model inference (~35 s/image on + # CPU), so the service defaults it OFF regardless of the config default; + # opt back in with BGFILTER_CROSS_CHECK=1. Other cross_check params (model, + # thresholds) still come from the config. + cross_check = os.environ.get("BGFILTER_CROSS_CHECK", "0").strip().lower() in ("1", "true", "yes", "on") + settings = override_settings(settings, cross_check=cross_check) return settings @@ -71,9 +77,10 @@ async def lifespan(app: FastAPI): settings = _build_settings() preload = os.environ.get("BGFILTER_PRELOAD", "1").strip().lower() not in ("0", "false", "no", "") logger.info( - "starting bgfilter service: device=%s default_seg=%s preload=%s max_pixels=%d", + "starting bgfilter service: device=%s default_seg=%s cross_check=%s preload=%s max_pixels=%d", settings.model.device, settings.segmentation.backend, + settings.cross_check.enabled, preload, MAX_IMAGE_PIXELS, ) From 8417653598f5963d152d5c7e5916685d47529465 Mon Sep 17 00:00:00 2001 From: lingh Date: Sat, 4 Jul 2026 22:34:08 +0800 Subject: [PATCH 4/5] Add per-request cross_check toggle to the HTTP service (lazy-loaded) POST /remove-background gains an optional cross_check form field (1/0/true/false/yes/no/on/off; empty = service default). The effective state is reported in the X-BGFilter-Cross-Check response header. The cross-check model is now never preloaded at startup: the first request that enables it pays the one-time load (measured ~8 s on top of the ~50 s cc inference), after which the instance is cached and shared. Verified with a functional test against PipelineManager: cc-off requests never load the model, cc=1 lazy-loads then reuses the cached instance, and omitting the field falls back to the service default. Co-Authored-By: Claude Fable 5 --- README.md | 16 +++++++++++++--- app.py | 23 ++++++++++++++++++++++- bgfilter/service.py | 23 ++++++++++++++++------- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3ca2b22..dcc5aad 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,9 @@ Two endpoints: - `file` (required) — the source image. The field name is fixed as `file`. - `screen_color` (optional) — `#RRGGBB` prior; omit/empty for auto-detect. - `seg_model` (optional) — `birefnet` (default) or `anime-seg`. + - `cross_check` (optional) — `1`/`0` (also `true`/`false`/`yes`/`no`/`on`/`off`); + per-request override of the cross-model veto. Omit/empty to use the service + default. The response reports the effective state in `X-BGFilter-Cross-Check`. ```bash # default (birefnet + auto background colour) @@ -166,6 +169,10 @@ curl -sS \ -F "screen_color=#CFEFFF" \ -F "seg_model=anime-seg" \ http://127.0.0.1:18083/remove-background -o output.png + +# enable the cross-model veto for this request only (see below; slow on CPU) +curl -sS -F "file=@input.png" -F "cross_check=1" \ + http://127.0.0.1:18083/remove-background -o output.png ``` Config comes from `BGFILTER_CONFIG` (defaults to `configs/default.yaml` when present); @@ -174,9 +181,12 @@ caps input size (default ~4MP → `413`); `BGFILTER_PRELOAD=1` loads the default startup. Run a single worker (`--workers 1`) — each worker loads its own copy of the models. Unlike the CLI, the service defaults the **cross-model veto to off** — it adds one -extra large-model inference (~35 s/image on CPU, 14 s → 49 s total). Opt in with -`BGFILTER_CROSS_CHECK=1` (worth it on GPU or when hair-gap residue matters more than -latency); `/healthz` reports the effective state as `crossCheck`. +extra large-model inference (~35 s/image on CPU, 14 s → 49 s total). Enable it +per request with the `cross_check` form field, or flip the service default with +`BGFILTER_CROSS_CHECK=1` (worth it on GPU or when hair-gap residue matters more +than latency); `/healthz` reports the default as `crossCheck`. The cross-check +model is lazy-loaded: it is never preloaded at startup, so the first request that +enables it also pays the one-time model load (~1 GB) on top of the inference. ## Quality Check diff --git a/app.py b/app.py index 9130d80..d1b8ff5 100644 --- a/app.py +++ b/app.py @@ -129,6 +129,20 @@ def _normalize_seg_model(value: str | None) -> str | None: return v +def _normalize_cross_check(value: str | None) -> bool | None: + """None/empty -> use the service default; otherwise a boolean string.""" + if value is None: + return None + v = value.strip().lower() + if v == "": + return None + if v in ("1", "true", "yes", "on"): + return True + if v in ("0", "false", "no", "off"): + return False + raise HTTPException(status_code=400, detail="invalid cross_check") + + def _decode_image(data: bytes) -> np.ndarray: if not data: raise HTTPException(status_code=400, detail="empty image") @@ -162,12 +176,15 @@ async def remove_background( file: UploadFile = File(...), screen_color: str | None = Form(None), seg_model: str | None = Form(None), + cross_check: str | None = Form(None), ): if _manager is None: raise HTTPException(status_code=503, detail="service not ready") normalized_color = _normalize_screen_color(screen_color) normalized_seg = _normalize_seg_model(seg_model) or _manager.default_seg_model + normalized_cc = _normalize_cross_check(cross_check) + effective_cc = normalized_cc if normalized_cc is not None else _manager.cross_check_enabled rgb = _decode_image(await file.read()) @@ -178,7 +195,10 @@ async def remove_background( result = await loop.run_in_executor( None, lambda: _manager.process( - rgb, screen_color=normalized_color, seg_model=normalized_seg + rgb, + screen_color=normalized_color, + seg_model=normalized_seg, + cross_check=normalized_cc, ), ) except RuntimeError as exc: @@ -200,5 +220,6 @@ async def remove_background( "X-BGFilter-Elapsed-Ms": str(elapsed_ms), "X-BGFilter-Seg-Model": normalized_seg, "X-BGFilter-Screen-Color": normalized_color if normalized_color else "auto", + "X-BGFilter-Cross-Check": "on" if effective_cc else "off", } return Response(content=png, media_type="image/png", headers=headers) diff --git a/bgfilter/service.py b/bgfilter/service.py index 1f4bea6..532d28f 100644 --- a/bgfilter/service.py +++ b/bgfilter/service.py @@ -24,9 +24,9 @@ class PipelineManager: ViTMatte is the matting model shared by both segmentation backends, so it is loaded once and reused. Each segmentation backend is lazily loaded once on first use and cached by name. Per-request parameters (``screen_color``, - ``seg_model``) never trigger a model reload -- ``screen_color`` only affects - this request's chroma/despill, and ``seg_model`` just selects an already-cached - (or first-time-loaded) segmenter. + ``seg_model``, ``cross_check``) never trigger a model reload -- ``screen_color`` + only affects this request's chroma/despill, and ``seg_model``/``cross_check`` + select an already-cached (or first-time-loaded, then cached) model. """ def __init__(self, base_settings: PipelineSettings, preload: bool = True): @@ -40,8 +40,8 @@ class PipelineManager: self._get_runner() if base_settings.segmentation.enabled: self._get_segmenter(base_settings.segmentation.backend) - if base_settings.cross_check.enabled: - self._get_cross_checker() + # The cross-check model is deliberately NOT preloaded: it is opt-in + # per request, so the first request that enables it pays the load. @property def default_seg_model(self) -> str: @@ -110,12 +110,21 @@ class PipelineManager: return self._cross_checker def process( - self, rgb: np.ndarray, *, screen_color: str | None, seg_model: str + self, + rgb: np.ndarray, + *, + screen_color: str | None, + seg_model: str, + cross_check: bool | None = None, ) -> MattingResult: if seg_model not in _SEG_MODELS: raise ValueError(f"invalid seg_model '{seg_model}'") settings = override_settings( - self._base, screen_color=screen_color, seg_backend=seg_model + self._base, + screen_color=screen_color, + seg_backend=seg_model, + # None = keep the service default; True/False = per-request override. + cross_check=cross_check, ) runner = ( self._get_runner() From 88987f508c9c5b54b0210411ccd340e41a2e45a0 Mon Sep 17 00:00:00 2001 From: lingh Date: Sat, 4 Jul 2026 22:41:11 +0800 Subject: [PATCH 5/5] Remove dead code: unused detect_background_color and ForegroundEstimate extras - chroma.detect_background_color had no callers anywhere (auto-detection goes through estimate_background_model / _background_border_cluster). - ForegroundEstimate.background and .correction were computed on every image but never consumed; only .rgb was used. estimate_foreground_rgb now returns the foreground array directly and the dataclass is gone, which also drops pymatting return_background work and a per-image correction-norm pass. Verified: CLI output on samples/TestImage3.png is byte-identical (SHA256) to the pre-cleanup baseline. Co-Authored-By: Claude Fable 5 --- bgfilter/chroma.py | 16 --------------- bgfilter/foreground.py | 44 ++++++++---------------------------------- bgfilter/pipeline.py | 2 +- 3 files changed, 9 insertions(+), 53 deletions(-) diff --git a/bgfilter/chroma.py b/bgfilter/chroma.py index 64d4afb..18a8827 100644 --- a/bgfilter/chroma.py +++ b/bgfilter/chroma.py @@ -112,22 +112,6 @@ def _background_border_cluster( return full, share, agree -def detect_background_color( - rgb: np.ndarray, settings: ChromaSettings | None = None -) -> tuple[float, float, float]: - """Detect a single flat background colour from the image border. - - Returns the colour as a 0..1 RGB triple. Raises RuntimeError (via - :func:`_background_border_cluster`) if the border is not dominated by one flat - colour, so callers fail loudly rather than matting against a wrong colour. - Pure: reads pixels only, with no matting side effects. - """ - settings = settings or ChromaSettings() - rgb_f, lab = convert_color_spaces(rgb) - mask, _, _ = _background_border_cluster(lab, settings) - return tuple(float(x) for x in np.median(rgb_f[mask], axis=0)) - - def estimate_background_model( rgb: np.ndarray, settings: ChromaSettings, diff --git a/bgfilter/foreground.py b/bgfilter/foreground.py index a26f2b3..b898761 100644 --- a/bgfilter/foreground.py +++ b/bgfilter/foreground.py @@ -1,7 +1,5 @@ from __future__ import annotations -from dataclasses import dataclass - import numpy as np from .chroma import BackgroundModel @@ -9,27 +7,16 @@ from .deps import require_cv2 from .settings import ForegroundSettings -@dataclass(frozen=True) -class ForegroundEstimate: - rgb: np.ndarray # estimated foreground colour F (uint8, HxWx3) - background: np.ndarray # estimated background colour B (uint8, HxWx3) - correction: np.ndarray # per-pixel |F - input| magnitude, 0..1 (float32, HxW) - - def estimate_foreground_rgb( rgb: np.ndarray, alpha: np.ndarray, bg_confidence: np.ndarray, model: BackgroundModel, settings: ForegroundSettings, -) -> ForegroundEstimate: +) -> np.ndarray: + """Estimated foreground colour F (uint8 HxWx3) for compositing over alpha.""" if not settings.enabled: - bg = np.broadcast_to(np.asarray(model.rgb_center, dtype=np.float32), rgb.shape) - return ForegroundEstimate( - rgb=rgb.copy(), - background=np.clip(bg * 255.0, 0, 255).astype(np.uint8), - correction=np.zeros(alpha.shape, dtype=np.float32), - ) + return rgb.copy() method = settings.method if method == "ml": @@ -48,7 +35,7 @@ def estimate_foreground_rgb( def _estimate_ml( rgb: np.ndarray, alpha: np.ndarray, settings: ForegroundSettings -) -> ForegroundEstimate: +) -> np.ndarray: """Multi-level closed-form foreground/background estimation (pymatting). Solves image = alpha * F + (1 - alpha) * B with spatial smoothness, so the @@ -66,17 +53,8 @@ def _estimate_ml( image = rgb.astype(np.float64) / 255.0 a = np.clip(alpha.astype(np.float64), 0.0, 1.0) - foreground, background = estimate_foreground_ml( - image, a, regularization=settings.ml_regularization, return_background=True - ) - foreground = np.clip(foreground, 0.0, 1.0) - background = np.clip(background, 0.0, 1.0) - correction = np.linalg.norm(foreground - image, axis=2) / np.sqrt(3.0) - return ForegroundEstimate( - rgb=np.clip(foreground * 255.0, 0, 255).astype(np.uint8), - background=np.clip(background * 255.0, 0, 255).astype(np.uint8), - correction=correction.astype(np.float32), - ) + foreground = estimate_foreground_ml(image, a, regularization=settings.ml_regularization) + return np.clip(foreground * 255.0, 0, 255).astype(np.uint8) def _weighted_blur(values: np.ndarray, weights: np.ndarray, radius: int) -> np.ndarray: @@ -97,7 +75,7 @@ def _estimate_unmix( bg_confidence: np.ndarray, model: BackgroundModel, settings: ForegroundSettings, -) -> ForegroundEstimate: +) -> np.ndarray: """Legacy heuristic: edge-band alpha unmix plus local foreground blur.""" rgb_f = rgb.astype(np.float32) / 255.0 r = rgb_f[..., 0] @@ -131,10 +109,4 @@ def _estimate_unmix( local_mask = base_mask * settings.local_strength * low_alpha_weight out = out * (1.0 - local_mask[..., None]) + local_fg * local_mask[..., None] - background = np.broadcast_to(bg, rgb_f.shape) - correction = np.linalg.norm(out - rgb_f, axis=2) / np.sqrt(3.0) - return ForegroundEstimate( - rgb=np.clip(out * 255.0, 0, 255).astype(np.uint8), - background=np.clip(background * 255.0, 0, 255).astype(np.uint8), - correction=correction.astype(np.float32), - ) + return np.clip(out * 255.0, 0, 255).astype(np.uint8) diff --git a/bgfilter/pipeline.py b/bgfilter/pipeline.py index 348df37..ce27d01 100644 --- a/bgfilter/pipeline.py +++ b/bgfilter/pipeline.py @@ -178,7 +178,7 @@ def _process_rgb(rgb: np.ndarray, pipeline: MattingPipeline) -> MattingResult: foreground = estimate_foreground_rgb( rgb, alpha, bg_confidence, model, settings.foreground ) - corrected_rgb, color_mask = despill(foreground.rgb, alpha, model, settings.despill) + corrected_rgb, color_mask = despill(foreground, alpha, model, settings.despill) metadata = { "alpha_source": alpha_source,