feat(jump-hop): optimize generated assets and runtime background

This commit is contained in:
2026-06-04 22:34:19 +08:00
parent c442c3c3f0
commit 0041b95f72
17 changed files with 1160 additions and 200 deletions

View File

@@ -2,13 +2,80 @@ use super::color::{
GENERATED_ASSET_SHEET_GREEN_SCREEN_MIN_SCORE, GENERATED_ASSET_SHEET_GREEN_SCREEN_SOFT_SCORE,
GENERATED_ASSET_SHEET_HARD_GREEN_SCREEN_SCORE, clamp_generated_asset_sheet_unit,
compute_generated_asset_sheet_green_screen_score,
compute_generated_asset_sheet_key_color_score,
compute_generated_asset_sheet_white_screen_score,
is_generated_asset_sheet_soft_green_matte_pixel, lerp_generated_asset_sheet_channel,
touches_generated_asset_sheet_background_mask,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GeneratedAssetSheetKeyColor {
pub red: u8,
pub green: u8,
pub blue: u8,
}
impl GeneratedAssetSheetKeyColor {
pub const GREEN_SCREEN: Self = Self {
red: 0,
green: 255,
blue: 0,
};
pub const MAGENTA_SCREEN: Self = Self {
red: 255,
green: 0,
blue: 255,
};
pub fn is_green_screen(self) -> bool {
self == Self::GREEN_SCREEN
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GeneratedAssetSheetAlphaOptions {
pub key_color: GeneratedAssetSheetKeyColor,
pub remove_near_white_background: bool,
pub remove_disconnected_hard_key_background: bool,
}
impl GeneratedAssetSheetAlphaOptions {
pub const fn green_screen() -> Self {
Self {
key_color: GeneratedAssetSheetKeyColor::GREEN_SCREEN,
remove_near_white_background: true,
remove_disconnected_hard_key_background: true,
}
}
pub const fn jump_hop_magenta_screen() -> Self {
Self {
key_color: GeneratedAssetSheetKeyColor::MAGENTA_SCREEN,
remove_near_white_background: false,
remove_disconnected_hard_key_background: false,
}
}
}
impl Default for GeneratedAssetSheetAlphaOptions {
fn default() -> Self {
Self::green_screen()
}
}
pub fn apply_generated_asset_sheet_green_screen_alpha(
source: image::DynamicImage,
) -> image::DynamicImage {
apply_generated_asset_sheet_alpha_with_options(
source,
GeneratedAssetSheetAlphaOptions::default(),
)
}
pub fn apply_generated_asset_sheet_alpha_with_options(
source: image::DynamicImage,
options: GeneratedAssetSheetAlphaOptions,
) -> image::DynamicImage {
let mut image = source.to_rgba8();
let (width, height) = image.dimensions();
@@ -16,6 +83,7 @@ pub fn apply_generated_asset_sheet_green_screen_alpha(
image.as_mut(),
width as usize,
height as usize,
options,
);
image::DynamicImage::ImageRgba8(image)
}
@@ -24,13 +92,14 @@ fn remove_generated_asset_sheet_green_screen_background(
pixels: &mut [u8],
width: usize,
height: usize,
options: GeneratedAssetSheetAlphaOptions,
) -> bool {
let pixel_count = width.saturating_mul(height);
if pixel_count == 0 || pixels.len() < pixel_count.saturating_mul(4) {
return false;
}
let mut green_scores = vec![0.0f32; pixel_count];
let mut key_scores = vec![0.0f32; pixel_count];
let mut white_scores = vec![0.0f32; pixel_count];
let mut background_hints = vec![0.0f32; pixel_count];
let mut background_mask = vec![0u8; pixel_count];
@@ -43,16 +112,19 @@ fn remove_generated_asset_sheet_green_screen_background(
let green = pixels[offset + 1];
let blue = pixels[offset + 2];
let alpha = pixels[offset + 3];
let green_score =
compute_generated_asset_sheet_green_screen_score([red, green, blue, alpha]);
let white_score =
compute_generated_asset_sheet_white_screen_score([red, green, blue, alpha]);
let key_score =
compute_generated_asset_sheet_key_score([red, green, blue, alpha], options.key_color);
let white_score = if options.remove_near_white_background {
compute_generated_asset_sheet_white_screen_score([red, green, blue, alpha])
} else {
0.0
};
let transparency_hint =
clamp_generated_asset_sheet_unit((56.0 - alpha as f32) / 56.0) * 0.75;
green_scores[pixel_index] = green_score;
key_scores[pixel_index] = key_score;
white_scores[pixel_index] = white_score;
background_hints[pixel_index] = green_score.max(white_score).max(transparency_hint);
background_hints[pixel_index] = key_score.max(white_score).max(transparency_hint);
}
let seed_background_pixel =
@@ -62,10 +134,10 @@ fn remove_generated_asset_sheet_green_screen_background(
}
let alpha = pixels[pixel_index * 4 + 3];
let strong_candidate = alpha < 40
|| green_scores[pixel_index] >= GENERATED_ASSET_SHEET_HARD_GREEN_SCREEN_SCORE
|| key_scores[pixel_index] >= GENERATED_ASSET_SHEET_HARD_GREEN_SCREEN_SCORE
|| (alpha < 224
&& green_scores[pixel_index] > GENERATED_ASSET_SHEET_GREEN_SCREEN_MIN_SCORE)
|| white_scores[pixel_index] > 0.32;
&& key_scores[pixel_index] > GENERATED_ASSET_SHEET_GREEN_SCREEN_MIN_SCORE)
|| (options.remove_near_white_background && white_scores[pixel_index] > 0.32);
if !strong_candidate {
return;
}
@@ -113,26 +185,34 @@ fn remove_generated_asset_sheet_green_screen_background(
}
let next_offset = next_pixel_index * 4;
let alpha = pixels[next_offset + 3];
let green_score = green_scores[next_pixel_index];
let key_score = key_scores[next_pixel_index];
let white_score = white_scores[next_pixel_index];
let hint = background_hints[next_pixel_index];
let reachable_soft_edge = hint > 0.08
&& alpha < 224
&& (green_score > 0.04 || white_score > 0.08 || alpha < 180);
let green_background = green_score >= GENERATED_ASSET_SHEET_HARD_GREEN_SCREEN_SCORE
|| (alpha < 224 && green_score > GENERATED_ASSET_SHEET_GREEN_SCREEN_MIN_SCORE);
if alpha < 40 || green_background || white_score > 0.32 || reachable_soft_edge {
&& (key_score > 0.04
|| (options.remove_near_white_background && white_score > 0.08)
|| alpha < 180);
let key_background = key_score >= GENERATED_ASSET_SHEET_HARD_GREEN_SCREEN_SCORE
|| (alpha < 224 && key_score > GENERATED_ASSET_SHEET_GREEN_SCREEN_MIN_SCORE);
if alpha < 40
|| key_background
|| (options.remove_near_white_background && white_score > 0.32)
|| reachable_soft_edge
{
background_mask[next_pixel_index] = 1;
queue.push(next_pixel_index);
}
}
}
for pixel_index in 0..pixel_count {
if background_mask[pixel_index] == 0
&& green_scores[pixel_index] >= GENERATED_ASSET_SHEET_HARD_GREEN_SCREEN_SCORE
{
background_mask[pixel_index] = 1;
if options.remove_disconnected_hard_key_background {
for pixel_index in 0..pixel_count {
if background_mask[pixel_index] == 0
&& key_scores[pixel_index] >= GENERATED_ASSET_SHEET_HARD_GREEN_SCREEN_SCORE
{
background_mask[pixel_index] = 1;
}
}
}
@@ -153,10 +233,14 @@ fn remove_generated_asset_sheet_green_screen_background(
pixels[offset + 2],
pixels[offset + 3],
];
let green_score = green_scores[pixel_index];
let key_score = key_scores[pixel_index];
let white_score = white_scores[pixel_index];
if !is_generated_asset_sheet_soft_green_matte_pixel(pixel, green_score, white_score)
{
if !is_generated_asset_sheet_soft_key_matte_pixel(
pixel,
key_score,
white_score,
options,
) {
continue;
}
if !touches_generated_asset_sheet_background_mask(
@@ -188,12 +272,12 @@ fn remove_generated_asset_sheet_green_screen_background(
continue;
}
let alpha = pixels[pixel_index * 4 + 3];
let green_score = green_scores[pixel_index];
let key_score = key_scores[pixel_index];
let white_score = white_scores[pixel_index];
let hint = background_hints[pixel_index];
let soft_matte_candidate = alpha < 224
|| white_score > 0.10
|| green_score >= GENERATED_ASSET_SHEET_HARD_GREEN_SCREEN_SCORE;
|| (options.remove_near_white_background && white_score > 0.10)
|| key_score >= GENERATED_ASSET_SHEET_HARD_GREEN_SCREEN_SCORE;
if hint < GENERATED_ASSET_SHEET_GREEN_SCREEN_SOFT_SCORE || !soft_matte_candidate {
continue;
}
@@ -278,9 +362,9 @@ fn remove_generated_asset_sheet_green_screen_background(
continue;
}
let green_score = green_scores[pixel_index];
let key_score = key_scores[pixel_index];
let white_score = white_scores[pixel_index];
let contamination = green_score.max(white_score).max(if alpha < 220 {
let contamination = key_score.max(white_score).max(if alpha < 220 {
((220 - alpha) as f32 / 220.0) * 0.25
} else {
0.0
@@ -308,23 +392,23 @@ fn remove_generated_asset_sheet_green_screen_background(
green = lerp_generated_asset_sheet_channel(green, sample_green as f32, blend);
blue = lerp_generated_asset_sheet_channel(blue, sample_blue as f32, blend);
if green_score > 0.04 {
if options.key_color.is_green_screen() && key_score > 0.04 {
green = green.min(sample_green as f32 + 18.0);
}
if white_score > 0.1 {
if options.remove_near_white_background && white_score > 0.1 {
red = red.min(sample_red as f32 + 26.0);
green = green.min(sample_green as f32 + 26.0);
blue = blue.min(sample_blue as f32 + 26.0);
}
} else {
if green_score > 0.04 {
if options.key_color.is_green_screen() && key_score > 0.04 {
let toned_green = (green - (green - red.max(blue)) * 0.78)
.round()
.max(red.max(blue));
green = green.min(toned_green).min(red.max(blue) + 18.0);
}
if white_score > 0.12 {
if options.remove_near_white_background && white_score > 0.12 {
let spread = red.max(green).max(blue) - red.min(green).min(blue);
if spread < 20.0 {
let toned_value = ((red + green + blue) / 3.0 * 0.88).round();
@@ -336,7 +420,7 @@ fn remove_generated_asset_sheet_green_screen_background(
}
let mut next_alpha = alpha;
let edge_fade = (green_score * 0.35).max(white_score * 0.28);
let edge_fade = (key_score * 0.35).max(white_score * 0.28);
if edge_fade > 0.08 {
next_alpha = ((alpha as f32) * (1.0 - edge_fade)).round() as u8;
if next_alpha < 10 {
@@ -364,6 +448,35 @@ fn remove_generated_asset_sheet_green_screen_background(
changed
}
fn compute_generated_asset_sheet_key_score(
pixel: [u8; 4],
key_color: GeneratedAssetSheetKeyColor,
) -> f32 {
if key_color.is_green_screen() {
return compute_generated_asset_sheet_green_screen_score(pixel);
}
compute_generated_asset_sheet_key_color_score(
pixel,
[key_color.red, key_color.green, key_color.blue],
)
}
fn is_generated_asset_sheet_soft_key_matte_pixel(
pixel: [u8; 4],
key_score: f32,
white_score: f32,
options: GeneratedAssetSheetAlphaOptions,
) -> bool {
if options.key_color.is_green_screen() {
return is_generated_asset_sheet_soft_green_matte_pixel(pixel, key_score, white_score);
}
pixel[3] != 0
&& key_score >= GENERATED_ASSET_SHEET_GREEN_SCREEN_MIN_SCORE
&& (!options.remove_near_white_background || white_score < 0.34)
}
fn collect_generated_asset_sheet_foreground_neighbor_color(
pixels: &[u8],
width: usize,

View File

@@ -139,6 +139,24 @@ pub(super) fn compute_generated_asset_sheet_green_screen_score(pixel: [u8; 4]) -
.clamp(0.0, 1.0)
}
pub(super) fn compute_generated_asset_sheet_key_color_score(
pixel: [u8; 4],
key_color: [u8; 3],
) -> f32 {
if pixel[3] == 0 {
return 1.0;
}
let color_distance = (pixel[0] as f32 - key_color[0] as f32).abs()
+ (pixel[1] as f32 - key_color[1] as f32).abs()
+ (pixel[2] as f32 - key_color[2] as f32).abs();
if color_distance >= 180.0 {
return 0.0;
}
clamp_generated_asset_sheet_unit(1.0 - color_distance / 180.0)
}
pub(super) fn compute_generated_asset_sheet_white_screen_score(pixel: [u8; 4]) -> f32 {
if pixel[3] == 0 {
return 1.0;

View File

@@ -5,7 +5,10 @@ pub mod persist;
pub mod prompt;
pub mod sheet;
pub use alpha::apply_generated_asset_sheet_green_screen_alpha;
pub use alpha::{
GeneratedAssetSheetAlphaOptions, GeneratedAssetSheetKeyColor,
apply_generated_asset_sheet_alpha_with_options, apply_generated_asset_sheet_green_screen_alpha,
};
pub use error::GeneratedAssetSheetError;
pub use persist::{
GeneratedAssetSheetPersistInput, GeneratedAssetSheetPersistPrompt, GeneratedAssetSheetUpload,
@@ -14,5 +17,6 @@ pub use persist::{
pub use prompt::{GeneratedAssetSheetPromptInput, build_generated_asset_sheet_prompt};
pub use sheet::{
GeneratedAssetSheetSliceImage, crop_generated_asset_sheet_view_edge_matte,
slice_generated_asset_sheet, slice_generated_asset_sheet_two_items_per_row,
crop_generated_asset_sheet_view_edge_matte_with_options, slice_generated_asset_sheet,
slice_generated_asset_sheet_two_items_per_row,
};

View File

@@ -1,6 +1,9 @@
use super::alpha::apply_generated_asset_sheet_green_screen_alpha;
use super::alpha::{
GeneratedAssetSheetAlphaOptions, apply_generated_asset_sheet_green_screen_alpha,
};
use super::color::{
is_generated_asset_sheet_foreground_pixel,
compute_generated_asset_sheet_key_color_score,
compute_generated_asset_sheet_white_screen_score, is_generated_asset_sheet_foreground_pixel,
is_generated_asset_sheet_green_contaminated_edge_pixel,
is_generated_asset_sheet_soft_edge_pixel, is_generated_asset_sheet_strong_green_contamination,
is_generated_asset_sheet_view_background_pixel, is_generated_asset_sheet_visible_pixel,
@@ -130,10 +133,25 @@ pub fn slice_generated_asset_sheet_two_items_per_row(
pub fn crop_generated_asset_sheet_view_edge_matte(
image: image::DynamicImage,
) -> image::DynamicImage {
crop_generated_asset_sheet_view_edge_matte_with_options(
image,
GeneratedAssetSheetAlphaOptions::default(),
)
}
pub fn crop_generated_asset_sheet_view_edge_matte_with_options(
image: image::DynamicImage,
options: GeneratedAssetSheetAlphaOptions,
) -> image::DynamicImage {
let mut image = image.to_rgba8();
let (width, height) = image.dimensions();
remove_generated_asset_sheet_view_edge_matte(image.as_mut(), width as usize, height as usize);
remove_generated_asset_sheet_view_edge_matte(
image.as_mut(),
width as usize,
height as usize,
options,
);
let bounds = detect_generated_asset_sheet_visible_bounds(&image).unwrap_or_else(|| {
GeneratedAssetSheetCellBounds {
x0: 0,
@@ -359,6 +377,7 @@ fn remove_generated_asset_sheet_view_edge_matte(
pixels: &mut [u8],
width: usize,
height: usize,
options: GeneratedAssetSheetAlphaOptions,
) -> bool {
let pixel_count = width.saturating_mul(height);
if pixel_count == 0 || pixels.len() < pixel_count.saturating_mul(4) {
@@ -403,7 +422,7 @@ fn remove_generated_asset_sheet_view_edge_matte(
pixels[offset + 2],
pixels[offset + 3],
];
if !is_generated_asset_sheet_view_background_pixel(pixel) {
if !is_generated_asset_sheet_view_background_pixel_with_options(pixel, options) {
continue;
}
background_mask[pixel_index] = 1;
@@ -434,7 +453,7 @@ fn remove_generated_asset_sheet_view_edge_matte(
pixels[offset + 2],
pixels[offset + 3],
];
if !is_generated_asset_sheet_view_background_pixel(pixel) {
if !is_generated_asset_sheet_view_background_pixel_with_options(pixel, options) {
continue;
}
background_mask[next_pixel_index] = 1;
@@ -452,12 +471,15 @@ fn remove_generated_asset_sheet_view_edge_matte(
continue;
}
let offset = pixel_index * 4;
if !is_generated_asset_sheet_view_background_pixel([
pixels[offset],
pixels[offset + 1],
pixels[offset + 2],
pixels[offset + 3],
]) {
if !is_generated_asset_sheet_view_background_pixel_with_options(
[
pixels[offset],
pixels[offset + 1],
pixels[offset + 2],
pixels[offset + 3],
],
options,
) {
continue;
}
@@ -526,7 +548,7 @@ fn remove_generated_asset_sheet_view_edge_matte(
pixels[offset + 2],
pixels[offset + 3],
];
if !is_generated_asset_sheet_green_contaminated_edge_pixel(pixel) {
if !is_generated_asset_sheet_key_contaminated_edge_pixel(pixel, options) {
continue;
}
if !touches_generated_asset_sheet_background_mask(
@@ -539,7 +561,7 @@ fn remove_generated_asset_sheet_view_edge_matte(
continue;
}
if is_generated_asset_sheet_strong_green_contamination(pixel) {
if is_generated_asset_sheet_strong_key_contamination(pixel, options) {
pixels[offset] = 0;
pixels[offset + 1] = 0;
pixels[offset + 2] = 0;
@@ -559,6 +581,7 @@ fn remove_generated_asset_sheet_view_edge_matte(
y,
&background_mask,
&visible_mask,
options,
)
.unwrap_or((
pixels[offset],
@@ -605,6 +628,7 @@ fn collect_generated_asset_sheet_visible_neighbor_color(
y: usize,
background_mask: &[u8],
visible_mask: &[u8],
options: GeneratedAssetSheetAlphaOptions,
) -> Option<(u8, u8, u8)> {
let mut total_weight = 0.0f32;
let mut total_red = 0.0f32;
@@ -638,8 +662,9 @@ fn collect_generated_asset_sheet_visible_neighbor_color(
pixels[next_offset + 2],
next_alpha,
];
if is_generated_asset_sheet_green_contaminated_edge_pixel(pixel)
|| is_generated_asset_sheet_soft_edge_pixel(pixel)
if is_generated_asset_sheet_key_contaminated_edge_pixel(pixel, options)
|| (options.key_color.is_green_screen()
&& is_generated_asset_sheet_soft_edge_pixel(pixel))
{
continue;
}
@@ -670,3 +695,73 @@ fn collect_generated_asset_sheet_visible_neighbor_color(
(total_blue / total_weight).round() as u8,
))
}
fn is_generated_asset_sheet_view_background_pixel_with_options(
pixel: [u8; 4],
options: GeneratedAssetSheetAlphaOptions,
) -> bool {
if options.key_color.is_green_screen() && options.remove_near_white_background {
return is_generated_asset_sheet_view_background_pixel(pixel);
}
if pixel[3] < 16 {
return true;
}
if options.key_color.is_green_screen() && is_generated_asset_sheet_soft_edge_pixel(pixel) {
return true;
}
if !options.key_color.is_green_screen()
&& compute_generated_asset_sheet_key_color_score(
pixel,
[
options.key_color.red,
options.key_color.green,
options.key_color.blue,
],
) > 0.18
{
return true;
}
options.remove_near_white_background
&& compute_generated_asset_sheet_white_screen_score(pixel) > 0.18
}
fn is_generated_asset_sheet_key_contaminated_edge_pixel(
pixel: [u8; 4],
options: GeneratedAssetSheetAlphaOptions,
) -> bool {
if options.key_color.is_green_screen() {
return is_generated_asset_sheet_green_contaminated_edge_pixel(pixel);
}
pixel[3] != 0
&& compute_generated_asset_sheet_key_color_score(
pixel,
[
options.key_color.red,
options.key_color.green,
options.key_color.blue,
],
) > 0.18
}
fn is_generated_asset_sheet_strong_key_contamination(
pixel: [u8; 4],
options: GeneratedAssetSheetAlphaOptions,
) -> bool {
if options.key_color.is_green_screen() {
return is_generated_asset_sheet_strong_green_contamination(pixel);
}
compute_generated_asset_sheet_key_color_score(
pixel,
[
options.key_color.red,
options.key_color.green,
options.key_color.blue,
],
) > 0.62
}

View File

@@ -2,9 +2,11 @@ use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use image::{DynamicImage, ImageFormat, Rgba, RgbaImage};
use platform_image::DownloadedImage;
use platform_image::generated_asset_sheets::{
GeneratedAssetSheetPersistInput, GeneratedAssetSheetPersistPrompt,
GeneratedAssetSheetPromptInput, apply_generated_asset_sheet_green_screen_alpha,
GeneratedAssetSheetAlphaOptions, GeneratedAssetSheetPersistInput,
GeneratedAssetSheetPersistPrompt, GeneratedAssetSheetPromptInput,
apply_generated_asset_sheet_alpha_with_options, apply_generated_asset_sheet_green_screen_alpha,
build_generated_asset_sheet_prompt, crop_generated_asset_sheet_view_edge_matte,
crop_generated_asset_sheet_view_edge_matte_with_options,
prepare_generated_asset_sheet_put_request, slice_generated_asset_sheet,
slice_generated_asset_sheet_two_items_per_row,
};
@@ -142,6 +144,68 @@ fn generated_asset_sheet_green_screen_alpha_removes_green_background() {
assert_eq!(cleaned.get_pixel(10, 10).0[3], 255);
}
#[test]
fn generated_asset_sheet_magenta_key_preserves_green_white_and_disconnected_key_subject() {
let mut sheet = RgbaImage::from_pixel(28, 28, Rgba([255, 0, 255, 255]));
for y in 6..22 {
for x in 6..14 {
sheet.put_pixel(x, y, Rgba([64, 188, 74, 255]));
}
}
for y in 6..22 {
for x in 14..22 {
sheet.put_pixel(x, y, Rgba([244, 244, 236, 255]));
}
}
for y in 12..16 {
for x in 12..16 {
sheet.put_pixel(x, y, Rgba([255, 0, 255, 255]));
}
}
let cleaned = apply_generated_asset_sheet_alpha_with_options(
DynamicImage::ImageRgba8(sheet),
GeneratedAssetSheetAlphaOptions::jump_hop_magenta_screen(),
)
.to_rgba8();
assert_eq!(cleaned.get_pixel(0, 0).0[3], 0);
assert_eq!(cleaned.get_pixel(8, 8).0[3], 255);
assert_eq!(cleaned.get_pixel(18, 8).0[3], 255);
assert_eq!(
cleaned.get_pixel(13, 13).0[3],
255,
"非边缘连通的 key 色像素不应被当成背景清掉"
);
}
#[test]
fn generated_asset_sheet_magenta_edge_matte_does_not_remove_white_subject() {
let mut sheet = RgbaImage::from_pixel(24, 24, Rgba([0, 0, 0, 0]));
for y in 2..22 {
for x in 2..22 {
sheet.put_pixel(x, y, Rgba([246, 246, 240, 255]));
}
}
for y in 0..24 {
sheet.put_pixel(0, y, Rgba([255, 0, 255, 255]));
sheet.put_pixel(23, y, Rgba([255, 0, 255, 255]));
}
let cleaned = crop_generated_asset_sheet_view_edge_matte_with_options(
DynamicImage::ImageRgba8(sheet),
GeneratedAssetSheetAlphaOptions::jump_hop_magenta_screen(),
)
.to_rgba8();
assert_eq!(cleaned.get_pixel(1, 1).0[3], 255);
assert!(
cleaned
.pixels()
.any(|pixel| pixel.0 == [246, 246, 240, 255])
);
}
#[test]
fn generated_asset_sheet_view_edge_matte_trims_transparent_border() {
let mut sheet = RgbaImage::from_pixel(20, 20, Rgba([0, 0, 0, 0]));