修复图片画布素材与宣发尺寸问题

修复画布上传素材持久化回填和项目重进资源元数据恢复

修复图片、音频、视频图层的私有资源换签与视频裸路径回退

修复宣发素材切换模型后卡片尺寸变成1024x1024的问题

修复音视频预览、媒体导出格式、音频默认提示词和前端去背边界

修复角色动作生成在编辑器路径缺少ffprobe时的已知时长兜底

补充图片画布、资源换签、宣发尺寸、媒体导出和chromaKey回归测试

更新图片画布编辑器技术文档
This commit is contained in:
2026-06-19 23:16:59 +08:00
parent 2be7eab8a7
commit 10dcc2b403
37 changed files with 1130 additions and 167 deletions
File diff suppressed because one or more lines are too long
@@ -47,6 +47,7 @@
- 承载中央画布工作区的视觉树:viewport / world DOM、图层渲染、生成占位框、选中图片浮动工具栏、空白和图片右键菜单、左下 dock、缩放菜单、背景设置面板、小地图和底部 AI 工具栏。
- 继续通过 props 调用主视图状态机,不接管拖拽 / 平移 / 缩放、画布坐标换算、历史 undo / redo、上传、登录、生成提交、素材持久化和右键命令实现。
- 保持 `canvasViewportRef` 由主视图传入,确保 pointer capture、drop 坐标、滚轮缩放和小地图拖拽仍使用同一套坐标源。
- 图片图层渲染必须和音频、视频一样走统一资源读取解析;图层存在 `objectKey` 时优先用它换签,避免重进项目后私有上传 / 生成资源只剩旧路径而无法显示。
第三阶段以后,主视图仍是画布编排入口。继续拆分前应优先选择能形成稳定边界的深模块,避免把上传链路、DataTransfer、画布坐标和历史快照拆成互相回调的小碎片。
@@ -84,6 +85,7 @@
- 承载图片画布工程持久化协调:项目加载、`projectId` 维护、未就绪资源队列、工程资源创建、资源创建后即时 layout 保存、450ms 自动保存和鉴权失败登录弹窗。
- 该 hook 以“项目持久化协调器”整体抽出,避免把加载、保存和资源创建拆成多个小 hook 后打散 `projectIdRef``pendingProjectResourceLayersRef``isProjectReady``saveTimerRef` 的时序约束。
- 主视图继续负责项目重命名 UI、素材库管理、上传流程和用户动作触发;新增图层仍通过 `appendCanvasLayersWithResources` 先写本地图层快照,再创建 project resource 并保存带真实 `resourceId` 的 layout。
- 项目加载 hydrate 时必须从 project resource 回填 `objectKey``assetObjectId``sourceResourceId` 等资源元数据;layout 快照保持轻量,但重进项目后的画布图层仍要保留换签和后续编辑所需线索。
## 第九阶段模块
@@ -0,0 +1,67 @@
import { describe, expect, it } from 'vitest';
import { removeBackgroundFromRgba } from './chromaKey';
function createSolidRgbaBuffer({
width,
height,
color,
}: {
width: number;
height: number;
color: [number, number, number, number];
}) {
const pixels = new Uint8ClampedArray(width * height * 4);
for (let index = 0; index < width * height; index += 1) {
pixels.set(color, index * 4);
}
return pixels;
}
function setPixel(
pixels: Uint8ClampedArray,
width: number,
x: number,
y: number,
color: [number, number, number, number],
) {
pixels.set(color, (y * width + x) * 4);
}
function alphaAt(
pixels: Uint8ClampedArray,
width: number,
x: number,
y: number,
) {
return pixels[(y * width + x) * 4 + 3] ?? 0;
}
describe('chromaKey', () => {
it('removes near-white canvas background without breaking an enclosed white character', () => {
const width = 9;
const height = 9;
const pixels = createSolidRgbaBuffer({
width,
height,
color: [250, 250, 250, 255],
});
for (let y = 2; y <= 6; y += 1) {
for (let x = 2; x <= 6; x += 1) {
setPixel(pixels, width, x, y, [92, 80, 72, 255]);
}
}
for (let y = 3; y <= 5; y += 1) {
for (let x = 3; x <= 5; x += 1) {
setPixel(pixels, width, x, y, [246, 246, 244, 255]);
}
}
expect(removeBackgroundFromRgba(pixels, width, height)).toBe(true);
expect(alphaAt(pixels, width, 0, 0)).toBe(0);
expect(alphaAt(pixels, width, 4, 4)).toBe(255);
expect(alphaAt(pixels, width, 2, 2)).toBeGreaterThan(200);
});
});
@@ -1661,6 +1661,7 @@ async fn extract_and_persist_editor_character_animation_frames(
request.frame_height,
extraction_settings,
&plan,
Some(request.duration_seconds as f64),
)
.await?;
@@ -1823,6 +1824,7 @@ async fn publish_single_animation_action(
frame_height,
extraction_settings,
&frame_plan,
None,
)
.await?
} else {
@@ -3058,6 +3060,7 @@ async fn extract_animation_frames_from_preview_video(
frame_height: u32,
extraction_settings: &BackendFrameExtractionSettings,
plan: &AnimationFrameExtractionPlan,
known_duration_seconds: Option<f64>,
) -> Result<Vec<FinalizedAnimationFrame>, AppError> {
let preview_payload = load_media_source_payload(state, preview_video_path).await?;
if !preview_payload.mime_type.starts_with("video/") {
@@ -3079,8 +3082,12 @@ async fn extract_animation_frames_from_preview_video(
})?;
let extraction_result = (|| {
let duration_seconds =
probe_video_duration_seconds(&input_path, extraction_settings)?.max(0.001);
let duration_seconds = probe_video_duration_seconds_or_known(
&input_path,
extraction_settings,
known_duration_seconds,
)?
.max(0.001);
let mut finalized_frames = Vec::with_capacity(plan.frame_count as usize);
for frame_index in 0..plan.frame_count {
let target_seconds = compute_sample_time_seconds(
@@ -3161,6 +3168,33 @@ fn probe_video_duration_seconds(
})
}
fn probe_video_duration_seconds_or_known(
input_path: &Path,
extraction_settings: &BackendFrameExtractionSettings,
known_duration_seconds: Option<f64>,
) -> Result<f64, AppError> {
match probe_video_duration_seconds(input_path, extraction_settings) {
Ok(duration_seconds) => Ok(duration_seconds),
Err(error)
if is_ffprobe_start_error(&error, extraction_settings.ffprobe_path.as_str()) =>
{
if let Some(duration_seconds) = known_duration_seconds
.filter(|duration_seconds| duration_seconds.is_finite() && *duration_seconds > 0.0)
{
return Ok(duration_seconds);
}
Err(error)
}
Err(error) => Err(error),
}
}
fn is_ffprobe_start_error(error: &AppError, ffprobe_path: &str) -> bool {
error.status_code() == StatusCode::SERVICE_UNAVAILABLE
&& error.body_text().contains("无法启动进程")
&& error.body_text().contains(ffprobe_path)
}
fn compute_sample_time_seconds(
duration_seconds: f64,
frame_index: u32,
@@ -4842,6 +4876,42 @@ mod tests {
);
}
#[test]
fn editor_character_animation_uses_known_duration_when_ffprobe_is_missing() {
let settings = BackendFrameExtractionSettings {
ffmpeg_path: "ffmpeg".to_string(),
ffprobe_path: "__missing_ffprobe_for_editor_animation_test__".to_string(),
timeout_ms: 1_000,
};
let duration = probe_video_duration_seconds_or_known(
Path::new("preview.mp4"),
&settings,
Some(6.0),
)
.expect("known duration should cover a missing ffprobe binary");
assert_eq!(duration, 6.0);
}
#[test]
fn editor_character_animation_keeps_ffprobe_error_without_known_duration() {
let settings = BackendFrameExtractionSettings {
ffmpeg_path: "ffmpeg".to_string(),
ffprobe_path: "__missing_ffprobe_for_editor_animation_test__".to_string(),
timeout_ms: 1_000,
};
let error = probe_video_duration_seconds_or_known(
Path::new("preview.mp4"),
&settings,
None,
)
.expect_err("missing ffprobe should still fail without a trusted duration");
assert!(error.body_text().contains("无法启动进程"));
}
#[test]
fn editor_video_normalizes_lovart_model_contract() {
let normalized = normalize_editor_video_request(EditorVideoGenerateRequest {
@@ -852,17 +852,17 @@ pub async fn generate_editor_image(
payload.image_size.as_deref(),
);
let has_dimension_options = payload.aspect_ratio.is_some() || payload.image_size.is_some();
let legacy_size = normalize_editor_image_generation_size(payload.size.as_deref());
let image_size = if has_dimension_options {
Cow::Owned(generation_options.size.clone())
} else {
legacy_size
};
let normalized_kind = payload.kind.as_deref().map(str::trim);
let is_character_generation = matches!(normalized_kind, Some("character"));
let is_ui_design_generation = matches!(normalized_kind, Some("ui-design"));
let is_publication_material_generation =
matches!(normalized_kind, Some("publication-material"));
let image_size = resolve_editor_image_request_size(
normalized_kind,
payload.size.as_deref(),
has_dimension_options,
&generation_options,
);
let submitted_prompt = if is_character_generation {
build_editor_character_image_prompt(role_setting.as_str())
} else if is_ui_design_generation {
@@ -1085,6 +1085,25 @@ fn normalize_editor_image_generation_size(size: Option<&str>) -> Cow<'static, st
}
}
fn resolve_editor_image_request_size(
normalized_kind: Option<&str>,
payload_size: Option<&str>,
has_dimension_options: bool,
generation_options: &EditorGenerationOptions,
) -> Cow<'static, str> {
let legacy_size = normalize_editor_image_generation_size(payload_size);
let has_explicit_payload_size = payload_size
.map(str::trim)
.is_some_and(|value| !value.is_empty());
if matches!(normalized_kind, Some("publication-material")) && has_explicit_payload_size {
return legacy_size;
}
if has_dimension_options {
return Cow::Owned(generation_options.size.clone());
}
legacy_size
}
fn is_editor_custom_image_size(value: &str) -> bool {
let Some((width, height)) = value.split_once('x') else {
return false;
@@ -1137,6 +1156,7 @@ fn normalize_editor_generation_aspect_ratio(aspect_ratio: Option<&str>) -> &'sta
{
Some("2:3") => "2:3",
Some("3:2") => "3:2",
Some("4:3") => "4:3",
Some("9:16") => "9:16",
Some("16:9") => "16:9",
_ => "1:1",
@@ -1169,8 +1189,10 @@ fn editor_generation_size_for_model(model: &str, aspect_ratio: &str, image_size:
("2K", "1:1") => "2048x2048",
// 中文注释:gpt-image-2 文档未列出 2K 竖版,竖版选择回落到文档明确支持的 1K 竖版。
("2K", "2:3") | ("2K", "9:16") => "1024x1536",
("2K", "4:3") => "2048x1536",
("2K", "16:9") | ("2K", "3:2") => "2048x1152",
("1K", "2:3") | ("1K", "9:16") => "1024x1536",
("1K", "4:3") => "1536x1024",
("1K", "3:2") | ("1K", "16:9") => "1536x1024",
_ => "1024x1024",
}
@@ -2274,6 +2296,18 @@ mod tests {
assert_eq!(gpt.image_size, "1K");
assert_eq!(gpt.provider_image_size, "1K");
let nanobanana_cover =
normalize_editor_generation_options(Some("nanobanana2"), Some("4:3"), Some("1K"));
assert_eq!(nanobanana_cover.model, EDITOR_IMAGE_MODEL_NANOBANANA2);
assert_eq!(nanobanana_cover.size, "1024");
assert_eq!(nanobanana_cover.aspect_ratio, "4:3");
let gpt_cover =
normalize_editor_generation_options(Some("gpt-image-2"), Some("4:3"), Some("1K"));
assert_eq!(gpt_cover.model, GPT_IMAGE_2_MODEL);
assert_eq!(gpt_cover.size, "1536x1024");
assert_eq!(gpt_cover.aspect_ratio, "4:3");
let gpt_landscape_2k =
normalize_editor_generation_options(Some("gpt-image-2"), Some("16:9"), Some("2K"));
assert_eq!(gpt_landscape_2k.model, GPT_IMAGE_2_MODEL);
@@ -2300,6 +2334,26 @@ mod tests {
assert_eq!(fallback.provider_image_size, "1K");
}
#[test]
fn publication_material_request_size_keeps_workflow_pixels() {
let options =
normalize_editor_generation_options(Some("gpt-image-2"), Some("4:3"), Some("1K"));
assert_eq!(
resolve_editor_image_request_size(
Some("publication-material"),
Some("720x540"),
true,
&options,
),
"720x540"
);
assert_eq!(
resolve_editor_image_request_size(Some("generate"), Some("720x540"), true, &options),
"1536x1024"
);
}
#[test]
fn editor_ui_design_prompt_uses_fixed_user_input_block_and_optional_icon_spec() {
let prompt = build_editor_ui_design_prompt("二消玩法,主界面和结算弹窗", true);
@@ -143,5 +143,25 @@ describe('ImageCanvasAssetRowView', () => {
expect(
document.querySelector('.image-canvas-editor__asset-media-overlay'),
).toBeTruthy();
expect(
document.querySelector('.image-canvas-editor__media-preview--audio'),
).toBeTruthy();
});
it('uses a stable visual preview for video asset cards without image thumbnails', () => {
renderAssetRow({
asset: createAsset({
label: '生成视频.mp4',
src: '/generated-character-drafts/editor/asset-library/video/demo.mp4',
mediaType: 'video',
}),
});
expect(screen.getByText('视频')).toBeTruthy();
expect(screen.queryByRole('img', { name: '素材:生成视频.mp4' })).toBeNull();
expect(
document.querySelector('.image-canvas-editor__media-preview--video')
?.className,
).toContain('image-canvas-editor__media-preview--variant-');
});
});
@@ -16,6 +16,10 @@ import type {
AssetPointerDragState,
EditorAsset,
} from './ImageCanvasEditorTypes';
import {
getCanvasMediaPreviewClassName,
getCanvasMediaPreviewMarker,
} from './ImageCanvasMediaModel';
import type { UploadFilesOptions } from './ImageCanvasAssetLibraryPanelView';
export type ImageCanvasAssetRowViewProps = {
@@ -65,6 +69,9 @@ export function ImageCanvasAssetRowView({
const isUploadingAsset = asset.uploadStatus === 'uploading';
const isFailedUpload = asset.uploadStatus === 'failed';
const uploadProgress = clamp(asset.uploadProgress ?? 0, 0, 100);
const mediaPreviewMarker = getCanvasMediaPreviewMarker(asset);
const mediaPreviewClassName =
getCanvasMediaPreviewClassName(mediaPreviewMarker);
const mediaDetail =
asset.mediaType === 'audio'
? '音频'
@@ -73,11 +80,15 @@ export function ImageCanvasAssetRowView({
: `${asset.width} x ${asset.height}`;
const mediaOverlay =
asset.mediaType === 'audio' ? (
<div className="image-canvas-editor__asset-media-overlay">
<div
className={`image-canvas-editor__asset-media-overlay ${mediaPreviewClassName}`}
>
<Music2 className="h-5 w-5" />
</div>
) : asset.mediaType === 'video' ? (
<div className="image-canvas-editor__asset-media-overlay">
<div
className={`image-canvas-editor__asset-media-overlay ${mediaPreviewClassName}`}
>
<Video className="h-5 w-5" />
</div>
) : undefined;
@@ -151,7 +162,6 @@ export function ImageCanvasAssetRowView({
<SidebarMediaItem
title={asset.label}
detail={mediaDetail}
imageSrc={asset.mediaType === 'image' ? asset.src : ''}
imageAlt={`素材:${asset.label}`}
primaryLabel={
isUploadingAsset
@@ -188,6 +198,11 @@ export function ImageCanvasAssetRowView({
primaryClassName="image-canvas-editor__asset-button"
thumbnailClassName="image-canvas-editor__asset-thumb"
metaClassName="image-canvas-editor__asset-meta"
imageSrc={
asset.mediaType === 'image'
? asset.src
: (asset.thumbnailSrc ?? '')
}
titleNode={
isUploadingAsset || isFailedUpload ? <span>{asset.label}</span> : titleNode
}
@@ -195,6 +195,43 @@ describe('ImageCanvasEditorModel', () => {
expect(hydrated?.generationInputs?.fields[0]?.value).toBe('骑士');
});
it('hydrates object metadata from project resources when the saved layer is lean', () => {
const hydrated = hydrateLayer(
{
layerId: 'layer-uploaded',
resourceId: 'resource-uploaded',
title: '上传图',
x: 10,
y: 20,
width: 640,
height: 360,
originalWidth: 640,
originalHeight: 360,
zIndex: 2,
sourceType: 'uploaded',
},
new Map([
[
'resource-uploaded',
{
imageSrc: '/generated-character-drafts/editor/asset-library/image.png',
objectKey: 'generated-character-drafts/editor/asset-library/image.png',
assetObjectId: 'asset-object-uploaded',
sourceResourceId: 'resource-source',
},
],
]),
);
expect(hydrated).toMatchObject({
id: 'layer-uploaded',
src: '/generated-character-drafts/editor/asset-library/image.png',
objectKey: 'generated-character-drafts/editor/asset-library/image.png',
assetObjectId: 'asset-object-uploaded',
sourceResourceId: 'resource-source',
});
});
it('hydrates manually selected publication material tags from saved layout', () => {
const hydrated = hydrateLayer(
{
@@ -418,12 +418,15 @@ export function hydrateLayer(
model: stringOrNull(snapshot.model),
provider: stringOrNull(snapshot.provider),
taskId: stringOrNull(snapshot.taskId),
objectKey: stringOrNull(snapshot.objectKey),
assetObjectId: stringOrNull(snapshot.assetObjectId),
objectKey: stringOrNull(snapshot.objectKey) ?? stringOrNull(resource?.objectKey),
assetObjectId:
stringOrNull(snapshot.assetObjectId) ?? stringOrNull(resource?.assetObjectId),
durationSeconds:
audioDurationOrNull(snapshot.durationSeconds) ??
audioDurationOrNull(resource?.durationSeconds),
sourceResourceId: stringOrNull(snapshot.sourceResourceId),
sourceResourceId:
stringOrNull(snapshot.sourceResourceId) ??
stringOrNull(resource?.sourceResourceId),
sourceAssetId: stringOrNull(snapshot.sourceAssetId),
groupId: stringOrNull(snapshot.groupId),
assetKind:
@@ -507,7 +510,10 @@ function inferAudioAssetKindFromLabel(label: string): CanvasAssetKind {
export type CanvasLayerResourceMetadata = {
imageSrc: string;
objectKey?: string | null;
assetObjectId?: string | null;
durationSeconds?: number | null;
sourceResourceId?: string | null;
assetKind?: string | null;
generationInputs?: unknown;
};
@@ -29,6 +29,7 @@ export type EditorAsset = {
label: string;
src: string;
mediaType?: CanvasMediaType;
thumbnailSrc?: string | null;
width: number;
height: number;
folderId: string;
@@ -72,6 +73,7 @@ export type CanvasLayer = {
title: string;
src: string;
mediaType?: CanvasMediaType;
thumbnailSrc?: string | null;
x: number;
y: number;
width: number;
@@ -431,6 +433,7 @@ export type CanvasAssetExportMetadata = {
layerId: string;
title: string;
file: string | null;
mediaType?: CanvasMediaType | null;
sourceType: CanvasSourceType;
prompt?: string | null;
actualPrompt?: string | null;
@@ -582,7 +582,7 @@ describe('ImageCanvasEditorView', () => {
);
fireEvent.click(screen.getByRole('button', { name: '下载按钮' }));
expect(downloadName).toBe('拼图素材.png');
expect(downloadName).toBe('拼图素材.webp');
expect(downloadHref).toContain('/creation-type-references/puzzle.webp');
expect(screen.getByAltText('画布图片:拼图素材')).toBeTruthy();
expect(screen.getByAltText('画布图片:大鱼素材')).toBeTruthy();
@@ -1,11 +1,13 @@
import { describe, expect, it } from 'vitest';
import { describe, expect, it, vi } from 'vitest';
import {
buildLayerExportMetadata,
dataUrlToBlob,
formatExportDate,
getImageExtensionFromTypeOrSrc,
getLayerAssetExtensionFromTypeOrSrc,
getLayerExportKey,
readLayerAssetBlob,
sanitizeExportFilePart,
} from './ImageCanvasExportModel';
import type { CanvasLayer } from './ImageCanvasEditorTypes';
@@ -47,6 +49,20 @@ describe('ImageCanvasExportModel', () => {
);
expect(getImageExtensionFromTypeOrSrc('', '/image.webp?x=1')).toBe('webp');
expect(getImageExtensionFromTypeOrSrc('', '/image.unknown')).toBe('png');
expect(
getLayerAssetExtensionFromTypeOrSrc(
'video',
'video/mp4',
'/video.png',
),
).toBe('mp4');
expect(
getLayerAssetExtensionFromTypeOrSrc(
'audio',
'',
'/generated-audios/effect.wav?token=1',
),
).toBe('wav');
});
it('converts data URLs and builds layer metadata for manifest files', async () => {
@@ -58,6 +74,7 @@ describe('ImageCanvasExportModel', () => {
layerId: 'layer-1',
title: '导出图层',
file: 'images/001-layer.png',
mediaType: 'image',
sourceType: 'generated',
prompt: '生成提示',
actualPrompt: '实际提示',
@@ -85,9 +102,57 @@ describe('ImageCanvasExportModel', () => {
},
});
});
it('reads private object-key assets through signed URLs before exporting', async () => {
const originalFetch = globalThis.fetch;
const fetchMock = vi.fn(async (url: string) => {
if (url.startsWith('/api/assets/read-url?')) {
return new Response(
JSON.stringify({
ok: true,
data: {
read: {
objectKey: 'generated/video.mp4',
signedUrl: 'https://oss.example.com/generated/video.mp4?x-oss-signature=1',
expiresAt: '2026-06-20T00:00:00.000Z',
},
},
error: null,
meta: { apiVersion: '2026-06-16' },
}),
);
}
if (url === 'https://oss.example.com/generated/video.mp4?x-oss-signature=1') {
return new Response(new Blob(['video'], { type: 'video/mp4' }));
}
return new Response(null, { status: 404 });
});
globalThis.fetch = fetchMock as typeof fetch;
try {
const blob = await readLayerAssetBlob(
buildLayer({
src: '/generated-editor-videos/video.mp4',
mediaType: 'video',
objectKey: 'generated/video.mp4',
}),
);
expect(blob.type).toBe('video/mp4');
expect(fetchMock).toHaveBeenCalledWith(
expect.stringContaining('/api/assets/read-url?objectKey=generated%2Fvideo.mp4'),
expect.any(Object),
);
expect(fetchMock).toHaveBeenCalledWith(
'https://oss.example.com/generated/video.mp4?x-oss-signature=1',
);
} finally {
globalThis.fetch = originalFetch;
}
});
});
function buildLayer(): CanvasLayer {
function buildLayer(overrides: Partial<CanvasLayer> = {}): CanvasLayer {
return {
id: 'layer-1',
resourceId: 'resource-1',
@@ -114,5 +179,6 @@ function buildLayer(): CanvasLayer {
locked: false,
flipX: true,
flipY: false,
...overrides,
};
}
@@ -1,7 +1,12 @@
import type {
CanvasAssetExportMetadata,
CanvasMediaType,
CanvasLayer,
} from './ImageCanvasEditorTypes';
import {
getSignedAssetReadUrl,
resolveAssetReadUrl,
} from '../../services/assetReadUrlService';
export function sanitizeExportFilePart(value: string, fallback: string) {
const safeValue = value
@@ -30,15 +35,80 @@ export function getLayerExportKey(layer: CanvasLayer) {
}
export function getImageExtensionFromTypeOrSrc(type: string, src: string) {
if (type.includes('jpeg') || /\.(jpe?g)(?:[?#].*)?$/iu.test(src)) {
return getLayerAssetExtensionFromTypeOrSrc('image', type, src);
}
function getExtensionFromSrc(src: string) {
const withoutQuery = src.split(/[?#]/u)[0] ?? '';
const match = /\.([a-z0-9]+)$/iu.exec(withoutQuery);
return match?.[1]?.toLowerCase() ?? null;
}
function pickKnownExtension(
extension: string | null,
allowedExtensions: string[],
fallback: string,
) {
return extension && allowedExtensions.includes(extension) ? extension : fallback;
}
export function getLayerAssetExtensionFromTypeOrSrc(
mediaType: CanvasMediaType | undefined,
type: string,
src: string,
) {
const normalizedType = type.toLowerCase();
const extensionFromSrc = getExtensionFromSrc(src);
if (mediaType === 'audio') {
if (normalizedType.includes('mpeg') || normalizedType.includes('mp3')) {
return 'mp3';
}
if (normalizedType.includes('wav')) {
return 'wav';
}
if (normalizedType.includes('ogg')) {
return 'ogg';
}
if (normalizedType.includes('aac')) {
return 'aac';
}
if (normalizedType.includes('flac')) {
return 'flac';
}
return pickKnownExtension(
extensionFromSrc,
['mp3', 'wav', 'ogg', 'aac', 'flac', 'm4a'],
'mp3',
);
}
if (mediaType === 'video') {
if (normalizedType.includes('mp4')) {
return 'mp4';
}
if (normalizedType.includes('webm')) {
return 'webm';
}
if (normalizedType.includes('quicktime')) {
return 'mov';
}
return pickKnownExtension(extensionFromSrc, ['mp4', 'webm', 'mov'], 'mp4');
}
if (
normalizedType.includes('jpeg') ||
extensionFromSrc === 'jpg' ||
extensionFromSrc === 'jpeg'
) {
return 'jpg';
}
if (type.includes('webp') || /\.webp(?:[?#].*)?$/iu.test(src)) {
if (normalizedType.includes('webp') || extensionFromSrc === 'webp') {
return 'webp';
}
if (type.includes('gif') || /\.gif(?:[?#].*)?$/iu.test(src)) {
if (normalizedType.includes('gif') || extensionFromSrc === 'gif') {
return 'gif';
}
if (normalizedType.includes('svg') || extensionFromSrc === 'svg') {
return 'svg';
}
return 'png';
}
@@ -59,17 +129,26 @@ export function dataUrlToBlob(dataUrl: string) {
return new Blob([bytes], { type });
}
export async function readLayerImageBlob(layer: CanvasLayer) {
export async function readLayerAssetBlob(layer: CanvasLayer) {
if (layer.src.startsWith('data:')) {
return dataUrlToBlob(layer.src);
}
const response = await fetch(layer.src);
const source = layer.objectKey
? await getSignedAssetReadUrl(
{ objectKey: layer.objectKey },
undefined,
{ cacheVersion: layer.taskId ?? layer.resourceId },
)
: await resolveAssetReadUrl(layer.src);
const response = await fetch(source);
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
return response.blob();
}
export const readLayerImageBlob = readLayerAssetBlob;
export async function blobToUint8Array(blob: Blob) {
if (typeof blob.arrayBuffer === 'function') {
return new Uint8Array(await blob.arrayBuffer());
@@ -98,6 +177,7 @@ export function buildLayerExportMetadata(
layerId: layer.id,
title: layer.title,
file,
mediaType: layer.mediaType ?? 'image',
sourceType: layer.sourceType,
prompt: layer.prompt,
actualPrompt: layer.actualPrompt,
@@ -225,6 +225,8 @@ describe('ImageCanvasGenerationDialogModel', () => {
expect(createSoundEffectGenerationDialogDraft({ canvasSize, viewport }))
.toMatchObject({
mode: 'audio-sound-effect',
prompt:
'短促清脆的游戏交互音效,带轻微闪光质感,干净收尾,适合按钮确认或收集奖励。',
status: 'idle',
composerOpen: true,
soundModel: 'audio1.0',
@@ -241,6 +243,8 @@ describe('ImageCanvasGenerationDialogModel', () => {
expect(createBackgroundMusicGenerationDialogDraft({ canvasSize, viewport }))
.toMatchObject({
mode: 'audio-background-music',
prompt:
'适合轻松冒险游戏的循环背景音乐,温暖明亮,旋律简洁,不包含人声。',
status: 'idle',
composerOpen: true,
makeInstrumental: true,
@@ -311,6 +315,37 @@ describe('ImageCanvasGenerationDialogModel', () => {
expect(createAudioRedrawGenerationDialogDraft(createLayer())).toBeNull();
});
it('uses audio prompt templates when remodeling old audio layers without prompts', () => {
expect(
createAudioRedrawGenerationDialogDraft(
createLayer({
mediaType: 'audio',
assetKind: 'sound-effect',
prompt: '',
generationInputs: { fields: [], references: [] },
}),
),
).toMatchObject({
mode: 'audio-sound-effect',
prompt:
'短促清脆的游戏交互音效,带轻微闪光质感,干净收尾,适合按钮确认或收集奖励。',
});
expect(
createAudioRedrawGenerationDialogDraft(
createLayer({
mediaType: 'audio',
assetKind: 'background-music',
title: '游戏背景音乐',
prompt: '',
generationInputs: { fields: [], references: [] },
}),
),
).toMatchObject({
mode: 'audio-background-music',
prompt: '适合轻松冒险游戏的循环背景音乐,温暖明亮,旋律简洁,不包含人声。',
});
});
it('creates edit, quick-edit, and character animation panel drafts', () => {
const sourceLayer = createLayer({
prompt: '原图提示',
@@ -40,6 +40,10 @@ const VIDEO_REFERENCE_LIMITS = {
video: 3,
audio: 3,
} as const;
const DEFAULT_SOUND_EFFECT_PROMPT =
'短促清脆的游戏交互音效,带轻微闪光质感,干净收尾,适合按钮确认或收集奖励。';
const DEFAULT_BACKGROUND_MUSIC_PROMPT =
'适合轻松冒险游戏的循环背景音乐,温暖明亮,旋律简洁,不包含人声。';
function getViewportWorldCenter({
canvasSize,
@@ -352,7 +356,7 @@ export function createSoundEffectGenerationDialogDraft({
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
return {
mode: 'audio-sound-effect',
prompt: '',
prompt: DEFAULT_SOUND_EFFECT_PROMPT,
status: 'idle',
composerOpen: true,
soundModel: DEFAULT_SOUND_EFFECT_MODEL,
@@ -378,7 +382,7 @@ export function createBackgroundMusicGenerationDialogDraft({
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
return {
mode: 'audio-background-music',
prompt: '',
prompt: DEFAULT_BACKGROUND_MUSIC_PROMPT,
status: 'idle',
composerOpen: true,
makeInstrumental: true,
@@ -405,7 +409,24 @@ function findGenerationInputFieldValue(
);
}
function resolveAudioRedrawPrompt(sourceLayer: CanvasLayer) {
function resolveDefaultAudioPrompt(
mode: Extract<
GenerateDialogState['mode'],
'audio-sound-effect' | 'audio-background-music'
>,
) {
return mode === 'audio-background-music'
? DEFAULT_BACKGROUND_MUSIC_PROMPT
: DEFAULT_SOUND_EFFECT_PROMPT;
}
function resolveAudioRedrawPrompt(
sourceLayer: CanvasLayer,
mode: Extract<
GenerateDialogState['mode'],
'audio-sound-effect' | 'audio-background-music'
>,
) {
return (
sourceLayer.prompt?.trim() ||
findGenerationInputFieldValue(sourceLayer, [
@@ -417,7 +438,7 @@ function resolveAudioRedrawPrompt(sourceLayer: CanvasLayer) {
'背景音乐提示词',
])?.trim() ||
sourceLayer.generationInputs?.fields[0]?.value?.trim() ||
''
resolveDefaultAudioPrompt(mode)
);
}
@@ -465,7 +486,7 @@ export function createAudioRedrawGenerationDialogDraft(
const baseDraft = {
mode,
sourceLayerId: sourceLayer.id,
prompt: resolveAudioRedrawPrompt(sourceLayer),
prompt: resolveAudioRedrawPrompt(sourceLayer, mode),
status: 'idle' as const,
composerOpen: true,
placeholder: {
@@ -14,9 +14,11 @@ import {
function ImageOptionsHarness({
initialDialog,
includeDimensions = true,
onRememberImageModel = vi.fn(),
}: {
initialDialog: GenerateDialogState;
includeDimensions?: boolean;
onRememberImageModel?: (model: string) => void;
}) {
const [dialog, setDialog] = useState<GenerateDialogState | null>(
@@ -28,7 +30,7 @@ function ImageOptionsHarness({
<ImageCanvasGenerationImageOptionsView
dialog={dialog}
setGenerateDialog={setDialog}
includeDimensions
includeDimensions={includeDimensions}
onRememberImageModel={onRememberImageModel}
cost={calculateEditorImageGenerationPrice({ model: dialog.imageModel })}
submitLabel="生成"
@@ -191,6 +193,53 @@ describe('ImageCanvasGenerationImageOptionsView', () => {
expect(submit.textContent).toBe('生成12泥点');
});
it('keeps fixed-card placeholders unchanged when model-only controls are shown', () => {
render(
<ImageOptionsHarness
includeDimensions={false}
initialDialog={{
mode: 'publication',
prompt: '',
status: 'idle',
imageModel: IMAGE_MODEL_NANOBANANA2,
aspectRatio: '4:3',
imageSize: '1K',
placeholder: {
x: 120,
y: 160,
width: 720,
height: 540,
originalWidth: 720,
originalHeight: 540,
},
}}
/>,
);
expect(
screen.queryByRole('button', { name: //u }),
).toBeNull();
fireEvent.click(
screen.getByRole('button', { name: '生成图片模型 nanobanana2' }),
);
fireEvent.click(
within(screen.getByRole('menu', { name: '生成图片模型选项' })).getByRole(
'button',
{ name: 'gpt-image-2' },
),
);
expect(screen.getByLabelText('当前模型').textContent).toBe(
IMAGE_MODEL_GPT_IMAGE_2,
);
expect(screen.getByLabelText('当前比例').textContent).toBe('4:3');
expect(screen.getByLabelText('当前尺寸').textContent).toBe('1K');
expect(screen.getByLabelText('当前占位').textContent).toBe(
'120:160:720:540:720:540',
);
});
it('keeps the canvas placeholder centered while syncing its ratio and size selection', () => {
render(
<ImageOptionsHarness
@@ -152,10 +152,16 @@ export function ImageCanvasGenerationImageOptionsView({
if (!currentDialog || currentDialog.mode !== dialog.mode) {
return currentDialog;
}
if (!includeDimensions) {
return {
...resetFailedDialogStatus(currentDialog),
imageModel: model,
};
}
const nextOptions = getImageDimensionOptions(model);
const nextAspectRatios = nextOptions.aspectRatios as readonly string[];
const nextImageSizes = nextOptions.imageSizes as readonly string[];
return resizeGenerationPlaceholderToImageSelection({
const nextDialog = {
...resetFailedDialogStatus(currentDialog),
imageModel: model,
aspectRatio:
@@ -169,7 +175,8 @@ export function ImageCanvasGenerationImageOptionsView({
? currentDialog.imageSize
: (nextOptions.imageSizes.find((size) => size === '1K') ??
nextOptions.imageSizes[0]),
});
};
return resizeGenerationPlaceholderToImageSelection(nextDialog);
});
};
@@ -98,6 +98,64 @@ describe('ImageCanvasGenerationLayerModel', () => {
expect(layer.generationInputs?.fields[0]?.value).toBe('生成提示词');
});
it('keeps publication material result dimensions constrained by the active placeholder', () => {
const layer = createGeneratedResultLayer({
generated: createGenerated({ width: 1024, height: 1024 }),
generatedIndex: 8,
canvasSize: { width: 900, height: 640 },
viewport: { x: 10, y: 20, scale: 2 },
frame: {
x: 180,
y: 120,
width: 360,
height: 640,
originalWidth: 720,
originalHeight: 1280,
},
assetKind: 'publication-material',
title: '8 宣发素材',
generationInputs: createGenerationInputs(),
});
expect(layer).toMatchObject({
id: 'layer-generated-8',
title: '8 宣发素材',
width: 720,
height: 1280,
originalWidth: 720,
originalHeight: 1280,
assetKind: 'publication-material',
});
});
it('keeps publication cover image results at the 4:3 workflow dimensions', () => {
const layer = createGeneratedResultLayer({
generated: createGenerated({ width: 1024, height: 1024 }),
generatedIndex: 9,
canvasSize: { width: 900, height: 640 },
viewport: { x: 10, y: 20, scale: 2 },
frame: {
x: 100,
y: 100,
width: 480,
height: 360,
originalWidth: 720,
originalHeight: 540,
},
assetKind: 'publication-material',
title: '9 宣发素材',
generationInputs: createGenerationInputs(),
});
expect(layer).toMatchObject({
width: 720,
height: 540,
originalWidth: 720,
originalHeight: 540,
assetKind: 'publication-material',
});
});
it('uses backend-persisted resource and asset snapshots for generated layers', () => {
const generationInputs = createGenerationInputs();
const layer = createGeneratedResultLayer({
@@ -133,8 +133,8 @@ export function createGeneratedResultLayer({
title,
generationInputs,
}: GeneratedResultLayerOptions): CanvasLayer {
const originalWidth = generated.width || 1024;
const originalHeight = generated.height || 1024;
const originalWidth = frame?.originalWidth || generated.width || 1024;
const originalHeight = frame?.originalHeight || generated.height || 1024;
const { width, height } = resolveLayerResolutionSize(
originalWidth,
originalHeight,
@@ -306,6 +306,8 @@ describe('ImageCanvasGenerationSubmissionModel', () => {
size: '720x540',
kind: 'publication-material',
model: 'gemini-3.1-flash-image-preview',
aspectRatio: '4:3',
imageSize: '1K',
referenceImageSrcs: ['data:image/png;base64,ref'],
},
result: {
@@ -359,6 +361,8 @@ describe('ImageCanvasGenerationSubmissionModel', () => {
prompt: expect.stringContaining('【宣发素材类型】详情页单图'),
size: '720x1280',
kind: 'publication-material',
aspectRatio: '9:16',
imageSize: '1K',
},
result: {
title: '11 宣发素材',
@@ -395,6 +399,8 @@ describe('ImageCanvasGenerationSubmissionModel', () => {
prompt: expect.stringContaining('【宣发素材类型】运营海报'),
size: '1280x720',
kind: 'publication-material',
aspectRatio: '16:9',
imageSize: '1K',
},
result: {
title: '12 宣发素材',
@@ -439,6 +445,8 @@ describe('ImageCanvasGenerationSubmissionModel', () => {
kind: 'publication-material',
model: 'gpt-image-2',
size: '1280x720',
aspectRatio: '16:9',
imageSize: '1K',
}),
);
expect(plan.rememberImageModel).toBe('gpt-image-2');
@@ -323,6 +323,8 @@ export function buildImageGenerationSubmissionPlan({
size: workflow.outputSize,
kind: 'publication-material',
model: imageModel,
aspectRatio: workflow.aspectRatio,
imageSize: workflow.imageSize,
...(dialog.publicationReferences?.length
? {
referenceImageSrcs: dialog.publicationReferences.map(
@@ -27,3 +27,54 @@ export function formatCanvasDurationMetric(value: unknown) {
// 中文注释:音频对象没有视觉分辨率语义,画布角标统一显示时长。
return `时长 ${formatCanvasDurationLabel(value)}`;
}
type MediaPreviewTone = 'audio' | 'video';
type MediaPreviewMarker = {
tone: MediaPreviewTone;
variant: number;
};
const MEDIA_PREVIEW_VARIANT_COUNT = 6;
function hashMediaPreviewSeed(seed: string) {
let hash = 0;
for (let index = 0; index < seed.length; index += 1) {
hash = (hash * 31 + seed.charCodeAt(index)) >>> 0;
}
return hash;
}
export function getCanvasMediaPreviewMarker({
mediaType,
assetKind,
title,
src,
}: {
mediaType?: string | null;
assetKind?: string | null;
title?: string | null;
src?: string | null;
}): MediaPreviewMarker | null {
if (mediaType !== 'audio' && mediaType !== 'video') {
return null;
}
const seed = `${assetKind ?? ''}|${title ?? ''}|${src ?? ''}`;
return {
tone: mediaType,
variant: hashMediaPreviewSeed(seed) % MEDIA_PREVIEW_VARIANT_COUNT,
};
}
export function getCanvasMediaPreviewClassName(
marker: MediaPreviewMarker | null,
) {
if (!marker) {
return '';
}
return [
'image-canvas-editor__media-preview',
`image-canvas-editor__media-preview--${marker.tone}`,
`image-canvas-editor__media-preview--variant-${marker.variant}`,
].join(' ');
}
@@ -8,6 +8,8 @@ export type PublicationMaterialsWorkflow = {
outputLabel: string;
sizeLabel: string;
outputSize: string;
aspectRatio: string;
imageSize: string;
scenario: string;
fields: string[];
framework: string[];
@@ -24,6 +26,8 @@ export const PUBLICATION_MATERIALS_WORKFLOWS: PublicationMaterialsWorkflow[] = [
outputLabel: '单张主视觉',
sizeLabel: '720 x 540',
outputSize: '720x540',
aspectRatio: '4:3',
imageSize: '1K',
scenario: '推荐流、作品卡和首屏传播位',
fields: ['游戏名称', '玩法分类标签', '一句话玩法', '主视觉关键词', '必要文字'],
framework: [
@@ -56,6 +60,8 @@ export const PUBLICATION_MATERIALS_WORKFLOWS: PublicationMaterialsWorkflow[] = [
outputLabel: '单张详情图',
sizeLabel: '720 x 1280',
outputSize: '720x1280',
aspectRatio: '9:16',
imageSize: '1K',
scenario: '作品详情、应用介绍和运营分发图组',
fields: ['游戏名称', '玩法分类标签', '卖点拆解', '图组顺序', '必要文字'],
framework: [
@@ -90,6 +96,8 @@ export const PUBLICATION_MATERIALS_WORKFLOWS: PublicationMaterialsWorkflow[] = [
outputLabel: '单张活动海报',
sizeLabel: '1280 x 720',
outputSize: '1280x720',
aspectRatio: '16:9',
imageSize: '1K',
scenario: '社群、活动、渠道投放和运营节日位',
fields: ['游戏名称', '活动主题', '玩法标签', '主文案', '行动指令'],
framework: [
@@ -1,3 +1,5 @@
import { removeBackgroundFromRgba } from '../../../packages/shared/src/assets/chromaKey';
export type CropExpandInsets = {
left: number;
top: number;
@@ -204,31 +206,6 @@ export async function renderCropExpandImage({
};
}
function colorDistance(
pixels: Uint8ClampedArray,
offset: number,
color: [number, number, number],
) {
const red = (pixels[offset] ?? 0) - color[0];
const green = (pixels[offset + 1] ?? 0) - color[1];
const blue = (pixels[offset + 2] ?? 0) - color[2];
return Math.sqrt(red * red + green * green + blue * blue);
}
function readPixelColor(
pixels: Uint8ClampedArray,
width: number,
x: number,
y: number,
): [number, number, number] {
const offset = (y * width + x) * 4;
return [
pixels[offset] ?? 0,
pixels[offset + 1] ?? 0,
pixels[offset + 2] ?? 0,
];
}
export async function removeImageBackground(
source: string,
): Promise<RasterEditResult> {
@@ -237,60 +214,7 @@ export async function removeImageBackground(
const { canvas, context } = createRasterCanvas(width, height);
context.drawImage(image, 0, 0, width, height);
const imageData = context.getImageData(0, 0, width, height);
const pixels = imageData.data;
const cornerColors = [
readPixelColor(pixels, width, 0, 0),
readPixelColor(pixels, width, width - 1, 0),
readPixelColor(pixels, width, 0, height - 1),
readPixelColor(pixels, width, width - 1, height - 1),
];
const visited = new Uint8Array(width * height);
const queue: number[] = [];
const threshold = 42;
const shouldClearPixel = (index: number) => {
const offset = index * 4;
const alpha = pixels[offset + 3] ?? 0;
if (alpha <= 8) {
return true;
}
return cornerColors.some(
(color) => colorDistance(pixels, offset, color) <= threshold,
);
};
const enqueue = (x: number, y: number) => {
if (x < 0 || y < 0 || x >= width || y >= height) {
return;
}
const index = y * width + x;
if (visited[index] || !shouldClearPixel(index)) {
return;
}
visited[index] = 1;
queue.push(index);
};
for (let x = 0; x < width; x += 1) {
enqueue(x, 0);
enqueue(x, height - 1);
}
for (let y = 0; y < height; y += 1) {
enqueue(0, y);
enqueue(width - 1, y);
}
while (queue.length) {
const index = queue.shift()!;
const offset = index * 4;
pixels[offset + 3] = 0;
const x = index % width;
const y = Math.floor(index / width);
enqueue(x + 1, y);
enqueue(x - 1, y);
enqueue(x, y + 1);
enqueue(x, y - 1);
}
removeBackgroundFromRgba(imageData.data, width, height);
context.putImageData(imageData, 0, 0);
return {
@@ -296,6 +296,7 @@ describe('ImageCanvasUploadModel', () => {
).toMatchObject([
{
id: 'layer-upload-1',
src: 'data:image/png;base64,persisted',
sourceAssetId: 'asset-1',
objectKey: 'object-key',
assetObjectId: 'asset-object',

Some files were not shown because too many files have changed in this diff Show More