Files
Genarrative/scripts/spacetime-migration-common.mjs
T
k88936 a03f13d0a5
Project CI / Repository checks (push) Successful in 1m28s
Project CI / Frontend tests (push) Successful in 3m19s
Project CI / Backend tests (push) Successful in 4m3s
Project CI / Native shell tests (push) Successful in 13m58s
Fix/动作 前端展示下载+后端数据结构+精选 的问题 (#117)
原问题

```
历史角色动作是一个兼容例外:
后端允许它没有 editor_project_resource 资源行,只要布局自身保存了完整图片序列帧。
```

- 把生成动作的原视频asstetkind改为 video, 只把图片序列作为action
- db asset新增字段image_sequence_frames_json image_seq_duration_ms等, (原来动作的这些数据存在generation-input中,并不合适)
修改了externaljob,把这部分数据正确填写到数据库中。
- 为避免到处fallback,做了数据库迁移, 脚本: `scripts/spacetime-normalize-editor-character-actions.mjs`   手动进行过画布+素材库中动作序列帧+原始视频 迁移的测试

- 移除preview_video_path字段,  reason:
>preview video
    → separate resource/asset with assetKind="video"
  final transparent action
    → assetKind="character-animation"
    → source_resource_id points to the preview-video resource
    → image_sequence_frames_json contains the actual playable result
>
> So preview_video_path on the final action duplicates the source video resource’s image_src.
- 移除每个frame的index字段 原因: 这个只用于后端内部处理时有一个并发请求, 每个赋一个index方便收集, 后续没有再用到,且与数组本身重复

- 清除副产品preview video的generation_input_json,  因为会影响改造功能, 迁移后预览视频不提供改造(参数), 只有序列帧动作有改造

仍存在的共性问题: #134

- 导出:
下载改为完整序列帧, 封面不再作为fallback, 部分帧读取失败时行为:仍生成 ZIP,并记录失败帧, 不变

画布放置:
before:
![shotmd-1785207959-compressed.webp](/attachments/e56454f0-49a3-4a8a-951f-c21960975f99)
after:
![shotmd-1785231365-compressed.webp](/attachments/886f07df-c25b-4ecc-acc9-160e1795a991)
精选模块: 主页展示, 审核部分UI:
![shotmd-1785500046-compressed.webp](/attachments/9ec3b679-0045-494a-9f20-134f7e73fecf)
![shotmd-1785499780-compressed.webp](/attachments/42396e63-5f26-4853-a367-0f3c15111add)
这两处序列帧们的加载设计为惰式的, 只有hover和单独preview才会全部加载

Reviewed-on: http://192.168.35.82/git/GenarrativeAI/Genarrative/pulls/117
Reviewed-by: 段舒康 <kdletters@qq.com>
Co-authored-by: 王德宇 <kvtodev@outlook.com>
Co-committed-by: 王德宇 <kvtodev@outlook.com>
2026-08-06 16:57:38 +08:00

623 lines
18 KiB
JavaScript

import { spawn } from 'node:child_process';
import { access, lstat, mkdir, readFile } from 'node:fs/promises';
import path from 'node:path';
export function parseArgs(argv) {
const options = {
chunkSize: parseOptionalPositiveInteger(
process.env.GENARRATIVE_SPACETIME_MIGRATION_CHUNK_SIZE,
'GENARRATIVE_SPACETIME_MIGRATION_CHUNK_SIZE',
),
database: process.env.GENARRATIVE_SPACETIME_DATABASE || '',
bootstrapSecret: process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET || '',
bootstrapSecretFile:
process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET_FILE || '',
includeTables: [],
operatorIdentity: process.env.GENARRATIVE_SPACETIME_MIGRATION_OPERATOR_IDENTITY || '',
passthrough: [],
note: '',
server: process.env.GENARRATIVE_SPACETIME_SERVER || '',
serverUrl: process.env.GENARRATIVE_SPACETIME_SERVER_URL || '',
token: process.env.GENARRATIVE_SPACETIME_TOKEN || '',
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
const readValue = (name) => {
const value = argv[index + 1];
if (!value || value.startsWith('--')) {
throw new Error(`${name} 缺少参数值。`);
}
index += 1;
return value;
};
if (arg === '--server') {
options.server = readValue(arg);
} else if (arg === '--use-http') {
options.useHttp = true;
} else if (arg === '--server-url') {
options.serverUrl = readValue(arg);
} else if (arg === '--token') {
options.token = readValue(arg);
} else if (arg === '--bootstrap-secret-file') {
options.bootstrapSecretFile = readValue(arg);
} else if (arg === '--chunk-size') {
options.chunkSize = parsePositiveInteger(readValue(arg), arg);
} else if (arg === '--operator-identity') {
options.operatorIdentity = readValue(arg);
} else if (arg === '--note') {
options.note = readValue(arg);
} else if (arg === '--root-dir') {
options.rootDir = readValue(arg);
} else if (arg === '--database') {
options.database = readValue(arg);
} else if (arg === '--out') {
options.out = readValue(arg);
} else if (arg === '--in') {
options.in = readValue(arg);
} else if (arg === '--include') {
options.includeTables = readValue(arg)
.split(',')
.map((value) => value.trim())
.filter(Boolean);
} else if (arg === '--replace-existing') {
options.replaceExisting = true;
} else if (arg === '--incremental') {
options.incremental = true;
} else if (arg === '--dry-run') {
options.dryRun = true;
} else if (arg === '--anonymous' || arg === '--no-config') {
options.passthrough.push(arg);
} else {
throw new Error(`未知参数: ${arg}`);
}
}
return options;
}
export async function resolveBootstrapSecret(options) {
if (options.bootstrapSecret && options.bootstrapSecretFile) {
throw new Error('bootstrap secret 明文与文件参数不能同时使用。');
}
if (!options.bootstrapSecretFile) {
return options.bootstrapSecret || '';
}
const secretPath = path.resolve(options.bootstrapSecretFile);
const metadata = await lstat(secretPath);
if (!metadata.isFile() || metadata.isSymbolicLink()) {
throw new Error('--bootstrap-secret-file 必须是普通文件且不能是符号链接。');
}
const secret = (await readFile(secretPath, 'utf8')).replace(/\r?\n$/u, '');
if (!/^[0-9a-f]{64}$/iu.test(secret)) {
throw new Error('bootstrap secret 必须是 64 位十六进制高熵值。');
}
return secret;
}
export function parsePositiveInteger(value, name) {
if (!/^[1-9][0-9]*$/u.test(String(value).trim())) {
throw new Error(`${name} 必须是正整数。`);
}
const parsed = Number.parseInt(String(value).trim(), 10);
if (!Number.isSafeInteger(parsed)) {
throw new Error(`${name} 超出安全整数范围。`);
}
return parsed;
}
export function encodeSpacetimeCliOption(value) {
return value === null || value === undefined ? null : [0, value];
}
function parseOptionalPositiveInteger(value, name) {
if (!value) {
return 0;
}
return parsePositiveInteger(value, name);
}
export function buildSpacetimeCallArgs(options, procedureName, input) {
if (!options.database) {
throw new Error('必须传入 --database。');
}
validateSpacetimeDatabaseName(options.database);
const args = [];
if (options.rootDir) {
args.push(`--root-dir=${options.rootDir}`);
}
args.push('call');
args.push('-s', resolveCliServer(options));
args.push(...options.passthrough);
if (!options.passthrough.includes('--no-config')) {
args.push('--no-config');
}
args.push(options.database, procedureName, JSON.stringify(input), '-y');
return args;
}
export async function callSpacetimeProcedure(options, procedureName, input) {
if (!options.database) {
throw new Error('必须传入 --database,或设置 GENARRATIVE_SPACETIME_DATABASE。');
}
validateSpacetimeDatabaseName(options.database);
const serverUrl = resolveServerUrl(options).replace(/\/+$/u, '');
const url = `${serverUrl}/v1/database/${encodeURIComponent(options.database)}/call/${encodeURIComponent(procedureName)}`;
const headers = {
Accept: 'application/json',
'Content-Type': 'application/json',
};
if (options.token) {
headers.Authorization = `Bearer ${options.token}`;
}
let response;
try {
response = await fetch(url, {
method: 'POST',
headers,
body: JSON.stringify([input]),
});
} catch (error) {
throw new Error(
`SpacetimeDB HTTP 请求失败: ${url}; ${error instanceof Error ? error.message : String(error)}`,
);
}
const text = await response.text();
if (!response.ok) {
throw new Error(
`SpacetimeDB HTTP ${response.status}: ${trimPreview(text)}${buildHttpAuthHint(text)}`,
);
}
return parseProcedureResult(text, procedureName);
}
export async function createSpacetimeWebIdentity(options) {
const serverUrl = resolveServerUrl(options).replace(/\/+$/u, '');
const url = `${serverUrl}/v1/identity`;
const headers = {
Accept: 'application/json',
'Content-Type': 'application/json',
};
let response;
try {
response = await fetch(url, { method: 'POST', headers });
} catch (error) {
throw new Error(
`SpacetimeDB identity 请求失败: ${url}; ${error instanceof Error ? error.message : String(error)}`,
);
}
const text = await response.text();
if (!response.ok) {
throw new Error(`SpacetimeDB identity HTTP ${response.status}: ${trimPreview(text)}`);
}
let payload;
try {
payload = JSON.parse(text);
} catch (error) {
throw new Error(
`SpacetimeDB identity 响应不是合法 JSON: ${error instanceof Error ? error.message : String(error)}`,
);
}
const identity =
payload.identity ?? payload.Identity ?? payload.identity_hex ?? payload.identityHex;
const token = payload.token ?? payload.Token;
if (typeof identity !== 'string' || typeof token !== 'string') {
throw new Error(`SpacetimeDB identity 响应缺少 identity/token: ${trimPreview(text)}`);
}
return { identity, token };
}
export async function callSpacetimeProcedureAuto(options, procedureName, input) {
if (options.useHttp) {
return callSpacetimeProcedure(options, procedureName, input);
}
return callSpacetimeProcedureViaCli(options, procedureName, input);
}
export async function callSpacetimeProcedureViaCli(options, procedureName, input) {
const args = buildSpacetimeCallArgs(options, procedureName, input);
const output = await runSpacetimeCli(args);
return parseProcedureResult(output, procedureName);
}
export function validateSpacetimeDatabaseName(database) {
if (!/^[a-z0-9]+(-[a-z0-9]+)*$/u.test(database)) {
throw new Error(
`SpacetimeDB 数据库名必须匹配 ^[a-z0-9]+(-[a-z0-9]+)*$,只能使用小写字母、数字,并用单个短横线分隔: ${database}`,
);
}
}
export function parseProcedureResult(output, procedureName = '') {
const candidates = [];
const trimmed = output.trim();
if (trimmed) {
candidates.push(trimmed);
}
for (const line of output.split(/\r?\n/u)) {
const value = line.trim();
if (value.startsWith('{') || value.startsWith('[')) {
candidates.push(value);
}
}
for (const candidate of candidates) {
try {
return normalizeProcedureResult(JSON.parse(candidate), procedureName);
} catch {
// SpacetimeDB CLI 在不同版本中可能附带说明文本,继续尝试后续候选。
}
}
throw new Error(`无法解析 procedure 返回值: ${trimmed}`);
}
export function ensureProcedureOk(result) {
if (!result.ok) {
throw new Error(result.error_message ?? '迁移 procedure 返回失败。');
}
}
export async function ensureParentDir(filePath) {
await mkdir(path.dirname(path.resolve(filePath)), { recursive: true });
}
export async function assertReadableFile(filePath) {
await access(path.resolve(filePath));
}
function normalizeProcedureResult(value, procedureName) {
if (value && typeof value === 'object' && !Array.isArray(value)) {
return normalizeSatsObject(value, procedureName);
}
if (Array.isArray(value)) {
return normalizeSatsProduct(value, procedureName);
}
throw new Error('procedure 返回值不是对象。');
}
function normalizeSatsObject(value, procedureName) {
const normalized = normalizeSatsValue(value);
if (
procedureName !==
'normalize_editor_character_animation_metadata_and_return' ||
!normalized ||
typeof normalized !== 'object' ||
Array.isArray(normalized)
) {
return normalized;
}
const { batch_sha_256: batchSha256, ...result } = normalized;
return {
...result,
batch_sha256: result.batch_sha256 ?? batchSha256,
};
}
function normalizeSatsProduct(value, procedureName) {
if (
procedureName === 'normalize_editor_character_animation_metadata_and_return' &&
value.length === 19
) {
return {
ok: normalizeSatsValue(value[0]),
scope: normalizeSatsValue(value[1]),
dry_run: normalizeSatsValue(value[2]),
scanned_count: normalizeSatsValue(value[3]),
matched_count: normalizeSatsValue(value[4]),
updated_count: normalizeSatsValue(value[5]),
backfilled_frames_count: normalizeSatsValue(value[6]),
backfilled_duration_count: normalizeSatsValue(value[7]),
reclassified_preview_count: normalizeSatsValue(value[8]),
cleaned_generation_inputs_count: normalizeSatsValue(value[9]),
cleaned_frame_index_count: normalizeSatsValue(value[10]),
cleaned_canvas_layer_count: normalizeSatsValue(value[11]),
materialized_resource_count: normalizeSatsValue(value[12]),
blocker_count: normalizeSatsValue(value[13]),
blocker_ids: normalizeSatsValue(value[14]),
next_cursor: normalizeSatsOption(value[15]),
has_more: normalizeSatsValue(value[16]),
batch_sha256: normalizeSatsValue(value[17]),
error_message: normalizeSatsOption(value[18]),
};
}
if (
procedureName === 'repair_editor_canvas_resources_and_return' &&
value.length === 3
) {
return {
ok: normalizeSatsValue(value[0]),
repair: normalizeEditorCanvasResourceRepairSnapshot(
normalizeSatsOption(value[1]),
),
error_message: normalizeSatsOption(value[2]),
};
}
if (
[
'backfill_editor_canvas_layout_and_return',
'activate_editor_canvas_layout_and_return',
'rollback_editor_canvas_layout_and_return',
].includes(procedureName) &&
value.length === 3
) {
return {
ok: normalizeSatsValue(value[0]),
migration: normalizeEditorCanvasLayoutMigrationSnapshot(
normalizeSatsOption(value[1]),
),
error_message: normalizeSatsOption(value[2]),
};
}
if (
procedureName === 'backfill_external_generation_job_summaries_and_return' &&
value.length === 8
) {
return {
ok: normalizeSatsValue(value[0]),
dry_run: normalizeSatsValue(value[1]),
scanned_count: normalizeSatsValue(value[2]),
selected_count: normalizeSatsValue(value[3]),
upserted_count: normalizeSatsValue(value[4]),
next_cursor_job_id: normalizeSatsOption(value[5]),
has_more: normalizeSatsValue(value[6]),
error_message: normalizeSatsOption(value[7]),
};
}
if (
procedureName === 'compact_external_generation_job_payloads_and_return' &&
value.length === 12
) {
return {
ok: normalizeSatsValue(value[0]),
dry_run: normalizeSatsValue(value[1]),
scanned_count: normalizeSatsValue(value[2]),
matched_count: normalizeSatsValue(value[3]),
updated_count: normalizeSatsValue(value[4]),
before_bytes: normalizeSatsValue(value[5]),
after_bytes: normalizeSatsValue(value[6]),
inline_media_count: normalizeSatsValue(value[7]),
invalid_json_count: normalizeSatsValue(value[8]),
next_cursor_job_id: normalizeSatsOption(value[9]),
has_more: normalizeSatsValue(value[10]),
error_message: normalizeSatsOption(value[11]),
};
}
if (value.length === 3) {
return {
ok: normalizeSatsValue(value[0]),
operator_identity_hex: normalizeSatsOption(value[1]),
error_message: normalizeSatsOption(value[2]),
};
}
if (value.length === 5) {
return {
ok: normalizeSatsValue(value[0]),
schema_version: normalizeSatsValue(value[1]),
migration_json: normalizeSatsOption(value[2]),
table_stats: normalizeTableStats(value[3]),
warnings: [],
error_message: normalizeSatsOption(value[4]),
};
}
return {
ok: normalizeSatsValue(value[0]),
schema_version: normalizeSatsValue(value[1]),
migration_json: normalizeSatsOption(value[2]),
table_stats: normalizeTableStats(value[3]),
warnings: normalizeMigrationWarnings(value[4]),
error_message: normalizeSatsOption(value[5]),
};
}
function normalizeEditorCanvasLayoutMigrationSnapshot(value) {
if (!Array.isArray(value) || value.length !== 11) {
return value;
}
return {
canvas_id: normalizeSatsValue(value[0]),
project_id: normalizeSatsValue(value[1]),
source_layout_sha256: normalizeSatsValue(value[2]),
structured_layout_sha256: normalizeSatsValue(value[3]),
verified_revision: normalizeSatsValue(value[4]),
layer_count: normalizeSatsValue(value[5]),
dialog_count: normalizeSatsValue(value[6]),
resource_refs_sha256: normalizeSatsValue(value[7]),
migration_version: normalizeSatsValue(value[8]),
status: normalizeSatsValue(value[9]),
revision: normalizeSatsValue(value[10]),
};
}
function normalizeEditorCanvasResourceRepairSnapshot(value) {
if (!Array.isArray(value) || value.length !== 7) {
return value;
}
return {
dry_run: normalizeSatsValue(value[0]),
already_repaired: normalizeSatsValue(value[1]),
matched_layer_count: normalizeSatsValue(value[2]),
remapped_layer_count: normalizeSatsValue(value[3]),
restored_resource_count: normalizeSatsValue(value[4]),
removed_source_resource_id_count: normalizeSatsValue(value[5]),
revision: normalizeSatsValue(value[6]),
};
}
function normalizeSatsValue(value) {
if (Array.isArray(value)) {
return value.map((item) => normalizeSatsValue(item));
}
if (value && typeof value === 'object') {
return Object.fromEntries(
Object.entries(value).map(([key, entry]) => [key, normalizeSatsValue(entry)]),
);
}
return value;
}
function normalizeSatsOption(value) {
if (Array.isArray(value)) {
if (value.length === 2 && value[0] === 0) {
return normalizeSatsValue(value[1]);
}
if (value.length === 0 || value[0] === 1) {
return null;
}
}
return normalizeSatsValue(value);
}
function normalizeTableStats(value) {
if (!Array.isArray(value)) {
return [];
}
return value.map((entry) => {
if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
return normalizeSatsValue(entry);
}
if (Array.isArray(entry)) {
return {
table_name: normalizeSatsValue(entry[0]),
exported_row_count: normalizeSatsValue(entry[1]),
imported_row_count: normalizeSatsValue(entry[2]),
skipped_row_count: normalizeSatsValue(entry[3]),
};
}
return entry;
});
}
function normalizeMigrationWarnings(value) {
if (!Array.isArray(value)) {
return [];
}
return value.map((entry) => {
if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
return normalizeSatsValue(entry);
}
if (Array.isArray(entry)) {
return {
table_name: normalizeSatsValue(entry[0]),
warning_kind: normalizeSatsValue(entry[1]),
message: normalizeSatsValue(entry[2]),
};
}
return entry;
});
}
export function resolveServerUrl(options) {
if (options.serverUrl) {
return options.serverUrl;
}
const server = (options.server || 'dev').trim();
if (server.startsWith('http://') || server.startsWith('https://')) {
return server;
}
if (server === 'dev') {
return 'http://127.0.0.1:3101';
}
if (server === 'local') {
return 'http://127.0.0.1:3000';
}
if (!server) {
return 'http://127.0.0.1:3101';
}
throw new Error(`未知 SpacetimeDB server: ${server}。请改用 --server-url 显式传入地址。`);
}
function resolveCliServer(options) {
if (options.serverUrl) {
return options.serverUrl;
}
const server = (options.server || '').trim();
if (!server || server === 'dev') {
return 'http://127.0.0.1:3101';
}
return server;
}
function trimPreview(text) {
const trimmed = text.trim();
if (trimmed.length <= 4000) {
return trimmed;
}
return `${trimmed.slice(0, 4000)}...`;
}
function buildHttpAuthHint(text) {
if (!text.includes('InvalidSignature') && !text.includes('TokenError')) {
return '';
}
return '。提示:这里需要 SpacetimeDB 客户端连接 token,不是 `spacetime login show --token` 输出的 CLI 登录 token;授权/撤销请直接使用 CLI 登录态,不要传 --token。';
}
function runSpacetimeCli(args) {
return new Promise((resolve, reject) => {
const child = spawn('spacetime', args, {
cwd: process.cwd(),
shell: false,
stdio: ['ignore', 'pipe', 'pipe'],
});
let output = '';
child.stdout.on('data', (chunk) => {
output += chunk.toString();
});
child.stderr.on('data', (chunk) => {
output += chunk.toString();
});
child.on('error', reject);
child.on('exit', (code, signal) => {
if (signal) {
reject(new Error(`spacetime call 被信号中断: ${signal}`));
return;
}
if (code !== 0) {
reject(new Error(`spacetime call 失败,退出码 ${code}: ${trimPreview(output)}`));
return;
}
resolve(output);
});
});
}