kdletters
324f99efdf
阻止图片编辑器将 Data URL 和超限 JSON 写入生成任务，复用 objectKey 等轻量引用 新增外部生成任务摘要投影与分批 payload 压缩回填流程，避免正式列表读取大字段 收紧 SpacetimeDB 发布为永不删数据并移除 Jenkins 普通清库入口，补齐维护窗口门禁 更新生成绑定、运维文档和回归测试，并完成生产数据副本迁移与压缩验证
513 lines
15 KiB
JavaScript
513 lines
15 KiB
JavaScript
import { spawn } from 'node:child_process';
|
|
import { access, lstat, mkdir, readFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
|
|
export function parseArgs(argv) {
|
|
const options = {
|
|
chunkSize: parseOptionalPositiveInteger(
|
|
process.env.GENARRATIVE_SPACETIME_MIGRATION_CHUNK_SIZE,
|
|
'GENARRATIVE_SPACETIME_MIGRATION_CHUNK_SIZE',
|
|
),
|
|
database: process.env.GENARRATIVE_SPACETIME_DATABASE || '',
|
|
bootstrapSecret: process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET || '',
|
|
bootstrapSecretFile:
|
|
process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET_FILE || '',
|
|
includeTables: [],
|
|
operatorIdentity: process.env.GENARRATIVE_SPACETIME_MIGRATION_OPERATOR_IDENTITY || '',
|
|
passthrough: [],
|
|
note: '',
|
|
server: process.env.GENARRATIVE_SPACETIME_SERVER || '',
|
|
serverUrl: process.env.GENARRATIVE_SPACETIME_SERVER_URL || '',
|
|
token: process.env.GENARRATIVE_SPACETIME_TOKEN || '',
|
|
};
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
const readValue = (name) => {
|
|
const value = argv[index + 1];
|
|
if (!value || value.startsWith('--')) {
|
|
throw new Error(`${name} 缺少参数值。`);
|
|
}
|
|
index += 1;
|
|
return value;
|
|
};
|
|
|
|
if (arg === '--server') {
|
|
options.server = readValue(arg);
|
|
} else if (arg === '--use-http') {
|
|
options.useHttp = true;
|
|
} else if (arg === '--server-url') {
|
|
options.serverUrl = readValue(arg);
|
|
} else if (arg === '--token') {
|
|
options.token = readValue(arg);
|
|
} else if (arg === '--bootstrap-secret-file') {
|
|
options.bootstrapSecretFile = readValue(arg);
|
|
} else if (arg === '--chunk-size') {
|
|
options.chunkSize = parsePositiveInteger(readValue(arg), arg);
|
|
} else if (arg === '--operator-identity') {
|
|
options.operatorIdentity = readValue(arg);
|
|
} else if (arg === '--note') {
|
|
options.note = readValue(arg);
|
|
} else if (arg === '--root-dir') {
|
|
options.rootDir = readValue(arg);
|
|
} else if (arg === '--database') {
|
|
options.database = readValue(arg);
|
|
} else if (arg === '--out') {
|
|
options.out = readValue(arg);
|
|
} else if (arg === '--in') {
|
|
options.in = readValue(arg);
|
|
} else if (arg === '--include') {
|
|
options.includeTables = readValue(arg)
|
|
.split(',')
|
|
.map((value) => value.trim())
|
|
.filter(Boolean);
|
|
} else if (arg === '--replace-existing') {
|
|
options.replaceExisting = true;
|
|
} else if (arg === '--incremental') {
|
|
options.incremental = true;
|
|
} else if (arg === '--dry-run') {
|
|
options.dryRun = true;
|
|
} else if (arg === '--anonymous' || arg === '--no-config') {
|
|
options.passthrough.push(arg);
|
|
} else {
|
|
throw new Error(`未知参数: ${arg}`);
|
|
}
|
|
}
|
|
|
|
return options;
|
|
}
|
|
|
|
export async function resolveBootstrapSecret(options) {
|
|
if (options.bootstrapSecret && options.bootstrapSecretFile) {
|
|
throw new Error('bootstrap secret 明文与文件参数不能同时使用。');
|
|
}
|
|
if (!options.bootstrapSecretFile) {
|
|
return options.bootstrapSecret || '';
|
|
}
|
|
|
|
const secretPath = path.resolve(options.bootstrapSecretFile);
|
|
const metadata = await lstat(secretPath);
|
|
if (!metadata.isFile() || metadata.isSymbolicLink()) {
|
|
throw new Error('--bootstrap-secret-file 必须是普通文件且不能是符号链接。');
|
|
}
|
|
const secret = (await readFile(secretPath, 'utf8')).replace(/\r?\n$/u, '');
|
|
if (!/^[0-9a-f]{64}$/iu.test(secret)) {
|
|
throw new Error('bootstrap secret 必须是 64 位十六进制高熵值。');
|
|
}
|
|
return secret;
|
|
}
|
|
|
|
export function parsePositiveInteger(value, name) {
|
|
if (!/^[1-9][0-9]*$/u.test(String(value).trim())) {
|
|
throw new Error(`${name} 必须是正整数。`);
|
|
}
|
|
|
|
const parsed = Number.parseInt(String(value).trim(), 10);
|
|
if (!Number.isSafeInteger(parsed)) {
|
|
throw new Error(`${name} 超出安全整数范围。`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
export function encodeSpacetimeCliOption(value) {
|
|
return value === null || value === undefined ? null : [0, value];
|
|
}
|
|
|
|
function parseOptionalPositiveInteger(value, name) {
|
|
if (!value) {
|
|
return 0;
|
|
}
|
|
return parsePositiveInteger(value, name);
|
|
}
|
|
|
|
export function buildSpacetimeCallArgs(options, procedureName, input) {
|
|
if (!options.database) {
|
|
throw new Error('必须传入 --database。');
|
|
}
|
|
validateSpacetimeDatabaseName(options.database);
|
|
|
|
const args = [];
|
|
if (options.rootDir) {
|
|
args.push(`--root-dir=${options.rootDir}`);
|
|
}
|
|
args.push('call');
|
|
args.push('-s', resolveCliServer(options));
|
|
args.push(...options.passthrough);
|
|
if (!options.passthrough.includes('--no-config')) {
|
|
args.push('--no-config');
|
|
}
|
|
args.push(options.database, procedureName, JSON.stringify(input), '-y');
|
|
return args;
|
|
}
|
|
|
|
export async function callSpacetimeProcedure(options, procedureName, input) {
|
|
if (!options.database) {
|
|
throw new Error('必须传入 --database，或设置 GENARRATIVE_SPACETIME_DATABASE。');
|
|
}
|
|
validateSpacetimeDatabaseName(options.database);
|
|
|
|
const serverUrl = resolveServerUrl(options).replace(/\/+$/u, '');
|
|
const url = `${serverUrl}/v1/database/${encodeURIComponent(options.database)}/call/${encodeURIComponent(procedureName)}`;
|
|
const headers = {
|
|
Accept: 'application/json',
|
|
'Content-Type': 'application/json',
|
|
};
|
|
if (options.token) {
|
|
headers.Authorization = `Bearer ${options.token}`;
|
|
}
|
|
|
|
let response;
|
|
try {
|
|
response = await fetch(url, {
|
|
method: 'POST',
|
|
headers,
|
|
body: JSON.stringify([input]),
|
|
});
|
|
} catch (error) {
|
|
throw new Error(
|
|
`SpacetimeDB HTTP 请求失败: ${url}; ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
const text = await response.text();
|
|
if (!response.ok) {
|
|
throw new Error(
|
|
`SpacetimeDB HTTP ${response.status}: ${trimPreview(text)}${buildHttpAuthHint(text)}`,
|
|
);
|
|
}
|
|
|
|
return parseProcedureResult(text, procedureName);
|
|
}
|
|
|
|
export async function createSpacetimeWebIdentity(options) {
|
|
const serverUrl = resolveServerUrl(options).replace(/\/+$/u, '');
|
|
const url = `${serverUrl}/v1/identity`;
|
|
const headers = {
|
|
Accept: 'application/json',
|
|
'Content-Type': 'application/json',
|
|
};
|
|
let response;
|
|
try {
|
|
response = await fetch(url, { method: 'POST', headers });
|
|
} catch (error) {
|
|
throw new Error(
|
|
`SpacetimeDB identity 请求失败: ${url}; ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
|
|
const text = await response.text();
|
|
if (!response.ok) {
|
|
throw new Error(`SpacetimeDB identity HTTP ${response.status}: ${trimPreview(text)}`);
|
|
}
|
|
|
|
let payload;
|
|
try {
|
|
payload = JSON.parse(text);
|
|
} catch (error) {
|
|
throw new Error(
|
|
`SpacetimeDB identity 响应不是合法 JSON: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
|
|
const identity =
|
|
payload.identity ?? payload.Identity ?? payload.identity_hex ?? payload.identityHex;
|
|
const token = payload.token ?? payload.Token;
|
|
if (typeof identity !== 'string' || typeof token !== 'string') {
|
|
throw new Error(`SpacetimeDB identity 响应缺少 identity/token: ${trimPreview(text)}`);
|
|
}
|
|
|
|
return { identity, token };
|
|
}
|
|
|
|
export async function callSpacetimeProcedureAuto(options, procedureName, input) {
|
|
if (options.useHttp) {
|
|
return callSpacetimeProcedure(options, procedureName, input);
|
|
}
|
|
|
|
return callSpacetimeProcedureViaCli(options, procedureName, input);
|
|
}
|
|
|
|
export async function callSpacetimeProcedureViaCli(options, procedureName, input) {
|
|
const args = buildSpacetimeCallArgs(options, procedureName, input);
|
|
const output = await runSpacetimeCli(args);
|
|
return parseProcedureResult(output, procedureName);
|
|
}
|
|
|
|
export function validateSpacetimeDatabaseName(database) {
|
|
if (!/^[a-z0-9]+(-[a-z0-9]+)*$/u.test(database)) {
|
|
throw new Error(
|
|
`SpacetimeDB 数据库名必须匹配 ^[a-z0-9]+(-[a-z0-9]+)*$，只能使用小写字母、数字，并用单个短横线分隔: ${database}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
export function parseProcedureResult(output, procedureName = '') {
|
|
const candidates = [];
|
|
const trimmed = output.trim();
|
|
if (trimmed) {
|
|
candidates.push(trimmed);
|
|
}
|
|
|
|
for (const line of output.split(/\r?\n/u)) {
|
|
const value = line.trim();
|
|
if (value.startsWith('{') || value.startsWith('[')) {
|
|
candidates.push(value);
|
|
}
|
|
}
|
|
|
|
for (const candidate of candidates) {
|
|
try {
|
|
return normalizeProcedureResult(JSON.parse(candidate), procedureName);
|
|
} catch {
|
|
// SpacetimeDB CLI 在不同版本中可能附带说明文本，继续尝试后续候选。
|
|
}
|
|
}
|
|
|
|
throw new Error(`无法解析 procedure 返回值: ${trimmed}`);
|
|
}
|
|
|
|
export function ensureProcedureOk(result) {
|
|
if (!result.ok) {
|
|
throw new Error(result.error_message ?? '迁移 procedure 返回失败。');
|
|
}
|
|
}
|
|
|
|
export async function ensureParentDir(filePath) {
|
|
await mkdir(path.dirname(path.resolve(filePath)), { recursive: true });
|
|
}
|
|
|
|
export async function assertReadableFile(filePath) {
|
|
await access(path.resolve(filePath));
|
|
}
|
|
|
|
function normalizeProcedureResult(value, procedureName) {
|
|
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
return value;
|
|
}
|
|
|
|
if (Array.isArray(value)) {
|
|
return normalizeSatsProduct(value, procedureName);
|
|
}
|
|
|
|
throw new Error('procedure 返回值不是对象。');
|
|
}
|
|
|
|
function normalizeSatsProduct(value, procedureName) {
|
|
if (
|
|
procedureName === 'backfill_external_generation_job_summaries_and_return' &&
|
|
value.length === 8
|
|
) {
|
|
return {
|
|
ok: normalizeSatsValue(value[0]),
|
|
dry_run: normalizeSatsValue(value[1]),
|
|
scanned_count: normalizeSatsValue(value[2]),
|
|
selected_count: normalizeSatsValue(value[3]),
|
|
upserted_count: normalizeSatsValue(value[4]),
|
|
next_cursor_job_id: normalizeSatsOption(value[5]),
|
|
has_more: normalizeSatsValue(value[6]),
|
|
error_message: normalizeSatsOption(value[7]),
|
|
};
|
|
}
|
|
|
|
if (
|
|
procedureName === 'compact_external_generation_job_payloads_and_return' &&
|
|
value.length === 12
|
|
) {
|
|
return {
|
|
ok: normalizeSatsValue(value[0]),
|
|
dry_run: normalizeSatsValue(value[1]),
|
|
scanned_count: normalizeSatsValue(value[2]),
|
|
matched_count: normalizeSatsValue(value[3]),
|
|
updated_count: normalizeSatsValue(value[4]),
|
|
before_bytes: normalizeSatsValue(value[5]),
|
|
after_bytes: normalizeSatsValue(value[6]),
|
|
inline_media_count: normalizeSatsValue(value[7]),
|
|
invalid_json_count: normalizeSatsValue(value[8]),
|
|
next_cursor_job_id: normalizeSatsOption(value[9]),
|
|
has_more: normalizeSatsValue(value[10]),
|
|
error_message: normalizeSatsOption(value[11]),
|
|
};
|
|
}
|
|
|
|
if (value.length === 3) {
|
|
return {
|
|
ok: normalizeSatsValue(value[0]),
|
|
operator_identity_hex: normalizeSatsOption(value[1]),
|
|
error_message: normalizeSatsOption(value[2]),
|
|
};
|
|
}
|
|
|
|
if (value.length === 5) {
|
|
return {
|
|
ok: normalizeSatsValue(value[0]),
|
|
schema_version: normalizeSatsValue(value[1]),
|
|
migration_json: normalizeSatsOption(value[2]),
|
|
table_stats: normalizeTableStats(value[3]),
|
|
warnings: [],
|
|
error_message: normalizeSatsOption(value[4]),
|
|
};
|
|
}
|
|
|
|
return {
|
|
ok: normalizeSatsValue(value[0]),
|
|
schema_version: normalizeSatsValue(value[1]),
|
|
migration_json: normalizeSatsOption(value[2]),
|
|
table_stats: normalizeTableStats(value[3]),
|
|
warnings: normalizeMigrationWarnings(value[4]),
|
|
error_message: normalizeSatsOption(value[5]),
|
|
};
|
|
}
|
|
|
|
function normalizeSatsValue(value) {
|
|
if (Array.isArray(value)) {
|
|
return value.map((item) => normalizeSatsValue(item));
|
|
}
|
|
|
|
if (value && typeof value === 'object') {
|
|
return Object.fromEntries(
|
|
Object.entries(value).map(([key, entry]) => [key, normalizeSatsValue(entry)]),
|
|
);
|
|
}
|
|
|
|
return value;
|
|
}
|
|
|
|
function normalizeSatsOption(value) {
|
|
if (Array.isArray(value)) {
|
|
if (value.length === 2 && value[0] === 0) {
|
|
return normalizeSatsValue(value[1]);
|
|
}
|
|
if (value.length === 0 || value[0] === 1) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
return normalizeSatsValue(value);
|
|
}
|
|
|
|
function normalizeTableStats(value) {
|
|
if (!Array.isArray(value)) {
|
|
return [];
|
|
}
|
|
|
|
return value.map((entry) => {
|
|
if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
|
|
return normalizeSatsValue(entry);
|
|
}
|
|
|
|
if (Array.isArray(entry)) {
|
|
return {
|
|
table_name: normalizeSatsValue(entry[0]),
|
|
exported_row_count: normalizeSatsValue(entry[1]),
|
|
imported_row_count: normalizeSatsValue(entry[2]),
|
|
skipped_row_count: normalizeSatsValue(entry[3]),
|
|
};
|
|
}
|
|
|
|
return entry;
|
|
});
|
|
}
|
|
|
|
function normalizeMigrationWarnings(value) {
|
|
if (!Array.isArray(value)) {
|
|
return [];
|
|
}
|
|
|
|
return value.map((entry) => {
|
|
if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
|
|
return normalizeSatsValue(entry);
|
|
}
|
|
|
|
if (Array.isArray(entry)) {
|
|
return {
|
|
table_name: normalizeSatsValue(entry[0]),
|
|
warning_kind: normalizeSatsValue(entry[1]),
|
|
message: normalizeSatsValue(entry[2]),
|
|
};
|
|
}
|
|
|
|
return entry;
|
|
});
|
|
}
|
|
|
|
export function resolveServerUrl(options) {
|
|
if (options.serverUrl) {
|
|
return options.serverUrl;
|
|
}
|
|
|
|
const server = (options.server || 'dev').trim();
|
|
if (server.startsWith('http://') || server.startsWith('https://')) {
|
|
return server;
|
|
}
|
|
if (server === 'dev') {
|
|
return 'http://127.0.0.1:3101';
|
|
}
|
|
if (server === 'local') {
|
|
return 'http://127.0.0.1:3000';
|
|
}
|
|
if (!server) {
|
|
return 'http://127.0.0.1:3101';
|
|
}
|
|
|
|
throw new Error(`未知 SpacetimeDB server: ${server}。请改用 --server-url 显式传入地址。`);
|
|
}
|
|
|
|
function resolveCliServer(options) {
|
|
if (options.serverUrl) {
|
|
return options.serverUrl;
|
|
}
|
|
|
|
const server = (options.server || '').trim();
|
|
if (!server || server === 'dev') {
|
|
return 'http://127.0.0.1:3101';
|
|
}
|
|
return server;
|
|
}
|
|
|
|
function trimPreview(text) {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length <= 4000) {
|
|
return trimmed;
|
|
}
|
|
|
|
return `${trimmed.slice(0, 4000)}...`;
|
|
}
|
|
|
|
function buildHttpAuthHint(text) {
|
|
if (!text.includes('InvalidSignature') && !text.includes('TokenError')) {
|
|
return '';
|
|
}
|
|
|
|
return '。提示：这里需要 SpacetimeDB 客户端连接 token，不是 `spacetime login show --token` 输出的 CLI 登录 token；授权/撤销请直接使用 CLI 登录态，不要传 --token。';
|
|
}
|
|
|
|
function runSpacetimeCli(args) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn('spacetime', args, {
|
|
cwd: process.cwd(),
|
|
shell: false,
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
});
|
|
let output = '';
|
|
|
|
child.stdout.on('data', (chunk) => {
|
|
output += chunk.toString();
|
|
});
|
|
child.stderr.on('data', (chunk) => {
|
|
output += chunk.toString();
|
|
});
|
|
child.on('error', reject);
|
|
child.on('exit', (code, signal) => {
|
|
if (signal) {
|
|
reject(new Error(`spacetime call 被信号中断: ${signal}`));
|
|
return;
|
|
}
|
|
if (code !== 0) {
|
|
reject(new Error(`spacetime call 失败，退出码 ${code}: ${trimPreview(output)}`));
|
|
return;
|
|
}
|
|
|
|
resolve(output);
|
|
});
|
|
});
|
|
}
|