修复链路上存在的若干阻塞性漏洞 (#199)
Project CI / Repository checks (push) Successful in 4m0s
Project CI / Frontend tests (push) Successful in 4m34s
Project CI / Backend tests (push) Successful in 6m48s
Project CI / Native shell tests (push) Successful in 16m12s

Reviewed-on: http://192.168.35.82/git/GenarrativeAI/Genarrative/pulls/199
Co-authored-by: Linghong <ink29535@proton.me>
Co-committed-by: Linghong <ink29535@proton.me>
This commit was merged in pull request #199.
This commit is contained in:
2026-08-28 17:09:02 +08:00
committed by 段舒康
parent 7d4c47ec0c
commit e31d8a2a5b
50 changed files with 5118 additions and 1658 deletions
File diff suppressed because it is too large Load Diff
+17 -1
View File
@@ -973,12 +973,14 @@ const checks = [
{
file: 'scripts/database-backup-to-oss.mjs',
includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})',
normalizeWhitespace: true,
reason: '生产冷备份必须先做工作目录剩余空间预检,避免停库后写满磁盘。',
},
{
file: 'scripts/database-backup-to-oss.mjs',
includes:
'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath})',
normalizeWhitespace: true,
reason: '生产冷备份打包失败时也必须恢复 SpacetimeDB 及依赖服务。',
},
{
@@ -7504,9 +7506,23 @@ const databaseTargetSourceStashes = [
let failed = false;
function includesGuardrail(content, check) {
if (!check.includes) {
return true;
}
if (!check.normalizeWhitespace) {
return content.includes(check.includes);
}
const normalizeSourceFragment = (value) =>
value.replace(/\s+/gu, '').replace(/,([})])/gu, '$1');
const normalizedContent = normalizeSourceFragment(content);
const normalizedExpected = normalizeSourceFragment(check.includes);
return normalizedContent.includes(normalizedExpected);
}
for (const check of checks) {
const content = readFileSync(check.file, 'utf8');
if (check.includes && !content.includes(check.includes)) {
if (!includesGuardrail(content, check)) {
failed = true;
console.error(
`[check:production-ops] ${check.file} 缺少 ${check.includes}${check.reason}`,
+124 -51
View File
@@ -1,4 +1,4 @@
import {spawn} from 'node:child_process';
import { spawn } from 'node:child_process';
import {
chmodSync,
copyFileSync,
@@ -13,7 +13,11 @@ import path from 'node:path';
const [, , rawCommand = 'help', ...rawArgs] = process.argv;
const projectRoot = process.cwd();
const composeFile = path.join('deploy', 'container', 'docker-compose.loadtest.yml');
const composeFile = path.join(
'deploy',
'container',
'docker-compose.loadtest.yml',
);
const smokeDir = path.join('deploy', 'container', 'worker-smoke');
const envPath = path.join(smokeDir, 'api-server.env');
const statePath = path.join(smokeDir, 'state.json');
@@ -22,13 +26,17 @@ const localImageDockerfilePath = path.join(localImageDir, 'Dockerfile.local');
const localImageBinaryPath = path.join(localImageDir, 'api-server');
const localCargoTargetDir = path.join('server-rs', 'target-worker-smoke');
const localSpacetimeImageDir = path.join(smokeDir, 'spacetimedb-image');
const localSpacetimeDockerfilePath = path.join(localSpacetimeImageDir, 'Dockerfile.local');
const localSpacetimeDockerfilePath = path.join(
localSpacetimeImageDir,
'Dockerfile.local',
);
const localSpacetimeBinaryPath = path.join(localSpacetimeImageDir, 'spacetime');
const localSpacetimeStandalonePath = path.join(
localSpacetimeImageDir,
'spacetimedb-standalone',
);
const projectName = process.env.GENARRATIVE_WORKER_SMOKE_PROJECT || 'genarrative-worker-smoke';
const projectName =
process.env.GENARRATIVE_WORKER_SMOKE_PROJECT || 'genarrative-worker-smoke';
const defaultDatabase =
process.env.GENARRATIVE_WORKER_SMOKE_DATABASE || 'genarrative-worker-smoke';
@@ -68,7 +76,7 @@ async function main() {
printHelp(false);
return;
case 'init':
await ensureStateAndEnv({force: rawArgs.includes('--force')});
await ensureStateAndEnv({ force: rawArgs.includes('--force') });
return;
case 'build':
await ensureStateAndEnv();
@@ -99,7 +107,7 @@ async function main() {
return;
case 'api-update':
await ensureStateAndEnv();
await apiOnlyUpdate({build: rawArgs.includes('--build')});
await apiOnlyUpdate({ build: rawArgs.includes('--build') });
return;
case 'scale':
await ensureStateAndEnv();
@@ -114,7 +122,7 @@ async function main() {
await dockerCompose(['ps', ...rawArgs]);
return;
case 'down':
await ensureStateAndEnv({create: false});
await ensureStateAndEnv({ create: false });
await dockerCompose(['down', ...rawArgs]);
return;
case 'smoke':
@@ -128,9 +136,9 @@ async function main() {
async function runSmoke() {
if (rawArgs.includes('--force')) {
await ensureStateAndEnv();
await dockerComposeCapture(['down', '-v'], {allowFailure: true});
await dockerComposeCapture(['down', '-v'], { allowFailure: true });
}
const state = await ensureStateAndEnv({force: rawArgs.includes('--force')});
const state = await ensureStateAndEnv({ force: rawArgs.includes('--force') });
await assertSavedPortsAvailableForNewProject(state);
console.log(
`[worker-smoke] 使用隔离环境 project=${projectName} database=${state.database}`,
@@ -147,10 +155,10 @@ async function runSmoke() {
const beforeWorkerIds = await getContainerIds('external-generation-worker');
console.log(`[worker-smoke] worker 容器: ${beforeWorkerIds.join(', ')}`);
const firstJobId = await enqueueSmokeJob({label: 'before-api-update'});
const firstJobId = await enqueueSmokeJob({ label: 'before-api-update' });
await waitForJobConsumed(firstJobId);
await apiOnlyUpdate({build: false});
await apiOnlyUpdate({ build: false });
const afterWorkerIds = await getContainerIds('external-generation-worker');
if (beforeWorkerIds.join('\n') !== afterWorkerIds.join('\n')) {
throw new Error(
@@ -159,10 +167,12 @@ async function runSmoke() {
}
console.log('[worker-smoke] api-only 更新未重建 worker 容器。');
const secondJobId = await enqueueSmokeJob({label: 'after-api-update'});
const secondJobId = await enqueueSmokeJob({ label: 'after-api-update' });
await waitForJobConsumed(secondJobId);
await printQueueStatus();
console.log('[worker-smoke] smoke 通过:worker 独立消费队列,API-only 更新未停止 worker。');
console.log(
'[worker-smoke] smoke 通过:worker 独立消费队列,API-only 更新未停止 worker。',
);
}
async function buildRuntimeImages() {
@@ -196,13 +206,19 @@ async function buildLocalBinaryRuntimeImages() {
process.env.GENARRATIVE_WORKER_SMOKE_CARGO_PROFILE === 'release'
? 'release'
: 'debug';
const buildArgs = ['build', '-p', 'api-server', '--manifest-path', 'server-rs/Cargo.toml'];
const buildArgs = [
'build',
'-p',
'api-server',
'--manifest-path',
'server-rs/Cargo.toml',
];
if (profile === 'release') {
buildArgs.push('--release');
}
const cargoImage = resolveLocalBinaryCargoImage();
const cargoHome = resolveLocalBinaryCargoHome();
mkdirSync(cargoHome, {recursive: true});
mkdirSync(cargoHome, { recursive: true });
console.log(
`[worker-smoke] 使用 ${cargoImage} 复用本机 Cargo 缓存构建 ${profile} api-server 二进制。`,
@@ -235,17 +251,27 @@ async function buildLocalBinaryRuntimeImages() {
...buildArgs,
]);
const sourceBinaryPath = path.join(localCargoTargetDir, profile, 'api-server');
const sourceBinaryPath = path.join(
localCargoTargetDir,
profile,
'api-server',
);
if (!existsSync(sourceBinaryPath)) {
throw new Error(`未找到 worker smoke api-server 二进制: ${sourceBinaryPath}`);
throw new Error(
`未找到 worker smoke api-server 二进制: ${sourceBinaryPath}`,
);
}
mkdirSync(localImageDir, {recursive: true});
mkdirSync(localImageDir, { recursive: true });
copyFileSync(sourceBinaryPath, localImageBinaryPath);
chmodSync(localImageBinaryPath, 0o755);
const baseImage = await resolveLocalBinaryBaseImage();
writeFileSync(localImageDockerfilePath, buildLocalBinaryDockerfile(baseImage), 'utf8');
writeFileSync(
localImageDockerfilePath,
buildLocalBinaryDockerfile(baseImage),
'utf8',
);
await run('docker', [
'build',
@@ -260,7 +286,9 @@ async function buildLocalBinaryRuntimeImages() {
}
function resolveLocalBinaryCargoImage() {
return process.env.GENARRATIVE_WORKER_SMOKE_CARGO_IMAGE || 'rust:1.93-bookworm';
return (
process.env.GENARRATIVE_WORKER_SMOKE_CARGO_IMAGE || 'rust:1.93-bookworm'
);
}
function resolveLocalBinaryCargoHome() {
@@ -274,42 +302,62 @@ function resolveLocalBinaryCargoHome() {
}
function currentUserSpec() {
if (typeof process.getuid === 'function' && typeof process.getgid === 'function') {
if (
typeof process.getuid === 'function' &&
typeof process.getgid === 'function'
) {
return `${process.getuid()}:${process.getgid()}`;
}
return '0:0';
}
async function ensureSpacetimeImage() {
if (process.env.GENARRATIVE_WORKER_SMOKE_SPACETIME_IMAGE_MODE === 'official') {
if (
process.env.GENARRATIVE_WORKER_SMOKE_SPACETIME_IMAGE_MODE === 'official'
) {
return;
}
const imageName = localSpacetimeImageName();
const existingImage = await runCapture('docker', ['image', 'inspect', imageName], {
allowFailure: true,
quiet: true,
});
const existingImage = await runCapture(
'docker',
['image', 'inspect', imageName],
{
allowFailure: true,
quiet: true,
},
);
if (existingImage.code === 0 && !rawArgs.includes('--force')) {
return;
}
const spacetimePath = await resolveSpacetimeBinaryPath();
if (!spacetimePath) {
throw new Error('未找到本机 spacetime CLI,无法构建隔离 SpacetimeDB 镜像。');
throw new Error(
'未找到本机 spacetime CLI,无法构建隔离 SpacetimeDB 镜像。',
);
}
mkdirSync(localSpacetimeImageDir, {recursive: true});
mkdirSync(localSpacetimeImageDir, { recursive: true });
copyFileSync(spacetimePath, localSpacetimeBinaryPath);
chmodSync(localSpacetimeBinaryPath, 0o755);
const standalonePath = path.join(path.dirname(spacetimePath), 'spacetimedb-standalone');
const standalonePath = path.join(
path.dirname(spacetimePath),
'spacetimedb-standalone',
);
if (!existsSync(standalonePath)) {
throw new Error(`未找到本机 spacetimedb-standalone: ${standalonePath}`);
}
copyFileSync(standalonePath, localSpacetimeStandalonePath);
chmodSync(localSpacetimeStandalonePath, 0o755);
writeFileSync(localSpacetimeDockerfilePath, buildLocalSpacetimeDockerfile(), 'utf8');
writeFileSync(
localSpacetimeDockerfilePath,
buildLocalSpacetimeDockerfile(),
'utf8',
);
console.log(`[worker-smoke] 使用本机 spacetime CLI 构建隔离镜像: ${imageName}`);
console.log(
`[worker-smoke] 使用本机 spacetime CLI 构建隔离镜像: ${imageName}`,
);
await run('docker', [
'build',
'-f',
@@ -337,12 +385,14 @@ async function resolveSpacetimeBinaryPath() {
if (process.env.GENARRATIVE_WORKER_SMOKE_SPACETIME_BIN) {
return process.env.GENARRATIVE_WORKER_SMOKE_SPACETIME_BIN;
}
const versionResult = await runCapture('spacetime', ['--version'], {quiet: true});
const versionResult = await runCapture('spacetime', ['--version'], {
quiet: true,
});
const pathMatch = versionResult.stdout.match(/^spacetime Path:\s*(.+)$/mu);
if (pathMatch?.[1]) {
return pathMatch[1].trim();
}
const whichResult = await runCapture('which', ['spacetime'], {quiet: true});
const whichResult = await runCapture('which', ['spacetime'], { quiet: true });
return whichResult.stdout.trim();
}
@@ -387,11 +437,11 @@ async function upRuntime() {
}
async function ensureStateAndEnv(options = {}) {
const {force = false, create = true} = options;
const { force = false, create = true } = options;
if (!create && !existsSync(statePath)) {
return defaultState();
}
mkdirSync(smokeDir, {recursive: true});
mkdirSync(smokeDir, { recursive: true });
if (!existsSync(statePath) || force) {
const state = {
@@ -419,7 +469,9 @@ async function ensureStateAndEnv(options = {}) {
}
console.log(`[worker-smoke] env=${envPath}`);
console.log(`[worker-smoke] state=${statePath}`);
console.log(`[worker-smoke] SpacetimeDB=http://127.0.0.1:${state.spacetimePort}`);
console.log(
`[worker-smoke] SpacetimeDB=http://127.0.0.1:${state.spacetimePort}`,
);
console.log(`[worker-smoke] Nginx=http://127.0.0.1:${state.httpPort}`);
return state;
}
@@ -552,7 +604,7 @@ async function enqueueSmokeJob(options = {}) {
source_module: 'editor-canvas',
source_entity_id: `worker-smoke-entity-${suffix}`,
request_label: `worker-smoke ${label}`,
request_payload_json: JSON.stringify({label, suffix}),
request_payload_json: JSON.stringify({ label, suffix }),
max_attempts: 1,
available_at_micros: nowMicros,
created_at_micros: nowMicros,
@@ -574,7 +626,9 @@ async function enqueueSmokeJob(options = {}) {
}
async function printQueueStatus() {
console.log('[worker-smoke] external_generation_job 是 private tablestatus 显示最近 worker 日志:');
console.log(
'[worker-smoke] external_generation_job 是 private tablestatus 显示最近 worker 日志:',
);
await printServiceLogs('external-generation-worker', 120);
}
@@ -584,17 +638,24 @@ async function waitForJobConsumed(jobId) {
while (Date.now() < deadline) {
const result = await dockerComposeCapture(
['logs', '--no-color', 'external-generation-worker'],
{allowFailure: true, quiet: true},
{ allowFailure: true, quiet: true },
);
lastOutput = `${result.stdout}\n${result.stderr}`;
if (lastOutput.includes(jobId) && lastOutput.includes('暂不支持的任务类型')) {
console.log(`[worker-smoke] job ${jobId} 已被 worker 领取并执行到 unsupported 分支。`);
if (
lastOutput.includes(jobId) &&
lastOutput.includes('暂不支持的任务类型')
) {
console.log(
`[worker-smoke] job ${jobId} 已被 worker 领取并执行到 unsupported 分支。`,
);
return;
}
await sleep(1000);
}
await printServiceLogs('external-generation-worker', 120);
throw new Error(`等待 worker 消费 job ${jobId} 超时,最后输出:\n${lastOutput}`);
throw new Error(
`等待 worker 消费 job ${jobId} 超时,最后输出:\n${lastOutput}`,
);
}
async function assertSavedPortsAvailableForNewProject(state) {
@@ -634,7 +695,7 @@ async function getProjectContainerIds() {
async function assertWorkersRunning() {
const result = await dockerComposeCapture(
['ps', '--status', 'running', '-q', 'external-generation-worker'],
{allowFailure: true, quiet: true},
{ allowFailure: true, quiet: true },
);
const workerIds = result.stdout
.split(/\r?\n/u)
@@ -644,7 +705,9 @@ async function assertWorkersRunning() {
return;
}
await printServiceLogs('external-generation-worker', 80);
throw new Error('external-generation-worker 未处于 running 状态,已输出最近日志。');
throw new Error(
'external-generation-worker 未处于 running 状态,已输出最近日志。',
);
}
async function printServiceLogs(service, tail = 80) {
@@ -663,8 +726,15 @@ async function waitForApi() {
const deadline = Date.now() + 120_000;
while (Date.now() < deadline) {
const result = await dockerComposeCapture(
['exec', '-T', 'api-server', 'curl', '-fsS', 'http://127.0.0.1:8082/healthz'],
{allowFailure: true, quiet: true},
[
'exec',
'-T',
'api-server',
'curl',
'-fsS',
'http://127.0.0.1:8082/healthz',
],
{ allowFailure: true, quiet: true },
);
if (result.code === 0) {
console.log('[worker-smoke] api-server 已就绪: api-server:8082/healthz');
@@ -690,7 +760,7 @@ async function waitForHttp(url, label) {
throw new Error(`${label} 等待超时: ${url}`);
}
async function apiOnlyUpdate({build}) {
async function apiOnlyUpdate({ build }) {
const beforeWorkerIds = await getContainerIds('external-generation-worker');
const args = ['up', '-d', '--no-deps', '--force-recreate'];
if (build) {
@@ -730,7 +800,7 @@ async function getContainerIds(service) {
}
async function dockerCompose(args) {
await run('docker', composeArgs(args), {env: composeEnv()});
await run('docker', composeArgs(args), { env: composeEnv() });
}
async function dockerComposeCapture(args, options = {}) {
@@ -750,7 +820,8 @@ function composeEnv() {
...process.env,
GENARRATIVE_CONTAINER_API_ENV_FILE: './worker-smoke/api-server.env',
GENARRATIVE_CONTAINER_SPACETIME_IMAGE:
process.env.GENARRATIVE_CONTAINER_SPACETIME_IMAGE || localSpacetimeImageName(),
process.env.GENARRATIVE_CONTAINER_SPACETIME_IMAGE ||
localSpacetimeImageName(),
GENARRATIVE_CONTAINER_SPACETIME_PORT: String(state.spacetimePort),
GENARRATIVE_CONTAINER_HTTP_PORT: String(state.httpPort),
GENARRATIVE_CONTAINER_OTLP_GRPC_PORT: String(state.otlpGrpcPort),
@@ -773,7 +844,9 @@ function sleep(ms) {
async function run(commandName, args, options = {}) {
const result = await runCapture(commandName, args, options);
if (result.code !== 0 && !options.allowFailure) {
throw new Error(`${commandName} ${args.join(' ')} 失败,exit=${result.code}`);
throw new Error(
`${commandName} ${args.join(' ')} 失败,exit=${result.code}`,
);
}
return result;
}
@@ -807,7 +880,7 @@ function runCapture(commandName, args, options = {}) {
reject(new Error(`${commandName} 被信号终止: ${signal}`));
return;
}
resolve({code: code ?? 0, stdout, stderr});
resolve({ code: code ?? 0, stdout, stderr });
});
});
}
File diff suppressed because it is too large Load Diff
+3 -1
View File
@@ -376,7 +376,9 @@ describe('dev scheduler api-server env', () => {
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_HOST).toBe('127.0.0.1');
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_PORT).toBe('18083');
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_CONCURRENCY).toBe('16');
expect(workerEnv.GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS).toBe('5000');
expect(workerEnv.GENARRATIVE_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS).toBe(
'5000',
);
expect(workerEnv.GENARRATIVE_BGFILTER_WORKER_MAX_REQUESTS).toBe('2048');
});
+41 -12
View File
@@ -9,11 +9,13 @@ export function parseArgs(argv) {
'GENARRATIVE_SPACETIME_MIGRATION_CHUNK_SIZE',
),
database: process.env.GENARRATIVE_SPACETIME_DATABASE || '',
bootstrapSecret: process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET || '',
bootstrapSecret:
process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET || '',
bootstrapSecretFile:
process.env.GENARRATIVE_SPACETIME_MIGRATION_BOOTSTRAP_SECRET_FILE || '',
includeTables: [],
operatorIdentity: process.env.GENARRATIVE_SPACETIME_MIGRATION_OPERATOR_IDENTITY || '',
operatorIdentity:
process.env.GENARRATIVE_SPACETIME_MIGRATION_OPERATOR_IDENTITY || '',
passthrough: [],
note: '',
server: process.env.GENARRATIVE_SPACETIME_SERVER || '',
@@ -142,7 +144,9 @@ export function buildSpacetimeCallArgs(options, procedureName, input) {
export async function callSpacetimeProcedure(options, procedureName, input) {
if (!options.database) {
throw new Error('必须传入 --database,或设置 GENARRATIVE_SPACETIME_DATABASE。');
throw new Error(
'必须传入 --database,或设置 GENARRATIVE_SPACETIME_DATABASE。',
);
}
validateSpacetimeDatabaseName(options.database);
@@ -196,7 +200,9 @@ export async function createSpacetimeWebIdentity(options) {
const text = await response.text();
if (!response.ok) {
throw new Error(`SpacetimeDB identity HTTP ${response.status}: ${trimPreview(text)}`);
throw new Error(
`SpacetimeDB identity HTTP ${response.status}: ${trimPreview(text)}`,
);
}
let payload;
@@ -209,16 +215,25 @@ export async function createSpacetimeWebIdentity(options) {
}
const identity =
payload.identity ?? payload.Identity ?? payload.identity_hex ?? payload.identityHex;
payload.identity ??
payload.Identity ??
payload.identity_hex ??
payload.identityHex;
const token = payload.token ?? payload.Token;
if (typeof identity !== 'string' || typeof token !== 'string') {
throw new Error(`SpacetimeDB identity 响应缺少 identity/token: ${trimPreview(text)}`);
throw new Error(
`SpacetimeDB identity 响应缺少 identity/token: ${trimPreview(text)}`,
);
}
return { identity, token };
}
export async function callSpacetimeProcedureAuto(options, procedureName, input) {
export async function callSpacetimeProcedureAuto(
options,
procedureName,
input,
) {
if (options.useHttp) {
return callSpacetimeProcedure(options, procedureName, input);
}
@@ -226,7 +241,11 @@ export async function callSpacetimeProcedureAuto(options, procedureName, input)
return callSpacetimeProcedureViaCli(options, procedureName, input);
}
export async function callSpacetimeProcedureViaCli(options, procedureName, input) {
export async function callSpacetimeProcedureViaCli(
options,
procedureName,
input,
) {
const args = buildSpacetimeCallArgs(options, procedureName, input);
const output = await runSpacetimeCli(args);
return parseProcedureResult(output, procedureName);
@@ -335,7 +354,8 @@ function normalizeSatsProduct(value, procedureName) {
}
if (
procedureName === 'normalize_editor_character_animation_metadata_and_return' &&
procedureName ===
'normalize_editor_character_animation_metadata_and_return' &&
value.length === 19
) {
return {
@@ -515,7 +535,10 @@ function normalizeSatsValue(value) {
if (value && typeof value === 'object') {
return Object.fromEntries(
Object.entries(value).map(([key, entry]) => [key, normalizeSatsValue(entry)]),
Object.entries(value).map(([key, entry]) => [
key,
normalizeSatsValue(entry),
]),
);
}
@@ -599,7 +622,9 @@ export function resolveServerUrl(options) {
return 'http://127.0.0.1:3101';
}
throw new Error(`未知 SpacetimeDB server: ${server}。请改用 --server-url 显式传入地址。`);
throw new Error(
`未知 SpacetimeDB server: ${server}。请改用 --server-url 显式传入地址。`,
);
}
function resolveCliServer(options) {
@@ -653,7 +678,11 @@ function runSpacetimeCli(args) {
return;
}
if (code !== 0) {
reject(new Error(`spacetime call 失败,退出码 ${code}: ${trimPreview(output)}`));
reject(
new Error(
`spacetime call 失败,退出码 ${code}: ${trimPreview(output)}`,
),
);
return;
}