合并 origin/master：保留 3D 契约与 provider checkpoint，旧玩法表随主线退役
- server-rs/crates/shared-contracts/src/lib.rs：采用主线的模块裁剪（删掉 cfg(any()) 遗留模块、补上 agc_analytics 与 game_distribution），同时保留本分支的 editor_canvas、model3d 模块与 EDITOR_GENERATION_OPERATION_KINDS 导出 - server-rs/crates/spacetime-module/src/migration.rs：主线删除的 410 行旧玩法表 normalize 段保持删除，保留本分支的 provider_kind / provider_task_id 兼容段与对应用例 - docs/【开发运维】本地开发验证与生产运维-2026-05-15.md：校验清单同时保留 Tripo 3D 生成任务与 editor_background_music_generation / model3d_text_to_model / model3d_image_to_model - .gitignore：补回本分支新增的 3D 模型文件忽略规则（*.glb / *.gltf 等 12 行），压测数据段随主线一并删除 - 共享记忆：本分支的 3D 决策与踩坑条目保留在主线重排后的 decision-log.md 与 pitfalls.md 中
This commit is contained in:
@@ -0,0 +1,410 @@
|
||||
#!/usr/bin/env node
|
||||
// 仅在临时 standalone 验证客户端埋点；显式传入带测试 bootstrap hash 的 WASM。
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { once } from 'node:events';
|
||||
import { access, chmod, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import net from 'node:net';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { createSpacetimeWebIdentity } from './spacetime-migration-common.mjs';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const database = 'agc-analytics-smoke';
|
||||
const testSecret = 'a'.repeat(64); // 公开测试值，绝不能用于正式部署。
|
||||
const sensitive = [testSecret];
|
||||
const redact = (value) =>
|
||||
sensitive.reduce(
|
||||
(text, secret) => text.replaceAll(secret, '[REDACTED]'),
|
||||
String(value),
|
||||
);
|
||||
|
||||
function command(args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('spacetime', args, {
|
||||
cwd: root,
|
||||
windowsHide: true,
|
||||
shell: false,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let output = '';
|
||||
for (const stream of [child.stdout, child.stderr])
|
||||
stream.on('data', (chunk) => {
|
||||
output = (output + chunk).slice(-16000);
|
||||
});
|
||||
const timer = setTimeout(() => {
|
||||
child.kill();
|
||||
reject(new Error('SpacetimeDB command timed out'));
|
||||
}, 120000);
|
||||
child.once('error', (error) => {
|
||||
clearTimeout(timer);
|
||||
reject(error);
|
||||
});
|
||||
child.once('exit', (code) => {
|
||||
clearTimeout(timer);
|
||||
if (code === 0) resolve(output);
|
||||
else reject(new Error(redact(output)));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function localPort() {
|
||||
const listener = net.createServer();
|
||||
listener.listen(0, '127.0.0.1');
|
||||
await once(listener, 'listening');
|
||||
const port = listener.address().port;
|
||||
await new Promise((resolve) => listener.close(resolve));
|
||||
return port;
|
||||
}
|
||||
|
||||
async function call(url, token, name, input) {
|
||||
const response = await fetch(`${url}/v1/database/${database}/call/${name}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify([input]),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
const text = await response.text();
|
||||
assert(response.ok, `${name}: HTTP ${response.status}: ${redact(text)}`);
|
||||
return JSON.parse(text);
|
||||
}
|
||||
|
||||
function ok(result) {
|
||||
assert.equal(result[0], 0, `Expected Ok, got ${JSON.stringify(result)}`);
|
||||
return JSON.parse(result[1]);
|
||||
}
|
||||
|
||||
const event = (user = 'smoke-user-a') => ({
|
||||
schema_version: 1,
|
||||
event_id: randomUUID(),
|
||||
event_name: 'editor_session_start',
|
||||
event_time: '2026-09-21T12:00:00.123Z',
|
||||
user_id: user,
|
||||
editor_session_id: randomUUID(),
|
||||
project_id: null,
|
||||
creative_task_id: null,
|
||||
agent_run_id: null,
|
||||
agent_turn_id: null,
|
||||
status: 'success',
|
||||
error_code: null,
|
||||
source: 'editor',
|
||||
client_version: 'smoke-1',
|
||||
properties: { entry_source: 'direct_launch', first_project_id: null },
|
||||
});
|
||||
const batch = (events) => ({
|
||||
schema_version: 1,
|
||||
batch_id: randomUUID(),
|
||||
destination_origin: 'https://analytics-smoke.invalid',
|
||||
user_id: events[0].user_id,
|
||||
events,
|
||||
});
|
||||
|
||||
function pricingInput() {
|
||||
const tiered = (model, unit, keys) => ({
|
||||
model,
|
||||
unit,
|
||||
price: [1, []],
|
||||
prices: keys.map((key) => ({ key, price: 1 })),
|
||||
});
|
||||
return {
|
||||
admin_user_id: 'smoke-bootstrap',
|
||||
updated_at_micros: 1,
|
||||
bootstrap_secret: testSecret,
|
||||
models: [
|
||||
tiered('gemini-3.1-flash-image-preview', 'perGeneration', [
|
||||
'0.5K',
|
||||
'1K',
|
||||
'2K',
|
||||
]),
|
||||
tiered('gpt-image-2', 'perGeneration', ['1K', '2K']),
|
||||
...['audio1.0', 'eleven_text_to_sound_v2', 'chirp-v5'].map((model) => ({
|
||||
model,
|
||||
unit: 'perGeneration',
|
||||
price: [0, 1],
|
||||
prices: [],
|
||||
})),
|
||||
...[
|
||||
'seedance2.0-fast',
|
||||
'seedance2.0',
|
||||
'kling3.0',
|
||||
'kling3.0-omni',
|
||||
'veo3.1',
|
||||
'veo3.1-fast',
|
||||
].map((model) => tiered(model, 'perSecond', ['480p', '720p', '1080p'])),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function verify(url, serviceToken, outsiderToken) {
|
||||
const upload = async (payload) =>
|
||||
call(
|
||||
url,
|
||||
serviceToken,
|
||||
'upload_agc_analytics_batch',
|
||||
JSON.stringify(payload),
|
||||
);
|
||||
const list = async (query = {}) =>
|
||||
ok(
|
||||
await call(
|
||||
url,
|
||||
serviceToken,
|
||||
'list_agc_tracking_events',
|
||||
JSON.stringify(query),
|
||||
),
|
||||
);
|
||||
const first = batch([event(), event(), event()]);
|
||||
first.events[0].event_time = '2026-09-21T12:01:00.123Z';
|
||||
assert.deepEqual(ok(await upload(first)), {
|
||||
acknowledged_batch_ids: [first.batch_id],
|
||||
event_count: 3,
|
||||
});
|
||||
const initial = await list();
|
||||
assert.equal(initial.entries.length, 3);
|
||||
ok(await upload(first));
|
||||
assert.deepEqual(
|
||||
await list(),
|
||||
initial,
|
||||
'Replay must preserve rows, first batch ID and receipt time',
|
||||
);
|
||||
const reordered = structuredClone(first);
|
||||
reordered.events[0].properties = {
|
||||
first_project_id: null,
|
||||
entry_source: 'direct_launch',
|
||||
};
|
||||
ok(await upload(reordered));
|
||||
assert.deepEqual(
|
||||
await list(),
|
||||
initial,
|
||||
'JSON property order must not cause conflict',
|
||||
);
|
||||
|
||||
const rollbackCandidate = event();
|
||||
const conflict = batch([
|
||||
rollbackCandidate,
|
||||
{ ...first.events[0], client_version: 'changed' },
|
||||
]);
|
||||
const rejected = await upload(conflict);
|
||||
assert.equal(rejected[0], 1);
|
||||
assert.equal(rejected[1], 'agc_event_conflict');
|
||||
assert.deepEqual(
|
||||
await list(),
|
||||
initial,
|
||||
'Earlier insert in failed batch must roll back',
|
||||
);
|
||||
const userB = batch([event('smoke-user-b')]);
|
||||
ok(await upload(userB));
|
||||
assert.equal(
|
||||
(await list({ userId: 'smoke-user-b' })).entries[0].eventId,
|
||||
userB.events[0].event_id,
|
||||
);
|
||||
|
||||
const page1 = await list({ limit: 2 });
|
||||
assert(page1.nextCursor);
|
||||
const late = batch([event()]);
|
||||
late.events[0].event_time = '2026-09-21T11:00:00.123Z';
|
||||
ok(await upload(late));
|
||||
const seen = [...page1.entries];
|
||||
let cursor = page1.nextCursor;
|
||||
while (cursor) {
|
||||
const page = await list({ limit: 2, cursor });
|
||||
seen.push(...page.entries);
|
||||
cursor = page.nextCursor;
|
||||
}
|
||||
assert.equal(seen.length, 4);
|
||||
assert.equal(new Set(seen.map((row) => row.eventId)).size, 4);
|
||||
assert(
|
||||
!seen.some((row) => row.eventId === late.events[0].event_id),
|
||||
'Cursor snapshot must exclude later insert',
|
||||
);
|
||||
for (let index = 1; index < seen.length; index++) {
|
||||
const previous = seen[index - 1];
|
||||
const current = seen[index];
|
||||
assert(
|
||||
previous.eventTime > current.eventTime ||
|
||||
(previous.eventTime === current.eventTime &&
|
||||
previous.eventId > current.eventId),
|
||||
'Stable event time/event ID descending order',
|
||||
);
|
||||
}
|
||||
const refreshed = await list();
|
||||
assert.equal(refreshed.entries.length, 5);
|
||||
assert.equal(refreshed.entries[0].eventId, first.events[0].event_id);
|
||||
assert.equal(refreshed.entries.at(-1).eventId, late.events[0].event_id);
|
||||
assert.equal(refreshed.entries[0].projectId, null);
|
||||
assert.equal(refreshed.entries[0].eventTime, '2026-09-21T12:01:00.123Z');
|
||||
for (const [name, input] of [
|
||||
['upload_agc_analytics_batch', first],
|
||||
['list_agc_tracking_events', {}],
|
||||
]) {
|
||||
const unauthorized = await call(
|
||||
url,
|
||||
outsiderToken,
|
||||
name,
|
||||
JSON.stringify(input),
|
||||
);
|
||||
assert.equal(unauthorized[0], 1, 'Nonservice identity must be rejected');
|
||||
assert.match(unauthorized[1], /无权/);
|
||||
}
|
||||
console.log(
|
||||
'[agc-analytics-smoke] PASS: first/replay, JSON order, atomic rollback, user filter, stable snapshot pagination, refresh, null/time, service authorization. 5 rows persisted.',
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const wasm = process.env.GENARRATIVE_AGC_ANALYTICS_SMOKE_WASM;
|
||||
assert(
|
||||
wasm,
|
||||
'Set GENARRATIVE_AGC_ANALYTICS_SMOKE_WASM to an isolated test WASM compiled with SHA256(a repeated 64 times) bootstrap hash.',
|
||||
);
|
||||
await access(wasm);
|
||||
const version = await command(['--version']);
|
||||
assert(
|
||||
version.includes('version 2.8.3') &&
|
||||
version.includes('8e410d2842147bd8e5a32a9589cc00c19f7478e2'),
|
||||
);
|
||||
const temp = await mkdtemp(
|
||||
path.join(os.tmpdir(), 'genarrative-agc-analytics-smoke-'),
|
||||
);
|
||||
let standalone;
|
||||
try {
|
||||
const port = await localPort();
|
||||
const url = `http://127.0.0.1:${port}`;
|
||||
standalone = spawn(
|
||||
'spacetime',
|
||||
[
|
||||
'start',
|
||||
'--data-dir',
|
||||
path.join(temp, 'data'),
|
||||
'--listen-addr',
|
||||
`127.0.0.1:${port}`,
|
||||
'--non-interactive',
|
||||
],
|
||||
{ windowsHide: true, shell: false, stdio: ['ignore', 'pipe', 'pipe'] },
|
||||
);
|
||||
let logs = '';
|
||||
for (const stream of [standalone.stdout, standalone.stderr])
|
||||
stream.on('data', (chunk) => {
|
||||
logs = (logs + chunk).slice(-8000);
|
||||
});
|
||||
const deadline = Date.now() + 30000;
|
||||
for (;;) {
|
||||
assert(standalone.exitCode === null, redact(logs));
|
||||
try {
|
||||
if (
|
||||
(await fetch(`${url}/v1/ping`, { signal: AbortSignal.timeout(1000) }))
|
||||
.ok
|
||||
)
|
||||
break;
|
||||
} catch {
|
||||
/* 等待启动 */
|
||||
}
|
||||
assert(
|
||||
Date.now() < deadline,
|
||||
`Standalone startup timeout: ${redact(logs)}`,
|
||||
);
|
||||
await delay(200);
|
||||
}
|
||||
const owner = await createSpacetimeWebIdentity({
|
||||
database,
|
||||
serverUrl: url,
|
||||
});
|
||||
const service = await createSpacetimeWebIdentity({
|
||||
database,
|
||||
serverUrl: url,
|
||||
});
|
||||
sensitive.push(owner.token, service.token);
|
||||
const config = path.join(temp, 'cli.toml');
|
||||
await command(['--config-path', config, 'login', '--token', owner.token]);
|
||||
await chmod(config, 0o600);
|
||||
await command([
|
||||
'--config-path',
|
||||
config,
|
||||
'publish',
|
||||
database,
|
||||
'--server',
|
||||
url,
|
||||
'--yes=all',
|
||||
'--no-config',
|
||||
'--bin-path',
|
||||
path.resolve(wasm),
|
||||
]);
|
||||
const bootstrap = await call(
|
||||
url,
|
||||
service.token,
|
||||
'initialize_editor_generation_pricing_config_if_missing_and_return',
|
||||
pricingInput(),
|
||||
);
|
||||
assert.equal(
|
||||
bootstrap[0],
|
||||
true,
|
||||
`Pricing service bootstrap failed: ${redact(JSON.stringify(bootstrap))}`,
|
||||
);
|
||||
await verify(url, service.token, owner.token);
|
||||
if (process.env.AGC_ANALYTICS_SMOKE_KEEP === '1') {
|
||||
const stopFile = path.join(temp, 'stop');
|
||||
const contextPath = path.join(temp, 'connection.json');
|
||||
await writeFile(
|
||||
contextPath,
|
||||
JSON.stringify({
|
||||
serverUrl: url,
|
||||
database,
|
||||
token: service.token,
|
||||
operatorToken: owner.token,
|
||||
configPath: config,
|
||||
stopFile,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
console.log(
|
||||
`[agc-analytics-smoke] Integration connection file: ${contextPath}`,
|
||||
);
|
||||
const stopDeadline = Date.now() + 45 * 60 * 1000;
|
||||
while (Date.now() < stopDeadline) {
|
||||
try {
|
||||
await access(stopFile);
|
||||
break;
|
||||
} catch {
|
||||
/* 联调结束后由调用方创建 stop 文件 */
|
||||
}
|
||||
await delay(1000);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (standalone && standalone.exitCode === null) {
|
||||
if (process.platform === 'win32') {
|
||||
const killer = spawn(
|
||||
'taskkill',
|
||||
['/PID', String(standalone.pid), '/T', '/F'],
|
||||
{ windowsHide: true, stdio: 'ignore' },
|
||||
);
|
||||
await once(killer, 'exit');
|
||||
} else {
|
||||
standalone.kill('SIGTERM');
|
||||
}
|
||||
if (standalone.exitCode === null) await once(standalone, 'exit');
|
||||
}
|
||||
// 仅删除本脚本 mkdtemp 创建的系统临时子目录。
|
||||
assert(
|
||||
path.dirname(path.resolve(temp)) === path.resolve(os.tmpdir()) &&
|
||||
path.basename(temp).startsWith('genarrative-agc-analytics-smoke-'),
|
||||
);
|
||||
await rm(temp, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
maxRetries: 5,
|
||||
retryDelay: 200,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(redact(error.stack ?? error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -5,6 +5,15 @@ set -euo pipefail
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
base_ref="${1:?usage: build-gitea-rust-cache.sh <verified-base-image> <candidate-tag>}"
|
||||
candidate_tag="${2:?candidate image tag is required}"
|
||||
requested_source_commit="${3:-}"
|
||||
if [[ "$#" -gt 3 ]]; then
|
||||
echo 'usage: build-gitea-rust-cache.sh <verified-base-image> <candidate-tag> [master-commit-sha]' >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ -n "${requested_source_commit}" && ! "${requested_source_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then
|
||||
echo 'master-commit-sha must be a complete 40-character SHA.' >&2
|
||||
exit 2
|
||||
fi
|
||||
# 与实际 Gitea checkout 路径一致；Rust 对象 key 包含编译 cwd，不能随意换临时根。
|
||||
workspace=/workspace/GenarrativeAI/Genarrative
|
||||
[[ "${CI:-}" != true ]] || { echo 'Run on the trusted image builder, outside CI jobs.' >&2; exit 1; }
|
||||
@@ -22,7 +31,17 @@ bash "${repo_root}/scripts/gitea-ci-job-image.sh" verify "${base_id}"
|
||||
|
||||
# 只归档远端 master 的确定提交；不复制当前工作区或本地凭据。
|
||||
git -C "${repo_root}" fetch --no-tags origin refs/heads/master
|
||||
source_commit="$(git -C "${repo_root}" rev-parse FETCH_HEAD^{commit})"
|
||||
master_commit="$(git -C "${repo_root}" rev-parse FETCH_HEAD^{commit})"
|
||||
if [[ -n "${requested_source_commit}" ]]; then
|
||||
git -C "${repo_root}" cat-file -e "${requested_source_commit}^{commit}"
|
||||
if ! git -C "${repo_root}" merge-base --is-ancestor "${requested_source_commit}" "${master_commit}"; then
|
||||
echo "master-commit-sha is not contained in fetched master: ${requested_source_commit}" >&2
|
||||
exit 1
|
||||
fi
|
||||
source_commit="$(git -C "${repo_root}" rev-parse "${requested_source_commit}^{commit}")"
|
||||
else
|
||||
source_commit="${master_commit}"
|
||||
fi
|
||||
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/gitea-rust-cache.XXXXXX")"
|
||||
container_id=''
|
||||
cleanup() {
|
||||
@@ -50,7 +69,7 @@ container_id="$(docker run --detach --cpus=4 --memory=12g --pids-limit=1024 \
|
||||
docker exec "${container_id}" mkdir -p "${workspace}" /opt/genarrative-ci/rust-cache/objects
|
||||
git -C "${repo_root}" archive "${source_commit}" | docker cp - "${container_id}:${workspace}"
|
||||
docker cp "${work_dir}/snapshot/." "${container_id}:/opt/genarrative-ci/rust-cache/"
|
||||
docker cp "${repo_root}/scripts/ci-rust-cache.sh" "${container_id}:/tmp/ci-rust-cache.sh"
|
||||
docker exec "${container_id}" cp "${workspace}/scripts/ci-rust-cache.sh" /tmp/ci-rust-cache.sh
|
||||
docker exec --interactive --workdir "${workspace}" "${container_id}" bash -s <<'WARM'
|
||||
set -euo pipefail
|
||||
rustc -vV > /opt/genarrative-ci/rust-cache/rustc.txt
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2602,12 +2602,13 @@ function assertAiGameCreatorShellUserDevBoundary() {
|
||||
{ url: 'https://*/api/*' },
|
||||
{ url: 'http://localhost:*/*' },
|
||||
{ url: 'http://127.0.0.1:*/*' },
|
||||
{ url: 'https://*.aliyuncs.com/*' },
|
||||
])
|
||||
) {
|
||||
throw new Error(
|
||||
// 更新清单与安装包下载已改由 tauri-plugin-updater 在原生侧完成，
|
||||
// 不再需要为 webview 的 http 插件放行 OSS 域名。
|
||||
'AI game creator native HTTP scope must match the release, dev, custom HTTPS, and loopback API boundary',
|
||||
// 更新清单与安装包下载由 tauri-plugin-updater 在原生侧完成；
|
||||
// 封面与截图仍通过 webview 的 http 插件向凭证指定的 OSS 地址直传。
|
||||
'AI game creator native HTTP scope must match the release, dev, custom HTTPS, loopback API, and OSS media upload boundary',
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2647,7 +2648,7 @@ function assertAiGameCreatorShellUserDevBoundary() {
|
||||
}
|
||||
for (const snippet of [
|
||||
"await invoke<LocalPreviewStatus>(\n 'activate_local_game_preview'",
|
||||
'已切换到客户端运行视图',
|
||||
'已载入客户端运行视图',
|
||||
]) {
|
||||
if (!sourceIncludesSnippet(aiGameCreatorShellAppSource, snippet)) {
|
||||
throw new Error(
|
||||
@@ -2665,6 +2666,14 @@ function assertAiGameCreatorShellUserDevBoundary() {
|
||||
);
|
||||
}
|
||||
}
|
||||
/*
|
||||
* 预览**不自动**开系统浏览器：启动与切换运行视图只走内置画面，两条老路必须一直挡着
|
||||
* （App.tsx 里的 `openPreviewInExternalBrowser` / 前端直调 `open_local_game_preview` /
|
||||
* Rust 侧 `.open_url(`）。
|
||||
*
|
||||
* 用户主动点「在浏览器打开」是另一回事：它走 opener 插件的 `openUrl`，入口只有顶栏那一枚
|
||||
* 按钮（见下面那条正向断言）。别把这条负向守卫推广成「任何地方都不许出现 openUrl」。
|
||||
*/
|
||||
if (
|
||||
aiGameCreatorShellAppSource.includes('openPreviewInExternalBrowser') ||
|
||||
aiGameCreatorShellAppSource.includes('open_local_game_preview') ||
|
||||
@@ -2674,6 +2683,31 @@ function assertAiGameCreatorShellUserDevBoundary() {
|
||||
'AI game creator preview must not invoke the external browser',
|
||||
);
|
||||
}
|
||||
for (const snippet of [
|
||||
"'@tauri-apps/plugin-opener'",
|
||||
'在浏览器打开',
|
||||
'await openUrl(embeddedPreviewUrl)',
|
||||
]) {
|
||||
if (
|
||||
!sourceIncludesSnippet(aiGameCreatorProjectDevelopmentSource, snippet)
|
||||
) {
|
||||
throw new Error(
|
||||
`AI game creator in-browser preview entry drifted: missing ${snippet}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
/*
|
||||
* 上面那条负向守卫只盯着 App.tsx 里的两条老路，而真正的浏览器出口在运行页视图里。
|
||||
* 这里再补一条「调用点只有一个」的判据：视图里冒出第二个 `openUrl(`（自动跳浏览器、
|
||||
* 或者拿它去开任意地址）时必须先显式改这条守卫，而不是顺手加一行。
|
||||
*/
|
||||
const openUrlCallCount =
|
||||
aiGameCreatorProjectDevelopmentSource.split('openUrl(').length - 1;
|
||||
if (openUrlCallCount !== 1) {
|
||||
throw new Error(
|
||||
`AI game creator must keep exactly one user-clicked browser-open call in the run view (found ${openUrlCallCount})`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
aiGameCreatorShellTauriSource.includes('fn open_developer_window(') ||
|
||||
aiGameCreatorShellTauriSource.includes(
|
||||
|
||||
@@ -54,6 +54,17 @@ const checks = [
|
||||
reason:
|
||||
'Copy Artifact Production 模式下，AGC 发号 Job 还必须授权手动发布管线读取总号，否则用户触发的手动发布会停在 copyArtifacts（SYSTEM 定时构建不受影响）。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.agc-global-version-issue',
|
||||
includes: 'Genarrative-Scheduled-Release-Trigger',
|
||||
reason:
|
||||
'Copy Artifact Production 模式下，AGC 发号 Job 必须授权每日 release 调度器读取总号。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/agc-global-version-issue-job-config.xml',
|
||||
includes: '<string>release-unified</string>',
|
||||
reason: 'AGC 发号 Job 配置必须保留 release 调度的统一发号用途标签。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-stdb-module-build',
|
||||
includes: 'npm run check:rustfmt',
|
||||
@@ -7733,6 +7744,14 @@ const scheduledRevisionTriggerJobConfig = readFileSync(
|
||||
'jenkins/scheduled-revision-trigger-job-config.xml',
|
||||
'utf8',
|
||||
);
|
||||
const scheduledReleaseTriggerContent = readFileSync(
|
||||
'jenkins/Jenkinsfile.scheduled-release-trigger',
|
||||
'utf8',
|
||||
);
|
||||
const scheduledReleaseTriggerJobConfig = readFileSync(
|
||||
'jenkins/scheduled-release-trigger-job-config.xml',
|
||||
'utf8',
|
||||
);
|
||||
|
||||
// 定时与版本比较统一收敛到调度管线，两个下游流水线不得再自带触发器。
|
||||
for (const [file, content] of [
|
||||
@@ -7742,7 +7761,7 @@ for (const [file, content] of [
|
||||
if (/\btriggers\s*\{/u.test(content) || content.includes('cron(')) {
|
||||
failed = true;
|
||||
console.error(
|
||||
`[check:production-ops] ${file} 不得自带定时触发器；版本检查与触发必须由 Genarrative-Scheduled-Revision-Trigger 统一负责。`,
|
||||
`[check:production-ops] ${file} 不得自带定时触发器；版本检查与触发必须由两条定时调度管线统一负责。`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -7883,6 +7902,123 @@ if (
|
||||
'[check:production-ops] scheduled-revision-trigger Job 必须指向调度 Jenkinsfile、使用本机 Git 入口与既有 SSH 凭据，并把定时器留在 Jenkinsfile。',
|
||||
);
|
||||
}
|
||||
|
||||
for (const [snippet, reason] of [
|
||||
["cron('0 4 * * *')", 'release 调度器必须在每天 04:00 检查远端版本'],
|
||||
['disableConcurrentBuilds()', '必须禁止 release 调度器并发触发'],
|
||||
['skipDefaultCheckout(true)', 'release 调度器不得依赖本地 SCM 工作区'],
|
||||
['git ls-remote', 'release 调度器必须直接从远端解析分支版本'],
|
||||
[
|
||||
"FULL_BUILD_JOB_NAME = 'Genarrative-Full-Build-And-Deploy'",
|
||||
'必须声明 release Full Build 目标 Job',
|
||||
],
|
||||
[
|
||||
"AGC_WINDOWS_BUILD_JOB_NAME = 'Genarrative-Agc-Windows-Build'",
|
||||
'必须声明 AGC Windows release 构建目标 Job',
|
||||
],
|
||||
[
|
||||
"AGC_MACOS_BUILD_JOB_NAME = 'Genarrative-Agc-MacOS-Build'",
|
||||
'必须声明 AGC macOS release 构建目标 Job',
|
||||
],
|
||||
[
|
||||
"FULL_REVISION_STATE_FILE = '.jenkins-last-release-full-revision'",
|
||||
'必须独立记录 Full Build 成功 revision',
|
||||
],
|
||||
[
|
||||
"AGC_REVISION_STATE_FILE = '.jenkins-last-release-agc-revision'",
|
||||
'必须独立记录 AGC 双平台成功 revision',
|
||||
],
|
||||
['full=changed', 'release 调度器必须计算服务端 Full Build scope'],
|
||||
[
|
||||
"env.FULL_BUILD_SCOPE = readScope('full')",
|
||||
'release 调度器必须解析 Full Build scope',
|
||||
],
|
||||
[
|
||||
"string(name: 'AGC_CHANNEL', value: 'release-unified')",
|
||||
'release 调度器必须先通过发号 Job 获取统一总版本号',
|
||||
],
|
||||
[
|
||||
"string(name: 'AGC_UPDATE_CHANNEL', value: 'release')",
|
||||
'release 调度器必须把 Windows 与 macOS 都发到 release 分区',
|
||||
],
|
||||
[
|
||||
"string(name: 'DEPLOY_TARGET', value: 'release')",
|
||||
'release Full Build 必须固定部署到 release',
|
||||
],
|
||||
[
|
||||
"booleanParam(name: 'CONFIRM_RELEASE_DEPLOY_AGENT', value: true)",
|
||||
'release Full Build 必须确认使用独立 release 部署 agent',
|
||||
],
|
||||
[
|
||||
"string(name: 'DATABASE_BACKUP_MODE', value: params.DATABASE_BACKUP_MODE)",
|
||||
'release Full Build 必须显式透传数据库备份策略',
|
||||
],
|
||||
[
|
||||
"string(name: 'STDB_API_ROLLOUT_MODE', value: params.STDB_API_ROLLOUT_MODE)",
|
||||
'release Full Build 必须显式透传 Stdb/API rollout 策略',
|
||||
],
|
||||
[
|
||||
'def runDownstream = { String jobName, List jobParameters, int timeoutMinutes ->',
|
||||
'release 调度器必须统一等待并收集三个下游结果',
|
||||
],
|
||||
[
|
||||
'build job: jobName,\n wait: true,\n propagate: false,',
|
||||
'release 下游必须等待完成并保留每个 Job 的结果',
|
||||
],
|
||||
[
|
||||
'branches[env.FULL_BUILD_JOB_NAME] = {',
|
||||
'release 调度器必须触发 Full Build',
|
||||
],
|
||||
[
|
||||
'branches[env.AGC_WINDOWS_BUILD_JOB_NAME] = {',
|
||||
'release 调度器必须触发 AGC Windows release 构建',
|
||||
],
|
||||
[
|
||||
'branches[env.AGC_MACOS_BUILD_JOB_NAME] = {',
|
||||
'release 调度器必须触发 AGC macOS release 构建',
|
||||
],
|
||||
[
|
||||
"booleanParam(name: 'SKIP_IF_SUPERSEDED', value: true)",
|
||||
'release 调度器必须让 macOS 节点恢复后跳过过期排队构建',
|
||||
],
|
||||
[
|
||||
'if (fullSucceeded) {',
|
||||
'只有 Full Build 成功后才能推进 Full Build revision',
|
||||
],
|
||||
['if (agcSucceeded) {', '只有 AGC 双平台成功后才能推进 AGC revision'],
|
||||
[
|
||||
'error("release 下游未全部成功: ${summary}")',
|
||||
'任一 release 下游失败时调度器必须失败',
|
||||
],
|
||||
]) {
|
||||
if (!scheduledReleaseTriggerContent.includes(snippet)) {
|
||||
failed = true;
|
||||
console.error(`[check:production-ops] release 调度管线${reason}。`);
|
||||
}
|
||||
}
|
||||
if (scheduledReleaseTriggerContent.includes('wait: false')) {
|
||||
failed = true;
|
||||
console.error(
|
||||
'[check:production-ops] release 调度管线不得 fire-and-forget，必须等待并汇总 Full Build 与 AGC 结果。',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!scheduledReleaseTriggerJobConfig.includes(
|
||||
'<scriptPath>jenkins/Jenkinsfile.scheduled-release-trigger</scriptPath>',
|
||||
) ||
|
||||
!scheduledReleaseTriggerJobConfig.includes(
|
||||
'ssh://git@127.0.0.1:2222/GenarrativeAI/Genarrative.git',
|
||||
) ||
|
||||
!scheduledReleaseTriggerJobConfig.includes(
|
||||
'<credentialsId>genarrative-local-gitea-ssh</credentialsId>',
|
||||
) ||
|
||||
!scheduledReleaseTriggerJobConfig.includes('<triggers/>')
|
||||
) {
|
||||
failed = true;
|
||||
console.error(
|
||||
'[check:production-ops] scheduled-release-trigger Job 必须指向 release 调度 Jenkinsfile、使用本机 Git 入口与既有 SSH 凭据，并把定时器留在 Jenkinsfile。',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!fullPipelineContent.includes(
|
||||
"string(name: 'COMMIT_HASH', value: env.SOURCE_COMMIT)",
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* 游戏发行来源配置门禁。
|
||||
*
|
||||
* 逐条校验 `deploy/nginx/genarrative-release-origin.conf`：
|
||||
* 1) 每游戏独立 origin 的按主机映射（命名捕获 `game_id` + 发行网关前缀）；
|
||||
* 2) 只暴露发行网关，不代理平台 API / 后台 / SPA；
|
||||
* 3) 发行来源不使用 Cookie（边缘 403 + 转发前清空）；
|
||||
* 4) 响应头策略仍由 api-server 发行网关负责（源码级交叉检查）。
|
||||
* 只要本机存在 nginx 与 openssl，还会用自签通配证书渲染一份临时配置执行
|
||||
* `nginx -t`，把语法与指令上下文一起验证掉。
|
||||
*/
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import {
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = join(scriptDir, '..');
|
||||
const templatePath = join(
|
||||
repoRoot,
|
||||
'deploy/nginx/genarrative-release-origin.conf',
|
||||
);
|
||||
const gatewayPath = join(
|
||||
repoRoot,
|
||||
'server-rs/crates/api-server/src/modules/game_distribution.rs',
|
||||
);
|
||||
|
||||
const failures = [];
|
||||
const notes = [];
|
||||
|
||||
function fail(message) {
|
||||
failures.push(message);
|
||||
}
|
||||
|
||||
function normalize(source) {
|
||||
return source.replace(/\s+/gu, ' ');
|
||||
}
|
||||
|
||||
function requireSnippet(source, snippet, message) {
|
||||
if (!normalize(source).includes(normalize(snippet))) {
|
||||
fail(message);
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
if (!existsSync(templatePath)) {
|
||||
fail(`缺少发行来源模板：${templatePath}`);
|
||||
return;
|
||||
}
|
||||
const template = readFileSync(templatePath, 'utf8');
|
||||
|
||||
requireSnippet(
|
||||
template,
|
||||
'server_name ~^(?<game_id>[a-z0-9_]+)\\.games\\.example\\.com$;',
|
||||
'发行来源必须用命名捕获 game_id 的子域匹配（每游戏独立 origin）',
|
||||
);
|
||||
requireSnippet(
|
||||
template,
|
||||
'ssl_certificate /etc/letsencrypt/live/games.example.com/fullchain.pem;',
|
||||
'发行来源必须使用通配 TLS 证书',
|
||||
);
|
||||
requireSnippet(
|
||||
template,
|
||||
'if ($http_cookie) { return 403; }',
|
||||
'发行来源必须拒绝携带平台 Cookie 的请求',
|
||||
);
|
||||
requireSnippet(
|
||||
template,
|
||||
'proxy_set_header Cookie "";',
|
||||
'发行来源转发前必须清空 Cookie',
|
||||
);
|
||||
requireSnippet(
|
||||
template,
|
||||
'proxy_pass http://genarrative_release_api/api/game-distribution/releases/$game_id$request_uri;',
|
||||
'发行来源必须按 game_id 映射到发行网关前缀',
|
||||
);
|
||||
requireSnippet(
|
||||
template,
|
||||
'location /.well-known/acme-challenge/',
|
||||
'发行来源必须保留 ACME challenge 路径',
|
||||
);
|
||||
|
||||
requireSnippet(
|
||||
template,
|
||||
'location = / {',
|
||||
'发行来源必须显式把子域根路径映射为该游戏的 index.html',
|
||||
);
|
||||
requireSnippet(
|
||||
template,
|
||||
'proxy_pass http://genarrative_release_api/api/game-distribution/releases/$game_id/index.html;',
|
||||
'子域根路径必须映射到该游戏的 index.html',
|
||||
);
|
||||
const proxyPassCount = (template.match(/proxy_pass\s/gu) ?? []).length;
|
||||
if (proxyPassCount !== 2) {
|
||||
fail(
|
||||
`发行来源只应存在两条 proxy_pass（子域根路径与发行网关前缀），实际 ${proxyPassCount} 条`,
|
||||
);
|
||||
}
|
||||
const cookieStripCount = (
|
||||
template.match(/proxy_set_header Cookie "";/gu) ?? []
|
||||
).length;
|
||||
if (cookieStripCount !== 2) {
|
||||
fail(`每条发行来源代理都必须清空 Cookie，实际 ${cookieStripCount} 处`);
|
||||
}
|
||||
const gatewayPrefixCount = (
|
||||
template.match(/api\/game-distribution\/releases\/\$game_id/gu) ?? []
|
||||
).length;
|
||||
if (gatewayPrefixCount !== 2) {
|
||||
fail(`发行来源代理必须都映射到发行网关前缀，实际 ${gatewayPrefixCount} 处`);
|
||||
}
|
||||
for (const forbidden of [
|
||||
'/api/auth',
|
||||
'/api/profile',
|
||||
'/admin/api',
|
||||
'/api/game-distribution/games',
|
||||
'/api/game-distribution/versions',
|
||||
]) {
|
||||
if (template.includes(forbidden)) {
|
||||
fail(`发行来源不得代理平台命名空间：${forbidden}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!existsSync(gatewayPath)) {
|
||||
fail(`缺少发行网关源码：${gatewayPath}`);
|
||||
} else {
|
||||
const gateway = readFileSync(gatewayPath, 'utf8');
|
||||
for (const [snippet, message] of [
|
||||
[
|
||||
'header::X_CONTENT_TYPE_OPTIONS',
|
||||
'发行网关必须继续设置 X-Content-Type-Options',
|
||||
],
|
||||
[
|
||||
'HeaderName::from_static("cross-origin-resource-policy")',
|
||||
'发行网关必须继续设置 CORP',
|
||||
],
|
||||
[
|
||||
'HeaderValue::from_static("cross-origin")',
|
||||
'CORP 必须是 cross-origin（opaque sandbox 才能加载自有脚本）',
|
||||
],
|
||||
[
|
||||
'header::ACCESS_CONTROL_ALLOW_ORIGIN',
|
||||
'发行网关必须继续设置无凭据 CORS',
|
||||
],
|
||||
['header::CONTENT_SECURITY_POLICY', '发行网关必须继续为 HTML 设置 CSP'],
|
||||
['StatusCode::FORBIDDEN', '发行网关必须继续拒绝携带 Cookie 的请求'],
|
||||
]) {
|
||||
if (!gateway.includes(snippet)) {
|
||||
fail(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
validateWithNginx(template);
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error('[check:release-origin-config] FAILED');
|
||||
for (const message of failures) {
|
||||
console.error(`- ${message}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
for (const note of notes) {
|
||||
console.log(`[check:release-origin-config] ${note}`);
|
||||
}
|
||||
console.log(
|
||||
'[check:release-origin-config] OK（发行来源模板、网关响应头策略与 nginx 语法一致）',
|
||||
);
|
||||
}
|
||||
|
||||
function binaryExists(binary) {
|
||||
try {
|
||||
execFileSync('sh', ['-c', `command -v ${binary}`], { stdio: 'ignore' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function validateWithNginx(template) {
|
||||
if (!binaryExists('nginx')) {
|
||||
notes.push('未找到 nginx，跳过渲染后的 nginx -t');
|
||||
return;
|
||||
}
|
||||
const workDir = mkdtempSync(join(tmpdir(), 'genarrative-release-origin-'));
|
||||
try {
|
||||
const certPath = join(workDir, 'wildcard.crt');
|
||||
const keyPath = join(workDir, 'wildcard.key');
|
||||
if (binaryExists('openssl')) {
|
||||
execFileSync(
|
||||
'openssl',
|
||||
[
|
||||
'req',
|
||||
'-x509',
|
||||
'-newkey',
|
||||
'rsa:2048',
|
||||
'-nodes',
|
||||
'-days',
|
||||
'1',
|
||||
'-subj',
|
||||
'/CN=games.example.com',
|
||||
'-addext',
|
||||
'subjectAltName=DNS:*.games.example.com,DNS:games.example.com',
|
||||
'-keyout',
|
||||
keyPath,
|
||||
'-out',
|
||||
certPath,
|
||||
],
|
||||
{ stdio: 'ignore' },
|
||||
);
|
||||
} else {
|
||||
notes.push('未找到 openssl，跳过渲染后的 nginx -t');
|
||||
return;
|
||||
}
|
||||
const rendered = template
|
||||
.replace(
|
||||
'/etc/letsencrypt/live/games.example.com/fullchain.pem',
|
||||
certPath,
|
||||
)
|
||||
.replace('/etc/letsencrypt/live/games.example.com/privkey.pem', keyPath)
|
||||
.replace(
|
||||
/\/var\/log\/nginx\/(genarrative-release\.[a-z]+\.log)/gu,
|
||||
join(workDir, '$1'),
|
||||
)
|
||||
// 非 root 环境无法绑定 80/443；语法检查用高位端口，不改生产模板本身。
|
||||
.replace('listen 80;', 'listen 18080;')
|
||||
.replace('listen 443 ssl http2;', 'listen 18443 ssl http2;');
|
||||
const renderedPath = join(workDir, 'release-origin.conf');
|
||||
writeFileSync(renderedPath, rendered);
|
||||
const wrapperPath = join(workDir, 'nginx.conf');
|
||||
writeFileSync(
|
||||
wrapperPath,
|
||||
[
|
||||
`pid ${join(workDir, 'nginx.pid')};`,
|
||||
`error_log ${join(workDir, 'error.log')} warn;`,
|
||||
'events { worker_connections 64; }',
|
||||
'http {',
|
||||
' access_log off;',
|
||||
' client_body_temp_path ' + join(workDir, 'client-body') + ';',
|
||||
' proxy_temp_path ' + join(workDir, 'proxy') + ';',
|
||||
' fastcgi_temp_path ' + join(workDir, 'fastcgi') + ';',
|
||||
' uwsgi_temp_path ' + join(workDir, 'uwsgi') + ';',
|
||||
' scgi_temp_path ' + join(workDir, 'scgi') + ';',
|
||||
` include ${renderedPath};`,
|
||||
'}',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
try {
|
||||
execFileSync('nginx', ['-t', '-c', wrapperPath], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
notes.push('渲染后的发行来源配置通过 nginx -t');
|
||||
} catch (error) {
|
||||
const stderr = error.stderr ? String(error.stderr) : '';
|
||||
fail(
|
||||
`渲染后的发行来源配置未通过 nginx -t：${stderr.trim() || error.message}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -12,6 +12,7 @@ const tableCatalogPath =
|
||||
'docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md';
|
||||
const bindingsRoot = 'server-rs/crates/spacetime-client/src/module_bindings/';
|
||||
const allowBreaking = process.env.SPACETIME_SCHEMA_GUARD_ALLOW_BREAKING === '1';
|
||||
|
||||
function normalizePath(path) {
|
||||
return path.replace(/\\/gu, '/');
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# 仅消费镜像内的可信快照；所有写入留在当前容器的可写层。
|
||||
# 消费镜像内可信快照；写入留在任务容器，master push 可在任务结束后导出。
|
||||
set -euo pipefail
|
||||
|
||||
cache_root="${GENARRATIVE_CI_RUST_CACHE_ROOT:-/opt/genarrative-ci/rust-cache}"
|
||||
@@ -22,7 +22,7 @@ configure_local_cache() {
|
||||
case "${1:-}" in
|
||||
prepare)
|
||||
: "${GITHUB_ENV:?GITHUB_ENV is required}"
|
||||
printf 'RUSTC_WRAPPER=\nCARGO_BUILD_RUSTC_WRAPPER=\nGENARRATIVE_CI_RUST_CACHE_STATE=\n' >> "${GITHUB_ENV}"
|
||||
printf 'RUSTC_WRAPPER=\nCARGO_BUILD_RUSTC_WRAPPER=\nGENARRATIVE_CI_RUST_CACHE_STATE=\nGENARRATIVE_CI_RUST_CACHE_EXPORT_READY=\n' >> "${GITHUB_ENV}"
|
||||
fallback() { printf '[rust-cache] mode=direct reason=%s\n' "$1"; exit 0; }
|
||||
[[ -x "${cache_binary}" && -d "${cache_root}/objects" && -f "${cache_root}/rustc.txt" && -f "${cache_root}/source-commit.txt" && -f "${cache_root}/workspace.txt" ]] \
|
||||
|| fallback snapshot-unavailable
|
||||
@@ -44,6 +44,10 @@ case "${1:-}" in
|
||||
fallback wrapper-probe-failed
|
||||
fi
|
||||
script_path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/$(basename "${BASH_SOURCE[0]}")"
|
||||
if [[ "${GITHUB_EVENT_NAME:-}" == push && "${GITHUB_REF:-}" == refs/heads/master ]]; then
|
||||
# 只记 key/大小/时间，不读取对象正文；PR 没有扫描与打包开销。
|
||||
python3 "$(dirname "${script_path}")/export-gitea-rust-cache.py" baseline "${state}/baseline.json"
|
||||
fi
|
||||
# wrapper 路径也参与 Rust cache key；固定容器内路径，隔离由 job 容器保证。
|
||||
wrapper_path="${cache_root}/rustc-wrapper"
|
||||
printf '#!/usr/bin/env bash\nexec bash %q "$@"\n' "${script_path}" > "${wrapper_path}"
|
||||
@@ -59,7 +63,13 @@ case "${1:-}" in
|
||||
if [[ -n "${state}" && -d "${state}" ]]; then
|
||||
configure_local_cache
|
||||
timeout --kill-after=2 5 "${cache_binary}" --show-stats || true
|
||||
timeout --kill-after=2 5 "${cache_binary}" --stop-server >/dev/null 2>&1 || true
|
||||
# 只有成功停服后才允许读取对象；失败不能把仍在写入的目录发布成完整快照。
|
||||
if timeout --kill-after=2 15 "${cache_binary}" --stop-server >/dev/null 2>&1; then
|
||||
if [[ "${GITHUB_EVENT_NAME:-}" == push && "${GITHUB_REF:-}" == refs/heads/master && ! -f "${state}/disabled" && -f "${state}/baseline.json" ]]; then
|
||||
mv -- "${state}/baseline.json" "${cache_root}/export-baseline.json"
|
||||
printf 'GENARRATIVE_CI_RUST_CACHE_EXPORT_READY=1\n' >> "${GITHUB_ENV}"
|
||||
fi
|
||||
fi
|
||||
rm -rf -- "${state}"
|
||||
else
|
||||
printf '[rust-cache] mode=direct\n'
|
||||
|
||||
@@ -35,7 +35,8 @@ function fixture(t) {
|
||||
`#!/bin/bash
|
||||
set -eu
|
||||
case "$1" in
|
||||
--stop-server|--show-stats) exit 0 ;;
|
||||
--stop-server) exit "\${STOP_FAILURE:-0}" ;;
|
||||
--show-stats) exit 0 ;;
|
||||
esac
|
||||
if [[ "$*" == *-vV ]]; then
|
||||
[[ "\${PROBE_FAILURE:-}" != 1 ]] || exit 1
|
||||
@@ -215,3 +216,25 @@ linuxTest('lost local cache state still invokes the real compiler', (t) => {
|
||||
const f = fixture(t);
|
||||
assert.equal(f.run(['/bin/bash', '-c', 'exit 43']).status, 43);
|
||||
});
|
||||
|
||||
linuxTest('only a cleanly stopped master cache can export objects', (t) => {
|
||||
const master = {
|
||||
GITHUB_EVENT_NAME: 'push',
|
||||
GITHUB_REF: 'refs/heads/master',
|
||||
};
|
||||
for (const stopFailure of ['0', '1']) {
|
||||
const f = fixture(t);
|
||||
const prepared = f.run(['prepare'], master);
|
||||
assert.equal(prepared.status, 0, prepared.stderr);
|
||||
const result = f.run(['report'], {
|
||||
...f.preparedEnv(),
|
||||
...master,
|
||||
STOP_FAILURE: stopFailure,
|
||||
});
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.equal(
|
||||
f.preparedEnv().GENARRATIVE_CI_RUST_CACHE_EXPORT_READY,
|
||||
stopFailure === '0' ? '1' : '',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env node
|
||||
// 持久下载缓存可以保留旧版本；交付给 CI 镜像的快照只带当前 lock 已下载的包。
|
||||
import fs from 'node:fs/promises';
|
||||
import { createRequire } from 'node:module';
|
||||
import path from 'node:path';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
|
||||
const [npmRoot, lockPath, source, destination] = process.argv.slice(2);
|
||||
if (!npmRoot || !lockPath || !source || !destination) {
|
||||
throw new Error(
|
||||
'usage: export-ci-npm-download-cache.mjs <npm-root> <lock> <source> <destination>',
|
||||
);
|
||||
}
|
||||
const require = createRequire(path.resolve(npmRoot, 'package.json'));
|
||||
const cacache = require('cacache');
|
||||
const lock = JSON.parse(await fs.readFile(lockPath, 'utf8'));
|
||||
const integrities = new Set(
|
||||
Object.values(lock.packages).flatMap((entry) =>
|
||||
typeof entry.integrity === 'string' ? [entry.integrity] : [],
|
||||
),
|
||||
);
|
||||
let count = 0;
|
||||
for await (const entry of cacache.ls.stream(source)) {
|
||||
if (!integrities.has(entry.integrity)) continue;
|
||||
await pipeline(
|
||||
cacache.get.stream(source, entry.key, { integrity: entry.integrity }),
|
||||
cacache.put.stream(destination, entry.key, {
|
||||
integrity: entry.integrity,
|
||||
metadata: entry.metadata,
|
||||
}),
|
||||
);
|
||||
count += 1;
|
||||
}
|
||||
console.log(
|
||||
`[ci-image] exported ${count} npm cache entries for the current lock`,
|
||||
);
|
||||
@@ -0,0 +1,83 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { existsSync } from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import { createRequire } from 'node:module';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const npmRoot = [
|
||||
path.resolve(path.dirname(process.execPath), 'node_modules/npm'),
|
||||
path.resolve(path.dirname(process.execPath), '../lib/node_modules/npm'),
|
||||
...(process.env.npm_execpath
|
||||
? [path.resolve(path.dirname(process.env.npm_execpath), '..')]
|
||||
: []),
|
||||
].find((root) => existsSync(path.join(root, 'node_modules/cacache')));
|
||||
assert.ok(npmRoot, 'tests require the cacache bundled with npm');
|
||||
const cacache = createRequire(path.join(npmRoot, 'package.json'))('cacache');
|
||||
const script = fileURLToPath(
|
||||
new URL('./export-ci-npm-download-cache.mjs', import.meta.url),
|
||||
);
|
||||
|
||||
async function fixture(t) {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ci-npm-snapshot-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const source = path.join(root, 'source');
|
||||
const destination = path.join(root, 'destination');
|
||||
const lock = path.join(root, 'package-lock.json');
|
||||
const key =
|
||||
'make-fetch-happen:request-cache:https://registry.npmjs.org/example/-/example-1.0.0.tgz';
|
||||
const metadata = {
|
||||
url: key.slice('make-fetch-happen:request-cache:'.length),
|
||||
};
|
||||
const integrity = String(
|
||||
await cacache.put(source, key, 'current-package', { metadata }),
|
||||
);
|
||||
await cacache.put(source, 'old-package', 'unused-old-version');
|
||||
await fs.writeFile(
|
||||
lock,
|
||||
JSON.stringify({ packages: { 'node_modules/example': { integrity } } }),
|
||||
);
|
||||
const run = () =>
|
||||
spawnSync(process.execPath, [script, npmRoot, lock, source, destination], {
|
||||
encoding: 'utf8',
|
||||
});
|
||||
return { source, destination, key, metadata, integrity, run };
|
||||
}
|
||||
|
||||
test('exports only current lock content, preserving npm request metadata for offline use', async (t) => {
|
||||
const f = await fixture(t);
|
||||
const result = f.run();
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.deepEqual(Object.keys(await cacache.ls(f.destination)), [f.key]);
|
||||
const output = await cacache.get(f.destination, f.key);
|
||||
assert.equal(output.data.toString(), 'current-package');
|
||||
assert.deepEqual(output.metadata, f.metadata);
|
||||
// 输出是独立快照；移走持久缓存仍可使用，不依赖挂载、链接或旧 builder。
|
||||
await fs.rm(f.source, { recursive: true });
|
||||
assert.equal(
|
||||
(await cacache.get(f.destination, f.key)).data.toString(),
|
||||
'current-package',
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a corrupted cached package instead of publishing it', async (t) => {
|
||||
const f = await fixture(t);
|
||||
const digest = Buffer.from(f.integrity.split('-')[1], 'base64').toString(
|
||||
'hex',
|
||||
);
|
||||
const contentPath = path.join(
|
||||
f.source,
|
||||
'content-v2',
|
||||
'sha512',
|
||||
digest.slice(0, 2),
|
||||
digest.slice(2, 4),
|
||||
digest.slice(4),
|
||||
);
|
||||
await fs.writeFile(contentPath, 'corrupted-package');
|
||||
const result = f.run();
|
||||
assert.notEqual(result.status, 0);
|
||||
assert.match(result.stderr, /EINTEGRITY|EBADSIZE/);
|
||||
});
|
||||
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env python3
|
||||
"""只把 master push 的任务内 sccache 对象导出为 Gitea Actions 产物。"""
|
||||
|
||||
import base64
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zipfile
|
||||
|
||||
|
||||
MAX_OBJECT_BYTES = 4 * 1024**3
|
||||
CHUNK_BYTES = 8 * 1024**2
|
||||
JOBS = {
|
||||
"ai-game-creator-shell-rust-lane-1",
|
||||
"ai-game-creator-shell-rust-lane-2",
|
||||
"ai-game-creator-shell-rust-smoke",
|
||||
"ai-game-creator-shell-rust-crates",
|
||||
"backend-tests",
|
||||
"native-shell-tests",
|
||||
}
|
||||
|
||||
|
||||
def object_path(name):
|
||||
parts = name.split("/")
|
||||
return (
|
||||
len(parts) == 3
|
||||
and re.fullmatch(r"[a-f0-9]{64}", parts[2]) is not None
|
||||
and parts[0] == parts[2][0]
|
||||
and parts[1] == parts[2][1]
|
||||
)
|
||||
|
||||
|
||||
class HashingReader:
|
||||
def __init__(self, stream):
|
||||
self.stream = stream
|
||||
self.digest = hashlib.sha256()
|
||||
|
||||
def read(self, size=-1):
|
||||
data = self.stream.read(size)
|
||||
self.digest.update(data)
|
||||
return data
|
||||
|
||||
|
||||
def scan_objects(root):
|
||||
objects = root / "objects"
|
||||
if objects.is_symlink() or not objects.is_dir():
|
||||
raise ValueError("cache object directory is unavailable")
|
||||
candidates = []
|
||||
for directory, directories, files in os.walk(objects, followlinks=False):
|
||||
directories[:] = [
|
||||
name for name in directories if not (Path(directory) / name).is_symlink()
|
||||
]
|
||||
for filename in files:
|
||||
path = Path(directory) / filename
|
||||
relative = path.relative_to(objects).as_posix()
|
||||
info = path.lstat()
|
||||
if object_path(relative) and stat.S_ISREG(info.st_mode):
|
||||
candidates.append((path, relative, info))
|
||||
return candidates
|
||||
|
||||
|
||||
def save_baseline(root, destination):
|
||||
# sccache 命中会更新 mtime，不能把时间变化当成内容变化，否则又变成全量上传。
|
||||
baseline = {
|
||||
relative: {"size": info.st_size, "mtime_ns": info.st_mtime_ns}
|
||||
for _, relative, info in scan_objects(root)
|
||||
}
|
||||
destination.write_text(json.dumps(baseline), encoding="utf-8")
|
||||
|
||||
|
||||
def pack_snapshot(root, destination, metadata, baseline, limit=MAX_OBJECT_BYTES):
|
||||
"""仅归档新 key；命中的已有对象只传递新近使用时间，不传输对象内容。"""
|
||||
candidates = scan_objects(root)
|
||||
candidates.sort(key=lambda item: (-item[2].st_mtime_ns, item[1]))
|
||||
entries = []
|
||||
touched = []
|
||||
total = 0
|
||||
with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_STORED) as bundle, \
|
||||
bundle.open("snapshot.tar", "w", force_zip64=True) as tar_stream, \
|
||||
tarfile.open(fileobj=tar_stream, mode="w|", format=tarfile.USTAR_FORMAT) as archive:
|
||||
for path, relative, info in candidates:
|
||||
previous = baseline.get(relative)
|
||||
if previous is not None and previous["size"] == info.st_size:
|
||||
if previous["mtime_ns"] != info.st_mtime_ns:
|
||||
touched.append({"path": "objects/" + relative, "mtime_ns": info.st_mtime_ns})
|
||||
continue
|
||||
if info.st_size <= 0 or total + info.st_size > limit:
|
||||
continue
|
||||
# report 已成功停止 daemon；打开后再核实对象，避免导出变化中的文件。
|
||||
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
with os.fdopen(descriptor, "rb") as stream:
|
||||
actual = os.fstat(stream.fileno())
|
||||
if not stat.S_ISREG(actual.st_mode) or (
|
||||
actual.st_size, actual.st_mtime_ns, actual.st_ino
|
||||
) != (info.st_size, info.st_mtime_ns, info.st_ino):
|
||||
raise ValueError("cache object changed during export")
|
||||
member = tarfile.TarInfo("objects/" + relative)
|
||||
member.size = info.st_size
|
||||
member.mode = 0o644
|
||||
member.mtime = int(info.st_mtime)
|
||||
reader = HashingReader(stream)
|
||||
archive.addfile(member, reader)
|
||||
entries.append({
|
||||
"path": member.name,
|
||||
"size": info.st_size,
|
||||
"sha256": reader.digest.hexdigest(),
|
||||
"mtime_ns": info.st_mtime_ns,
|
||||
})
|
||||
total += info.st_size
|
||||
manifest = dict(metadata, schema=1, mode="delta", objects=entries, touched=touched)
|
||||
data = json.dumps(manifest, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
member = tarfile.TarInfo("manifest.json")
|
||||
member.size = len(data)
|
||||
member.mode = 0o644
|
||||
archive.addfile(member, io.BytesIO(data))
|
||||
return len(entries), total
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
# runtime token 只允许发往明确配置的 Gitea 地址。
|
||||
return None
|
||||
|
||||
|
||||
def request(method, url, token, data=None, headers=None, attempts=3):
|
||||
opener = urllib.request.build_opener(NoRedirect())
|
||||
for attempt in range(attempts):
|
||||
req = urllib.request.Request(url, data=data, method=method, headers={
|
||||
**({"Authorization": "Bearer " + token} if token else {}),
|
||||
"Content-Type": "application/json",
|
||||
**(headers or {}),
|
||||
})
|
||||
try:
|
||||
with opener.open(req, timeout=120) as response:
|
||||
body = response.read(1024 * 1024)
|
||||
return json.loads(body) if body else {}
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code not in (408, 429, 500, 502, 503, 504) or attempt == attempts - 1:
|
||||
raise RuntimeError(f"artifact {method} failed: HTTP {error.code}") from None
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError):
|
||||
if attempt == attempts - 1:
|
||||
raise RuntimeError(f"artifact {method} connection failed") from None
|
||||
time.sleep(2 ** attempt)
|
||||
|
||||
|
||||
def artifact_base_url(env):
|
||||
# .runner.address 可指向仅支持 RPC 的领取网关，不能用 GITHUB_SERVER_URL。
|
||||
repository = env["GITHUB_REPOSITORY"]
|
||||
clone_url = env["GENARRATIVE_GITEA_REPOSITORY_URL"]
|
||||
suffix = "/" + repository + ".git"
|
||||
parsed = urllib.parse.urlsplit(clone_url)
|
||||
if (
|
||||
parsed.scheme not in ("http", "https")
|
||||
or not parsed.netloc
|
||||
or parsed.username or parsed.password or parsed.query or parsed.fragment
|
||||
or not parsed.path.endswith(suffix)
|
||||
):
|
||||
raise ValueError("a credential-free HTTP Gitea repository URL is required")
|
||||
return urllib.parse.urlunsplit((
|
||||
parsed.scheme, parsed.netloc, parsed.path[:-len(suffix)], "", ""
|
||||
))
|
||||
|
||||
|
||||
def upload_snapshot(path, name, run_id, base_url, token):
|
||||
"""Gitea 1.26.4 原生 v4：只有完成全部块并校验 SHA256 后才发布。"""
|
||||
producer_match = re.fullmatch(r"rust-cache-v1-([a-z0-9-]+)-attempt-(0|[1-9][0-9]*)", name)
|
||||
if not producer_match or producer_match[1] not in JOBS:
|
||||
raise ValueError("unexpected cache artifact name")
|
||||
producer = f"{producer_match[1]}:{producer_match[2]}"
|
||||
endpoint = f"{base_url}/twirp/github.actions.results.api.v1.ArtifactService"
|
||||
# Gitea 从任务凭据确定 job，只要求请求中的 run ID 与任务所属 run 一致。
|
||||
identity = {"workflowRunBackendId": str(run_id), "name": name}
|
||||
created = request("POST", endpoint + "/CreateArtifact", token, json.dumps({
|
||||
**identity, "version": 4,
|
||||
# Gitea 将剩余小时向下取整成天；多给一天，服务端才实际保留至少七天。
|
||||
"expiresAt": (datetime.now(timezone.utc) + timedelta(days=8)).isoformat(),
|
||||
}).encode())
|
||||
received = urllib.parse.urlsplit(created.get("signedUploadUrl", ""))
|
||||
expected_path = urllib.parse.urlsplit(endpoint).path + "/UploadArtifact"
|
||||
if not created.get("ok") or received.path != expected_path or not received.query:
|
||||
raise ValueError("unexpected artifact upload path")
|
||||
# 服务可能返回外部 AppURL；保留签名，但只连接配置的内部 Gitea 地址。
|
||||
upload_url = endpoint + "/UploadArtifact?" + received.query
|
||||
size = path.stat().st_size
|
||||
digest = hashlib.sha256()
|
||||
blocks = []
|
||||
with path.open("rb") as source:
|
||||
while chunk := source.read(CHUNK_BYTES):
|
||||
# 双层编码中的内层保留所属任务；宿主只回收带此前缀的过期未完成块。
|
||||
block = base64.b64encode(
|
||||
f"genarrative-rust-cache-v1:{producer}:{len(blocks):08d}".encode()
|
||||
).decode()
|
||||
blocks.append(block)
|
||||
digest.update(chunk)
|
||||
request("PUT", upload_url + "&" + urllib.parse.urlencode({
|
||||
"comp": "block", "blockid": block,
|
||||
}), None, chunk, {
|
||||
"Content-Type": "application/octet-stream",
|
||||
})
|
||||
blocklist = "<BlockList>" + "".join(f"<Latest>{block}</Latest>" for block in blocks) + "</BlockList>"
|
||||
request("PUT", upload_url + "&comp=blocklist", None, blocklist.encode(), {
|
||||
"Content-Type": "application/xml",
|
||||
})
|
||||
# Finalize 会消耗服务端块列表；响应不确定时不盲目重试、也不宣告成功。
|
||||
finalized = request("POST", endpoint + "/FinalizeArtifact", token, json.dumps({
|
||||
**identity, "size": str(size), "hash": "sha256:" + digest.hexdigest(),
|
||||
}).encode(), attempts=1)
|
||||
if not finalized.get("ok"):
|
||||
raise ValueError("artifact was not finalized")
|
||||
|
||||
|
||||
def export(env):
|
||||
# 双重限定：手工调用、PR 和其它分支不会扫描、打包或上传对象。
|
||||
if env.get("GITHUB_EVENT_NAME") != "push" or env.get("GITHUB_REF") != "refs/heads/master":
|
||||
return
|
||||
if env.get("GENARRATIVE_CI_RUST_CACHE_EXPORT_READY") != "1":
|
||||
print("[rust-cache] export skipped: cache daemon did not stop cleanly")
|
||||
return
|
||||
job = env.get("GITHUB_JOB")
|
||||
if job not in JOBS:
|
||||
raise ValueError("unexpected Rust cache job")
|
||||
sha = env["GITHUB_SHA"]
|
||||
if not re.fullmatch(r"[a-f0-9]{40}", sha):
|
||||
raise ValueError("a complete source SHA is required")
|
||||
run_id, attempt = int(env["GITHUB_RUN_ID"]), int(env["GITHUB_RUN_ATTEMPT"])
|
||||
if run_id <= 0 or attempt < 0:
|
||||
raise ValueError("invalid run identity")
|
||||
token = env.get("ACTIONS_RUNTIME_TOKEN") or env["GENARRATIVE_GITEA_TOKEN"]
|
||||
base_url = artifact_base_url(env)
|
||||
root = Path(env.get("GENARRATIVE_CI_RUST_CACHE_ROOT", "/opt/genarrative-ci/rust-cache"))
|
||||
baseline_path = root / "export-baseline.json"
|
||||
baseline = json.loads(baseline_path.read_text(encoding="utf-8"))
|
||||
rustc = subprocess.check_output(["rustc", "-vV"], text=True)
|
||||
workspace = str(Path.cwd().resolve())
|
||||
if rustc != (root / "rustc.txt").read_text() or workspace != (root / "workspace.txt").read_text().strip():
|
||||
raise ValueError("cache provenance changed after prepare")
|
||||
metadata = {
|
||||
"repository": env["GITHUB_REPOSITORY"],
|
||||
"run_id": run_id, "run_attempt": attempt, "job": job, "source_sha": sha,
|
||||
"rustc": rustc, "workspace": workspace,
|
||||
"inherited_source_sha": (root / "source-commit.txt").read_text().strip(),
|
||||
"sccache_version": subprocess.check_output([str(root / "sccache"), "--version"], text=True).strip(),
|
||||
}
|
||||
if (root / "base-image.txt").is_file():
|
||||
metadata["base_image"] = (root / "base-image.txt").read_text().strip()
|
||||
name = f"rust-cache-v1-{job}-attempt-{attempt}"
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="ci-rust-cache-export-") as directory:
|
||||
path = Path(directory) / "snapshot.zip"
|
||||
count, size = pack_snapshot(root, path, metadata, baseline)
|
||||
upload_snapshot(path, name, run_id, base_url, token)
|
||||
finally:
|
||||
baseline_path.unlink(missing_ok=True)
|
||||
print(f"[rust-cache] artifact={name} objects={count} bytes={size} complete=true")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "baseline":
|
||||
save_baseline(Path(os.environ.get("GENARRATIVE_CI_RUST_CACHE_ROOT", "/opt/genarrative-ci/rust-cache")), Path(sys.argv[2]))
|
||||
else:
|
||||
export(os.environ)
|
||||
@@ -6,12 +6,31 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
dockerfile_context_path="deploy/container/gitea-ci-job.Dockerfile"
|
||||
image_tag="${GENARRATIVE_GITEA_CI_IMAGE_TAG:-genarrative/gitea-project-ci:20260920.2}"
|
||||
runner_container="${GENARRATIVE_GITEA_RUNNER_CONTAINER:-gitea-runner}"
|
||||
builder_name="genarrative-ci-images"
|
||||
|
||||
prepare_builder() {
|
||||
if ! docker buildx version >/dev/null 2>&1; then
|
||||
echo 'Gitea CI image builds require the Docker Buildx plugin; see deploy/container/README.md.' >&2
|
||||
return 1
|
||||
fi
|
||||
if ! docker buildx inspect "${builder_name}" >/dev/null 2>&1; then
|
||||
docker buildx create --name "${builder_name}" --driver docker-container \
|
||||
--driver-opt image=moby/buildkit:v0.23.2@sha256:ddd1ca44b21eda906e81ab14a3d467fa6c39cd73b9a39df1196210edcb8db59e \
|
||||
--buildkitd-config "${repo_root}/deploy/container/gitea-ci-buildkitd.toml"
|
||||
fi
|
||||
if [[ "$(docker buildx inspect "${builder_name}" | awk '$1 == "Driver:" { print $2 }')" != docker-container ]]; then
|
||||
echo "${builder_name} must use the isolated docker-container driver" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
write_build_context_file_list() {
|
||||
printf '%s\0' \
|
||||
deploy/container/gitea-ci-job.Dockerfile \
|
||||
deploy/container/gitea-ci-job.Dockerfile.dockerignore \
|
||||
deploy/container/gitea-ci-buildkitd.toml \
|
||||
deploy/container/gitea-ci-checkout.sh \
|
||||
scripts/export-ci-npm-download-cache.mjs \
|
||||
package.json \
|
||||
package-lock.json \
|
||||
apps/admin-web/package.json \
|
||||
@@ -29,8 +48,9 @@ write_build_context_file_list() {
|
||||
server-rs/Cargo.lock \
|
||||
apps/desktop-shell/src-tauri/Cargo.toml \
|
||||
apps/desktop-shell/src-tauri/Cargo.lock
|
||||
find server-rs/crates plugins/agc-*-editor/native/*-editor-bridge \
|
||||
\( -name Cargo.toml -o -path 'plugins/agc-*-editor/native/*-editor-bridge/*' \) \
|
||||
find server-rs/crates apps/ai-game-creator-shell/src-tauri/vendor \
|
||||
plugins/agc-*-editor/native/*-editor-bridge \
|
||||
-name Cargo.toml \
|
||||
-type f -print0 \
|
||||
| sort -z
|
||||
}
|
||||
@@ -39,6 +59,8 @@ usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/gitea-ci-job-image.sh build
|
||||
bash scripts/gitea-ci-job-image.sh seed-downloads <可信 CI 镜像完整 Image ID>
|
||||
bash scripts/gitea-ci-job-image.sh revision
|
||||
bash scripts/gitea-ci-job-image.sh verify [镜像引用]
|
||||
bash scripts/gitea-ci-job-image.sh load-runner [镜像引用]
|
||||
bash scripts/gitea-ci-job-image.sh export <归档路径> [镜像引用]
|
||||
@@ -65,40 +87,38 @@ verify_image() {
|
||||
|
||||
command_name="${1:-}"
|
||||
case "${command_name}" in
|
||||
build)
|
||||
image_revision="$(
|
||||
seed-downloads)
|
||||
# 运维显式指定的可信镜像只贡献下载包，不作为新基础镜像的父层。
|
||||
seed_image="${2:-}"
|
||||
[[ "${seed_image}" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo 'seed requires a full trusted Image ID' >&2; exit 2; }
|
||||
prepare_builder
|
||||
seed_dir="$(mktemp -d)"
|
||||
seed_container=""
|
||||
cleanup_seed() {
|
||||
if [[ -n "${seed_container}" ]]; then docker rm --volumes "${seed_container}" >/dev/null; fi
|
||||
rm -rf -- "${seed_dir}"
|
||||
}
|
||||
trap cleanup_seed EXIT
|
||||
seed_container="$(docker create "${seed_image}")"
|
||||
mkdir -p "${seed_dir}/cargo-cache" "${seed_dir}/cargo-index" "${seed_dir}/npm"
|
||||
docker cp "${seed_container}:/usr/local/cargo/registry/cache/." "${seed_dir}/cargo-cache/"
|
||||
docker cp "${seed_container}:/usr/local/cargo/registry/index/." "${seed_dir}/cargo-index/"
|
||||
docker cp "${seed_container}:/root/.npm/_cacache/." "${seed_dir}/npm/"
|
||||
docker buildx build --builder "${builder_name}" --progress plain \
|
||||
--target download-cache-seed --no-cache-filter download-cache-seed \
|
||||
--build-context "download-seed=${seed_dir}" \
|
||||
--file "${repo_root}/${dockerfile_context_path}" "${seed_dir}"
|
||||
;;
|
||||
revision)
|
||||
# 与 build 的 IMAGE_REVISION 使用同一份输入顺序，用于维护器判断基础镜像是否过期。
|
||||
(
|
||||
cd "${repo_root}"
|
||||
{
|
||||
sha256sum \
|
||||
deploy/container/gitea-ci-job.Dockerfile \
|
||||
deploy/container/gitea-ci-job.Dockerfile.dockerignore \
|
||||
deploy/container/gitea-ci-checkout.sh \
|
||||
package.json \
|
||||
package-lock.json \
|
||||
apps/admin-web/package.json \
|
||||
apps/ai-game-creator-shell/package.json \
|
||||
apps/desktop-shell/package.json \
|
||||
apps/mobile-shell/package.json \
|
||||
apps/preview-deployer-web/package.json \
|
||||
packages/image-canvas-core/package.json \
|
||||
packages/image-canvas-react/package.json \
|
||||
packages/shared/package.json \
|
||||
tools/spine-json-export-validator/package.json \
|
||||
apps/ai-game-creator-shell/src-tauri/Cargo.toml \
|
||||
apps/ai-game-creator-shell/src-tauri/Cargo.lock \
|
||||
server-rs/Cargo.toml \
|
||||
server-rs/Cargo.lock \
|
||||
apps/desktop-shell/src-tauri/Cargo.toml \
|
||||
apps/desktop-shell/src-tauri/Cargo.lock
|
||||
find server-rs/crates plugins/agc-*-editor/native/*-editor-bridge \
|
||||
\( -name Cargo.toml -o -path 'plugins/agc-*-editor/native/*-editor-bridge/*' \) \
|
||||
-type f -print0 \
|
||||
| sort -z \
|
||||
| xargs -0 -r sha256sum
|
||||
} \
|
||||
| sha256sum \
|
||||
| awk '{ print $1 }'
|
||||
)"
|
||||
write_build_context_file_list | xargs -0 -r sha256sum | sha256sum | awk '{ print $1 }'
|
||||
)
|
||||
;;
|
||||
build)
|
||||
prepare_builder
|
||||
image_revision="$(bash "${BASH_SOURCE[0]}" revision)"
|
||||
npm_lock_sha256="$(sha256sum "${repo_root}/package-lock.json")"
|
||||
npm_lock_sha256="${npm_lock_sha256%% *}"
|
||||
server_rust_lock_sha256="$(sha256sum "${repo_root}/server-rs/Cargo.lock")"
|
||||
@@ -111,7 +131,7 @@ case "${command_name}" in
|
||||
cd "${repo_root}"
|
||||
write_build_context_file_list \
|
||||
| tar --null --create --file - --files-from=- \
|
||||
| docker build \
|
||||
| docker buildx build --builder "${builder_name}" --load --progress plain \
|
||||
--pull=false \
|
||||
--build-arg "IMAGE_REVISION=${image_revision}" \
|
||||
--build-arg "NPM_LOCK_SHA256=${npm_lock_sha256}" \
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gitea Runner 领取屏障；控制 socket 仅供宿主缓存维护任务使用。"""
|
||||
|
||||
import http.client
|
||||
import http.server
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import zlib
|
||||
|
||||
MAX_BODY = 32 * 1024 * 1024
|
||||
HOP_HEADERS = {
|
||||
"connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
|
||||
"te", "trailer", "transfer-encoding", "upgrade", "host", "content-length",
|
||||
}
|
||||
RPC_PREFIX = "/api/actions/runner.v1.RunnerService/"
|
||||
|
||||
|
||||
def protobuf_fields(data):
|
||||
"""只解码已核对的 actions-proto-go v0.4.1 字段，不引入 protobuf 运行时。"""
|
||||
fields = {}
|
||||
offset = 0
|
||||
|
||||
def varint():
|
||||
nonlocal offset
|
||||
value = 0
|
||||
for shift in range(0, 70, 7):
|
||||
if offset >= len(data):
|
||||
raise ValueError("truncated protobuf")
|
||||
byte = data[offset]
|
||||
offset += 1
|
||||
value |= (byte & 127) << shift
|
||||
if byte < 128:
|
||||
return value
|
||||
raise ValueError("invalid protobuf varint")
|
||||
|
||||
while offset < len(data):
|
||||
tag = varint()
|
||||
number, wire = tag >> 3, tag & 7
|
||||
if not number:
|
||||
raise ValueError("invalid protobuf tag")
|
||||
if wire == 0:
|
||||
value = varint()
|
||||
elif wire in (1, 2, 5):
|
||||
length = varint() if wire == 2 else (8 if wire == 1 else 4)
|
||||
if offset + length > len(data):
|
||||
raise ValueError("truncated protobuf field")
|
||||
value = data[offset:offset + length]
|
||||
offset += length
|
||||
else:
|
||||
raise ValueError("unsupported protobuf wire type")
|
||||
fields.setdefault(number, []).append(value)
|
||||
return fields
|
||||
|
||||
|
||||
def single(fields, number, default=None):
|
||||
values = fields.get(number, [])
|
||||
if len(values) > 1:
|
||||
raise ValueError("ambiguous tracking field")
|
||||
return values[0] if values else default
|
||||
|
||||
|
||||
def rpc_fields(body, headers):
|
||||
encoding = headers.get("Content-Encoding", "identity").lower()
|
||||
if encoding == "gzip":
|
||||
with gzip.GzipFile(fileobj=io.BytesIO(body)) as stream:
|
||||
body = stream.read(MAX_BODY + 1)
|
||||
elif encoding != "identity":
|
||||
raise ValueError("unsupported RPC encoding")
|
||||
if len(body) > MAX_BODY:
|
||||
raise ValueError("decoded RPC too large")
|
||||
if headers.get("Content-Type", "").split(";", 1)[0] != "application/proto":
|
||||
raise ValueError("task tracking requires Connect protobuf")
|
||||
return protobuf_fields(body)
|
||||
|
||||
|
||||
def task_state(fields):
|
||||
nested = single(fields, 1)
|
||||
if not isinstance(nested, bytes):
|
||||
raise ValueError("missing task state")
|
||||
state = protobuf_fields(nested)
|
||||
task_id = positive_id(single(state, 1))
|
||||
result = single(state, 2, 0)
|
||||
if type(result) is not int or result not in range(5):
|
||||
raise ValueError("unknown task result")
|
||||
return task_id, result
|
||||
|
||||
|
||||
def positive_id(value):
|
||||
if type(value) is not int or not 0 < value < 2 ** 63:
|
||||
raise ValueError("invalid task ID")
|
||||
return str(value)
|
||||
|
||||
|
||||
class Gate:
|
||||
def __init__(self, directory):
|
||||
self.directory = Path(directory)
|
||||
self.directory.mkdir(parents=True, exist_ok=True)
|
||||
self.lock = threading.Lock()
|
||||
self.inflight = 0
|
||||
self.last_fetch_peer = None
|
||||
self.last_fetch_at = None
|
||||
self.tasks = {}
|
||||
self.uncertain = (self.directory / "uncertain").exists()
|
||||
try:
|
||||
if (self.directory / "tasks.json").exists():
|
||||
tasks = json.loads((self.directory / "tasks.json").read_text())
|
||||
if (not isinstance(tasks, dict) or any(
|
||||
positive_id(int(key)) != key or type(value) is not bool
|
||||
for key, value in tasks.items())):
|
||||
raise ValueError("invalid task ledger")
|
||||
self.tasks = tasks
|
||||
except (ValueError, TypeError, OSError):
|
||||
self.uncertain = True
|
||||
self.mark("uncertain")
|
||||
if (self.directory / "inflight").exists():
|
||||
self.uncertain = True
|
||||
self.mark("uncertain")
|
||||
self.paused = (self.directory / "paused").exists() or self.uncertain
|
||||
|
||||
def mark(self, name):
|
||||
with (self.directory / name).open("w", encoding="ascii") as stream:
|
||||
stream.write("1\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
self.sync_directory()
|
||||
|
||||
def sync_directory(self):
|
||||
descriptor = os.open(self.directory, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
def snapshot(self):
|
||||
return {"paused": self.paused, "inflight": self.inflight,
|
||||
"active_tasks": len(self.tasks), "task_ids": sorted(self.tasks),
|
||||
"uncertain": self.uncertain, "last_fetch_peer": self.last_fetch_peer,
|
||||
"last_fetch_at": self.last_fetch_at}
|
||||
|
||||
def save_tasks(self):
|
||||
temporary = self.directory / "tasks.json.tmp"
|
||||
with temporary.open("w", encoding="ascii") as stream:
|
||||
json.dump(self.tasks, stream, sort_keys=True)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, self.directory / "tasks.json")
|
||||
self.sync_directory()
|
||||
|
||||
def fail_closed(self):
|
||||
with self.lock:
|
||||
self.uncertain = self.paused = True
|
||||
self.mark("uncertain")
|
||||
|
||||
def assigned(self, fields):
|
||||
task = single(fields, 1)
|
||||
if task is None:
|
||||
return
|
||||
if not isinstance(task, bytes):
|
||||
raise ValueError("invalid fetched task")
|
||||
task_id = positive_id(single(protobuf_fields(task), 1))
|
||||
with self.lock:
|
||||
self.tasks[task_id] = False
|
||||
# 必须先持久化，再把领取结果交给 runner；仅保存 ID，不保存 secrets。
|
||||
self.save_tasks()
|
||||
|
||||
def reported(self, method, request, response):
|
||||
if method == "UpdateLog":
|
||||
task_id = positive_id(single(request, 1))
|
||||
index = single(request, 2, 0)
|
||||
ack = single(response, 1, 0)
|
||||
no_more = single(request, 4, 0)
|
||||
if (type(index) is not int or type(ack) is not int
|
||||
or index < 0 or ack < 0 or no_more not in (0, 1)):
|
||||
raise ValueError("invalid log acknowledgement")
|
||||
finalized = no_more == 1 and ack == index + len(request.get(3, []))
|
||||
else:
|
||||
task_id, result = task_state(request)
|
||||
response_id, response_result = task_state(response)
|
||||
if response_id != task_id:
|
||||
raise ValueError("mismatched task acknowledgement")
|
||||
# 取消响应可能出现在任务执行途中，必须等 runner 自己报告终态。
|
||||
finalized = result != 0 and response_result != 0
|
||||
output_keys = {single(protobuf_fields(entry), 1, b"")
|
||||
for entry in request.get(2, [])}
|
||||
finalized = finalized and output_keys.issubset(set(response.get(2, [])))
|
||||
with self.lock:
|
||||
if task_id not in self.tasks:
|
||||
# 客户端可能未读到已成功发出的终态响应而重试；v2.0.0 仅在
|
||||
# executor 清理后的 Close 中发送终态，不为幂等重报增加墓碑账本。
|
||||
if method == "UpdateTask" and finalized:
|
||||
return
|
||||
raise ValueError("report for untracked task; idle bootstrap required")
|
||||
if method == "UpdateLog" and finalized:
|
||||
self.tasks[task_id] = True
|
||||
self.save_tasks()
|
||||
elif method == "UpdateTask" and finalized and self.tasks[task_id]:
|
||||
# act_runner Reporter.Close 在 executor 清理之后先封存日志再报终态。
|
||||
del self.tasks[task_id]
|
||||
self.save_tasks()
|
||||
|
||||
def record_fetch(self, peer):
|
||||
with self.lock:
|
||||
self.last_fetch_peer = peer
|
||||
self.last_fetch_at = time.time()
|
||||
|
||||
def control(self, action):
|
||||
with self.lock:
|
||||
if action == "pause":
|
||||
self.mark("paused")
|
||||
self.paused = True
|
||||
elif action == "resume":
|
||||
if self.uncertain:
|
||||
return {**self.snapshot(), "error": "upstream completion uncertain; operator recovery required"}
|
||||
(self.directory / "paused").unlink(missing_ok=True)
|
||||
self.sync_directory()
|
||||
self.paused = False
|
||||
elif action != "status":
|
||||
return {**self.snapshot(), "error": "unknown action"}
|
||||
return self.snapshot()
|
||||
|
||||
def enter(self):
|
||||
with self.lock:
|
||||
if self.paused or self.uncertain:
|
||||
return False
|
||||
# 必须先落盘再转发；崩溃后不能把遗留的领取请求误认为已完成。
|
||||
self.mark("inflight")
|
||||
self.inflight += 1
|
||||
return True
|
||||
|
||||
def leave(self, completed):
|
||||
with self.lock:
|
||||
self.inflight -= 1
|
||||
if not completed:
|
||||
self.uncertain = self.paused = True
|
||||
self.mark("uncertain")
|
||||
if self.inflight == 0 and not self.uncertain:
|
||||
(self.directory / "inflight").unlink(missing_ok=True)
|
||||
self.sync_directory()
|
||||
|
||||
|
||||
def read_body(stream, headers):
|
||||
"""解码 HTTP 请求；拒绝含糊 framing，不依赖下游连接关闭。"""
|
||||
encodings = headers.get_all("Transfer-Encoding", [])
|
||||
lengths = headers.get_all("Content-Length", [])
|
||||
if encodings and lengths:
|
||||
raise ValueError("ambiguous request framing")
|
||||
if len(lengths) > 1 or len(encodings) > 1:
|
||||
raise ValueError("duplicate request framing")
|
||||
|
||||
def exact(length):
|
||||
data = stream.read(length)
|
||||
if len(data) != length:
|
||||
raise ValueError("incomplete request body")
|
||||
return data
|
||||
|
||||
if not encodings:
|
||||
length = int(lengths[0]) if lengths else 0
|
||||
if not 0 <= length <= MAX_BODY:
|
||||
raise ValueError("request too large")
|
||||
return exact(length)
|
||||
if encodings[0].strip().lower() != "chunked":
|
||||
raise ValueError("unsupported transfer encoding")
|
||||
body = bytearray()
|
||||
while True:
|
||||
line = stream.readline(8193)
|
||||
if len(line) > 8192 or not line.endswith(b"\r\n"):
|
||||
raise ValueError("invalid chunk header")
|
||||
length = int(line.split(b";", 1)[0].strip(), 16)
|
||||
if length < 0 or len(body) + length > MAX_BODY:
|
||||
raise ValueError("request too large")
|
||||
if length == 0:
|
||||
trailer_size = 0
|
||||
while True:
|
||||
line = stream.readline(8193)
|
||||
trailer_size += len(line)
|
||||
if trailer_size > 8192 or not line.endswith(b"\r\n"):
|
||||
raise ValueError("invalid request trailers")
|
||||
if line == b"\r\n":
|
||||
return bytes(body)
|
||||
body.extend(exact(length))
|
||||
if exact(2) != b"\r\n":
|
||||
raise ValueError("invalid chunk terminator")
|
||||
|
||||
|
||||
class Proxy(http.server.BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def log_message(self, *_args):
|
||||
pass # 不输出 RPC 认证头、请求内容或带认证信息的 URL。
|
||||
|
||||
def reply(self, status, body, headers=()):
|
||||
delivered = False
|
||||
try:
|
||||
self.send_response(status)
|
||||
excluded = HOP_HEADERS | {
|
||||
item.strip().lower() for key, value in headers if key.lower() == "connection"
|
||||
for item in value.split(",")
|
||||
}
|
||||
for key, value in headers:
|
||||
if key.lower() not in excluded:
|
||||
self.send_header(key, value)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
self.wfile.flush()
|
||||
delivered = True
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
self.close_connection = True
|
||||
return delivered
|
||||
|
||||
def do_POST(self):
|
||||
path = urllib.parse.urlsplit(self.path)
|
||||
if (path.scheme or path.netloc or not path.path.startswith("/api/actions/")
|
||||
or "%" in path.path or any(p in (".", "..") for p in path.path.split("/"))):
|
||||
self.reply(404, b"runner RPC only\n")
|
||||
return
|
||||
try:
|
||||
body = read_body(self.rfile, self.headers)
|
||||
except (ValueError, OSError):
|
||||
self.reply(400, b"invalid request body\n")
|
||||
return
|
||||
method = path.path.removeprefix(RPC_PREFIX) if path.path.startswith(RPC_PREFIX) else ""
|
||||
is_fetch = method == "FetchTask"
|
||||
if is_fetch:
|
||||
# 包括暂停时被拒绝的请求；只记录网络来源与时间以证明 daemon 路由。
|
||||
self.server.gate.record_fetch(self.client_address[0])
|
||||
if not self.server.gate.enter():
|
||||
self.reply(503, b"runner maintenance\n")
|
||||
return
|
||||
completed = False
|
||||
connection = None
|
||||
try:
|
||||
upstream = self.server.upstream
|
||||
cls = http.client.HTTPSConnection if upstream.scheme == "https" else http.client.HTTPConnection
|
||||
connection = cls(upstream.hostname, upstream.port, timeout=None)
|
||||
try:
|
||||
connection.connect()
|
||||
except OSError:
|
||||
# TCP/TLS 连接阶段尚未发送 RPC，不存在服务端分配事务。
|
||||
completed = True
|
||||
raise
|
||||
excluded = HOP_HEADERS | {
|
||||
name.strip().lower() for name in self.headers.get("Connection", "").split(",")
|
||||
}
|
||||
headers = {key: value for key, value in self.headers.items() if key.lower() not in excluded}
|
||||
headers["Content-Length"] = str(len(body))
|
||||
connection.request("POST", self.path, body=body, headers=headers)
|
||||
response = connection.getresponse()
|
||||
result = bytearray()
|
||||
oversized = False
|
||||
# 客户端即使断开，也继续读取上游完整响应，再释放领取屏障。
|
||||
while block := response.read(65536):
|
||||
if len(result) + len(block) <= MAX_BODY and not oversized:
|
||||
result.extend(block)
|
||||
else:
|
||||
oversized = True
|
||||
result.clear()
|
||||
if response.length not in (None, 0):
|
||||
raise http.client.IncompleteRead(bytes(result), response.length)
|
||||
completed = True
|
||||
if is_fetch:
|
||||
try:
|
||||
if oversized or response.status != 200:
|
||||
raise ValueError("unknown FetchTask result")
|
||||
self.server.gate.assigned(rpc_fields(bytes(result), response.headers))
|
||||
except (ValueError, TypeError, OSError, EOFError, zlib.error):
|
||||
self.server.gate.fail_closed()
|
||||
self.server.gate.leave(True)
|
||||
is_fetch = False
|
||||
if oversized:
|
||||
self.reply(502, b"upstream response too large\n")
|
||||
else:
|
||||
# 日志封存先记账再返回，避免 runner 立即发终态时抢先读到旧账本。
|
||||
if method == "UpdateLog" and response.status == 200:
|
||||
self.observe_report(method, body, bytes(result), response.headers)
|
||||
delivered = self.reply(response.status, bytes(result), response.getheaders())
|
||||
if method == "UpdateTask" and response.status == 200 and delivered:
|
||||
self.observe_report(method, body, bytes(result), response.headers)
|
||||
except (OSError, http.client.HTTPException, ValueError):
|
||||
self.reply(502, b"runner upstream unavailable\n")
|
||||
finally:
|
||||
if connection is not None:
|
||||
connection.close()
|
||||
if is_fetch:
|
||||
self.server.gate.leave(completed)
|
||||
|
||||
def observe_report(self, method, request, response, headers):
|
||||
try:
|
||||
self.server.gate.reported(method, rpc_fields(request, self.headers),
|
||||
rpc_fields(response, headers))
|
||||
except (ValueError, TypeError, OSError, EOFError, zlib.error):
|
||||
self.server.gate.fail_closed()
|
||||
|
||||
|
||||
class Control(socketserver.StreamRequestHandler):
|
||||
def handle(self):
|
||||
try:
|
||||
line = self.rfile.readline(4097)
|
||||
if len(line) > 4096 or not line.endswith(b"\n"):
|
||||
raise ValueError("invalid control request")
|
||||
request = json.loads(line)
|
||||
result = self.server.gate.control(request["action"])
|
||||
except (ValueError, KeyError, TypeError, OSError):
|
||||
result = {"error": "invalid control request"}
|
||||
self.wfile.write(json.dumps(result).encode("utf-8") + b"\n")
|
||||
|
||||
|
||||
class ControlServer(socketserver.ThreadingUnixStreamServer):
|
||||
daemon_threads = True
|
||||
|
||||
|
||||
def create_proxy(address, upstream, gate):
|
||||
parsed = urllib.parse.urlsplit(upstream)
|
||||
if (parsed.scheme not in ("http", "https") or not parsed.hostname
|
||||
or parsed.username or parsed.password or parsed.path not in ("", "/")
|
||||
or parsed.query or parsed.fragment):
|
||||
raise ValueError("upstream must be an HTTP(S) origin")
|
||||
server = http.server.ThreadingHTTPServer(address, Proxy)
|
||||
server.upstream = parsed
|
||||
server.gate = gate
|
||||
return server
|
||||
|
||||
|
||||
def main():
|
||||
gate = Gate(os.environ.get("GITEA_GATE_CONTROL_DIR", "/control"))
|
||||
socket_path = gate.directory / "gate.sock"
|
||||
socket_path.unlink(missing_ok=True)
|
||||
control = ControlServer(str(socket_path), Control)
|
||||
control.gate = gate
|
||||
os.chmod(socket_path, 0o600)
|
||||
proxy = create_proxy(("0.0.0.0", 8080), os.environ.get(
|
||||
"GITEA_RUNNER_UPSTREAM", "http://gitea:3000"), gate)
|
||||
threading.Thread(target=control.serve_forever, daemon=True).start()
|
||||
proxy.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,115 @@
|
||||
"""定向回收 Gitea 1.26.4 不会自动过期的本仓库缓存上传块。"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
|
||||
|
||||
JOBS = {
|
||||
"ai-game-creator-shell-rust-lane-1",
|
||||
"ai-game-creator-shell-rust-lane-2",
|
||||
"ai-game-creator-shell-rust-smoke",
|
||||
"ai-game-creator-shell-rust-crates",
|
||||
"backend-tests",
|
||||
"native-shell-tests",
|
||||
}
|
||||
CHUNK = re.compile(r"block-([1-9][0-9]*)-([1-9][0-9]*)-(0|[1-9][0-9]*)-([A-Za-z0-9_-]+={0,2})")
|
||||
BLOCKLIST = re.compile(r"([1-9][0-9]*)-([1-9][0-9]*)-blocklist")
|
||||
MARKER = re.compile(r"genarrative-rust-cache-v1:([a-z0-9-]+):(0|[1-9][0-9]*):([0-9]{8})")
|
||||
|
||||
|
||||
def decode_owner(encoded):
|
||||
"""Gitea 文件名编码为 URL-base64(生产者上传的标准 base64 blockid)。"""
|
||||
try:
|
||||
block_id = base64.b64decode(encoded, altchars=b"-_", validate=True)
|
||||
if base64.urlsafe_b64encode(block_id).decode() != encoded:
|
||||
return None
|
||||
marker = base64.b64decode(block_id, validate=True)
|
||||
if base64.b64encode(marker) != block_id:
|
||||
return None
|
||||
match = MARKER.fullmatch(marker.decode("ascii"))
|
||||
if match and match[1] in JOBS:
|
||||
return match[1], int(match[2])
|
||||
except (binascii.Error, UnicodeError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _unchanged_old_regular(path, previous, cutoff):
|
||||
try:
|
||||
current = path.lstat()
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
return (
|
||||
stat.S_ISREG(current.st_mode)
|
||||
and current.st_mtime < cutoff
|
||||
and (current.st_ino, current.st_size, current.st_mtime_ns)
|
||||
== (previous.st_ino, previous.st_size, previous.st_mtime_ns)
|
||||
)
|
||||
|
||||
|
||||
def cleanup_upload_chunks(storage_root: Path, eligible_runs: set[int], cutoff_timestamp: float) -> int:
|
||||
"""调用方先用仓库 API 确认这些 master run 已结束超过保留期限。"""
|
||||
storage_root = Path(storage_root)
|
||||
if not storage_root.is_absolute() or storage_root.resolve(strict=True) != storage_root:
|
||||
raise ValueError("artifact storage root must be an existing absolute real directory")
|
||||
if not stat.S_ISDIR(storage_root.lstat().st_mode):
|
||||
raise ValueError("artifact storage root must be a directory")
|
||||
if not math.isfinite(cutoff_timestamp) or cutoff_timestamp <= 0:
|
||||
raise ValueError("invalid artifact cleanup cutoff")
|
||||
if any(type(run_id) is not int or run_id <= 0 for run_id in eligible_runs):
|
||||
raise ValueError("invalid eligible run ID")
|
||||
temporary = storage_root / "tmp-upload"
|
||||
if not temporary.exists() and not temporary.is_symlink():
|
||||
return 0
|
||||
if temporary.is_symlink() or not temporary.is_dir():
|
||||
raise ValueError("artifact temporary path must be a real directory")
|
||||
|
||||
deleted = 0
|
||||
for run_id in sorted(eligible_runs):
|
||||
directory = temporary / f"run-{run_id}-v4"
|
||||
if directory.is_symlink() or not directory.is_dir():
|
||||
continue
|
||||
groups = {}
|
||||
unknown = False
|
||||
for path in directory.iterdir():
|
||||
chunk = CHUNK.fullmatch(path.name)
|
||||
blocklist = BLOCKLIST.fullmatch(path.name)
|
||||
match = chunk or blocklist
|
||||
if match and int(match[1]) == run_id:
|
||||
artifact_id = int(match[2])
|
||||
owner = decode_owner(chunk[4]) if chunk else None
|
||||
owned = owner is not None if chunk else True
|
||||
else:
|
||||
# 若仍能识别 artifact ID，仅保护该组；无法识别时保护整个 run。
|
||||
prefix = re.match(rf"(?:block-)?{run_id}-([1-9][0-9]*)-", path.name)
|
||||
if not prefix:
|
||||
unknown = True
|
||||
break
|
||||
artifact_id, owner, owned = int(prefix[1]), None, False
|
||||
group = groups.setdefault(artifact_id, {"files": [], "owners": set(), "safe": True})
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError:
|
||||
group["safe"] = False
|
||||
continue
|
||||
group["safe"] &= owned and stat.S_ISREG(info.st_mode) and info.st_mtime < cutoff_timestamp
|
||||
if owner:
|
||||
group["owners"].add(owner)
|
||||
group["files"].append((path, info))
|
||||
if unknown:
|
||||
continue
|
||||
for group in groups.values():
|
||||
if not group["safe"] or len(group["owners"]) != 1:
|
||||
continue
|
||||
files = group["files"]
|
||||
if not all(_unchanged_old_regular(path, info, cutoff_timestamp) for path, info in files):
|
||||
continue
|
||||
# 保留目录，不使用递归删除；没有所属块证明的孤立 blocklist 也不会删除。
|
||||
for path, _ in files:
|
||||
path.unlink()
|
||||
deleted += 1
|
||||
return deleted
|
||||
@@ -1,353 +0,0 @@
|
||||
# Genarrative 作品列表 K6 压测
|
||||
|
||||
> 2026-07-18 退役：本文与本目录只保留旧公开作品 / gallery 压测历史，`container:k6` 和 compose `k6` 运行目标已经下线；不得把这些脚本用于当前容量验收或恢复旧业务接口。
|
||||
|
||||
本目录用于对“作品列表/公开广场”读接口做本地压测。数据源来自私有 SpacetimeDB migration，但提取脚本只输出作品 profile 白名单表，并对用户、作者、作品号、asset id 等标识做稳定映射。
|
||||
|
||||
## 文件
|
||||
|
||||
- `extract-works-list-data.mjs`：从 migration JSON 提取作品列表压测数据；本地输出也会脱敏路由 ID，因此默认用于列表接口压测，详情接口需先把同一份脱敏数据导入目标环境。
|
||||
- `k6-works-list.js`：K6 压测脚本。
|
||||
- `data/spacetime-migration-7.local.json`：本地私有原始数据副本，已被 `.gitignore` 忽略，不要提交。
|
||||
- `data/works-list.local.json`：本地脱敏压测数据，已被 `.gitignore` 忽略，不要提交。
|
||||
- `data/works-list.sample.json`：可提交的少量脱敏样例。
|
||||
|
||||
## 数据边界
|
||||
|
||||
允许导入的表：
|
||||
|
||||
- `puzzle_work_profile`
|
||||
- `custom_world_profile`
|
||||
- `match3d_work_profile`
|
||||
- `square_hole_work_profile`
|
||||
- `big_fish_work_profile`
|
||||
- `visual_novel_work_profile`
|
||||
|
||||
明确不导入：
|
||||
|
||||
- 账号/认证：`user_account`、`auth_identity`、`refresh_session`、`auth_store_snapshot`
|
||||
- 钱包/邀请：`profile_wallet_ledger`、`profile_redeem_*`、`profile_invite_*`
|
||||
- 游玩历史/埋点/存档：`public_work_play_daily_stat`、`profile_played_world`、`puzzle_runtime_run`、`profile_save_archive`、`runtime_snapshot`
|
||||
- AI 任务过程：`ai_task`、`ai_task_stage`、`ai_text_chunk`
|
||||
- asset 二进制：`asset_object`、`asset_entity_binding`
|
||||
|
||||
提取脚本会移除 `source_session_id` / `source_agent_session_id` 等会话派生字段；这些字段不属于作品列表卡片压测必要字段。
|
||||
|
||||
## 重新提取数据
|
||||
|
||||
从仓库根目录执行：
|
||||
|
||||
```bash
|
||||
npm run loadtest:extract-works -- \
|
||||
--input scripts/loadtest/data/spacetime-migration-7.local.json \
|
||||
--output scripts/loadtest/data/works-list.local.json \
|
||||
--sample-output scripts/loadtest/data/works-list.sample.json
|
||||
```
|
||||
|
||||
也可以直接执行：
|
||||
|
||||
```bash
|
||||
node scripts/loadtest/extract-works-list-data.mjs \
|
||||
--input scripts/loadtest/data/spacetime-migration-7.local.json \
|
||||
--output scripts/loadtest/data/works-list.local.json \
|
||||
--sample-output scripts/loadtest/data/works-list.sample.json
|
||||
```
|
||||
|
||||
当前 local 全量提取结果：
|
||||
|
||||
- `puzzle_work_profile`: 80
|
||||
- `custom_world_profile`: 1
|
||||
- `match3d_work_profile`: 0
|
||||
- `normalizedWorks`: 81
|
||||
|
||||
当前可提交 sample 结果：
|
||||
|
||||
- `puzzle_work_profile`: 3
|
||||
- `custom_world_profile`: 1
|
||||
- `match3d_work_profile`: 0
|
||||
- `normalizedWorks`: 4
|
||||
|
||||
## 真实接口
|
||||
|
||||
已从 `server-rs/crates/api-server/src/app.rs` 确认的读接口：
|
||||
|
||||
公开接口，无需 Bearer token：
|
||||
|
||||
- `GET /api/runtime/puzzle/gallery`
|
||||
- `GET /api/runtime/puzzle/gallery/{profile_id}`
|
||||
- `GET /api/runtime/custom-world-gallery`
|
||||
- `GET /api/runtime/custom-world-gallery/{owner_user_id}/{profile_id}`
|
||||
- `GET /api/runtime/custom-world-gallery/by-code/{code}`
|
||||
|
||||
需要 Bearer token 的个人作品列表接口：
|
||||
|
||||
- `GET /api/runtime/puzzle/works`
|
||||
- `GET /api/runtime/puzzle/works/{profile_id}`
|
||||
- `GET /api/runtime/custom-world/works`
|
||||
|
||||
K6 脚本默认只跑公开列表接口；传入 `AUTH_TOKEN` 后会额外跑需要登录态的个人作品列表接口。当前真实列表 handler 未暴露分页/排序 query 参数，因此脚本不追加 `limit/offset`；若后续接口增加分页参数，再在 K6 中补随机分页。
|
||||
|
||||
详情接口默认不压测，因为本地数据中的 `profile_id` / `owner_user_id` 已脱敏，直接请求未导入脱敏数据的目标服务会 404。只有在目标环境已导入同一份脱敏数据，或改用真实 ID 本地文件时，才设置 `DETAIL_RATIO` 大于 0；详情请求不把 404 视为成功。
|
||||
|
||||
## 启动服务
|
||||
|
||||
按项目约定启动本地 dev 栈：
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
注意端口可能漂移。以启动日志中的实际 api-server 端口为准，然后传给 K6。
|
||||
|
||||
注意：K6 的 `open()` 会按 `k6-works-list.js` 所在目录解析相对路径，因此 `WORKS_DATA` 应写成 `data/works-list.local.json`，不要写成 `scripts/loadtest/data/works-list.local.json`。
|
||||
|
||||
Bash / Git Bash：
|
||||
|
||||
```bash
|
||||
BASE_URL=http://127.0.0.1:<actual-api-port> WORKS_DATA=data/works-list.local.json npm run loadtest:k6:works -- --summary-trend-stats="avg,min,med,p(90),p(95),p(99),max"
|
||||
```
|
||||
|
||||
PowerShell：
|
||||
|
||||
```powershell
|
||||
$env:BASE_URL="http://127.0.0.1:<actual-api-port>"
|
||||
$env:WORKS_DATA="data/works-list.local.json"
|
||||
npm run loadtest:k6:works -- --summary-trend-stats="avg,min,med,p(90),p(95),p(99),max"
|
||||
```
|
||||
|
||||
## 50 HTTP req/s 口径
|
||||
|
||||
`k6-works-list.js` 默认一次 iteration 会依次请求两个公开列表接口：`/api/runtime/puzzle/gallery` 和 `/api/runtime/custom-world-gallery`。因此目标约 50 HTTP req/s 时，`ramping-arrival-rate` 的 `PEAK_RPS` 应设置为 `25`。如果传入 `AUTH_TOKEN` 或把 `DETAIL_RATIO` 设为大于 0，每次 iteration 的请求数会增加，需要重新折算。
|
||||
|
||||
验收目标：
|
||||
|
||||
- `http_req_failed < 1%`
|
||||
- `http_req_duration p95 < 2000ms`
|
||||
- `dropped_iterations = 0`
|
||||
- 压测窗口内 Nginx 无新增 502
|
||||
|
||||
## Smoke
|
||||
|
||||
```bash
|
||||
BASE_URL=http://127.0.0.1:8787 \
|
||||
WORKS_DATA=data/works-list.local.json \
|
||||
SCENARIO=smoke \
|
||||
DETAIL_RATIO=0 \
|
||||
npm run loadtest:k6:works
|
||||
```
|
||||
|
||||
默认：1 VU / 30s。
|
||||
|
||||
## Baseline
|
||||
|
||||
```bash
|
||||
BASE_URL=http://127.0.0.1:8787 \
|
||||
WORKS_DATA=data/works-list.local.json \
|
||||
SCENARIO=baseline \
|
||||
VUS=10 \
|
||||
DURATION=3m \
|
||||
DETAIL_RATIO=0 \
|
||||
npm run loadtest:k6:works
|
||||
```
|
||||
|
||||
默认阈值：
|
||||
|
||||
- `http_req_failed < 1%`
|
||||
- `http_req_duration p95 < 800ms`
|
||||
- `http_req_duration p99 < 1500ms`
|
||||
- `works_list_shape_error_rate < 1%`
|
||||
|
||||
## Spike
|
||||
|
||||
```bash
|
||||
BASE_URL=http://127.0.0.1:8787 \
|
||||
WORKS_DATA=data/works-list.local.json \
|
||||
SCENARIO=spike \
|
||||
START_RPS=5 \
|
||||
PEAK_RPS=25 \
|
||||
HOLD=60s \
|
||||
DETAIL_RATIO=0 \
|
||||
npm run loadtest:k6:works
|
||||
```
|
||||
|
||||
默认阈值：
|
||||
|
||||
- `http_req_failed < 1%`
|
||||
- `http_req_duration p95 < 2000ms`
|
||||
- `dropped_iterations = 0`
|
||||
- `works_list_shape_error_rate < 1%`
|
||||
|
||||
PowerShell：
|
||||
|
||||
```powershell
|
||||
$env:BASE_URL="https://genarrative.world"
|
||||
$env:WORKS_DATA="data/works-list.local.json"
|
||||
$env:SCENARIO="spike"
|
||||
$env:START_RPS="5"
|
||||
$env:PEAK_RPS="25"
|
||||
$env:HOLD="60s"
|
||||
$env:END_RPS="5"
|
||||
$env:DETAIL_RATIO="0"
|
||||
npm run loadtest:k6:works -- --summary-trend-stats="avg,min,med,p(90),p(95),p(99),max"
|
||||
```
|
||||
|
||||
线上 release 回归可使用同一组环境变量：
|
||||
|
||||
```bash
|
||||
SCENARIO=spike START_RPS=5 PEAK_RPS=25 HOLD=60s END_RPS=5 DETAIL_RATIO=0 npm run loadtest:k6:works
|
||||
```
|
||||
|
||||
## 带登录态压测个人作品列表
|
||||
|
||||
先通过本地登录或接口获取 access token，然后传入：
|
||||
|
||||
```bash
|
||||
BASE_URL=http://127.0.0.1:8787 \
|
||||
AUTH_TOKEN='<access-token>' \
|
||||
SCENARIO=smoke \
|
||||
DETAIL_RATIO=0 \
|
||||
npm run loadtest:k6:works
|
||||
```
|
||||
|
||||
不要把 token 写入仓库文件、README 或 shell history 中可共享的位置。
|
||||
|
||||
## 详情接口压测
|
||||
|
||||
仅当目标环境存在 `WORKS_DATA` 中的同一批 `profileId/ownerUserId` 时启用：
|
||||
|
||||
```bash
|
||||
BASE_URL=http://127.0.0.1:8787 \
|
||||
WORKS_DATA=data/works-list.local.json \
|
||||
SCENARIO=smoke \
|
||||
DETAIL_RATIO=0.35 \
|
||||
npm run loadtest:k6:works
|
||||
```
|
||||
|
||||
如果详情请求返回 404，说明压测数据 ID 未导入目标环境或目标服务数据不一致，应先修正数据源，不要把 404 当成功。
|
||||
|
||||
## 排障
|
||||
|
||||
- 如果公开 gallery 返回 `creation_entry_disabled` 或 503，检查本地 creation entry 配置是否禁用了对应入口。
|
||||
- 如果高压下返回 429，优先确认目标环境是否设置了 `GENARRATIVE_API_MAX_CONCURRENT_REQUESTS` 以及 `GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS`、`GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS`、`GENARRATIVE_API_ADMIN_MAX_CONCURRENT_REQUESTS`。429 表示 Nginx 或 api-server 背压已生效，不等同于业务错误；继续看内存、p95、`http_req_failed` 和 OTLP / Nginx timing 判断阈值是否偏低。
|
||||
- 如果直连 `api-server` 压测出现 `connection refused` 或 status 0，说明压力已经打到 TCP 监听 / accept 层；此时同时检查 `GENARRATIVE_API_LISTEN_BACKLOG`、Nginx upstream keepalive 和是否需要在 Nginx 前置限流，不能只靠应用层背压解释。
|
||||
- 如果个人作品列表返回 401，确认 `AUTH_TOKEN` 是当前 api-server 可识别的 access token。
|
||||
- 如果详情全部 404，确认是否已向目标环境导入与 `WORKS_DATA` 一致的数据。
|
||||
|
||||
## 压测窗口采集
|
||||
|
||||
Nginx upstream timing：
|
||||
|
||||
```bash
|
||||
sudo tail -f /var/log/nginx/genarrative.access.log
|
||||
sudo tail -f /var/log/nginx/genarrative.error.log
|
||||
```
|
||||
|
||||
api-server 与 SpacetimeDB 日志：
|
||||
|
||||
```bash
|
||||
sudo journalctl -u genarrative-api.service -f
|
||||
sudo journalctl -u spacetimedb.service -f
|
||||
```
|
||||
|
||||
api-server 的 OpenTelemetry 在生产与容器模板里默认开启。需要临时关闭时，显式把 `GENARRATIVE_OTEL_ENABLED=false`；需要验证 OTLP traces / metrics / logs 时，先在服务器本机启动只监听 `127.0.0.1` 的 `otelcol-contrib` debug exporter：
|
||||
|
||||
```bash
|
||||
npm run otel:debug
|
||||
```
|
||||
|
||||
如果要把本机数据转发给 Rider OpenTelemetry 面板，先在 Rider 的 OpenTelemetry 设置中启用固定 OTLP server port，例如 `17011`，再运行：
|
||||
|
||||
```bash
|
||||
RIDER_OTLP_GRPC_ENDPOINT=127.0.0.1:17011 npm run otel:rider
|
||||
```
|
||||
|
||||
脚本会在 `.codex-temp/otelcol/` 生成临时 collector 配置，默认接收 api-server 发到 `http://127.0.0.1:4318` 的 OTLP HTTP 数据。需要改端口时可设置：
|
||||
|
||||
- `OTELCOL_OTLP_HTTP_ENDPOINT`，默认 `127.0.0.1:4318`
|
||||
- `OTELCOL_OTLP_GRPC_ENDPOINT`，默认 `127.0.0.1:4317`
|
||||
- `RIDER_OTLP_GRPC_ENDPOINT`，默认 `127.0.0.1:17011`
|
||||
- `OTELCOL_BIN`，默认 `otelcol-contrib`
|
||||
|
||||
等价的 debug collector 配置如下：
|
||||
|
||||
```yaml
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 127.0.0.1:4317
|
||||
http:
|
||||
endpoint: 127.0.0.1:4318
|
||||
|
||||
exporters:
|
||||
debug:
|
||||
verbosity: detailed
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
exporters: [debug]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
exporters: [debug]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
exporters: [debug]
|
||||
```
|
||||
|
||||
```bash
|
||||
otelcol-contrib --config /etc/otelcol-contrib/genarrative-debug.yaml
|
||||
```
|
||||
|
||||
然后在 `/etc/genarrative/api-server.env` 中打开：
|
||||
|
||||
```env
|
||||
GENARRATIVE_OTEL_ENABLED=true
|
||||
OTEL_SERVICE_NAME=genarrative-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318
|
||||
```
|
||||
|
||||
注意 `api-server` 当前使用 OTLP HTTP exporter，`OTEL_EXPORTER_OTLP_ENDPOINT` 必须指向 Collector 的 HTTP base endpoint `http://127.0.0.1:4318`。不要把它改成 Collector gRPC 端口 `4317`，也不要直接指向 Rider 的 gRPC 端口；Rider 只由 `npm run otel:rider` 启动的 Collector 通过 `RIDER_OTLP_GRPC_ENDPOINT` 转发。
|
||||
|
||||
OTLP logs 是远端观测增量，不替代本地日志；api-server 日志仍看 `journalctl` / `logs/api-server/`，Nginx 日志仍看文件。日志等级继续用 `GENARRATIVE_API_LOG` / `RUST_LOG` 控制，例如 `info,tower_http=info,spacetime_client=info`。
|
||||
|
||||
Rider 的 Logs 面板展示的是 OTLP log event 自身字段，不会自动把父 span 的全部 attributes 摊平到每一条日志。请求完成日志会直接携带 `request_id`、`http.request.method`、`http.route`、`url.scheme`、`url.path`、`http.response.status_code`、`status_class`、`latency_ms` 和 `slow_request`；更完整的请求链路仍在 Traces 面板中按同一个 trace/span 关联查看。
|
||||
|
||||
压测期间可在 Metrics 面板或 debug exporter 中观察进程内存指标：
|
||||
|
||||
- `process.memory.usage`：进程常驻内存 / RSS。
|
||||
- `process.memory.virtual`：进程虚拟内存；Windows 当前按 `PrivateUsage` 上报，Linux 取 `VmSize`。
|
||||
- `genarrative.process.memory.private`：进程私有内存，Windows 来自 `PrivateUsage`，Linux 近似取 `/proc/self/status` 的 `VmData`。
|
||||
- `process.cpu.time`：进程 user + system 累计 CPU 秒数。
|
||||
- `genarrative.process.cpu.usage_percent`：两次指标采集之间的进程 CPU 使用率；100% 约等于占满 1 个 CPU core。
|
||||
- `process.thread.count`：线程数。
|
||||
- `process.windows.handle.count`：Windows 句柄数。
|
||||
- `process.unix.file_descriptor.count`：Linux 文件描述符数。
|
||||
- `genarrative.http.server.response_bodies.in_flight`：Axum / Hyper 仍持有的响应 body 数；如果内存高但该值很低，说明热点不在业务 handler 生命周期内。
|
||||
- `genarrative.http.server.request_permits.available`：应用层 HTTP 背压剩余 permit 数，带 `pool=default|gallery|detail|admin`；如果目标 pool 未接近 0，说明没有打满对应 `GENARRATIVE_API_*_MAX_CONCURRENT_REQUESTS`。
|
||||
- `genarrative.puzzle_gallery.cache.hits` / `genarrative.puzzle_gallery.cache.stale_hits` / `genarrative.puzzle_gallery.cache.misses` / `genarrative.puzzle_gallery.cache.refreshes_started` / `genarrative.puzzle_gallery.cache.refreshes_failed` / `genarrative.puzzle_gallery.cache.rebuilds`：拼图广场响应缓存 fresh 命中、stale 命中、未命中、后台刷新和重建次数。
|
||||
- `genarrative.puzzle_gallery.cache.rebuild.duration`：拼图广场缓存重建耗时。
|
||||
- `genarrative.puzzle_gallery.cache.data_json_bytes`：拼图广场缓存内预序列化 data JSON 大小。
|
||||
- `genarrative.spacetime.read.calls` / `genarrative.spacetime.read.duration_ms`：SpacetimeDB 订阅本地 cache 读次数和耗时；`read=list_puzzle_gallery` 表示当前路径走 view / local cache，不是 procedure。
|
||||
|
||||
若 `/api/runtime/puzzle/gallery` 单接口压测出现 GB 级瞬时内存峰值，先区分“持续泄漏”和“请求期分配峰值”：关闭 OTEL 后若峰值仍复现且压测结束后回落，主因通常不是 Collector / exporter。当前拼图广场列表命中缓存时应复用 `PuzzleGalleryCache` 中的预序列化 data JSON，只按请求拼接 envelope meta，不应每个请求重新深拷贝 `PuzzleGalleryResponse` 或构造完整 `serde_json::Value`。
|
||||
|
||||
本地 Windows 直连 `api-server` 压测还要单独看 K6 的 VU / 连接模型。已验证在 250 RPS、`PREALLOCATED_VUS=300` 时，哪怕打 `/healthz` 这种小响应，也可能因为本地 300 个 Established 连接触发 `api-server` private memory 瞬时升到约 7GB，压测结束后回落到 100MB 级；同样 250 RPS 改成 `PREALLOCATED_VUS=20 MAX_VUS=40` 后，拼图广场 p95 约 9ms，峰值降到约 600MB。这个现象说明高水位主要来自本机直连连接 / 发送链路，不等价于 SpacetimeDB 或拼图 JSON 缓存泄漏。做本地容量判断时优先让 VU 接近真实并发，避免用过高预分配 VU 把测试变成 Windows 本机连接缓冲压力测试；生产仍以 Nginx upstream keepalive、系统内存和 OTLP 指标一起判断。
|
||||
|
||||
线上回归辅助命令：
|
||||
|
||||
```bash
|
||||
systemctl show genarrative-api.service -p LimitNOFILE -p TasksMax
|
||||
cat /proc/$(pidof api-server)/limits
|
||||
tr '\0' '\n' < /proc/$(pidof api-server)/environ | grep 'GENARRATIVE_API_.*MAX_CONCURRENT_REQUESTS'
|
||||
ss -ltnp | grep 8082
|
||||
curl -sS http://127.0.0.1:8082/healthz
|
||||
```
|
||||
|
||||
## 验证命令
|
||||
|
||||
```bash
|
||||
npx vitest run scripts/loadtest/extract-works-list-data.test.ts
|
||||
npx eslint scripts/loadtest/extract-works-list-data.mjs scripts/loadtest/extract-works-list-data.test.ts scripts/loadtest/k6-works-list.js
|
||||
```
|
||||
@@ -1,208 +0,0 @@
|
||||
{
|
||||
"source": "spacetime-migration-1.json",
|
||||
"generatedAt": "2026-05-16T13:35:40.282Z",
|
||||
"counts": {
|
||||
"puzzle_work_profile": 3,
|
||||
"custom_world_profile": 1,
|
||||
"match3d_work_profile": 0,
|
||||
"square_hole_work_profile": 0,
|
||||
"visual_novel_work_profile": 0
|
||||
},
|
||||
"tables": {
|
||||
"puzzle_work_profile": [
|
||||
{
|
||||
"profile_id": "profile-001",
|
||||
"work_id": "work-001",
|
||||
"owner_user_id": "user-001",
|
||||
"author_display_name": "author-001",
|
||||
"cover_asset_id": "asset-001",
|
||||
"cover_image_src": "/generated-puzzle-assets/puzzle-session-f38101d7277040fcb6fbc41fea8b714a/puzzle-session-f38101d7277040fcb6fbc41fea8b714a-candidate-2/asset-1777649330373133/image.png",
|
||||
"work_title": "化学家",
|
||||
"level_name": "文学家",
|
||||
"summary": "几个文学家正站在山上面对着瀑布侃侃而谈",
|
||||
"work_description": "一个穿着白大褂的化学家正在做酷炫的化学实验，背景是化学实验室",
|
||||
"levels_json": "[{\"level_id\":\"puzzle-level-1777649242577-7\",\"level_name\":\"文学家\",\"picture_description\":\"几个文学家正站在山上面对着瀑布侃侃而谈\",\"candidates\":[{\"candidate_id\":\"puzzle-session-f38101d7277040fcb6fbc41fea8b714a-candidate-2\",\"image_src\":\"/generated-puzzle-assets/puzzle-session-f38101d7277040fcb6fbc41fea8b714a/puzzle-session-f38101d7277040fcb6fbc41fea8b714a-candidate-2/asset-1777649330373133/image.png\",\"asset_id\":\"asset-1777649330373133\",\"prompt\":\"几个文学家正站在山上面对着瀑布侃侃而谈\",\"actual_prompt\":\"请生成一张高清插画。画面主体：几个文学家正站在山上面对着瀑布侃侃而谈。画面…",
|
||||
"anchor_pack_json": "{\"theme_promise\":{\"key\":\"themePromise\",\"label\":\"题材承诺\",\"value\":\"化学家\",\"status\":\"Locked\"},\"visual_subject\":{\"key\":\"visualSubject\",\"label\":\"画面主体\",\"value\":\"一个穿着白大褂的化学家正在做酷炫的化学实验，背景是化学实验室\",\"status\":\"Locked\"},\"visual_mood\":{\"key\":\"visualMood\",\"label\":\"视觉气质\",\"value\":\"清晰、适合拼图切块\",\"status\":\"Inferred\"},\"composition_hooks\":{\"key\":\"compositionHooks\",\"label\":\"拼图记忆点\",\"value\":\"主体轮廓、色块分区、局部细节\",\"status\":\"Inferred\"},\"tags_and_forbidden\":{\"key\":\"tagsAndForbidden\",\"label\":\"标签与禁忌\",\"value\":\"化学家、拼图、插画；禁止标题字\",\"status\":\"I…",
|
||||
"theme_tags_json": "[\"化学家\",\"拼图\",\"插画\",\"禁止标题字\"]",
|
||||
"publication_status": {
|
||||
"Published": []
|
||||
},
|
||||
"play_count": 1,
|
||||
"like_count": 0,
|
||||
"remix_count": 1,
|
||||
"updated_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777703338322544
|
||||
},
|
||||
"created_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777648804043558
|
||||
},
|
||||
"published_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777649364112270
|
||||
}
|
||||
},
|
||||
{
|
||||
"profile_id": "profile-002",
|
||||
"work_id": "work-002",
|
||||
"owner_user_id": "user-002",
|
||||
"author_display_name": "author-002",
|
||||
"work_title": "我不知道",
|
||||
"level_name": "",
|
||||
"summary": "你猜我是谁",
|
||||
"work_description": "你猜我是谁",
|
||||
"levels_json": "[{\"level_id\":\"puzzle-level-1\",\"level_name\":\"\",\"picture_description\":\"真不知道\",\"candidates\":[],\"selected_candidate_id\":null,\"cover_image_src\":null,\"cover_asset_id\":null,\"generation_status\":\"idle\"}]",
|
||||
"anchor_pack_json": "{\"theme_promise\":{\"key\":\"themePromise\",\"label\":\"题材承诺\",\"value\":\"我不知道\",\"status\":\"Locked\"},\"visual_subject\":{\"key\":\"visualSubject\",\"label\":\"画面主体\",\"value\":\"真不知道\",\"status\":\"Locked\"},\"visual_mood\":{\"key\":\"visualMood\",\"label\":\"视觉气质\",\"value\":\"清晰、适合拼图切块\",\"status\":\"Inferred\"},\"composition_hooks\":{\"key\":\"compositionHooks\",\"label\":\"拼图记忆点\",\"value\":\"主体轮廓、色块分区、局部细节\",\"status\":\"Inferred\"},\"tags_and_forbidden\":{\"key\":\"tagsAndForbidden\",\"label\":\"标签与禁忌\",\"value\":\"我不知道、拼图、插画；禁止标题字\",\"status\":\"Inferred\"}}",
|
||||
"theme_tags_json": "[\"我不知道\"]",
|
||||
"publication_status": {
|
||||
"Draft": []
|
||||
},
|
||||
"play_count": 0,
|
||||
"like_count": 0,
|
||||
"remix_count": 0,
|
||||
"updated_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777619351714201
|
||||
},
|
||||
"created_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777619336673245
|
||||
}
|
||||
},
|
||||
{
|
||||
"profile_id": "profile-003",
|
||||
"work_id": "work-003",
|
||||
"owner_user_id": "user-003",
|
||||
"author_display_name": "author-002",
|
||||
"work_title": "",
|
||||
"level_name": "",
|
||||
"summary": "",
|
||||
"work_description": "",
|
||||
"levels_json": "[{\"level_id\":\"puzzle-level-1\",\"level_name\":\"\",\"picture_description\":\"\",\"candidates\":[],\"selected_candidate_id\":null,\"cover_image_src\":null,\"cover_asset_id\":null,\"generation_status\":\"idle\"}]",
|
||||
"anchor_pack_json": "{\"theme_promise\":{\"key\":\"themePromise\",\"label\":\"题材承诺\",\"value\":\"\",\"status\":\"Missing\"},\"visual_subject\":{\"key\":\"visualSubject\",\"label\":\"画面主体\",\"value\":\"\",\"status\":\"Missing\"},\"visual_mood\":{\"key\":\"visualMood\",\"label\":\"视觉气质\",\"value\":\"\",\"status\":\"Missing\"},\"composition_hooks\":{\"key\":\"compositionHooks\",\"label\":\"拼图记忆点\",\"value\":\"\",\"status\":\"Missing\"},\"tags_and_forbidden\":{\"key\":\"tagsAndForbidden\",\"label\":\"标签与禁忌\",\"value\":\"\",\"status\":\"Missing\"}}",
|
||||
"theme_tags_json": "[\"拼图\",\"插画\",\"清晰构图\"]",
|
||||
"publication_status": {
|
||||
"Draft": []
|
||||
},
|
||||
"play_count": 0,
|
||||
"like_count": 0,
|
||||
"remix_count": 0,
|
||||
"updated_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777622285252380
|
||||
},
|
||||
"created_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777622285252380
|
||||
}
|
||||
}
|
||||
],
|
||||
"custom_world_profile": [
|
||||
{
|
||||
"profile_id": "profile-081",
|
||||
"owner_user_id": "user-002",
|
||||
"author_display_name": "author-012",
|
||||
"author_public_user_code": "author-code-001",
|
||||
"world_name": "青春飞扬校园",
|
||||
"summary_text": "在现代校园中，玩家摆脱内卷，追求真实成长",
|
||||
"subtitle": "反内卷的自由学习之旅",
|
||||
"profile_payload_json": "{\"anchorContent\":null,\"anchorPack\":null,\"attributeSchema\":{\"generatedFrom\":{\"conflictCore\":\"与传统教育模式的冲突\",\"settingSummary\":\"在现代校园中，玩家摆脱内卷，追求真实成长\",\"tone\":\"积极向上，充满活力与创新\",\"worldName\":\"青春飞扬校园\",\"worldType\":\"CUSTOM\"},\"id\":\"schema:rpg-agent:1e15b44d:v1\",\"schemaVersion\":1,\"slots\":[{\"name\":\"知识储备\",\"slotId\":\"axis_a\"},{\"name\":\"创新思维\",\"slotId\":\"axis_b\"},{\"name\":\"社交能力\",\"slotId\":\"axis_c\"},{\"name\":\"抗压能力\",\"slotId\":\"axis_d\"},{\"name\":\"自我认知\",\"slotId\":\"axis_e\"},{\"name\":\"团队协作\",\"slotId\":\"axis_f\"}],\"worldId\":\"custom:青春飞扬校…",
|
||||
"publication_status": {
|
||||
"Draft": []
|
||||
},
|
||||
"play_count": 0,
|
||||
"like_count": 0,
|
||||
"remix_count": 0,
|
||||
"updated_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777532006629209
|
||||
},
|
||||
"created_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777531745887256
|
||||
}
|
||||
}
|
||||
],
|
||||
"match3d_work_profile": [],
|
||||
"square_hole_work_profile": [],
|
||||
"visual_novel_work_profile": []
|
||||
},
|
||||
"profileIds": {
|
||||
"puzzle": ["profile-001", "profile-002", "profile-003"],
|
||||
"customWorld": ["profile-081"],
|
||||
"match3d": [],
|
||||
"squareHole": [],
|
||||
"bigFish": [],
|
||||
"visualNovel": []
|
||||
},
|
||||
"workIds": {
|
||||
"puzzle": ["work-001", "work-002", "work-003"],
|
||||
"customWorld": [],
|
||||
"match3d": [],
|
||||
"squareHole": [],
|
||||
"bigFish": [],
|
||||
"visualNovel": []
|
||||
},
|
||||
"normalizedWorks": [
|
||||
{
|
||||
"type": "puzzle",
|
||||
"workId": "work-001",
|
||||
"profileId": "profile-001",
|
||||
"ownerUserId": "user-001",
|
||||
"title": "化学家",
|
||||
"subtitle": "几个文学家正站在山上面对着瀑布侃侃而谈",
|
||||
"publicationStatus": {
|
||||
"Published": []
|
||||
},
|
||||
"playCount": 1,
|
||||
"likeCount": 0,
|
||||
"remixCount": 1,
|
||||
"coverImageSrc": "/generated-puzzle-assets/puzzle-session-f38101d7277040fcb6fbc41fea8b714a/puzzle-session-f38101d7277040fcb6fbc41fea8b714a-candidate-2/asset-1777649330373133/image.png",
|
||||
"updatedAt": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777703338322544
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "puzzle",
|
||||
"workId": "work-002",
|
||||
"profileId": "profile-002",
|
||||
"ownerUserId": "user-002",
|
||||
"title": "我不知道",
|
||||
"subtitle": "你猜我是谁",
|
||||
"publicationStatus": {
|
||||
"Draft": []
|
||||
},
|
||||
"playCount": 0,
|
||||
"likeCount": 0,
|
||||
"remixCount": 0,
|
||||
"updatedAt": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777619351714201
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "puzzle",
|
||||
"workId": "work-003",
|
||||
"profileId": "profile-003",
|
||||
"ownerUserId": "user-003",
|
||||
"title": "",
|
||||
"subtitle": "",
|
||||
"publicationStatus": {
|
||||
"Draft": []
|
||||
},
|
||||
"playCount": 0,
|
||||
"likeCount": 0,
|
||||
"remixCount": 0,
|
||||
"updatedAt": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777622285252380
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "customWorld",
|
||||
"profileId": "profile-081",
|
||||
"ownerUserId": "user-002",
|
||||
"title": "青春飞扬校园",
|
||||
"subtitle": "反内卷的自由学习之旅",
|
||||
"publicationStatus": {
|
||||
"Draft": []
|
||||
},
|
||||
"playCount": 0,
|
||||
"likeCount": 0,
|
||||
"remixCount": 0,
|
||||
"updatedAt": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777532006629209
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
{
|
||||
"source": "spacetime-migration-1.json",
|
||||
"generatedAt": "2026-05-18T11:54:04.280Z",
|
||||
"counts": {
|
||||
"puzzle_work_profile": 3,
|
||||
"custom_world_profile": 1,
|
||||
"match3d_work_profile": 0,
|
||||
"square_hole_work_profile": 0,
|
||||
"visual_novel_work_profile": 0
|
||||
},
|
||||
"tables": {
|
||||
"puzzle_work_profile": [
|
||||
{
|
||||
"profile_id": "profile-001",
|
||||
"work_id": "work-001",
|
||||
"owner_user_id": "user-001",
|
||||
"author_display_name": "author-001",
|
||||
"cover_asset_id": "asset-001",
|
||||
"cover_image_src": "/generated-puzzle-assets/puzzle-session-f38101d7277040fcb6fbc41fea8b714a/puzzle-session-f38101d7277040fcb6fbc41fea8b714a-candidate-2/asset-1777649330373133/image.png",
|
||||
"work_title": "化学家",
|
||||
"level_name": "文学家",
|
||||
"summary": "几个文学家正站在山上面对着瀑布侃侃而谈",
|
||||
"work_description": "一个穿着白大褂的化学家正在做酷炫的化学实验，背景是化学实验室",
|
||||
"levels_json": "[{\"level_id\":\"puzzle-level-1777649242577-7\",\"level_name\":\"文学家\",\"picture_description\":\"几个文学家正站在山上面对着瀑布侃侃而谈\",\"candidates\":[{\"candidate_id\":\"puzzle-session-f38101d7277040fcb6fbc41fea8b714a-candidate-2\",\"image_src\":\"/generated-puzzle-assets/puzzle-session-f38101d7277040fcb6fbc41fea8b714a/puzzle-session-f38101d7277040fcb6fbc41fea8b714a-candidate-2/asset-1777649330373133/image.png\",\"asset_id\":\"asset-1777649330373133\",\"prompt\":\"几个文学家正站在山上面对着瀑布侃侃而谈\",\"actual_prompt\":\"请生成一张高清插画。画面主体：几个文学家正站在山上面对着瀑布侃侃而谈。画面…",
|
||||
"anchor_pack_json": "{\"theme_promise\":{\"key\":\"themePromise\",\"label\":\"题材承诺\",\"value\":\"化学家\",\"status\":\"Locked\"},\"visual_subject\":{\"key\":\"visualSubject\",\"label\":\"画面主体\",\"value\":\"一个穿着白大褂的化学家正在做酷炫的化学实验，背景是化学实验室\",\"status\":\"Locked\"},\"visual_mood\":{\"key\":\"visualMood\",\"label\":\"视觉气质\",\"value\":\"清晰、适合拼图切块\",\"status\":\"Inferred\"},\"composition_hooks\":{\"key\":\"compositionHooks\",\"label\":\"拼图记忆点\",\"value\":\"主体轮廓、色块分区、局部细节\",\"status\":\"Inferred\"},\"tags_and_forbidden\":{\"key\":\"tagsAndForbidden\",\"label\":\"标签与禁忌\",\"value\":\"化学家、拼图、插画；禁止标题字\",\"status\":\"I…",
|
||||
"theme_tags_json": "[\"化学家\",\"拼图\",\"插画\",\"禁止标题字\"]",
|
||||
"publication_status": {
|
||||
"Published": []
|
||||
},
|
||||
"play_count": 1,
|
||||
"like_count": 0,
|
||||
"remix_count": 1,
|
||||
"updated_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777703338322544
|
||||
},
|
||||
"created_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777648804043558
|
||||
},
|
||||
"published_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777649364112270
|
||||
}
|
||||
},
|
||||
{
|
||||
"profile_id": "profile-002",
|
||||
"work_id": "work-002",
|
||||
"owner_user_id": "user-002",
|
||||
"author_display_name": "author-002",
|
||||
"work_title": "我不知道",
|
||||
"level_name": "",
|
||||
"summary": "你猜我是谁",
|
||||
"work_description": "你猜我是谁",
|
||||
"levels_json": "[{\"level_id\":\"puzzle-level-1\",\"level_name\":\"\",\"picture_description\":\"真不知道\",\"candidates\":[],\"selected_candidate_id\":null,\"cover_image_src\":null,\"cover_asset_id\":null,\"generation_status\":\"idle\"}]",
|
||||
"anchor_pack_json": "{\"theme_promise\":{\"key\":\"themePromise\",\"label\":\"题材承诺\",\"value\":\"我不知道\",\"status\":\"Locked\"},\"visual_subject\":{\"key\":\"visualSubject\",\"label\":\"画面主体\",\"value\":\"真不知道\",\"status\":\"Locked\"},\"visual_mood\":{\"key\":\"visualMood\",\"label\":\"视觉气质\",\"value\":\"清晰、适合拼图切块\",\"status\":\"Inferred\"},\"composition_hooks\":{\"key\":\"compositionHooks\",\"label\":\"拼图记忆点\",\"value\":\"主体轮廓、色块分区、局部细节\",\"status\":\"Inferred\"},\"tags_and_forbidden\":{\"key\":\"tagsAndForbidden\",\"label\":\"标签与禁忌\",\"value\":\"我不知道、拼图、插画；禁止标题字\",\"status\":\"Inferred\"}}",
|
||||
"theme_tags_json": "[\"我不知道\"]",
|
||||
"publication_status": {
|
||||
"Draft": []
|
||||
},
|
||||
"play_count": 0,
|
||||
"like_count": 0,
|
||||
"remix_count": 0,
|
||||
"updated_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777619351714201
|
||||
},
|
||||
"created_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777619336673245
|
||||
}
|
||||
},
|
||||
{
|
||||
"profile_id": "profile-003",
|
||||
"work_id": "work-003",
|
||||
"owner_user_id": "user-003",
|
||||
"author_display_name": "author-002",
|
||||
"work_title": "",
|
||||
"level_name": "",
|
||||
"summary": "",
|
||||
"work_description": "",
|
||||
"levels_json": "[{\"level_id\":\"puzzle-level-1\",\"level_name\":\"\",\"picture_description\":\"\",\"candidates\":[],\"selected_candidate_id\":null,\"cover_image_src\":null,\"cover_asset_id\":null,\"generation_status\":\"idle\"}]",
|
||||
"anchor_pack_json": "{\"theme_promise\":{\"key\":\"themePromise\",\"label\":\"题材承诺\",\"value\":\"\",\"status\":\"Missing\"},\"visual_subject\":{\"key\":\"visualSubject\",\"label\":\"画面主体\",\"value\":\"\",\"status\":\"Missing\"},\"visual_mood\":{\"key\":\"visualMood\",\"label\":\"视觉气质\",\"value\":\"\",\"status\":\"Missing\"},\"composition_hooks\":{\"key\":\"compositionHooks\",\"label\":\"拼图记忆点\",\"value\":\"\",\"status\":\"Missing\"},\"tags_and_forbidden\":{\"key\":\"tagsAndForbidden\",\"label\":\"标签与禁忌\",\"value\":\"\",\"status\":\"Missing\"}}",
|
||||
"theme_tags_json": "[\"拼图\",\"插画\",\"清晰构图\"]",
|
||||
"publication_status": {
|
||||
"Draft": []
|
||||
},
|
||||
"play_count": 0,
|
||||
"like_count": 0,
|
||||
"remix_count": 0,
|
||||
"updated_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777622285252380
|
||||
},
|
||||
"created_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777622285252380
|
||||
}
|
||||
}
|
||||
],
|
||||
"custom_world_profile": [
|
||||
{
|
||||
"profile_id": "profile-081",
|
||||
"owner_user_id": "user-002",
|
||||
"author_display_name": "author-012",
|
||||
"author_public_user_code": "author-code-001",
|
||||
"world_name": "青春飞扬校园",
|
||||
"summary_text": "在现代校园中，玩家摆脱内卷，追求真实成长",
|
||||
"subtitle": "反内卷的自由学习之旅",
|
||||
"profile_payload_json": "{\"anchorContent\":null,\"anchorPack\":null,\"attributeSchema\":{\"generatedFrom\":{\"conflictCore\":\"与传统教育模式的冲突\",\"settingSummary\":\"在现代校园中，玩家摆脱内卷，追求真实成长\",\"tone\":\"积极向上，充满活力与创新\",\"worldName\":\"青春飞扬校园\",\"worldType\":\"CUSTOM\"},\"id\":\"schema:rpg-agent:1e15b44d:v1\",\"schemaVersion\":1,\"slots\":[{\"name\":\"知识储备\",\"slotId\":\"axis_a\"},{\"name\":\"创新思维\",\"slotId\":\"axis_b\"},{\"name\":\"社交能力\",\"slotId\":\"axis_c\"},{\"name\":\"抗压能力\",\"slotId\":\"axis_d\"},{\"name\":\"自我认知\",\"slotId\":\"axis_e\"},{\"name\":\"团队协作\",\"slotId\":\"axis_f\"}],\"worldId\":\"custom:青春飞扬校…",
|
||||
"publication_status": {
|
||||
"Draft": []
|
||||
},
|
||||
"play_count": 0,
|
||||
"like_count": 0,
|
||||
"remix_count": 0,
|
||||
"updated_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777532006629209
|
||||
},
|
||||
"created_at": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777531745887256
|
||||
}
|
||||
}
|
||||
],
|
||||
"match3d_work_profile": [],
|
||||
"square_hole_work_profile": [],
|
||||
"visual_novel_work_profile": []
|
||||
},
|
||||
"profileIds": {
|
||||
"puzzle": ["profile-001", "profile-002", "profile-003"],
|
||||
"customWorld": ["profile-081"],
|
||||
"match3d": [],
|
||||
"squareHole": [],
|
||||
"bigFish": [],
|
||||
"visualNovel": []
|
||||
},
|
||||
"workIds": {
|
||||
"puzzle": ["work-001", "work-002", "work-003"],
|
||||
"customWorld": [],
|
||||
"match3d": [],
|
||||
"squareHole": [],
|
||||
"bigFish": [],
|
||||
"visualNovel": []
|
||||
},
|
||||
"normalizedWorks": [
|
||||
{
|
||||
"type": "puzzle",
|
||||
"workId": "work-001",
|
||||
"profileId": "profile-001",
|
||||
"ownerUserId": "user-001",
|
||||
"title": "化学家",
|
||||
"subtitle": "几个文学家正站在山上面对着瀑布侃侃而谈",
|
||||
"publicationStatus": {
|
||||
"Published": []
|
||||
},
|
||||
"playCount": 1,
|
||||
"likeCount": 0,
|
||||
"remixCount": 1,
|
||||
"coverImageSrc": "/generated-puzzle-assets/puzzle-session-f38101d7277040fcb6fbc41fea8b714a/puzzle-session-f38101d7277040fcb6fbc41fea8b714a-candidate-2/asset-1777649330373133/image.png",
|
||||
"updatedAt": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777703338322544
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "puzzle",
|
||||
"workId": "work-002",
|
||||
"profileId": "profile-002",
|
||||
"ownerUserId": "user-002",
|
||||
"title": "我不知道",
|
||||
"subtitle": "你猜我是谁",
|
||||
"publicationStatus": {
|
||||
"Draft": []
|
||||
},
|
||||
"playCount": 0,
|
||||
"likeCount": 0,
|
||||
"remixCount": 0,
|
||||
"updatedAt": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777619351714201
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "puzzle",
|
||||
"workId": "work-003",
|
||||
"profileId": "profile-003",
|
||||
"ownerUserId": "user-003",
|
||||
"title": "",
|
||||
"subtitle": "",
|
||||
"publicationStatus": {
|
||||
"Draft": []
|
||||
},
|
||||
"playCount": 0,
|
||||
"likeCount": 0,
|
||||
"remixCount": 0,
|
||||
"updatedAt": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777622285252380
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "customWorld",
|
||||
"profileId": "profile-081",
|
||||
"ownerUserId": "user-002",
|
||||
"title": "青春飞扬校园",
|
||||
"subtitle": "反内卷的自由学习之旅",
|
||||
"publicationStatus": {
|
||||
"Draft": []
|
||||
},
|
||||
"playCount": 0,
|
||||
"likeCount": 0,
|
||||
"remixCount": 0,
|
||||
"updatedAt": {
|
||||
"__timestamp_micros_since_unix_epoch__": 1777532006629209
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,428 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { basename } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ALLOWED_TABLES = new Set([
|
||||
'puzzle_work_profile',
|
||||
'custom_world_profile',
|
||||
'match3d_work_profile',
|
||||
'square_hole_work_profile',
|
||||
'big_fish_work_profile',
|
||||
'visual_novel_work_profile',
|
||||
]);
|
||||
|
||||
const WORK_TABLE_TYPES = {
|
||||
puzzle_work_profile: 'puzzle',
|
||||
custom_world_profile: 'customWorld',
|
||||
match3d_work_profile: 'match3d',
|
||||
square_hole_work_profile: 'squareHole',
|
||||
big_fish_work_profile: 'bigFish',
|
||||
visual_novel_work_profile: 'visualNovel',
|
||||
};
|
||||
|
||||
const TABLE_OUTPUT_ORDER = [
|
||||
'puzzle_work_profile',
|
||||
'custom_world_profile',
|
||||
'match3d_work_profile',
|
||||
'square_hole_work_profile',
|
||||
'big_fish_work_profile',
|
||||
'visual_novel_work_profile',
|
||||
];
|
||||
|
||||
const WORK_TYPES = [
|
||||
'puzzle',
|
||||
'customWorld',
|
||||
'match3d',
|
||||
'squareHole',
|
||||
'bigFish',
|
||||
'visualNovel',
|
||||
];
|
||||
const SHORT_TEXT_LIMIT = 120;
|
||||
const LONG_TEXT_LIMIT = 500;
|
||||
const SENSITIVE_PATTERN =
|
||||
/(token|secret|password|passwd|phone|wallet|credential|authorization|auth[_-]?key|api[_-]?key)/giu;
|
||||
|
||||
class StableMapper {
|
||||
constructor(prefix) {
|
||||
this.prefix = prefix;
|
||||
this.values = new Map();
|
||||
}
|
||||
|
||||
map(value) {
|
||||
if (value === undefined || value === null || value === '') return value;
|
||||
const key = String(value);
|
||||
if (!this.values.has(key)) {
|
||||
this.values.set(
|
||||
key,
|
||||
`${this.prefix}-${String(this.values.size + 1).padStart(3, '0')}`,
|
||||
);
|
||||
}
|
||||
return this.values.get(key);
|
||||
}
|
||||
}
|
||||
|
||||
function createContext() {
|
||||
return {
|
||||
user: new StableMapper('user'),
|
||||
session: new StableMapper('session'),
|
||||
author: new StableMapper('author'),
|
||||
authorCode: new StableMapper('author-code'),
|
||||
publicWorkCode: new StableMapper('public-work-code'),
|
||||
coverAsset: new StableMapper('asset'),
|
||||
work: new StableMapper('work'),
|
||||
profile: new StableMapper('profile'),
|
||||
};
|
||||
}
|
||||
|
||||
function createWorkTypeBuckets() {
|
||||
return Object.fromEntries(WORK_TYPES.map((type) => [type, []]));
|
||||
}
|
||||
|
||||
function unwrapSpacetimeOption(value) {
|
||||
if (
|
||||
value &&
|
||||
typeof value === 'object' &&
|
||||
!Array.isArray(value) &&
|
||||
Object.keys(value).length === 1
|
||||
) {
|
||||
if (Object.prototype.hasOwnProperty.call(value, 'some')) return value.some;
|
||||
if (Object.prototype.hasOwnProperty.call(value, 'none')) return undefined;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function truncateText(value, limit) {
|
||||
if (value === undefined || value === null) return value;
|
||||
const text = String(value).replace(/\s+/g, ' ').trim();
|
||||
if (text.length <= limit) return text;
|
||||
return `${text.slice(0, limit)}…`;
|
||||
}
|
||||
|
||||
function redactSensitiveText(value) {
|
||||
if (value === undefined || value === null) return value;
|
||||
return String(value).replace(SENSITIVE_PATTERN, '[redacted]');
|
||||
}
|
||||
|
||||
function sanitizeCoverImageSrc(value) {
|
||||
const unwrapped = unwrapSpacetimeOption(value);
|
||||
if (unwrapped === undefined || unwrapped === null || unwrapped === '')
|
||||
return unwrapped;
|
||||
const text = String(unwrapped);
|
||||
if (text.startsWith('data:image/')) return '[redacted-data-image]';
|
||||
let withoutQuery = text.split('?')[0].split('#')[0];
|
||||
if (withoutQuery.length > 180)
|
||||
withoutQuery = `${withoutQuery.slice(0, 180)}…`;
|
||||
return withoutQuery;
|
||||
}
|
||||
|
||||
function sanitizeLargeJson(value) {
|
||||
const unwrapped = unwrapSpacetimeOption(value);
|
||||
if (unwrapped === undefined || unwrapped === null) return unwrapped;
|
||||
if (typeof unwrapped === 'string') {
|
||||
return truncateText(redactSensitiveText(unwrapped), LONG_TEXT_LIMIT);
|
||||
}
|
||||
try {
|
||||
return truncateText(
|
||||
redactSensitiveText(JSON.stringify(unwrapped)),
|
||||
LONG_TEXT_LIMIT,
|
||||
);
|
||||
} catch {
|
||||
return truncateText(
|
||||
redactSensitiveText(String(unwrapped)),
|
||||
LONG_TEXT_LIMIT,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function firstDefined(row, keys) {
|
||||
for (const key of keys) {
|
||||
if (row[key] !== undefined && row[key] !== null) return row[key];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function sanitizeShortField(row, sanitized, key) {
|
||||
if (row[key] !== undefined) {
|
||||
sanitized[key] = truncateText(
|
||||
unwrapSpacetimeOption(row[key]),
|
||||
SHORT_TEXT_LIMIT,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizeWorkRow(row, ctx) {
|
||||
const sanitized = {};
|
||||
const profileId = unwrapSpacetimeOption(
|
||||
firstDefined(row, ['profile_id', 'profileId']),
|
||||
);
|
||||
const workId = unwrapSpacetimeOption(
|
||||
firstDefined(row, ['work_id', 'workId']),
|
||||
);
|
||||
|
||||
if (profileId !== undefined)
|
||||
sanitized.profile_id = ctx.profile.map(profileId);
|
||||
if (workId !== undefined) sanitized.work_id = ctx.work.map(workId);
|
||||
if (row.owner_user_id !== undefined) {
|
||||
sanitized.owner_user_id = ctx.user.map(
|
||||
unwrapSpacetimeOption(row.owner_user_id),
|
||||
);
|
||||
}
|
||||
if (row.user_id !== undefined)
|
||||
sanitized.user_id = ctx.user.map(unwrapSpacetimeOption(row.user_id));
|
||||
|
||||
if (row.author_display_name !== undefined) {
|
||||
sanitized.author_display_name = ctx.author.map(
|
||||
unwrapSpacetimeOption(row.author_display_name),
|
||||
);
|
||||
}
|
||||
if (row.public_work_code !== undefined) {
|
||||
sanitized.public_work_code = ctx.publicWorkCode.map(
|
||||
unwrapSpacetimeOption(row.public_work_code),
|
||||
);
|
||||
}
|
||||
if (row.author_public_user_code !== undefined) {
|
||||
sanitized.author_public_user_code = ctx.authorCode.map(
|
||||
unwrapSpacetimeOption(row.author_public_user_code),
|
||||
);
|
||||
}
|
||||
if (row.cover_asset_id !== undefined) {
|
||||
sanitized.cover_asset_id = ctx.coverAsset.map(
|
||||
unwrapSpacetimeOption(row.cover_asset_id),
|
||||
);
|
||||
}
|
||||
if (row.cover_image_src !== undefined)
|
||||
sanitized.cover_image_src = sanitizeCoverImageSrc(row.cover_image_src);
|
||||
|
||||
for (const key of [
|
||||
'title',
|
||||
'work_title',
|
||||
'level_name',
|
||||
'world_name',
|
||||
'summary',
|
||||
'summary_text',
|
||||
'description',
|
||||
'work_description',
|
||||
'subtitle',
|
||||
]) {
|
||||
sanitizeShortField(row, sanitized, key);
|
||||
}
|
||||
|
||||
for (const key of [
|
||||
'levels_json',
|
||||
'profile_payload_json',
|
||||
'anchor_pack_json',
|
||||
'theme_tags_json',
|
||||
]) {
|
||||
if (row[key] !== undefined) sanitized[key] = sanitizeLargeJson(row[key]);
|
||||
}
|
||||
|
||||
const passthroughKeys = [
|
||||
'publication_status',
|
||||
'publicationStatus',
|
||||
'play_count',
|
||||
'playCount',
|
||||
'like_count',
|
||||
'likeCount',
|
||||
'remix_count',
|
||||
'remixCount',
|
||||
'updated_at',
|
||||
'created_at',
|
||||
'published_at',
|
||||
'visibility',
|
||||
'status',
|
||||
'category',
|
||||
'tags',
|
||||
];
|
||||
for (const key of passthroughKeys) {
|
||||
if (row[key] !== undefined)
|
||||
sanitized[key] = unwrapSpacetimeOption(row[key]);
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
function normalizeWork(tableName, row) {
|
||||
const type = WORK_TABLE_TYPES[tableName];
|
||||
return {
|
||||
type,
|
||||
workId: row.work_id,
|
||||
profileId: row.profile_id,
|
||||
ownerUserId: row.owner_user_id,
|
||||
publicWorkCode: row.public_work_code,
|
||||
title: row.title ?? row.work_title ?? row.level_name ?? row.world_name,
|
||||
subtitle:
|
||||
row.subtitle ??
|
||||
row.summary_text ??
|
||||
row.summary ??
|
||||
row.work_description ??
|
||||
row.description,
|
||||
publicationStatus:
|
||||
row.publicationStatus ?? row.publication_status ?? row.status,
|
||||
playCount: row.playCount ?? row.play_count ?? 0,
|
||||
likeCount: row.likeCount ?? row.like_count ?? 0,
|
||||
remixCount: row.remixCount ?? row.remix_count ?? 0,
|
||||
coverImageSrc: row.cover_image_src,
|
||||
updatedAt: row.updated_at,
|
||||
};
|
||||
}
|
||||
|
||||
function toRowsByTable(input) {
|
||||
const tables = Array.isArray(input?.tables) ? input.tables : [];
|
||||
const result = new Map();
|
||||
for (const table of tables) {
|
||||
if (!ALLOWED_TABLES.has(table?.name)) continue;
|
||||
result.set(table.name, Array.isArray(table.rows) ? table.rows : []);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function extractWorksListData(input, options = {}) {
|
||||
const ctx = createContext();
|
||||
const rowsByTable = toRowsByTable(input);
|
||||
const outputTables = {};
|
||||
const counts = {};
|
||||
const profileIds = createWorkTypeBuckets();
|
||||
const workIds = createWorkTypeBuckets();
|
||||
const normalizedWorks = [];
|
||||
|
||||
for (const tableName of TABLE_OUTPUT_ORDER) {
|
||||
const sourceRows = rowsByTable.get(tableName);
|
||||
if (!sourceRows) continue;
|
||||
const sanitizedRows = sourceRows.map((row) => sanitizeWorkRow(row, ctx));
|
||||
outputTables[tableName] = sanitizedRows;
|
||||
counts[tableName] = sanitizedRows.length;
|
||||
|
||||
const type = WORK_TABLE_TYPES[tableName];
|
||||
if (type) {
|
||||
for (const row of sanitizedRows) {
|
||||
if (row.profile_id) profileIds[type].push(row.profile_id);
|
||||
if (row.work_id) workIds[type].push(row.work_id);
|
||||
normalizedWorks.push(normalizeWork(tableName, row));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: options.source ?? 'unknown',
|
||||
generatedAt: options.generatedAt ?? new Date().toISOString(),
|
||||
counts,
|
||||
tables: outputTables,
|
||||
profileIds,
|
||||
workIds,
|
||||
normalizedWorks,
|
||||
};
|
||||
}
|
||||
|
||||
function createSampleOutput(output, maxRowsPerTable = 3) {
|
||||
const tables = {};
|
||||
const counts = {};
|
||||
const allowedWorkIds = new Set();
|
||||
const allowedProfileIds = new Set();
|
||||
|
||||
for (const [tableName, rows] of Object.entries(output.tables)) {
|
||||
tables[tableName] = rows.slice(0, maxRowsPerTable);
|
||||
counts[tableName] = tables[tableName].length;
|
||||
const type = WORK_TABLE_TYPES[tableName];
|
||||
if (type) {
|
||||
for (const row of tables[tableName]) {
|
||||
if (row.work_id) allowedWorkIds.add(row.work_id);
|
||||
if (row.profile_id) allowedProfileIds.add(row.profile_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const profileIds = Object.fromEntries(
|
||||
Object.entries(output.profileIds).map(([type, ids]) => [
|
||||
type,
|
||||
ids.filter((id) => allowedProfileIds.has(id)).slice(0, maxRowsPerTable),
|
||||
]),
|
||||
);
|
||||
const workIds = Object.fromEntries(
|
||||
Object.entries(output.workIds).map(([type, ids]) => [
|
||||
type,
|
||||
ids.filter((id) => allowedWorkIds.has(id)).slice(0, maxRowsPerTable),
|
||||
]),
|
||||
);
|
||||
const normalizedWorks = output.normalizedWorks
|
||||
.filter(
|
||||
(work) =>
|
||||
allowedWorkIds.has(work.workId) ||
|
||||
allowedProfileIds.has(work.profileId),
|
||||
)
|
||||
.slice(0, maxRowsPerTable * 6);
|
||||
|
||||
return {
|
||||
...output,
|
||||
counts,
|
||||
tables,
|
||||
profileIds,
|
||||
workIds,
|
||||
normalizedWorks,
|
||||
};
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--input' || arg === '--output' || arg === '--sample-output') {
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith('--'))
|
||||
throw new Error(`${arg} requires a value`);
|
||||
args[arg.slice(2)] = value;
|
||||
index += 1;
|
||||
} else if (arg === '--help' || arg === '-h') {
|
||||
args.help = true;
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function usage() {
|
||||
return 'Usage: node scripts/loadtest/extract-works-list-data.mjs --input <migration.json> --output <works-list.local.json> [--sample-output <works-list.sample.json>]';
|
||||
}
|
||||
|
||||
export async function runCli(argv = process.argv.slice(2)) {
|
||||
const args = parseArgs(argv);
|
||||
if (args.help) {
|
||||
console.log(usage());
|
||||
return;
|
||||
}
|
||||
if (!args.input) throw new Error('Missing required --input. ' + usage());
|
||||
if (!args.output) throw new Error('Missing required --output. ' + usage());
|
||||
|
||||
const raw = await readFile(args.input, 'utf8');
|
||||
const migration = JSON.parse(raw);
|
||||
const output = extractWorksListData(migration, {
|
||||
source: basename(args.input),
|
||||
});
|
||||
await writeFile(args.output, `${JSON.stringify(output, null, 2)}\n`, 'utf8');
|
||||
|
||||
if (args['sample-output']) {
|
||||
const sample = createSampleOutput(output);
|
||||
await writeFile(
|
||||
args['sample-output'],
|
||||
`${JSON.stringify(sample, null, 2)}\n`,
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`works-list extracted: source=${output.source}, tables=${Object.keys(output.tables).length}, normalizedWorks=${output.normalizedWorks.length}`,
|
||||
);
|
||||
for (const [tableName, count] of Object.entries(output.counts)) {
|
||||
console.log(` ${tableName}: ${count}`);
|
||||
}
|
||||
}
|
||||
|
||||
const isDirectRun =
|
||||
process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
|
||||
if (isDirectRun) {
|
||||
runCli().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -1,296 +0,0 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { extractWorksListData } from './extract-works-list-data.mjs';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const scriptPath = fileURLToPath(
|
||||
new URL('./extract-works-list-data.mjs', import.meta.url),
|
||||
);
|
||||
|
||||
const fixtureMigration = {
|
||||
schema_version: 7,
|
||||
tables: [
|
||||
{
|
||||
name: 'puzzle_work_profile',
|
||||
rows: [
|
||||
{
|
||||
profile_id: 'profile-real-aaa',
|
||||
work_id: 'work-real-aaa',
|
||||
owner_user_id: 'owner-secret-123',
|
||||
author_display_name: 'Alice Secret',
|
||||
author_public_user_code: 'author-code-secret',
|
||||
public_work_code: 'public-code-secret',
|
||||
title: '超长标题'.repeat(20),
|
||||
summary: 'summary '.repeat(80),
|
||||
description: 'description '.repeat(120),
|
||||
publication_status: 'published',
|
||||
play_count: 42,
|
||||
like_count: 7,
|
||||
cover_asset_id: { some: 'asset-secret-cover' },
|
||||
cover_image_src: {
|
||||
some: 'https://cdn.example.test/cover.png?token=***&sig=abc',
|
||||
},
|
||||
levels_json: JSON.stringify({
|
||||
secret: 'level-token-value',
|
||||
data: 'x'.repeat(2000),
|
||||
}),
|
||||
theme_tags_json: JSON.stringify(['化学家', '实验室']),
|
||||
remix_count: 2,
|
||||
updated_at: '2026-05-01T00:00:00Z',
|
||||
},
|
||||
{
|
||||
profile_id: 'profile-real-bbb',
|
||||
work_id: 'work-real-bbb',
|
||||
owner_user_id: 'owner-secret-123',
|
||||
author_display_name: 'Alice Secret',
|
||||
publication_status: 'draft',
|
||||
play_count: 3,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'custom_world_profile',
|
||||
rows: [
|
||||
{
|
||||
profile_id: 'world-profile-secret',
|
||||
work_id: 'world-work-secret',
|
||||
owner_user_id: 'world-owner-secret',
|
||||
title: '世界作品',
|
||||
profile_payload_json: '{"large":"' + 'y'.repeat(2000) + '"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'public_work_play_daily_stat',
|
||||
rows: [
|
||||
{
|
||||
source_type: 'puzzle',
|
||||
profile_id: 'profile-real-aaa',
|
||||
owner_user_id: 'owner-secret-123',
|
||||
user_id: 'player-secret-456',
|
||||
source_session_id: 'session-secret-789',
|
||||
played_day: '2026-05-01',
|
||||
play_count: 12,
|
||||
updated_at: '2026-05-02T00:00:00Z',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'user_account',
|
||||
rows: [
|
||||
{
|
||||
user_id: 'owner-secret-123',
|
||||
phone: '+8613800138000',
|
||||
auth_token: 'auth-token-secret',
|
||||
wallet_balance: 999,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'refresh_session',
|
||||
rows: [
|
||||
{
|
||||
token: 'refresh-token-secret',
|
||||
source_session_id: 'session-secret-789',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'profile_wallet_ledger',
|
||||
rows: [{ wallet_id: 'wallet-secret', amount: 100 }],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
async function withTempDir(fn) {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), 'works-list-test-'));
|
||||
try {
|
||||
return await fn(dir);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe('extractWorksListData', () => {
|
||||
it('只保留作品 profile 白名单表，禁用的行为/敏感表不会出现在输出 JSON 字符串中', () => {
|
||||
const output = extractWorksListData(fixtureMigration, {
|
||||
source: 'fixture.local.json',
|
||||
});
|
||||
const serialized = JSON.stringify(output);
|
||||
|
||||
expect(Object.keys(output.tables).sort()).toEqual([
|
||||
'custom_world_profile',
|
||||
'puzzle_work_profile',
|
||||
]);
|
||||
expect(serialized).not.toContain('public_work_play_daily_stat');
|
||||
expect(serialized).not.toContain('user_account');
|
||||
expect(serialized).not.toContain('refresh_session');
|
||||
expect(serialized).not.toContain('profile_wallet_ledger');
|
||||
expect(serialized).not.toContain('+8613800138000');
|
||||
expect(serialized).not.toContain('auth-token-secret');
|
||||
expect(serialized).not.toContain('wallet-secret');
|
||||
});
|
||||
|
||||
it('不会输出 owner/user/session/auth/token/phone/wallet 等敏感原值，owner 稳定映射', () => {
|
||||
const output = extractWorksListData(fixtureMigration, {
|
||||
source: 'fixture.local.json',
|
||||
});
|
||||
const serialized = JSON.stringify(output);
|
||||
|
||||
for (const secret of [
|
||||
'owner-secret-123',
|
||||
'player-secret-456',
|
||||
'session-secret-789',
|
||||
'Alice Secret',
|
||||
'author-code-secret',
|
||||
'public-code-secret',
|
||||
'asset-secret-cover',
|
||||
'SECRET_TOKEN',
|
||||
]) {
|
||||
expect(serialized).not.toContain(secret);
|
||||
}
|
||||
|
||||
expect(output.tables.puzzle_work_profile[0].owner_user_id).toBe('user-001');
|
||||
expect(output.tables.puzzle_work_profile[1].owner_user_id).toBe('user-001');
|
||||
expect(output.tables.puzzle_work_profile[0].author_display_name).toBe(
|
||||
'author-001',
|
||||
);
|
||||
expect(serialized).not.toContain('level-token-value');
|
||||
});
|
||||
|
||||
it('puzzle 数据生成 profileIds/workIds 和 normalizedWorks，并保留列表展示字段', () => {
|
||||
const output = extractWorksListData(fixtureMigration, {
|
||||
source: 'fixture.local.json',
|
||||
});
|
||||
|
||||
expect(output.source).toBe('fixture.local.json');
|
||||
expect(output.generatedAt).toEqual(expect.any(String));
|
||||
expect(output.counts.puzzle_work_profile).toBe(2);
|
||||
expect(output.profileIds.puzzle).toEqual(['profile-001', 'profile-002']);
|
||||
expect(output.workIds.puzzle).toEqual(['work-001', 'work-002']);
|
||||
expect(output.normalizedWorks).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
type: 'puzzle',
|
||||
workId: 'work-001',
|
||||
profileId: 'profile-001',
|
||||
publicationStatus: 'published',
|
||||
playCount: 42,
|
||||
title: expect.any(String),
|
||||
remixCount: 2,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(output.tables.puzzle_work_profile[0].cover_image_src).toBe(
|
||||
'https://cdn.example.test/cover.png',
|
||||
);
|
||||
expect(output.tables.puzzle_work_profile[0].theme_tags_json).toBe(
|
||||
'["化学家","实验室"]',
|
||||
);
|
||||
});
|
||||
|
||||
it('data image、URL token 和绝对输入路径不会泄露到输出', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const input = path.join(dir, 'migration.local.json');
|
||||
const output = path.join(dir, 'works-list.local.json');
|
||||
await writeFile(
|
||||
input,
|
||||
JSON.stringify({
|
||||
tables: [
|
||||
{
|
||||
name: 'puzzle_work_profile',
|
||||
rows: [
|
||||
{
|
||||
profile_id: 'profile-real',
|
||||
work_id: 'work-real',
|
||||
cover_image_src: {
|
||||
some: 'data:image/png;base64,SECRET_IMAGE_BYTES',
|
||||
},
|
||||
levels_json: JSON.stringify({
|
||||
token: 'SECRET_TOKEN_VALUE',
|
||||
title: 'safe',
|
||||
}),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
await execFileAsync(process.execPath, [
|
||||
scriptPath,
|
||||
'--input',
|
||||
input,
|
||||
'--output',
|
||||
output,
|
||||
]);
|
||||
const extracted = JSON.parse(await readFile(output, 'utf8'));
|
||||
const serialized = JSON.stringify(extracted);
|
||||
|
||||
expect(extracted.source).toBe('migration.local.json');
|
||||
expect(serialized).not.toContain(dir);
|
||||
expect(serialized).not.toContain('SECRET_IMAGE_BYTES');
|
||||
expect(serialized).not.toContain('SECRET_TOKEN_VALUE');
|
||||
expect(extracted.tables.puzzle_work_profile[0].cover_image_src).toBe(
|
||||
'[redacted-data-image]',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('sample-output 只输出少量脱敏样例', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const input = path.join(dir, 'migration.local.json');
|
||||
const output = path.join(dir, 'works-list.local.json');
|
||||
const sampleOutput = path.join(dir, 'works-list.sample.json');
|
||||
const manyRows = Array.from({ length: 5 }, (_, index) => ({
|
||||
profile_id: `profile-real-${index}`,
|
||||
work_id: `work-real-${index}`,
|
||||
owner_user_id: `owner-secret-${index}`,
|
||||
title: `作品 ${index}`,
|
||||
publication_status: 'published',
|
||||
play_count: index,
|
||||
}));
|
||||
await writeFile(
|
||||
input,
|
||||
JSON.stringify({
|
||||
tables: [{ name: 'puzzle_work_profile', rows: manyRows }],
|
||||
}),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
await execFileAsync(process.execPath, [
|
||||
scriptPath,
|
||||
'--input',
|
||||
input,
|
||||
'--output',
|
||||
output,
|
||||
'--sample-output',
|
||||
sampleOutput,
|
||||
]);
|
||||
const sample = JSON.parse(await readFile(sampleOutput, 'utf8'));
|
||||
const serialized = JSON.stringify(sample);
|
||||
|
||||
expect(sample.tables.puzzle_work_profile).toHaveLength(3);
|
||||
expect(sample.normalizedWorks).toHaveLength(3);
|
||||
expect(serialized).not.toContain('owner-secret-0');
|
||||
expect(serialized).not.toContain('work-real-0');
|
||||
});
|
||||
});
|
||||
|
||||
it('CLI 参数缺失时退出非 0 并输出清晰错误', async () => {
|
||||
await expect(
|
||||
execFileAsync(process.execPath, [scriptPath, '--input', 'missing.json']),
|
||||
).rejects.toMatchObject({
|
||||
code: 1,
|
||||
stderr: expect.stringContaining('--output'),
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,278 +0,0 @@
|
||||
/* global __ENV */
|
||||
import { check, sleep } from 'k6';
|
||||
import { SharedArray } from 'k6/data';
|
||||
import http from 'k6/http';
|
||||
import { Rate, Trend } from 'k6/metrics';
|
||||
|
||||
// k6 resolves open() paths relative to this script file, not the shell cwd.
|
||||
const DEFAULT_WORKS_DATA = 'data/works-list.local.json';
|
||||
const WORKS_DATA = __ENV.WORKS_DATA || DEFAULT_WORKS_DATA;
|
||||
const BASE_URL = (__ENV.BASE_URL || 'http://127.0.0.1:8787').replace(
|
||||
/\/+$/u,
|
||||
'',
|
||||
);
|
||||
const AUTH_TOKEN = __ENV.AUTH_TOKEN || '';
|
||||
const SCENARIO = __ENV.SCENARIO || 'smoke';
|
||||
const REQUEST_TIMEOUT = __ENV.REQUEST_TIMEOUT || '30s';
|
||||
const SLEEP_MIN_SECONDS = Number(__ENV.SLEEP_MIN_SECONDS || '0.5');
|
||||
const SLEEP_MAX_SECONDS = Number(__ENV.SLEEP_MAX_SECONDS || '2');
|
||||
const DETAIL_RATIO = Number(__ENV.DETAIL_RATIO || '0');
|
||||
|
||||
const worksListShapeErrorRate = new Rate('works_list_shape_error_rate');
|
||||
const worksDetailShapeErrorRate = new Rate('works_detail_shape_error_rate');
|
||||
const worksListDuration = new Trend('works_list_duration');
|
||||
const worksDetailDuration = new Trend('works_detail_duration');
|
||||
|
||||
const data = new SharedArray('works-list-data', () => [
|
||||
JSON.parse(open(WORKS_DATA)),
|
||||
])[0];
|
||||
const normalizedWorks = Array.isArray(data.normalizedWorks)
|
||||
? data.normalizedWorks
|
||||
: [];
|
||||
|
||||
const scenarioOptions = {
|
||||
smoke: {
|
||||
scenarios: {
|
||||
smoke: {
|
||||
executor: 'constant-vus',
|
||||
vus: Number(__ENV.VUS || 1),
|
||||
duration: __ENV.DURATION || '30s',
|
||||
},
|
||||
},
|
||||
thresholds: {
|
||||
http_req_failed: ['rate<0.01'],
|
||||
http_req_duration: ['p(95)<800'],
|
||||
works_list_shape_error_rate: ['rate<0.01'],
|
||||
},
|
||||
},
|
||||
baseline: {
|
||||
scenarios: {
|
||||
baseline: {
|
||||
executor: 'constant-vus',
|
||||
vus: Number(__ENV.VUS || 10),
|
||||
duration: __ENV.DURATION || '3m',
|
||||
},
|
||||
},
|
||||
thresholds: {
|
||||
http_req_failed: ['rate<0.01'],
|
||||
http_req_duration: ['p(95)<800', 'p(99)<1500'],
|
||||
works_list_shape_error_rate: ['rate<0.01'],
|
||||
},
|
||||
},
|
||||
spike: {
|
||||
scenarios: {
|
||||
spike: {
|
||||
executor: 'ramping-arrival-rate',
|
||||
startRate: Number(__ENV.START_RPS || 5),
|
||||
preAllocatedVUs: Number(__ENV.PREALLOCATED_VUS || 50),
|
||||
maxVUs: Number(__ENV.MAX_VUS || 200),
|
||||
timeUnit: '1s',
|
||||
stages: [
|
||||
{
|
||||
target: Number(__ENV.PEAK_RPS || 25),
|
||||
duration: __ENV.RAMP_UP || '30s',
|
||||
},
|
||||
{
|
||||
target: Number(__ENV.PEAK_RPS || 25),
|
||||
duration: __ENV.HOLD || '2m',
|
||||
},
|
||||
{
|
||||
target: Number(__ENV.END_RPS || 5),
|
||||
duration: __ENV.RAMP_DOWN || '30s',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
thresholds: {
|
||||
http_req_failed: ['rate<0.01'],
|
||||
http_req_duration: ['p(95)<2000'],
|
||||
dropped_iterations: ['count==0'],
|
||||
works_list_shape_error_rate: ['rate<0.01'],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const options = scenarioOptions[SCENARIO] || scenarioOptions.smoke;
|
||||
|
||||
const PUBLIC_ENDPOINTS = [
|
||||
{
|
||||
name: 'puzzle_gallery_list',
|
||||
method: 'GET',
|
||||
path: '/api/runtime/puzzle/gallery',
|
||||
expectCollectionKeys: ['items', 'works', 'entries'],
|
||||
},
|
||||
{
|
||||
name: 'custom_world_gallery_list',
|
||||
method: 'GET',
|
||||
path: '/api/runtime/custom-world-gallery',
|
||||
expectCollectionKeys: ['entries', 'items', 'works'],
|
||||
},
|
||||
];
|
||||
|
||||
const AUTH_ENDPOINTS = [
|
||||
{
|
||||
name: 'puzzle_works_list',
|
||||
method: 'GET',
|
||||
path: '/api/runtime/puzzle/works',
|
||||
expectCollectionKeys: ['items', 'works'],
|
||||
},
|
||||
{
|
||||
name: 'custom_world_works_list',
|
||||
method: 'GET',
|
||||
path: '/api/runtime/custom-world/works',
|
||||
expectCollectionKeys: ['items', 'entries', 'works'],
|
||||
},
|
||||
];
|
||||
|
||||
function requestParams(endpointName) {
|
||||
const headers = { 'x-genarrative-response-envelope': 'v1' };
|
||||
if (AUTH_TOKEN) headers.Authorization = `Bearer ${AUTH_TOKEN}`;
|
||||
return {
|
||||
headers,
|
||||
timeout: REQUEST_TIMEOUT,
|
||||
tags: { endpoint: endpointName },
|
||||
};
|
||||
}
|
||||
|
||||
function buildUrl(path) {
|
||||
return `${BASE_URL}${path}`;
|
||||
}
|
||||
|
||||
function parseJson(response) {
|
||||
try {
|
||||
return response.json();
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function unwrapPayload(json) {
|
||||
if (!json || typeof json !== 'object') return null;
|
||||
if (json.data && typeof json.data === 'object') return json.data;
|
||||
return json;
|
||||
}
|
||||
|
||||
function hasCollection(payload, keys) {
|
||||
return Boolean(payload) && keys.some((key) => Array.isArray(payload[key]));
|
||||
}
|
||||
|
||||
function firstCollection(payload, keys) {
|
||||
for (const key of keys) {
|
||||
if (payload && Array.isArray(payload[key])) return payload[key];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function hasListItemShape(payload, keys) {
|
||||
const collection = firstCollection(payload, keys);
|
||||
if (collection.length === 0) return true;
|
||||
const item = collection[0];
|
||||
const hasId = Boolean(
|
||||
item &&
|
||||
(item.profileId ||
|
||||
item.profile_id ||
|
||||
item.workId ||
|
||||
item.work_id ||
|
||||
item.publicWorkCode),
|
||||
);
|
||||
const hasTitle = Boolean(
|
||||
item &&
|
||||
(item.title ||
|
||||
item.workTitle ||
|
||||
item.work_title ||
|
||||
item.levelName ||
|
||||
item.worldName),
|
||||
);
|
||||
return hasId && hasTitle;
|
||||
}
|
||||
|
||||
function randomItem(items) {
|
||||
if (!items.length) return null;
|
||||
return items[Math.floor(Math.random() * items.length)];
|
||||
}
|
||||
|
||||
function listEndpoints() {
|
||||
return AUTH_TOKEN
|
||||
? PUBLIC_ENDPOINTS.concat(AUTH_ENDPOINTS)
|
||||
: PUBLIC_ENDPOINTS;
|
||||
}
|
||||
|
||||
function detailEndpointFor(work) {
|
||||
if (!work || !work.profileId) return null;
|
||||
if (work.type === 'puzzle') {
|
||||
return {
|
||||
name: 'puzzle_gallery_detail',
|
||||
path: `/api/runtime/puzzle/gallery/${encodeURIComponent(work.profileId)}`,
|
||||
expectKeys: ['item', 'work', 'entry'],
|
||||
};
|
||||
}
|
||||
if (work.type === 'customWorld' && work.profileId && work.ownerUserId) {
|
||||
return {
|
||||
name: 'custom_world_gallery_detail',
|
||||
path: `/api/runtime/custom-world-gallery/${encodeURIComponent(work.ownerUserId)}/${encodeURIComponent(work.profileId)}`,
|
||||
expectKeys: ['entry', 'item', 'work'],
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function performListRequest(endpoint) {
|
||||
const url = buildUrl(endpoint.path);
|
||||
const response = http.request(
|
||||
endpoint.method,
|
||||
url,
|
||||
null,
|
||||
requestParams(endpoint.name),
|
||||
);
|
||||
worksListDuration.add(response.timings.duration, { endpoint: endpoint.name });
|
||||
const json = parseJson(response);
|
||||
const payload = unwrapPayload(json);
|
||||
const ok = check(response, {
|
||||
[`${endpoint.name} status is 200`]: (res) => res.status === 200,
|
||||
[`${endpoint.name} returns json object`]: () => Boolean(payload),
|
||||
[`${endpoint.name} has collection`]: () =>
|
||||
hasCollection(payload, endpoint.expectCollectionKeys),
|
||||
[`${endpoint.name} list item shape`]: () =>
|
||||
hasListItemShape(payload, endpoint.expectCollectionKeys),
|
||||
});
|
||||
worksListShapeErrorRate.add(!ok, { endpoint: endpoint.name });
|
||||
}
|
||||
|
||||
function performDetailRequest() {
|
||||
const endpoint = detailEndpointFor(randomItem(normalizedWorks));
|
||||
if (!endpoint) return;
|
||||
|
||||
const response = http.get(
|
||||
buildUrl(endpoint.path),
|
||||
requestParams(endpoint.name),
|
||||
);
|
||||
worksDetailDuration.add(response.timings.duration, {
|
||||
endpoint: endpoint.name,
|
||||
});
|
||||
const json = parseJson(response);
|
||||
const payload = unwrapPayload(json);
|
||||
const ok = check(response, {
|
||||
[`${endpoint.name} status is 200`]: (res) => res.status === 200,
|
||||
[`${endpoint.name} has detail payload`]: () =>
|
||||
Boolean(payload) && endpoint.expectKeys.some((key) => payload[key]),
|
||||
});
|
||||
worksDetailShapeErrorRate.add(!ok, { endpoint: endpoint.name });
|
||||
}
|
||||
|
||||
export default function () {
|
||||
for (const endpoint of listEndpoints()) {
|
||||
performListRequest(endpoint);
|
||||
}
|
||||
if (
|
||||
normalizedWorks.length &&
|
||||
DETAIL_RATIO > 0 &&
|
||||
Math.random() < DETAIL_RATIO
|
||||
) {
|
||||
performDetailRequest();
|
||||
}
|
||||
|
||||
const jitter =
|
||||
SLEEP_MIN_SECONDS +
|
||||
Math.random() * Math.max(0, SLEEP_MAX_SECONDS - SLEEP_MIN_SECONDS);
|
||||
sleep(jitter);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -142,6 +142,38 @@ function backendStepIndex(stepName: string) {
|
||||
}
|
||||
|
||||
describe('project CI workflow', () => {
|
||||
it('publishes cache deltas only after Rust reporting on non-cancelled master pushes', () => {
|
||||
const producers = [
|
||||
'ai-game-creator-shell-rust-lane-1',
|
||||
'ai-game-creator-shell-rust-lane-2',
|
||||
'ai-game-creator-shell-rust-smoke',
|
||||
'ai-game-creator-shell-rust-crates',
|
||||
'backend-tests',
|
||||
'native-shell-tests',
|
||||
];
|
||||
for (const job of jobNames) {
|
||||
if (!producers.includes(job)) {
|
||||
expect(jobSection(job)).not.toContain('export-gitea-rust-cache.py');
|
||||
continue;
|
||||
}
|
||||
const publish = stepSection(job, 'Publish master Rust cache artifact');
|
||||
expect(publish).toContain(
|
||||
"if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/master' }}",
|
||||
);
|
||||
expect(publish).toContain('GENARRATIVE_GITEA_TOKEN: ${{ github.token }}');
|
||||
expect(publish).toContain('continue-on-error: true');
|
||||
expect(publish).toContain(
|
||||
'run: python3 scripts/export-gitea-rust-cache.py',
|
||||
);
|
||||
const section = jobSection(job);
|
||||
expect(
|
||||
section.indexOf('Publish master Rust cache artifact'),
|
||||
).toBeGreaterThan(
|
||||
section.indexOf('run: bash scripts/ci-rust-cache.sh report'),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('uses isolated compilation caching for every Rust test job', () => {
|
||||
const firstRustSteps = {
|
||||
'ai-game-creator-shell-rust-lane-1':
|
||||
@@ -331,7 +363,8 @@ describe('project CI workflow', () => {
|
||||
const rustLock = 'apps/ai-game-creator-shell/src-tauri/Cargo.lock';
|
||||
|
||||
for (const workspaceManifest of workspaceManifests) {
|
||||
expect(imageBuildScript.split(workspaceManifest)).toHaveLength(3);
|
||||
// 构建上下文与 revision 共用一份清单，不重复枚举 manifest。
|
||||
expect(imageBuildScript.split(workspaceManifest)).toHaveLength(2);
|
||||
expect(imageDockerignore).toContain(`!${workspaceManifest}`);
|
||||
expect(imageDockerfile).toContain(
|
||||
`COPY ${workspaceManifest} /usr/local/share/genarrative-ci/npm/${workspaceManifest}`,
|
||||
@@ -339,16 +372,15 @@ describe('project CI workflow', () => {
|
||||
}
|
||||
|
||||
for (const [path, expectedCount] of [
|
||||
[rustManifest, 2],
|
||||
[rustLock, 3],
|
||||
[rustManifest, 1],
|
||||
[rustLock, 2],
|
||||
] as const) {
|
||||
expect(imageBuildScript.split(path)).toHaveLength(expectedCount + 1);
|
||||
expect(imageDockerignore).toContain(`!${path}`);
|
||||
}
|
||||
|
||||
// AGC 通过本地 path 依赖引用三个编辑器 bridge crate。镜像预热会对
|
||||
// AGC manifest 执行 cargo fetch --locked，构建上下文与 dockerignore
|
||||
// 必须同时放行这些 crate，否则镜像在 cargo fetch 阶段必然失败。
|
||||
// AGC 通过本地 path 依赖引用三个编辑器 bridge crate。Cargo fetch 只需要
|
||||
// manifest；完整源码不得进入镜像构建上下文，实际清单闭包由 Python tar 测试核验。
|
||||
for (const bridgeDir of [
|
||||
'plugins/agc-cocos-editor/native/cocos-editor-bridge',
|
||||
'plugins/agc-unity-editor/native/unity-editor-bridge',
|
||||
@@ -361,6 +393,18 @@ describe('project CI workflow', () => {
|
||||
`COPY ${bridgeDir} /tmp/genarrative-cargo-cache/${bridgeDir}`,
|
||||
);
|
||||
}
|
||||
expect(imageBuildScript).toContain(
|
||||
'apps/ai-game-creator-shell/src-tauri/vendor',
|
||||
);
|
||||
expect(imageDockerignore).toContain(
|
||||
'!apps/ai-game-creator-shell/src-tauri/vendor/*/Cargo.toml',
|
||||
);
|
||||
expect(imageDockerignore).toContain(
|
||||
'!plugins/agc-*-editor/native/*-editor-bridge/Cargo.toml',
|
||||
);
|
||||
expect(imageDockerfile).toContain(
|
||||
'COPY apps/ai-game-creator-shell/src-tauri /tmp/genarrative-cargo-cache/apps/ai-game-creator-shell/src-tauri',
|
||||
);
|
||||
|
||||
expect(imageBuildScript).toContain(
|
||||
'--build-arg "AGC_RUST_LOCK_SHA256=${agc_rust_lock_sha256}"',
|
||||
@@ -398,6 +442,25 @@ describe('project CI workflow', () => {
|
||||
expect(imageCheckScript).toContain(
|
||||
'::warning title=CI dependency cache is partial::',
|
||||
);
|
||||
|
||||
// 下载缓存由 BuildKit 的固定 ID 独占写入，最终镜像只复制受控快照，不继承旧镜像层。
|
||||
for (const mount of [
|
||||
'id=genarrative-ci-cargo-cache-v1,target=/usr/local/cargo/registry/cache,sharing=locked',
|
||||
'id=genarrative-ci-cargo-index-v1,target=/usr/local/cargo/registry/index,sharing=locked',
|
||||
'id=genarrative-ci-npm-v1,target=/var/cache/genarrative-ci-npm,sharing=locked',
|
||||
]) {
|
||||
expect(imageDockerfile).toContain(mount);
|
||||
}
|
||||
expect(imageDockerfile).toContain(
|
||||
'FROM rust-toolchain AS download-cache-seed',
|
||||
);
|
||||
expect(imageBuildScript).toContain('--target download-cache-seed');
|
||||
expect(imageDockerfile).toContain(
|
||||
'COPY --from=rust-dependency-cache /opt/ci-downloads/registry /usr/local/cargo/registry',
|
||||
);
|
||||
expect(imageDockerfile).toContain(
|
||||
'/var/cache/genarrative-ci-npm/_cacache /root/.npm/_cacache',
|
||||
);
|
||||
});
|
||||
|
||||
it('copies every workspace manifest before the API image web-builder clean install', () => {
|
||||
|
||||
@@ -1,202 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
export const DEFAULT_ORPHAN_WORK_OWNER_USER_ID = 'wx-openid-placeholder';
|
||||
|
||||
export const WORK_OWNER_TABLES = [
|
||||
'custom_world_profile',
|
||||
'custom_world_gallery_entry',
|
||||
'custom_world_session',
|
||||
'custom_world_agent_session',
|
||||
'custom_world_draft_card',
|
||||
'puzzle_agent_session',
|
||||
'puzzle_work_profile',
|
||||
'bark_battle_draft_config',
|
||||
'bark_battle_published_config',
|
||||
'match3d_agent_session',
|
||||
'match3d_work_profile',
|
||||
'jump_hop_agent_session',
|
||||
'jump_hop_work_profile',
|
||||
'wooden_fish_agent_session',
|
||||
'wooden_fish_work_profile',
|
||||
'square_hole_agent_session',
|
||||
'square_hole_work_profile',
|
||||
'visual_novel_agent_session',
|
||||
'visual_novel_work_profile',
|
||||
'big_fish_creation_session',
|
||||
];
|
||||
|
||||
const ROW_KEY_FIELDS = [
|
||||
'profile_id',
|
||||
'work_id',
|
||||
'session_id',
|
||||
'draft_id',
|
||||
'gallery_entry_id',
|
||||
'id',
|
||||
];
|
||||
|
||||
if (isCliEntry()) {
|
||||
runCli(process.argv.slice(2)).catch((error) => {
|
||||
console.error(
|
||||
`[rebind-orphan-work-owners] ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export function rebindOrphanWorkOwnersInMigration(
|
||||
migration,
|
||||
{
|
||||
placeholderUserId = DEFAULT_ORPHAN_WORK_OWNER_USER_ID,
|
||||
validUserIds = [],
|
||||
} = {},
|
||||
) {
|
||||
if (!migration || !Array.isArray(migration.tables)) {
|
||||
throw new Error('迁移 JSON 必须包含 tables 数组。');
|
||||
}
|
||||
|
||||
const normalizedPlaceholderUserId = placeholderUserId.trim();
|
||||
const validUserIdSet = new Set(
|
||||
(Array.isArray(validUserIds) ? validUserIds : [])
|
||||
.map((value) => String(value).trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
validUserIdSet.add(normalizedPlaceholderUserId);
|
||||
|
||||
const reboundRows = [];
|
||||
for (const table of migration.tables) {
|
||||
if (
|
||||
!table ||
|
||||
!WORK_OWNER_TABLES.includes(table.name) ||
|
||||
!Array.isArray(table.rows)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const row of table.rows) {
|
||||
if (!row || typeof row !== 'object') {
|
||||
continue;
|
||||
}
|
||||
const currentOwner =
|
||||
typeof row.owner_user_id === 'string' ? row.owner_user_id.trim() : '';
|
||||
if (
|
||||
currentOwner === normalizedPlaceholderUserId ||
|
||||
validUserIdSet.has(currentOwner)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const originalOwner =
|
||||
typeof row.owner_user_id === 'string' ? row.owner_user_id : '';
|
||||
row.owner_user_id = normalizedPlaceholderUserId;
|
||||
reboundRows.push({
|
||||
table: table.name,
|
||||
rowKey: resolveRowKey(row),
|
||||
from: originalOwner,
|
||||
to: normalizedPlaceholderUserId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { reboundRows, validUserCount: validUserIdSet.size };
|
||||
}
|
||||
|
||||
function resolveRowKey(row) {
|
||||
for (const field of ROW_KEY_FIELDS) {
|
||||
const value = row[field];
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return '<unknown>';
|
||||
}
|
||||
|
||||
async function runCli(argv) {
|
||||
const options = parseCliArgs(argv);
|
||||
const inputPath = path.resolve(options.in);
|
||||
const outputPath = path.resolve(options.out);
|
||||
const migration = JSON.parse(await readFile(inputPath, 'utf8'));
|
||||
const result = rebindOrphanWorkOwnersInMigration(migration, {
|
||||
placeholderUserId: options.placeholderUserId,
|
||||
validUserIds: collectValidUserIds(migration),
|
||||
});
|
||||
|
||||
if (!options.dryRun) {
|
||||
await writeFile(
|
||||
outputPath,
|
||||
`${JSON.stringify(migration, null, 2)}\n`,
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[rebind-orphan-work-owners] ${options.dryRun ? 'dry-run' : `已写入 ${outputPath}`}，回填 ${result.reboundRows.length} 行`,
|
||||
);
|
||||
}
|
||||
|
||||
function parseCliArgs(argv) {
|
||||
const options = {
|
||||
in: '',
|
||||
out: '',
|
||||
placeholderUserId: DEFAULT_ORPHAN_WORK_OWNER_USER_ID,
|
||||
dryRun: false,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const readValue = (name) => {
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith('--')) {
|
||||
throw new Error(`${name} 缺少参数值。`);
|
||||
}
|
||||
index += 1;
|
||||
return value;
|
||||
};
|
||||
|
||||
if (arg === '--in') {
|
||||
options.in = readValue(arg);
|
||||
} else if (arg === '--out') {
|
||||
options.out = readValue(arg);
|
||||
} else if (arg === '--placeholder-user-id') {
|
||||
options.placeholderUserId = readValue(arg);
|
||||
} else if (arg === '--dry-run') {
|
||||
options.dryRun = true;
|
||||
} else {
|
||||
throw new Error(`未知参数: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!options.in) {
|
||||
throw new Error('必须传入 --in。');
|
||||
}
|
||||
if (!options.out && !options.dryRun) {
|
||||
throw new Error('非 dry-run 必须传入 --out。');
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function collectValidUserIds(migration) {
|
||||
const result = new Set();
|
||||
for (const table of migration.tables ?? []) {
|
||||
if (!table || !Array.isArray(table.rows)) {
|
||||
continue;
|
||||
}
|
||||
if (table.name === 'user_account') {
|
||||
for (const row of table.rows) {
|
||||
if (typeof row?.user_id === 'string' && row.user_id.trim()) {
|
||||
result.add(row.user_id.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function isCliEntry() {
|
||||
const entry = process.argv[1];
|
||||
return entry
|
||||
? import.meta.url === `file://${entry.replace(/\\/gu, '/')}`
|
||||
: false;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user