diff --git a/.gitea/workflows/project-ci.yml b/.gitea/workflows/project-ci.yml index d796f3d91..b4e69e78f 100644 --- a/.gitea/workflows/project-ci.yml +++ b/.gitea/workflows/project-ci.yml @@ -27,9 +27,26 @@ env: RUSTC_WRAPPER: '' CARGO_BUILD_RUSTC_WRAPPER: '' +# job 声明顺序就是 runner 领取顺序，因此把最长尾的客户端 Rust 门禁排在前面， +# 让它在最少的等待下占用并发槽位；其余 job 按时长递减排列。 +# +# 客户端（微信壳 / Expo 移动壳 / Tauri 桌面壳 / AI 游戏创作壳）门禁原先全部串在 +# `Native shell tests` 一个 job 里，实测 18 分 37 秒。现在按门禁组拆成三个 job： +# `Native shell tests`（契约 + H5 / 微信 / 移动 / 桌面壳门禁 + 发布构建 smoke）、 +# `AI game creator shell web tests`（typecheck + 壳内测试）与 +# `AI game creator shell Rust tests`（AGC 壳 bin 单测分片并行 + agent-run smoke）， +# 再把 AGC 壳依赖的共享 / 平台 crate 测试拆成 `AI game creator shell Rust crates`。 +# 各自的命令与拆分前逐一对应，本地 `npm run check:native-shells` 仍是同一条串行序列。 +# +# AGC 壳的 bin 单测（2466 条）按名单分 4 片、每片一个进程并行执行，片内保持 +# `--test-threads=1`：当年线程并行会互相干扰的是进程内后台锁与异步终态，进程分片 +# 天然隔离，每片另有独立 TMPDIR，因此不必再让整套用例串成一小时级的尾巴。 jobs: - repository-checks: - name: Repository checks + # 客户端 AGC 壳自身的 Rust 门禁：bin target 单测按名单分 4 片并行（片内仍保持 + # `--test-threads=1`）加 agent-run smoke。这里不装 npm 依赖：壳 Rust 门禁与 smoke + # 只用 cargo 与 node 内建模块，也只需要 AGC 壳自己那份锁定依赖。 + ai-game-creator-shell-rust-tests: + name: AI game creator shell Rust tests runs-on: genarrative-ci steps: - name: Checkout full history from Gitea @@ -41,76 +58,94 @@ jobs: - name: Validate preinstalled CI job image and sandbox run: GENARRATIVE_GITEA_CI_CHECK_RUNTIME=1 bash scripts/check-gitea-ci-job-image.sh - - name: Resolve comparison base + - name: Prepare AI game creator shell Rust dependencies shell: bash run: | set -euo pipefail - base_ref="$(node -e ' - const fs = require("node:fs"); - const event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); - process.stdout.write(event.pull_request?.base?.sha ?? event.before ?? ""); - ')" - if [[ -n "${base_ref}" && ! "${base_ref}" =~ ^0+$ ]]; then - git cat-file -e "${base_ref}^{commit}" 2>/dev/null || { - echo "comparison base commit is unavailable: ${base_ref}" >&2 + # AGC 壳有独立 Cargo.lock，其 path 依赖已含 platform-llm、platform-agent、 + # agent-runtime-core 与 shared-contracts，因此只锁这一份 manifest 就能覆盖壳测试 + # 与 smoke 的全部第三方依赖；server-rs 那次预热归 crate 级 job，不在这里重复。 + for attempt in $(seq 1 5); do + if cargo fetch --locked \ + --target x86_64-unknown-linux-gnu \ + --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml; then + break + fi + if [[ "${attempt}" -eq 5 ]]; then + echo 'AI game creator shell Cargo dependency fetch failed after 5 attempts.' >&2 exit 1 - } - else - base_ref="$(git merge-base HEAD origin/master 2>/dev/null || git rev-parse HEAD)" - fi - resolved_base_ref="$(git rev-parse --verify "${base_ref}^{commit}" 2>/dev/null || true)" - head_ref="$(git rev-parse HEAD)" - if [[ "${resolved_base_ref}" == "${head_ref}" ]]; then - resolved_base_ref="$(git rev-parse --verify HEAD^ 2>/dev/null || true)" - fi - if [[ -z "${resolved_base_ref}" ]]; then - echo 'comparison base must resolve to a commit distinct from HEAD.' >&2 - exit 1 - fi - base_ref="${resolved_base_ref}" - if [[ "${GITHUB_EVENT_NAME:-}" == 'pull_request' ]] \ - && ! git merge-base --is-ancestor "${base_ref}" HEAD; then - echo 'pull request head does not contain the latest base commit; update the branch and rerun CI.' >&2 - exit 1 - fi - echo "SPACETIME_SCHEMA_BASE_REF=${base_ref}" >> "${GITHUB_ENV}" + fi + sleep $((attempt * 2)) + done - - name: Install npm dependencies - run: bash scripts/ci-npm-ci-with-retry.sh + - name: Run AI game creator shell Rust gates + run: npm run check:native-shells:agc-rust-shell - - name: Run repository checks - run: npm run check:repository-ci - - frontend-tests: - name: Frontend tests + # AGC 壳依赖的共享 / 平台 crate 测试用的是 server-rs workspace 与两个无锁独立 crate + # 的 manifest，属另一套依赖图，因此单独一个 job 预热、单独跑。 + ai-game-creator-shell-rust-crates: + name: AI game creator shell Rust crates runs-on: genarrative-ci steps: - - name: Checkout source from Gitea + - name: Checkout full history from Gitea env: - GENARRATIVE_GITEA_FETCH_DEPTH: '1' + GENARRATIVE_GITEA_FETCH_DEPTH: '0' GENARRATIVE_GITEA_TOKEN: ${{ github.token }} run: genarrative-gitea-checkout - name: Validate preinstalled CI job image and sandbox run: GENARRATIVE_GITEA_CI_CHECK_RUNTIME=1 bash scripts/check-gitea-ci-job-image.sh - - name: Install npm dependencies - run: bash scripts/ci-npm-ci-with-retry.sh + - name: Prepare server-rs Rust dependencies + shell: bash + run: | + set -euo pipefail + for attempt in $(seq 1 5); do + if cargo fetch --locked \ + --target x86_64-unknown-linux-gnu \ + --manifest-path server-rs/Cargo.toml; then + break + fi + if [[ "${attempt}" -eq 5 ]]; then + echo 'server-rs Cargo dependency fetch failed after 5 attempts.' >&2 + exit 1 + fi + sleep $((attempt * 2)) + done - - name: Run frontend and script tests - run: npm run test + - name: Prepare standalone Rust crate dependencies + shell: bash + run: | + set -euo pipefail + # agent-runtime-core / agent-runtime-orchestration 被 server-rs/Cargo.toml 的 + # exclude 排除，不参与上面的 workspace 锁文件，因此上面那次锁定 fetch 覆盖不到它们； + # 而 `npm run ai-game-creator-shell:check:rust:crates` 会用 + # `cargo test --manifest-path` 单独跑这两个 crate。不在这里预热的话，这两条测试 + # 会在测试阶段自己 `Updating crates.io index`，crates.io 一抖动整条 job 就红 + # （见 #327 / PR #316 run 1950）。 + # 两个 crate 都没有提交 Cargo.lock，所以这里只能做不带锁标志的 fetch： + # 加锁标志会因为缺少锁文件直接失败。生成的 Cargo.lock 落在两个 crate 目录内， + # 已被各自的 .gitignore 忽略，只留在容器里；随后的测试阶段因此能用锁定版本 + # 解析，不再触碰 registry index。 + for manifest_path in \ + server-rs/crates/agent-runtime-core/Cargo.toml \ + server-rs/crates/agent-runtime-orchestration/Cargo.toml; do + for attempt in $(seq 1 5); do + if cargo fetch \ + --target x86_64-unknown-linux-gnu \ + --manifest-path "${manifest_path}"; then + break + fi + if [[ "${attempt}" -eq 5 ]]; then + echo "standalone crate dependency fetch failed after 5 attempts: ${manifest_path}" >&2 + exit 1 + fi + sleep $((attempt * 2)) + done + done - - name: Run BgFilter worker smoke harness tests - run: npm run bgfilter-worker:smoke-test - - - name: Validate production health patrol behavior - run: npm run check:production-health-patrol - - - name: Validate production API release behavior - run: npm run check:production-api-release - - - name: Validate production API deploy behavior - run: npm run check:production-api-deploy + - name: Run AI game creator shell shared crate gates + run: npm run check:native-shells:agc-rust-crates backend-tests: name: Backend tests @@ -194,6 +229,8 @@ jobs: - name: Check SpacetimeDB module run: cargo check --locked -p spacetime-module --manifest-path server-rs/Cargo.toml + # 客户端的壳级与契约门禁：静态契约断言、H5 / 微信 / 移动 / 桌面壳运行时门禁， + # 以及依赖发布产物的构建 smoke。 native-shell-tests: name: Native shell tests runs-on: genarrative-ci @@ -215,7 +252,6 @@ jobs: run: | set -euo pipefail for manifest_path in \ - server-rs/Cargo.toml \ apps/desktop-shell/src-tauri/Cargo.toml \ apps/ai-game-creator-shell/src-tauri/Cargo.toml; do for attempt in $(seq 1 5); do @@ -232,39 +268,118 @@ jobs: done done - - name: Prepare standalone Rust crate dependencies - shell: bash - run: | - set -euo pipefail - # agent-runtime-core / agent-runtime-orchestration 被 server-rs/Cargo.toml 的 - # exclude 排除，不参与上面的 workspace 锁文件，因此上面那次锁定 fetch 覆盖不到它们； - # 而 check:native-shells 会经 agent-runtime-*:check 用 `cargo test --manifest-path` - # 单独跑这两个 crate。不在这里预热的话，这两条测试会在测试阶段自己 - # `Updating crates.io index`，crates.io 一抖动整条 native shell 作业就红 - # （见 #327 / PR #316 run 1950）。 - # 两个 crate 都没有提交 Cargo.lock，所以这里只能做不带锁标志的 fetch： - # 加锁标志会因为缺少锁文件直接失败。生成的 Cargo.lock 落在两个 crate 目录内， - # 已被各自的 .gitignore 忽略，只留在容器里；随后的测试阶段因此能用锁定版本 - # 解析，不再触碰 registry index。 - for manifest_path in \ - server-rs/crates/agent-runtime-core/Cargo.toml \ - server-rs/crates/agent-runtime-orchestration/Cargo.toml; do - for attempt in $(seq 1 5); do - if cargo fetch \ - --target x86_64-unknown-linux-gnu \ - --manifest-path "${manifest_path}"; then - break - fi - if [[ "${attempt}" -eq 5 ]]; then - echo "standalone crate dependency fetch failed after 5 attempts: ${manifest_path}" >&2 - exit 1 - fi - sleep $((attempt * 2)) - done - done + - name: Run native shell contract gates + run: npm run check:native-shells:contract - name: Run native shell gates - run: npm run check:native-shells + run: npm run check:native-shells:shells + + - name: Run native shell release build smoke + run: npm run check:native-shells:release - name: Ensure native lockfiles are unchanged run: git diff --exit-code -- apps/desktop-shell/src-tauri/Cargo.lock apps/ai-game-creator-shell/src-tauri/Cargo.lock + + frontend-tests: + name: Frontend tests + runs-on: genarrative-ci + steps: + - name: Checkout source from Gitea + env: + GENARRATIVE_GITEA_FETCH_DEPTH: '1' + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: genarrative-gitea-checkout + + - name: Validate preinstalled CI job image and sandbox + run: GENARRATIVE_GITEA_CI_CHECK_RUNTIME=1 bash scripts/check-gitea-ci-job-image.sh + + - name: Install npm dependencies + run: bash scripts/ci-npm-ci-with-retry.sh + + - name: Run frontend and script tests + run: npm run test + + - name: Run BgFilter worker smoke harness tests + run: npm run bgfilter-worker:smoke-test + + - name: Validate production health patrol behavior + run: npm run check:production-health-patrol + + - name: Validate production API release behavior + run: npm run check:production-api-release + + - name: Validate production API deploy behavior + run: npm run check:production-api-deploy + + repository-checks: + name: Repository checks + runs-on: genarrative-ci + steps: + - name: Checkout full history from Gitea + env: + GENARRATIVE_GITEA_FETCH_DEPTH: '0' + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: genarrative-gitea-checkout + + - name: Validate preinstalled CI job image and sandbox + run: GENARRATIVE_GITEA_CI_CHECK_RUNTIME=1 bash scripts/check-gitea-ci-job-image.sh + + - name: Resolve comparison base + shell: bash + run: | + set -euo pipefail + base_ref="$(node -e ' + const fs = require("node:fs"); + const event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); + process.stdout.write(event.pull_request?.base?.sha ?? event.before ?? ""); + ')" + if [[ -n "${base_ref}" && ! "${base_ref}" =~ ^0+$ ]]; then + git cat-file -e "${base_ref}^{commit}" 2>/dev/null || { + echo "comparison base commit is unavailable: ${base_ref}" >&2 + exit 1 + } + else + base_ref="$(git merge-base HEAD origin/master 2>/dev/null || git rev-parse HEAD)" + fi + resolved_base_ref="$(git rev-parse --verify "${base_ref}^{commit}" 2>/dev/null || true)" + head_ref="$(git rev-parse HEAD)" + if [[ "${resolved_base_ref}" == "${head_ref}" ]]; then + resolved_base_ref="$(git rev-parse --verify HEAD^ 2>/dev/null || true)" + fi + if [[ -z "${resolved_base_ref}" ]]; then + echo 'comparison base must resolve to a commit distinct from HEAD.' >&2 + exit 1 + fi + base_ref="${resolved_base_ref}" + if [[ "${GITHUB_EVENT_NAME:-}" == 'pull_request' ]] \ + && ! git merge-base --is-ancestor "${base_ref}" HEAD; then + echo 'pull request head does not contain the latest base commit; update the branch and rerun CI.' >&2 + exit 1 + fi + echo "SPACETIME_SCHEMA_BASE_REF=${base_ref}" >> "${GITHUB_ENV}" + + - name: Install npm dependencies + run: bash scripts/ci-npm-ci-with-retry.sh + + - name: Run repository checks + run: npm run check:repository-ci + + # 客户端的 AI 游戏创作壳前端门禁：typecheck 与壳内测试，不触碰 Cargo。 + ai-game-creator-shell-web-tests: + name: AI game creator shell web tests + runs-on: genarrative-ci + steps: + - name: Checkout full history from Gitea + env: + GENARRATIVE_GITEA_FETCH_DEPTH: '0' + GENARRATIVE_GITEA_TOKEN: ${{ github.token }} + run: genarrative-gitea-checkout + + - name: Validate preinstalled CI job image and sandbox + run: GENARRATIVE_GITEA_CI_CHECK_RUNTIME=1 bash scripts/check-gitea-ci-job-image.sh + + - name: Install npm dependencies + run: bash scripts/ci-npm-ci-with-retry.sh + + - name: Run AI game creator shell web gates + run: npm run check:native-shells:agc-web diff --git a/apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs b/apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs new file mode 100644 index 000000000..7cefb0ed2 --- /dev/null +++ b/apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs @@ -0,0 +1,426 @@ +#!/usr/bin/env node +// AGC 壳 Rust 套件的分片运行器。 +// +// 背景：AGC 壳的 Rust 单测集中在 src/main.rs 的 bin target（实测 2466 条），仓库口径用 +// `--test-threads=1` 跑，理由是并行调度会让共享 Agent Runtime 后台锁与异步终态的用例在 +// **同一进程内**互相干扰（见 development-workflow 的 Tauri suite 单线程口径）。代价是 +// 整套用例串行跑满 507 秒，占掉 CI 上 `AI game creator shell Rust tests` job 的大头。 +// +// 这里保留「片内串行」的既有口径，只把用例集合切成 N 片、让每片在**独立进程**里并行： +// 当年线程并行的两个根因（进程内全局锁、异步终态）在多进程下不存在，每片还会拿到自己的 +// TMPDIR，避免 tempfile 目录互相踩。片并集必须等于全集、且不得重复，数量不符即失败， +// 防止分片规则改动后静默漏跑。 +// +// 用法： +// node scripts/run-rust-shell-test-shards.mjs [--shards=4] [--concurrency=4] +// node scripts/run-rust-shell-test-shards.mjs --manifest= --target-kind=lib --no-locked +// +// 参数： +// --shards= 分片数，默认 4 +// --concurrency= 同时运行的片数，默认等于分片数 +// --manifest= Cargo.toml，默认 ../src-tauri/Cargo.toml（相对本脚本） +// --target-kind= bin | lib，默认 bin（本地自测小 crate 时用 lib） +// --bin= bin target 名，默认 genarrative-ai-game-creator-shell +// --package= target-kind=lib 时要跑的包名（配合 lib 目标使用） +// --no-locked 传给 cargo 时不带 --locked（只对没有提交 Cargo.lock 的 crate 需要） +// --shard-tmp-root= 片专属 TMPDIR 的父目录，默认 <系统临时目录>/agc-rust-shards + +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); +const shellRoot = path.resolve(scriptDirectory, '..'); + +function fail(message) { + console.error(`[rust-shards] ${message}`); + process.exit(1); +} + +function parsePositiveInteger(name, rawValue) { + if (!/^[1-9][0-9]*$/.test(rawValue)) { + fail(`${name} must be a positive integer, received: ${rawValue}`); + } + + return Number(rawValue); +} + +const options = { + shards: 4, + concurrency: undefined, + manifestPath: path.join(shellRoot, 'src-tauri', 'Cargo.toml'), + targetKind: 'bin', + binName: 'genarrative-ai-game-creator-shell', + packageName: undefined, + locked: true, + shardTmpRoot: path.join(os.tmpdir(), 'agc-rust-shards'), +}; + +for (const rawArgument of process.argv.slice(2)) { + if (rawArgument === '--no-locked') { + options.locked = false; + continue; + } + + const separatorIndex = rawArgument.indexOf('='); + if (!rawArgument.startsWith('--') || separatorIndex === -1) { + fail(`unexpected argument: ${rawArgument}`); + } + + const name = rawArgument.slice(2, separatorIndex); + const value = rawArgument.slice(separatorIndex + 1); + switch (name) { + case 'shards': + options.shards = parsePositiveInteger('--shards', value); + break; + case 'concurrency': + options.concurrency = parsePositiveInteger('--concurrency', value); + break; + case 'manifest': + options.manifestPath = path.resolve(process.cwd(), value); + break; + case 'target-kind': + if (value !== 'bin' && value !== 'lib') { + fail(`--target-kind must be bin or lib, received: ${value}`); + } + options.targetKind = value; + break; + case 'bin': + options.binName = value; + break; + case 'package': + options.packageName = value; + break; + case 'shard-tmp-root': + options.shardTmpRoot = path.resolve(process.cwd(), value); + break; + default: + fail(`unexpected argument: ${rawArgument}`); + } +} + +if (!fs.existsSync(options.manifestPath)) { + fail(`manifest does not exist: ${options.manifestPath}`); +} + +const concurrency = options.concurrency ?? options.shards; +const crateRoot = path.dirname(options.manifestPath); + +function buildCargoArguments(target) { + const cargoArguments = ['test']; + if (options.locked) { + cargoArguments.push('--locked'); + } + cargoArguments.push('--manifest-path', options.manifestPath); + if (target.kind === 'lib') { + if (options.packageName !== undefined) { + cargoArguments.push('-p', options.packageName); + } + cargoArguments.push('--lib'); + return cargoArguments; + } + cargoArguments.push('--bin', target.name); + return cargoArguments; +} + +function formatDuration(milliseconds) { + return `${(milliseconds / 1000).toFixed(1)}s`; +} + +// 编译一次，直接拿到测试可执行文件：后续每片都运行同一个二进制，不再各自调用 cargo， +// 免得 N 个 cargo 去争 package cache 与 target 目录锁。 +function resolveTestExecutable() { + return new Promise((resolve, reject) => { + const cargoArguments = buildCargoArguments({ + kind: options.targetKind, + name: options.binName, + }); + cargoArguments.push('--no-run', '--message-format=json'); + console.log(`[rust-shards] cargo ${cargoArguments.join(' ')}`); + + const child = spawn('cargo', cargoArguments, { + cwd: crateRoot, + env: process.env, + stdio: ['ignore', 'pipe', 'inherit'], + }); + + let buffered = ''; + const executables = []; + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + buffered += chunk; + const lines = buffered.split('\n'); + buffered = lines.pop() ?? ''; + for (const line of lines) { + if (!line.startsWith('{')) { + continue; + } + let message; + try { + message = JSON.parse(line); + } catch { + continue; + } + if ( + message.reason === 'compiler-artifact' && + typeof message.executable === 'string' + ) { + executables.push(message.executable); + } + } + }); + + child.on('error', (error) => { + reject(new Error(`unable to start cargo: ${error.message}`)); + }); + child.on('close', (code) => { + if (code !== 0) { + reject( + new Error( + `cargo ${cargoArguments.join(' ')} failed with exit code ${code}`, + ), + ); + return; + } + const uniqueExecutables = [...new Set(executables)]; + if (uniqueExecutables.length !== 1) { + reject( + new Error( + `expected exactly one test executable for the ${options.targetKind} target, found ${uniqueExecutables.length}: ${uniqueExecutables.join(', ')}`, + ), + ); + return; + } + resolve(uniqueExecutables[0]); + }); + }); +} + +function listTestNames(executable) { + return new Promise((resolve, reject) => { + const child = spawn(executable, ['--list'], { + cwd: crateRoot, + env: process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + stdout += chunk; + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + child.on('error', (error) => { + reject(new Error(`unable to list tests: ${error.message}`)); + }); + child.on('close', (code) => { + if (code !== 0) { + reject( + new Error( + `listing tests failed with exit code ${code}: ${stderr.trim()}`, + ), + ); + return; + } + const names = []; + for (const line of stdout.split('\n')) { + const match = /^(.*): test$/.exec(line.trim()); + if (match !== null && match[1].length > 0) { + names.push(match[1]); + } + } + resolve(names); + }); + }); +} + +function splitTestNames(testNames, shardCount) { + const sortedNames = [...testNames].sort(); + const shards = Array.from({ length: shardCount }, () => []); + sortedNames.forEach((testName, index) => { + shards[index % shardCount].push(testName); + }); + return shards; +} + +function assertShardsCoverEveryTest(testNames, shards) { + const flattened = shards.flat(); + if (flattened.length !== testNames.length) { + fail( + `shard split covered ${flattened.length} of ${testNames.length} tests; the split rule must be exhaustive`, + ); + } + if (new Set(flattened).size !== flattened.length) { + fail( + 'shard split selected the same test more than once; the split rule must be disjoint', + ); + } + const missing = testNames.filter((testName) => !flattened.includes(testName)); + if (missing.length > 0) { + fail( + `shard split is missing tests, for example: ${missing.slice(0, 5).join(', ')}`, + ); + } +} + +function runShard(executable, shardIndex, shardCount, shardTestNames) { + const label = `shard ${shardIndex + 1}/${shardCount}`; + const shardTmpDirectory = path.join( + options.shardTmpRoot, + `shard-${shardIndex + 1}`, + ); + fs.rmSync(shardTmpDirectory, { recursive: true, force: true }); + fs.mkdirSync(shardTmpDirectory, { recursive: true }); + const startedAt = Date.now(); + + return new Promise((resolve) => { + const child = spawn( + executable, + ['--exact', ...shardTestNames, '--test-threads=1'], + { + cwd: crateRoot, + env: { + ...process.env, + TMPDIR: shardTmpDirectory, + TMP: shardTmpDirectory, + TEMP: shardTmpDirectory, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + + const failureLines = []; + let inFailureList = false; + let stderr = ''; + const consumeLine = (rawLine) => { + const line = rawLine.replace(/\r$/, ''); + if (line.includes('failures:')) { + inFailureList = true; + return; + } + if (inFailureList) { + if (line.trim().length === 0) { + inFailureList = false; + return; + } + failureLines.push(line.trim()); + } + }; + + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + let stdoutBuffer = ''; + child.stdout.on('data', (chunk) => { + stdoutBuffer += chunk; + const lines = stdoutBuffer.split('\n'); + stdoutBuffer = lines.pop() ?? ''; + for (const line of lines) { + consumeLine(line); + } + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + + child.on('error', (error) => { + resolve({ + label, + ok: false, + durationMs: Date.now() - startedAt, + testCount: shardTestNames.length, + failures: [`unable to start test binary: ${error.message}`], + stderr, + }); + }); + child.on('close', (code) => { + resolve({ + label, + ok: code === 0, + durationMs: Date.now() - startedAt, + testCount: shardTestNames.length, + failures: failureLines, + stderr, + }); + }); + }); +} + +async function runWithConcurrency(shards, runner) { + const results = new Array(shards.length); + let nextIndex = 0; + const workers = Array.from( + { length: Math.min(concurrency, shards.length) }, + async () => { + while (nextIndex < shards.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await runner(shards[index], index); + } + }, + ); + await Promise.all(workers); + return results; +} + +async function main() { + const executable = await resolveTestExecutable(); + const testNames = await listTestNames(executable); + if (testNames.length === 0) { + fail( + `no tests discovered in ${options.manifestPath} (${options.targetKind})`, + ); + } + + const shards = splitTestNames(testNames, options.shards); + assertShardsCoverEveryTest(testNames, shards); + + console.log( + `[rust-shards] ${testNames.length} tests, ${shards.length} shard(s), concurrency ${Math.min(concurrency, shards.length)}`, + ); + shards.forEach((shardTestNames, index) => { + console.log( + `[rust-shards] shard ${index + 1}/${shards.length}: ${shardTestNames.length} test(s)`, + ); + }); + + const results = await runWithConcurrency(shards, (shardTestNames, index) => + runShard(executable, index, shards.length, shardTestNames), + ); + + let failed = false; + for (const result of results) { + if (result.ok) { + console.log( + `[rust-shards] ${result.label} ok: ${result.testCount} test(s) in ${formatDuration(result.durationMs)}`, + ); + continue; + } + failed = true; + console.error( + `[rust-shards] ${result.label} FAILED: ${result.testCount} test(s) in ${formatDuration(result.durationMs)}`, + ); + for (const failure of result.failures) { + console.error(`[rust-shards] ${failure}`); + } + if (result.stderr.trim().length > 0) { + console.error( + `[rust-shards] stderr: ${result.stderr.trim().split('\n').slice(-20).join('\n[rust-shards] ')}`, + ); + } + } + + if (failed) { + process.exit(1); + } + console.log('[rust-shards] OK'); +} + +main().catch((error) => { + fail(error instanceof Error ? error.message : String(error)); +}); diff --git a/apps/ai-game-creator-shell/src-tauri/src/project/write_lock.rs b/apps/ai-game-creator-shell/src-tauri/src/project/write_lock.rs index 9ed36a8a9..bdf5bd33d 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project/write_lock.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project/write_lock.rs @@ -14,16 +14,59 @@ const PROJECT_WRITE_LOCK_UNWRITTEN_GRACE_SECONDS: u64 = 30; const PROJECT_WRITE_LOCK_PID_REUSE_TOLERANCE_SECONDS: u64 = 5; const PROJECT_WRITE_LOCK_MAX_BYTES: u64 = 4 * 1024; +/// 本进程内真正落盘持有项目写锁的线程登记表。 +/// +/// `.agent/project.lock` 的 `pid` 只能证明“锁由本进程的某条写通道持有”，它分不清 +/// 两种完全不同的局面： +/// - **同一条调用链再次取锁**：持锁方就是自己，必须放行，否则每次嵌套项目写入都要 +/// 白等一个等待预算再报“项目正在被其他写操作占用”； +/// - **本进程另一条写通道正在写**：项目 revision 侧车、steer 序号、一致快照读、 +/// pending sidecar 复核和恢复安装都靠这把锁串行化，必须照旧等待。 +/// +/// 复用判据因此不能停在 `pid`：只有**当前线程**就是真实持锁线程时才返回 advisory +/// guard，本进程其余争用继续走有界等待与终态占用。登记按路径进行、按路径注销： +/// guard 可能被移到别的线程再 Drop（例如写入路径把锁交给阻塞线程池的持有者）， +/// 按线程注销会漏项，让后续的重入判断失真。 +static PROJECT_WRITE_LOCK_THREAD_OWNERS: std::sync::Mutex> = + std::sync::Mutex::new(Vec::new()); + +fn project_write_lock_thread_owners( +) -> std::sync::MutexGuard<'static, Vec<(PathBuf, std::thread::ThreadId)>> { + // 登记表只是复用判据的加速器：中毒时继续用内部值，不能让一次取锁失败升级成 + // 整个进程再也写不了项目。 + PROJECT_WRITE_LOCK_THREAD_OWNERS + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn register_project_write_lock_thread_owner(path: &Path) { + let mut owners = project_write_lock_thread_owners(); + if owners.iter().any(|(owner, _)| owner == path) { + return; + } + owners.push((path.to_path_buf(), std::thread::current().id())); +} + +fn unregister_project_write_lock_thread_owner(path: &Path) { + project_write_lock_thread_owners().retain(|(owner, _)| owner != path); +} + +/// 当前线程是否就是这条锁路径上真实落盘的持有者（同线程重入）。 +fn project_write_lock_reentered_by_current_thread(path: &Path) -> bool { + let thread = std::thread::current().id(); + project_write_lock_thread_owners() + .iter() + .any(|(owner, owner_thread)| owner == path && *owner_thread == thread) +} + #[derive(Debug)] pub(crate) struct ProjectWriteLock { path: PathBuf, content: String, - /// In the free-form autonomous lane a single Runtime process may have - /// several specialist actions in flight at once. A file lock is still - /// useful across processes, but making same-process contenders fail turns - /// ordinary parallel work into a dead run (and can deadlock nested tool - /// calls). Such a contender receives an in-process/advisory guard instead - /// of deleting the real holder's lock on drop. + /// 两种“本进程持锁但不必自等”的争用会拿到 advisory guard：同一线程重入（同一条 + /// 调用链再次取锁）和自主游戏构建流水线（它有意让并行专家动作同时在飞）。这两种 + /// 情况下争用是进程内重叠而不是另一个客户端在改项目，返回的 guard 不拥有 + /// `.agent/project.lock`，Drop 时也不得删除真实持有者的锁。 bypassed_same_process: bool, } @@ -47,6 +90,7 @@ impl Drop for ProjectWriteLock { if self.bypassed_same_process { return; } + unregister_project_write_lock_thread_owner(&self.path); if fs::read_to_string(&self.path).is_ok_and(|content| content == self.content) { let _ = fs::remove_file(&self.path); } @@ -815,6 +859,7 @@ pub(crate) fn acquire_project_write_lock_failure( path.display() ))); } + register_project_write_lock_thread_owner(&path); return Ok(ProjectWriteLock { path, content: content.clone(), @@ -860,11 +905,15 @@ pub(crate) fn acquire_project_write_lock_failure( } } } - if project_write_lock_is_owned_by_current_process(&path) { - // A project lock is the client-use lock. Nested calls in - // the same client process must reuse that ownership instead - // of waiting on their own durable marker. Cross-process - // contenders still take the normal retryable path. + if project_write_lock_is_owned_by_current_process(&path) + && (crate::agent::autonomous_game_build_root_run_active_at(root) + || project_write_lock_reentered_by_current_thread(&path)) + { + // 持锁方就是本进程自己时必须区分重入与并发：同一条调用链（同一 + // 线程）再次取锁，以及自主流水线有意并行专家动作，返回 advisory + // guard、不自等、不动真实锁；本进程**其它线程**正在写则继续走 + // 有界等待，保住 revision 侧车、steer 序号、一致快照读与恢复安装 + // 的串行化。 return Ok(ProjectWriteLock { path, content: String::new(), diff --git a/apps/ai-game-creator-shell/src-tauri/src/tests/project_tools.rs b/apps/ai-game-creator-shell/src-tauri/src/tests/project_tools.rs index 7e39fccbe..8709bc5b0 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/tests/project_tools.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/tests/project_tools.rs @@ -5818,8 +5818,27 @@ async fn agent_runtime_file_write_lock_failure_redacts_project_path() { }, ) .expect("allow direct file write"); - let lock = acquire_project_write_lock(&root, "persistent-writer") - .expect("acquire persistent project writer"); + // 持锁方必须是**另一条线程**：本用例验证的是“别的写通道正在写时 file.write 必须 + // 走满等待预算并失败关闭”，同一条调用链自持锁属于重入复用，不会失败。 + let holder_root = root.clone(); + let (release_sender, release_receiver) = mpsc::channel::<()>(); + let holder = std::thread::spawn(move || { + let lock = acquire_project_write_lock(&holder_root, "persistent-writer") + .expect("acquire persistent project writer"); + let _ = release_receiver.recv(); + drop(lock); + }); + let lock_path = root.join(PROJECT_WRITE_LOCK_PATH); + for _ in 0..400 { + if lock_path.is_file() { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + lock_path.is_file(), + "persistent writer must hold the project write lock" + ); let observation = execute_game_creator_agent_runtime_tool_action( &root, @@ -5837,7 +5856,8 @@ async fn agent_runtime_file_write_lock_failure_redacts_project_path() { ) .await; - drop(lock); + let _ = release_sender.send(()); + holder.join().expect("join persistent project writer"); assert_eq!(observation.status, "failed"); assert!(!observation .summary diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs index 04b72c0b3..bc7d08d88 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs @@ -1214,8 +1214,28 @@ mod tests { .expect("resolve primary"); fs::write(&primary, b"{broken").expect("corrupt primary"); - let project_lock = acquire_project_write_lock(directory.path(), "test.concurrent-save") - .expect("hold project write lock"); + // 持锁方必须是**另一条线程**：本用例验证的是“另一个写者持锁时恢复安装必须失败 + // 关闭”，同一条调用链自持锁属于重入复用，不再产生占用失败。 + let holder_root = directory.path().to_path_buf(); + let (release_sender, release_receiver) = std::sync::mpsc::channel::<()>(); + let holder = std::thread::spawn(move || { + let lock = acquire_project_write_lock(&holder_root, "test.concurrent-save") + .expect("hold project write lock"); + let _ = release_receiver.recv(); + drop(lock); + }); + let lock_path = resolve_local_project_path(directory.path(), PROJECT_WRITE_LOCK_PATH) + .expect("resolve project write lock path"); + for _ in 0..400 { + if lock_path.is_file() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!( + lock_path.is_file(), + "concurrent writer must hold the project write lock" + ); let error = load_ui_design_state_at(LoadUiDesignStateInput { project_path: directory.path().to_string_lossy().into_owned(), expected_project_id: PROJECT_ID.to_string(), @@ -1224,7 +1244,8 @@ mod tests { .expect_err("recovery must not install while another writer holds the lock"); assert!(error.contains("项目正在被其他写操作占用")); assert!(read_ui_design_document_path(&primary).is_err()); - drop(project_lock); + let _ = release_sender.send(()); + holder.join().expect("join concurrent writer"); let recovered = load_ui_design_state_at(LoadUiDesignStateInput { project_path: directory.path().to_string_lossy().into_owned(), diff --git a/docs/project-memory/plans/【实施计划】项目客户端占用锁收敛-2026-09-14.md b/docs/project-memory/plans/【实施计划】项目客户端占用锁收敛-2026-09-14.md index f1ffb1b07..e579a6514 100644 --- a/docs/project-memory/plans/【实施计划】项目客户端占用锁收敛-2026-09-14.md +++ b/docs/project-memory/plans/【实施计划】项目客户端占用锁收敛-2026-09-14.md @@ -15,14 +15,16 @@ Milestone: `【里程碑】项目客户端占用锁收敛-2026-09-14.md` ## 修改顺序 1. 统一同进程嵌套调用的项目锁语义，禁止自等待。 -2. 盘点并迁移 Runner 的项目级 owner 文件到统一锁，保留诊断投影与跨 boot 恢复。 -3. 删除重复项目级锁路径及其专属调用，保留底层原子写和 Git 锁。 -4. 补齐同进程重入、跨进程占用、崩溃恢复和锁释放测试。 +2. 收窄复用判据：按 `pid` 放行会放过本进程其它线程的并行写，改为按“当前线程就是真实持锁线程”判定重入，并保住同进程跨线程的等待与终态占用。 +3. 盘点并迁移 Runner 的项目级 owner 文件到统一锁，保留诊断投影与跨 boot 恢复。 +4. 删除重复项目级锁路径及其专属调用，保留底层原子写和 Git 锁。 +5. 补齐同进程重入、同进程跨线程争用、跨进程占用、崩溃恢复和锁释放测试。 ## 验证命令 - `cargo fmt --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml -- --check` -- `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml project_write_lock_reuses_same_process_owner_and_releases_on_drop --no-default-features` +- `cargo test --locked --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml -- --test-threads=1` +- `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml project_write_lock --no-default-features` - Runner owner 与 response stream 相关定向测试 - `npm run check:encoding` - `git diff --check` @@ -31,4 +33,5 @@ Milestone: `【里程碑】项目客户端占用锁收敛-2026-09-14.md` - Runner 与 GUI 可能是不同进程；统一锁前必须验证同一客户端不会互相阻塞。 - 旧 `.agent/runtime/execution-owner.lock` 残留需要按 PID/启动身份安全回收，不能直接删除。 +- 复用判据按线程判定：出现同进程跨线程重入的现场时先按 `*_locked` 入口处置，不要把判据退回按 `pid` 一律放行（那会放过并行写，见里程碑「边界」末条）。 - 若跨 boot 恢复或 GUI/Runner 联动回归，回滚统一路径迁移，保留已验证的同进程重入修复。 diff --git a/docs/project-memory/plans/【里程碑】项目客户端占用锁收敛-2026-09-14.md b/docs/project-memory/plans/【里程碑】项目客户端占用锁收敛-2026-09-14.md index ed1c643a2..218b10182 100644 --- a/docs/project-memory/plans/【里程碑】项目客户端占用锁收敛-2026-09-14.md +++ b/docs/project-memory/plans/【里程碑】项目客户端占用锁收敛-2026-09-14.md @@ -7,22 +7,24 @@ Parent Spec: `docs/technical/【技术方案】AI游戏创作智能体App实施 ## 目标 -项目只保留一个面向客户端占用的项目级跨进程锁，防止多个客户端同时打开同一项目；同一客户端进程内的嵌套调用复用既有项目锁，不因自身持锁进入等待。 +项目只保留一个面向客户端占用的项目级跨进程锁，防止多个客户端同时打开同一项目；同一客户端进程内**同一条写调用链（同一线程）的嵌套调用**复用既有项目锁，不因自身持锁进入等待。 ## 边界 - 项目客户端占用锁与项目写入调用的职责统一，跨进程竞争仍返回占用语义。 - Agent DB、session lane、manifest 原子写和 Git 自身的底层一致性机制不在本里程碑删除范围内。 -- 不改变项目 revision、权限、幂等、恢复和数据格式合同。 +- 不改变项目 revision、权限、幂等、恢复和数据格式合同。**本进程其它线程的并发写入必须继续串行化**：按 `pid` 一律返回 advisory guard 会放过并行写，直接违反本边界（见验收标准第 2 条）。 ## 验收标准 -- 同一进程内嵌套取得项目锁立即返回 advisory guard，不等待、不删除真实持有者锁。 +- 同一线程（同一条写调用链）嵌套取得项目锁立即返回 advisory guard，不等待、不删除真实持有者锁。 +- 本进程另一条线程持锁（模拟“另一个写通道/另一个客户端”的既有用例形态）时仍保持等待与终态占用：项目 revision 侧车、steer 序号分配、一致快照读、pending sidecar 复核和恢复安装不得被复用判据放过。 - 不同进程持有项目锁时仍保持占用失败与残留回收判据。 - 客户端项目占用入口与 Runtime 写入入口不会各自维护第二个项目级锁文件。 - 锁释放后下一客户端可重新取得锁。 -- 定向 Rust 锁测试、`cargo fmt --check`、`npm run check:encoding`、`git diff --check` 通过。 +- 定向 Rust 锁测试、`cargo fmt --check`、`npm run check:encoding`、`git diff --check` 通过；锁语义变更必须跑 `cargo test --locked --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml -- --test-threads=1` 全量，定向用例覆盖不到 `project_tools` / `command_runtime` / `parallel_actions` / `runtime_state` / `response_stream` / `direct_tool_bridge` / `ui_editor::persistence` 里的锁不变量。 ## 未决事项 - Runner 的 `execution-owner.lock` 如何迁移到统一客户端占用锁，需要补充跨进程启动、恢复和诊断测试后再落地。 +- 同进程**跨线程**重入（持锁调用链在 `await` / `spawn_blocking` 之后于其它线程再次取锁）仍会走有界等待，预算耗尽时报“项目正在被其他写操作占用”。发现这类现场时按 2026-08-27 的既有处置改用 `*_locked` 入口复用已有 guard（`project-memory/shared-memory/pitfalls.md`「持锁调用链二次取锁」），不放宽整条锁的串行化语义。 diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index ef516ae27..93338917b 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -3,6 +3,27 @@ > 用途：记录已经确认、会影响后续开发的长期技术/产品/协作决策。短期讨论不要写在这里。 > 当前口径：历史条目的旧路径、旧版本和已退役对象只用于追溯，不构成现行实现依据；如与当前代码或 `docs/README.md` 冲突，以当前代码和最新专题文档为准。 +## 2026-09-14 AGC 壳 Rust 套件改为「分片并行 + 片内串行」，客户端 Rust 关键路径压到 7 分钟以内 + +- 背景：`AI game creator shell Rust tests` 是客户端 CI 的关键路径（run 2097 实测 15 分 27 秒）。拆开来看：前置 5 分 30 秒（checkout 10s + `npm ci` 2m45s + Cargo fetch 2m35s）、编译 1m39s、**AGC 壳 bin target 的 2466 条单测串行 507s**、`agent-run` smoke 51s。这 2466 条全在 `apps/ai-game-creator-shell/src-tauri/src/main.rs` 的 bin target 里，一条 `cargo test … -- --test-threads=1` 跑完。 +- 为什么原本整套串行：2026-07-21 的 `a273377b1`（「稳定AI原生壳全量测试」）把 Tauri suite 固定为 `--test-threads=1`，理由是**共享 Agent Runtime 后台锁与异步终态在 libtest 并行调度下互相干扰**——即同进程内的全局锁、异步终态与进程级 static 被交叉触发；当时的口径是「修正 suite 调度口径，不放宽断言」。另有少量用例自身会 spawn `cargo test`，需要独占 cargo 的 package cache / target 锁。 +- 决策：**只把「整套串行」放宽到「片内串行」**。新增 `apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs`：`cargo test --no-run` 编译一次拿到测试可执行文件，用 `--list` 取全部用例名，排序后按 `index % shards` 切 4 片，每片作为**独立进程**执行 ` --exact <名单> --test-threads=1`，并给每片独立 `TMPDIR`。进程分片不共享当年出问题的进程内状态，也不共享 tempfile 目录，因此可以并行；片内仍严格串行，不退回整套并行，也不需要放宽任何断言。 +- 不变量：片并集必须等于 `--list` 的全集且互斥，数量或成员不符立即失败（`assertShardsCoverEveryTest`），防止分片规则改动后静默漏跑门禁。 +- 配套拆分：`npm run ai-game-creator-shell:check:rust` 拆成 `:rust:crates`（`agent-runtime-core`、`agent-runtime-orchestration`、`platform-llm`、`shared-contracts`）与 `:rust:shell`（分片运行器），聚合脚本保持同序，因而 `ai-game-creator-shell:check` 与本地 `npm run check:native-shells` 语义不变。CI 相应新增 `AI game creator shell Rust crates` job（第 7 个），`AI game creator shell Rust tests` 只保留壳分片与 smoke。 +- 前置瘦身：AGC 壳有独立 `Cargo.lock`，其 path 依赖已包含 `platform-llm` / `platform-agent` / `agent-runtime-core` / `shared-contracts`，所以壳 job 只需预热 AGC 壳这一份 manifest；壳 Rust 门禁与 `agent-run` smoke 只用 cargo 与 node 内建模块，因此 **`AI game creator shell Rust tests` 与 `AI game creator shell Rust crates` 都不再执行 `npm ci`**（各省 1~3 分钟）。 +- 影响范围：`.gitea/workflows/project-ci.yml`（七个 job）、`scripts/check-native-shells.mjs`（分组由五个变六个：新增 `agc-rust-crates`、`agc-rust-shell`，移除 `agc-rust`）、根 `package.json`、`scripts/project-ci-workflow.test.ts`（新增纯 cargo job 免 `npm ci` 与分片运行器断言）、开发运维文档与共享记忆。Gitea `master` 分支保护的 required context 是追加式的，需补上 `Project CI / AI game creator shell Rust crates (pull_request)`。 +- 验证方式：`npx vitest run scripts/project-ci-workflow.test.ts`；分片运行器本地以 `agent-runtime-core`（7 条）与 `platform-llm`（146 条）验证分片、`--exact` 与片 TMPDIR 隔离；`node scripts/check-native-shells.mjs --groups=contract` 回归。预期 `AI game creator shell Rust tests` 收敛到 6 分钟左右（前置 1m30s + 编译 1m39s + 分片约 2 分钟 + smoke），整轮 wall clock 由 `Backend tests`（8 分 36 秒）与 runner 并发（4）决定。 +- 关联文档：[开发运维](../../【开发运维】本地开发验证与生产运维-2026-05-15.md)、[踩坑记录](pitfalls.md)。 + +## 2026-09-14 客户端 CI 按门禁组拆成三个 job，AGC 的 web / rust 两段并行 + +- 背景：`Project CI / Native shell tests` 把微信壳、Expo 移动壳、Tauri 桌面壳、H5 HostBridge 与 AI 游戏创作壳的全部门禁串在一个 job 里，实测 18 分 37 秒；同一次运行的 Repository / Frontend / Backend 分别只要 3 分 21 秒、4 分 16 秒、6 分 14 秒，其余三个 job 结束后客户端 job 还要再跑十几分钟。日志时间戳显示门禁段 932 秒里：AGC `ai-game-creator-shell:check` 占 654 秒（其中壳内 Rust 套件 2451 个用例 `--test-threads=1` 单跑 441.58 秒、编译 79 秒），AGC vitest 75 秒，两个发布构建 smoke 加落盘断言 230 秒，而 h5 / 微信 / 移动 / 桌面壳的全部运行时门禁加起来不到 50 秒。 +- 决策：`scripts/check-native-shells.mjs` 引入 `--groups=`，把门禁分成 `contract`（静态契约断言）、`shells`（H5 / 微信 / Expo / 桌面壳运行时门禁）、`agc-web`（AGC typecheck 与壳内测试）、`agc-rust`（共享 / 平台 crate 测试、AGC 串行壳测试、agent-run smoke）、`release`（AGC 与桌面壳发布构建 smoke、落盘产物断言）五组，每组暴露一个 `check:native-shells:` 根脚本；不带 `--groups=` 时仍然串行跑全部分组，本地 `npm run check:native-shells` 语义不变。CI 据此把原客户端 job 拆成 `Native shell tests`（contract + shells + release）、`AI game creator shell web tests`（agc-web）、`AI game creator shell Rust tests`（agc-rust）三个 job，并把最长的 AGC Rust job 声明在最前，使 runner 领取顺序与关键路径一致。 +- 命令等价：`npm run ai-game-creator-shell:check` 拆成 `:check:web`（typecheck + 壳内测试）与 `:check:rust`（agent-runtime 两个独立 crate + `platform-llm` + `shared-contracts` + AGC 壳串行测试），聚合脚本仍是 `web && rust && agent-run:smoke` 同序同命令，本地与文档入口不变。`agent-run:smoke` 会用 `src-tauri/Cargo.toml` spawn `cargo`，因此归入 `agc-rust` 分组，与 AGC 依赖预热同 job。 +- 影响范围：`.gitea/workflows/project-ci.yml`（六个 job）、`scripts/check-native-shells.mjs`、根 `package.json` 门禁脚本、`scripts/project-ci-workflow.test.ts`（校验分组清单、根脚本内容与 job 覆盖，防止新增分组时静默漏跑）、开发运维文档与开发流程记忆。门禁覆盖不变，只有执行位置改变；Gitea `master` 分支保护的 required context 是追加式的（旧四个继续上报，需补上两个新 AGC context）。 +- 验证方式：`npx vitest run scripts/project-ci-workflow.test.ts`（11 条）；`node scripts/check-native-shells.mjs --groups=contract` 本地 0.6 秒通过；`--groups=` 未知组与空组都要报错关闭。拆分前同一类运行的 wall-clock 是 22 分 15 秒（run 2094，`Native shell tests` 单 job 19 分 50 秒）；拆分后 run 2097 六 job 全绿、wall-clock 15 分 27 秒，关键路径转移到 `AI game creator shell Rust tests`（15 分 27 秒 = 前置 5 分 30 秒 + 门禁 11 分 43 秒），其余五个 job 3 分 45 秒 ~ 8 分 36 秒。AGC 壳内串行套件（2451 用例）实测 507 秒，是这条关键路径的硬底，再切 job 只会重复 `npm ci` 与 Cargo 预热。 +- 关联文档：[开发运维](../../【开发运维】本地开发验证与生产运维-2026-05-15.md)、[踩坑记录](pitfalls.md)。 + ## 2026-09-10 策划 Agent 迁移只复用生产基建 - 决策：待实施的生产迁移以自由协作策划原型为行为基线，仅复用 Provider、恢复、文件操作、审计和 UI 通信；不继承旧 Planning V2 的强制工具、问询轮数、GDD 内容校验和版本审批。保留五阶段与顾问态、当前阶段资源注入和产物存在性检查，系统阶段空必需清单不增加解析或登记功能。 @@ -8637,3 +8658,10 @@ CI 上 `background_agent_runtime_recovers_stale_running_before_pending_task` 在 - 决策：DirectProject app-server thread 改为 `sandbox="danger-full-access"`，turn 改为 `sandboxPolicy.type="dangerFullAccess"`，不再发送 `writableRoots` 或 workspace 网络开关，原生命令网络随完整 sandbox 开放；app-server 交互请求不再按 grant root 做白名单裁剪，直接项目会话统一接受文件变更、命令执行和权限请求。首页只读对话、AGC `agc_tools` 业务授权、Provider 凭据隔离、Runtime 审计和客户端受控文件工具合同继续保留。 - 提示词同步：DirectProject 不再把路径范围描述成 Codex 原生能力禁区，但仍禁止主动输出 Token、Cookie、auth.json、`.env` 和 Runtime 私有控制面。 - 验证：Rust 定向单测覆盖 `danger-full-access` / `dangerFullAccess`、无 `writableRoots`、外部 grant root 仍接受，以及 DirectHome 继续只读拒绝。 + +## 2026-09-14 项目写锁的同进程复用收窄为同线程重入 + +- 背景：`write_lock.rs` 的 advisory 复用判据曾放宽为「`.agent/project.lock` 的 `pid` 等于当前进程」，使本进程所有写通道都不再等待。`Project CI` 的 Rust 全量门禁因此出现 12 条失败：另一线程持锁时一致快照读 / `project.diff` / `action_history` / `command.output_read` / steer 不再等待，4 路并行直写撞项目 revision 侧车（`File exists (os error 17)`），8 线程并发 steer 拿到重复序号，`file.write` 锁失败脱敏与恢复安装的失败关闭变成成功。 +- 决策：复用判据收窄为**同一条写调用链（同一线程）重入**——按锁路径登记真实持锁线程，只有当前线程就是持锁线程时才返回 advisory guard；本进程其它线程的争用继续走有界等待与终态占用。自主游戏构建流水线的并行专家动作豁免保持不变；跨进程占用、残留回收、权限分类、等待预算和错误文案不变。 +- 边界：锁定这些不变量的既有用例（`project_tools` / `command_runtime` / `parallel_actions` / `runtime_state` / `response_stream` / `direct_tool_bridge` / `ui_editor::persistence`）不得为了让锁语义通过而改写；用「同线程自持锁」模拟「另一个写者」的两条用例改为**在另一条线程持锁**，断言语义不变。同进程跨线程重入（持锁链在 `await` / `spawn_blocking` 后于其它线程再取锁）仍会等满预算，出现现场时按 2026-08-27 的既有处置改用 `*_locked` 入口，不放宽判据。 +- 关联文档：[项目客户端占用锁收敛里程碑](../plans/【里程碑】项目客户端占用锁收敛-2026-09-14.md)、[踩坑记录](pitfalls.md)。 diff --git a/docs/project-memory/shared-memory/development-workflow.md b/docs/project-memory/shared-memory/development-workflow.md index 6409631f4..9447f8b7f 100644 --- a/docs/project-memory/shared-memory/development-workflow.md +++ b/docs/project-memory/shared-memory/development-workflow.md @@ -74,4 +74,4 @@ SpacetimeDB 任务统一先读取 `.codex/skills/genarrative-spacetimedb/SKILL.m ## Gitea CI 依赖闭合 -`.gitea/workflows/project-ci.yml` 的 `Native shell tests` 在运行原生壳门禁前，必须使用 `cargo fetch --locked` 预取 `server-rs/Cargo.toml`、桌面壳和 AGC 壳三份依赖。Backend host workspace tests 使用 `cargo test --locked --workspace --exclude spacetime-module --no-fail-fast`，避免 `spacetime-module` 的 `spacetime-types` feature 统一污染普通领域 crate 的 host 测试；随后单独执行 `cargo test --locked -p spacetime-module --no-fail-fast`，由 `spacetime-module/src/active.rs` 在 host 测试构建期间提供仅测试期的 SpacetimeDB ABI 链接支持，使该 crate 的纯单元测试也纳入 Backend 门禁。`spacetime-module` 的 reducer / procedure 运行时行为仍必须通过真实 SpacetimeDB runtime/integration harness 验证，host 链接支持不得被当作运行时替身。Backend 另外执行 `cargo check --locked -p spacetime-module` 验证模块源码。AGC 壳检查还会运行 `platform-llm` 与 `shared-contracts` 的 server-rs workspace 测试，这些命令以及 AGC 壳测试必须带 `--locked`，避免在测试阶段重新解析 registry index；锁文件发生变化时应先更新受信任 CI 镜像缓存，再重跑门禁。 +`.gitea/workflows/project-ci.yml` 的客户端门禁拆成四个 job，每个 job 只预热自己会构建的那几份依赖：`AI game creator shell Rust tests` 只预取 AGC 壳 manifest（AGC 壳那份 `Cargo.lock` 的 path 依赖已含 `platform-llm`、`platform-agent`、`agent-runtime-core` 与 `shared-contracts`；`agent-run` smoke 会用 `src-tauri/Cargo.toml` spawn `cargo run`，因此必须同 job），`AI game creator shell Rust crates` 预取 `server-rs/Cargo.toml` 与两个独立 crate，`Native shell tests` 预取桌面壳与 AGC 壳 manifest，`AI game creator shell web tests` 不触碰 Cargo，不预热。前两个 job 只用 cargo 与 node 内建模块，因此不执行 `npm ci`。两个被 `server-rs/Cargo.toml` 排除、且没有提交 `Cargo.lock` 的独立 crate（`agent-runtime-core`、`agent-runtime-orchestration`）只能在 `AI game creator shell Rust crates` 里用不带锁标志的 fetch。AGC 壳的 bin target 单测（约 2466 条）由 `apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs` 编译一次后按 `--list` 名单分 4 片、每片一个进程并行跑，片内保持 `--test-threads=1` 并各自使用独立 `TMPDIR`：当年 libtest 线程并行会互相干扰的是进程内后台锁与异步终态，进程分片不共享这些状态，因此可以并行而无需放宽断言。Backend host workspace tests 使用 `cargo test --locked --workspace --exclude spacetime-module --no-fail-fast`，避免 `spacetime-module` 的 `spacetime-types` feature 统一污染普通领域 crate 的 host 测试；随后单独执行 `cargo test --locked -p spacetime-module --no-fail-fast`，由 `spacetime-module/src/active.rs` 在 host 测试构建期间提供仅测试期的 SpacetimeDB ABI 链接支持，使该 crate 的纯单元测试也纳入 Backend 门禁。`spacetime-module` 的 reducer / procedure 运行时行为仍必须通过真实 SpacetimeDB runtime/integration harness 验证，host 链接支持不得被当作运行时替身。Backend 另外执行 `cargo check --locked -p spacetime-module` 验证模块源码。AGC 壳检查还会运行 `platform-llm` 与 `shared-contracts` 的 server-rs workspace 测试，这些命令以及 AGC 壳测试必须带 `--locked`，避免在测试阶段重新解析 registry index；锁文件发生变化时应先更新受信任 CI 镜像缓存，再重跑门禁。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 4df507394..3015f6be4 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -1,5 +1,48 @@ # 踩坑与排障记录 +## 2026-09-14 AGC 壳 Rust 套件的「整套单线程」只放宽到「片内单线程」，且分片必须自校验覆盖 + +- **现象**：`AI game creator shell Rust tests` 一直是客户端 CI 的关键路径。run 2097 实测 15 分 27 秒，其中 `apps/ai-game-creator-shell/src-tauri/src/main.rs` 的 bin target 单测（2466 条）一条 `cargo test -- --test-threads=1` 串行占 507 秒。 +- **为什么原本是整个 suite 串行**：2026-07-21 `a273377b1` 的判据是「共享 Agent Runtime 后台锁与异步终态在 libtest 并行调度下互相干扰」，即**同进程内**的全局后台锁、异步终态与进程级 static 被交叉触发（另有少数用例自身 spawn `cargo test`，要独占 cargo 的 package cache/target 锁）。这是进程内并行的问题，不是用例之间的数据依赖。 +- **处理**：新增 `apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs`，`cargo test --no-run` 编译一次后用 `--list` 名单把用例切成 4 片，每片一个**独立进程**跑 `--exact <名单> --test-threads=1`，片内串行不变。配套把 `ai-game-creator-shell:check:rust` 拆成 `:rust:crates` 与 `:rust:shell`，CI 新增第 7 个 job `AI game creator shell Rust crates`。 +- **易错点**：① 分片规则必须自校验「片并集等于 `--list` 全集且互斥」，否则改分片方式会静默漏跑门禁；② 每片要拿独立 `TMPDIR`，`tempfile::tempdir()` 默认落在它下面，否则同容器内多片会互踩临时目录（测试里的硬编码 `/tmp/...` 多是「必须拒绝」的负向断言，不是真实读写）；③ 不要因为"反正要跑测试"就给分片 job 装 `npm ci`——AGC 壳 Rust 门禁与 `agent-run` smoke 只用 cargo 与 node 内建模块，两份 `npm ci` 正好是达标 7 分钟的主要障碍；④ 壳 job 只需预热 AGC 壳自己的 manifest（其 `Cargo.lock` 的 path 依赖已覆盖 `platform-llm` / `platform-agent` / `agent-runtime-core` / `shared-contracts`），`server-rs` 那份预热属于 crate 级 job；⑤ 分片后 `--test-threads=1` 不再出现在 workflow 里，但它是分片运行器的片内参数，别再往 workflow 里补整套串行命令。 +- **不要做的事**：不要退回「整套 `--test-threads=1`」（507 秒长尾回来了），也不要放开成整套并行（进程内后台锁与异步终态会再互相干扰），更不要用逐项单线程通过来替代整套门禁的稳定性结论。 +- **关联**：`apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.mjs`、`.gitea/workflows/project-ci.yml`、`scripts/check-native-shells.mjs`（`agc-rust-shell` / `agc-rust-crates` 分组）、`package.json`。 + +## 2026-09-14 根门禁的 `[check:native-shells]