修复 AGC Ctrl+C 残留上个工作树后端导致切换工作树复用旧后端 (#315)
Project CI / Frontend tests (push) Successful in 3m14s
Project CI / Backend tests (push) Successful in 8m54s
Project CI / Repository checks (push) Successful in 2m29s
Project CI / Native shell tests (push) Successful in 19m23s

closes #314

## 现象

`npm run agc` 按 Ctrl+C 后有概率残留上个工作树的 `api-server.exe` / SpacetimeDB,切换 worktree 再启动时 AGC 复用旧后端,改过数据库 / schema 的工作树会串库。

## 根因

1. Windows 下长驻服务都经 Node `shell: true` 的 `cmd.exe /d /s /c` 包装层启动,Ctrl+C 先杀包装层(`0xC000013A`);`dev.mjs` 的 `stopProcess` 见到直接子进程已退出就 return,`taskkill /PID <已退出 PID> /T /F` 也只会失败,深处的 `cargo → api-server.exe` 无人清理。
2. 按根 PID 遍历依赖快照里的父子链,中间层先消失时链断,只能拿到根 PID。
3. 复用判据只看 `.app/dev-stack.json` status 与 `/healthz`、`/readyz`、`/v1/ping`,不校验端口上的进程属于哪个工作树,残留后端照样被判为健康并复用。

## 改动

- 新增 `scripts/dev-windows-process.mjs`:按根 PID 遍历 + 按身份匹配(`server-rs/target/debug/api-server.exe` 绝对路径、SpacetimeDB `--data-dir`)两条独立清理路径,带 1s 快照缓存避免清理被拖慢。
- `scripts/dev.mjs`:直接子进程已退出时仍按记录 PID 清理后代;退出时按身份兜底清扫本工作树后端(复用他人 standalone 时跳过);启动前清理旧 api-server 保留 `Wait-Process` 语义,避免 `failed to remove file`。
- `apps/ai-game-creator-shell/scripts/start-dev-stack.mjs`:复用前校验端口监听进程归属,无法证明归属就不复用、改为启动本工作树后端并允许端口漂移;信号与 `finally` 各兜底清扫一次;`taskkill` 失败时降级按 PID 遍历;等待就绪时输出归属校验失败原因,避免静默超时。探测不可用时退化为旧行为,不阻断本地启动。
- 测试与文档:新增 `scripts/dev-windows-process.test.ts`、扩充 AGC 复用门禁用例;同步 `docs/project-memory/shared-memory/pitfalls.md` 与本地开发运维文档。

## 验证

- 伪造 `api-server.exe` 进程:按身份精确命中并杀掉(`matched=[17284] stopped=[17284]`)。
- 3 个真实监听进程下归属判定:`owned` / `api-server-owner-mismatch` / `spacetime-owner-mismatch` 均正确。
- `npx vitest run scripts/dev.test.ts scripts/dev-windows-process.test.ts scripts/dev-stack-port-utils.test.ts apps/ai-game-creator-shell/tests/...`:119 passed(唯一失败为 Windows 文件权限用例,已确认在合并基线 `origin/master` 上同样失败)。
- `node --check`、`eslint --max-warnings 0`、`prettier --check`、`npm run check:encoding`、`git diff --check` 全部通过。

## 备注

Rust 侧 `api-server` 的 `with_graceful_shutdown` 没有超时上限,是「有概率」的来源之一;本次只在 Node 侧收口,是否给优雅退出加 deadline 可另行评估。

Reviewed-on: https://git.genarrative.world/git/GenarrativeAI/Genarrative/pulls/315
Co-authored-by: Suzumiya <suzmii@qq.com>
Co-committed-by: Suzumiya <suzmii@qq.com>
This commit was merged in pull request #315.
This commit is contained in:
2026-09-09 20:08:04 +08:00
committed by 孔令弘
parent 7c49e7e51c
commit a1b9b24891
7 changed files with 992 additions and 103 deletions
@@ -1,10 +1,16 @@
import { spawn } from 'node:child_process';
import { spawn, spawnSync } from 'node:child_process';
import { existsSync, readdirSync, readFileSync } from 'node:fs';
import http from 'node:http';
import net from 'node:net';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
normalizeWindowsPath,
parseWindowsProcessSnapshot,
stopWindowsProcessTree,
stopWindowsWorktreeProcesses,
} from '../../../scripts/dev-windows-process.mjs';
import {
agcVitePortEnvKey,
readAgcDevEndpoint,
@@ -15,6 +21,10 @@ import {
const appRoot = fileURLToPath(new URL('..', import.meta.url));
const repoRoot = resolve(appRoot, '../..');
const devStackStatePath = resolve(repoRoot, '.app/dev-stack.json');
const apiServerExePath = resolve(
repoRoot,
'server-rs/target/debug/api-server.exe',
);
const defaultApiTarget =
process.env.RUST_SERVER_TARGET || 'http://127.0.0.1:8082';
const backendDatabase = 'genarrative-game-creator-dev';
@@ -160,19 +170,184 @@ function readBackendServiceFailure(
return null;
}
function urlPort(url) {
try {
const port = Number(new URL(url).port);
return Number.isInteger(port) && port > 0 ? port : 0;
} catch {
return 0;
}
}
// 读取端口当前真正的监听进程身份。返回 null 表示探测本身不可用(例如缺少
// Get-NetTCPConnection),此时调用方必须退化为旧行为,不能让本地启动直接失败。
function readWindowsPortOwnerIdentities(
ports,
{ spawnImpl = spawnSync, env = process.env } = {},
) {
const uniquePorts = [...new Set(ports.filter((port) => port > 0))];
if (uniquePorts.length === 0) {
return null;
}
const command = [
'$ErrorActionPreference = "SilentlyContinue"',
'$ports = ($env:GENARRATIVE_QUERY_PORTS -split ",") | Where-Object { $_ }',
'$result = @()',
'foreach ($port in $ports) {',
' $connection = Get-NetTCPConnection -State Listen -LocalPort ([int]$port) -ErrorAction SilentlyContinue | Select-Object -First 1',
' if (-not $connection) { continue }',
' $owner = Get-CimInstance Win32_Process -Filter ("ProcessId=" + $connection.OwningProcess) -ErrorAction SilentlyContinue',
' $result += [pscustomobject]@{ port = [int]$port; processId = [int]$connection.OwningProcess; name = $owner.Name; executablePath = $owner.ExecutablePath; commandLine = $owner.CommandLine }',
'}',
'ConvertTo-Json -InputObject @($result) -Compress',
].join('\n');
const result = spawnImpl(
'powershell.exe',
['-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', command],
{
encoding: 'utf8',
env: { ...env, GENARRATIVE_QUERY_PORTS: uniquePorts.join(',') },
maxBuffer: 8 * 1024 * 1024,
},
);
if (result?.error || result?.status !== 0) {
return null;
}
const owners = new Map();
for (const entry of parseWindowsProcessSnapshot(result.stdout)) {
const port = Number(entry?.port);
if (Number.isInteger(port) && port > 0) {
owners.set(port, entry);
}
}
return owners;
}
function isWorktreeApiServerOwner(
owner,
{ expectedExePath = apiServerExePath } = {},
) {
if (!owner) {
return false;
}
const expected = normalizeWindowsPath(expectedExePath);
const actual = normalizeWindowsPath(owner.executablePath);
return Boolean(expected) && actual === expected;
}
function isWorktreeSpacetimeOwner(
owner,
{ expectedDataDir = backendSpacetimeDataDir } = {},
) {
if (!owner) {
return false;
}
const expected = normalizeWindowsPath(expectedDataDir);
if (!expected) {
return false;
}
const name = String(owner.name ?? '').toLowerCase();
if (!name.startsWith('spacetime')) {
return false;
}
return normalizeWindowsPath(owner.commandLine).includes(expected);
}
// 端口健康不代表后端属于当前工作树:上个工作树 Ctrl+C 残留的 api-server 仍会
// 应答 /healthz。复用前必须证明端口上的进程就是本工作树的可执行文件与数据目录。
function verifyAgcBackendOwnership({
apiUrl,
spacetimeUrl,
bgfilterWorkerUrl,
platform = process.platform,
expectedExePath = apiServerExePath,
expectedDataDir = backendSpacetimeDataDir,
readPortOwners = readWindowsPortOwnerIdentities,
} = {}) {
if (platform !== 'win32') {
return { ok: true, reason: 'platform-unsupported', owners: new Map() };
}
const ports = [
urlPort(apiUrl),
urlPort(bgfilterWorkerUrl),
urlPort(spacetimeUrl),
];
const owners = readPortOwners(ports);
if (!owners) {
return { ok: true, reason: 'owner-probe-unavailable', owners: new Map() };
}
const apiOwner = owners.get(urlPort(apiUrl));
if (!isWorktreeApiServerOwner(apiOwner, { expectedExePath })) {
return { ok: false, reason: 'api-server-owner-mismatch', owners, apiOwner };
}
const workerOwner = owners.get(urlPort(bgfilterWorkerUrl));
if (!isWorktreeApiServerOwner(workerOwner, { expectedExePath })) {
return {
ok: false,
reason: 'bgfilter-worker-owner-mismatch',
owners,
workerOwner,
};
}
const spacetimeOwner = owners.get(urlPort(spacetimeUrl));
if (!isWorktreeSpacetimeOwner(spacetimeOwner, { expectedDataDir })) {
return {
ok: false,
reason: 'spacetime-owner-mismatch',
owners,
spacetimeOwner,
};
}
return { ok: true, reason: 'owned', owners };
}
function formatOwnerLabel(owner) {
if (!owner) {
return '未知进程';
}
const pid = Number(owner.processId);
const label = owner.executablePath || owner.commandLine || owner.name || '';
return `${Number.isInteger(pid) ? `pid=${pid} ` : ''}${String(label).trim()}`.trim();
}
async function isBackendReady({
state = readJson(devStackStatePath),
isReady = isHttpReady,
verifyOwnership = verifyAgcBackendOwnership,
onOwnershipRejected = null,
} = {}) {
const { apiUrl, spacetimeUrl, bgfilterWorkerUrl, hasMatchingBackend } =
resolveBackendTargetsFromState(state, {
requireAgcBackend: true,
});
if (!hasMatchingBackend || !apiUrl || !spacetimeUrl || !bgfilterWorkerUrl) {
return false;
}
const ownership = await verifyOwnership({
apiUrl,
spacetimeUrl,
bgfilterWorkerUrl,
});
if (!ownership?.ok) {
onOwnershipRejected?.(ownership);
return false;
}
if (ownership.reason === 'owner-probe-unavailable') {
console.warn(
'[ai-game-creator-shell] 无法读取端口监听进程归属,本次按旧行为复用配套后端。',
);
}
return (
hasMatchingBackend &&
Boolean(apiUrl) &&
Boolean(spacetimeUrl) &&
Boolean(bgfilterWorkerUrl) &&
(await isReady(`${apiUrl}/healthz`)) &&
(await isReady(`${spacetimeUrl}/v1/ping`)) &&
(await isReady(`${bgfilterWorkerUrl}/readyz`))
@@ -485,14 +660,18 @@ async function terminateChildTree(
return { stopped: true, forced: false };
}
const result = await taskkillImpl(child.pid);
return {
stopped:
!result?.timedOut &&
!result?.error &&
[0, 128].includes(result?.code ?? 0),
forced: true,
result,
};
const taskkillStopped =
!result?.timedOut &&
!result?.error &&
[0, 128].includes(result?.code ?? 0);
if (taskkillStopped) {
return { stopped: true, forced: true, result };
}
// 包装层(cmd.exe / npm.cmd)先被 Ctrl+C 杀掉时 taskkill 拿不到活着的 PID
// 这里继续按记录下来的根 PID 遍历,尽量收掉更深的后端进程。
const treeStopped = stopWindowsProcessTree(child.pid);
return { stopped: treeStopped.length > 0, forced: true, result };
}
const processGroupId = childLifecycles.get(child)?.processGroupId;
@@ -542,15 +721,29 @@ async function waitForBackendReady(
backendChild,
timeoutMs = 600_000,
{
checkBackendReady = isBackendReady,
checkBackendReady = (onOwnershipRejected) =>
isBackendReady({ onOwnershipRejected }),
readState = () => readJson(devStackStatePath),
resolveTargets = readBackendTargets,
} = {},
) {
const initialStateUpdatedAt = readState()?.updatedAt ?? '';
const startedAt = Date.now();
let lastOwnershipReason = '';
while (Date.now() - startedAt < timeoutMs) {
if (await checkBackendReady()) {
if (
await checkBackendReady((ownership) => {
if (ownership.reason === lastOwnershipReason) {
return;
}
lastOwnershipReason = ownership.reason;
// 本次自己拉起的后端如果归属校验一直不通过,必须把原因打出来,
// 否则只会表现为等待 600 秒后超时。
console.warn(
`[ai-game-creator-shell] 等待配套后端就绪时归属校验未通过(${ownership.reason}: ${formatOwnerLabel(ownership.apiOwner ?? ownership.spacetimeOwner ?? ownership.workerOwner)})。`,
);
})
) {
return resolveTargets();
}
const state = readState();
@@ -576,7 +769,14 @@ async function waitForBackendReady(
async function ensureBackend({
onBackendChild = () => {},
checkBackendReady = isBackendReady,
checkBackendReady = () =>
isBackendReady({
onOwnershipRejected(ownership) {
console.warn(
`[ai-game-creator-shell] 端口上的配套后端不属于当前工作树(${ownership.reason}: ${formatOwnerLabel(ownership.apiOwner ?? ownership.spacetimeOwner ?? ownership.workerOwner)}),改为启动本工作树自己的后端。`,
);
},
}),
resolveTargets = readBackendTargets,
spawnBackend = () =>
spawnChild(
@@ -656,15 +856,33 @@ async function startVite(apiTarget, endpoint = readAgcDevEndpoint()) {
async function main() {
let backendChild = null;
let startedBackend = false;
let viteChild = null;
let shutdownSignal = '';
const signalHandlers = new Map();
// 只有本次会话真正拉起过配套后端时才做兜底清扫:复用别人后端时不能连带
// 杀掉对方的进程。dev.mjs 的清理依赖它的 shell 包装层仍然活着,而 Ctrl+C
// 往往先杀掉包装层,所以这里必须按本工作树 api-server.exe 的身份再收一次。
const sweepStartedBackend = () => {
if (!startedBackend || process.platform !== 'win32') {
return;
}
const stopped = stopWindowsWorktreeProcesses({ apiServerExePath });
if (stopped.length > 0) {
console.log(
`[ai-game-creator-shell] 已清理残留后端进程: ${stopped.join(', ')}`,
);
}
};
for (const signal of ['SIGINT', 'SIGTERM']) {
const handler = () => {
shutdownSignal = signal;
stopChild(viteChild, signal);
stopChild(backendChild, signal);
// 立刻清扫,避免外层 taskkill /F 抢在 finally 之前把本进程杀掉。
sweepStartedBackend();
};
signalHandlers.set(signal, handler);
process.on(signal, handler);
@@ -683,6 +901,7 @@ async function main() {
},
});
backendChild = backend.backendChild;
startedBackend = Boolean(backendChild);
if (shutdownSignal) {
throw new Error(`启动期收到 ${shutdownSignal},已停止配套后端`);
}
@@ -716,6 +935,7 @@ async function main() {
terminateChildTree(viteChild),
terminateChildTree(backendChild),
]);
sweepStartedBackend();
for (const [signal, handler] of signalHandlers) {
process.off(signal, handler);
}
@@ -732,20 +952,25 @@ function isDirectModuleExecution() {
export {
ensureBackend,
formatChildFailure,
formatOwnerLabel,
isAiGameCreatorServer,
isBackendReady,
isDirectModuleExecution,
isProcessGroupAlive,
isWorktreeApiServerOwner,
isWorktreeSpacetimeOwner,
preflightExistingVite,
readBackendServiceFailure,
readChildFailure,
readExistingViteServer,
readLinuxProcessGroupAlive,
readWindowsPortOwnerIdentities,
resolveBackendTargetsFromState,
runWindowsTaskkill,
spawnChild,
stopChild,
terminateChildTree,
verifyAgcBackendOwnership,
waitForBackendReady,
waitForChildTermination,
};
@@ -7,22 +7,33 @@ import { describe, expect, test, vi } from 'vitest';
import {
ensureBackend,
formatOwnerLabel,
isBackendReady,
isProcessGroupAlive,
isWorktreeApiServerOwner,
isWorktreeSpacetimeOwner,
preflightExistingVite,
readBackendServiceFailure,
readLinuxProcessGroupAlive,
readWindowsPortOwnerIdentities,
resolveBackendTargetsFromState,
runWindowsTaskkill,
spawnChild,
stopChild,
terminateChildTree,
verifyAgcBackendOwnership,
waitForBackendReady,
waitForChildTermination,
} from '../scripts/start-dev-stack.mjs';
const expectedDatabase = 'genarrative-game-creator-dev';
const expectedDataDir = resolve('server-rs/.spacetimedb/ai-game-creator/data');
const expectedExePath = resolve('server-rs/target/debug/api-server.exe');
const ownedBackend = async () => ({
ok: true,
reason: 'owned',
owners: new Map(),
});
function backendState(spacetimeDataDir?: string, includeBgfilterWorker = true) {
return {
@@ -109,6 +120,7 @@ describe('AI 游戏创作配套后端复用门禁', () => {
isBackendReady({
state: backendState(expectedDataDir, false),
isReady,
verifyOwnership: ownedBackend,
}),
).resolves.toBe(false);
expect(isReady).not.toHaveBeenCalled();
@@ -118,6 +130,7 @@ describe('AI 游戏创作配套后端复用门禁', () => {
isBackendReady({
state: backendState(expectedDataDir),
isReady,
verifyOwnership: ownedBackend,
}),
).resolves.toBe(false);
expect(isReady).toHaveBeenCalledWith('http://127.0.0.1:8082/healthz');
@@ -127,6 +140,7 @@ describe('AI 游戏创作配套后端复用门禁', () => {
isBackendReady({
state: backendState(expectedDataDir),
isReady,
verifyOwnership: ownedBackend,
}),
).resolves.toBe(true);
});
@@ -171,6 +185,232 @@ describe('AI 游戏创作配套后端复用门禁', () => {
}),
).rejects.toThrow('配套后端启动失败: bgfilter-worker code=98');
});
test('端口上的后端不属于当前工作树时拒绝复用', async () => {
const isReady = vi.fn(async () => true);
const onOwnershipRejected = vi.fn();
await expect(
isBackendReady({
state: backendState(expectedDataDir),
isReady,
verifyOwnership: async () => ({
ok: false,
reason: 'api-server-owner-mismatch',
apiOwner: { processId: 4321, name: 'api-server.exe' },
}),
onOwnershipRejected,
}),
).resolves.toBe(false);
expect(onOwnershipRejected).toHaveBeenCalledWith(
expect.objectContaining({ reason: 'api-server-owner-mismatch' }),
);
expect(isReady).not.toHaveBeenCalled();
});
test('归属探测不可用时退化为旧行为而不是让本地启动失败', async () => {
const isReady = vi.fn(async () => true);
await expect(
isBackendReady({
state: backendState(expectedDataDir),
isReady,
verifyOwnership: async () => ({
ok: true,
reason: 'owner-probe-unavailable',
owners: new Map(),
}),
}),
).resolves.toBe(true);
expect(isReady).toHaveBeenCalledWith('http://127.0.0.1:8082/healthz');
});
});
describe('AI 游戏创作配套后端归属校验', () => {
const urls = {
apiUrl: 'http://127.0.0.1:8082',
spacetimeUrl: 'http://127.0.0.1:3101',
bgfilterWorkerUrl: 'http://127.0.0.1:8083',
};
function ownerMap({
apiExe = expectedExePath,
dataDir = expectedDataDir,
} = {}) {
return new Map([
[
8082,
{
port: 8082,
processId: 11,
name: 'api-server.exe',
executablePath: apiExe,
commandLine: null,
},
],
[
8083,
{
port: 8083,
processId: 12,
name: 'api-server.exe',
executablePath: expectedExePath,
commandLine: null,
},
],
[
3101,
{
port: 3101,
processId: 13,
name: 'spacetimedb-standalone.exe',
executablePath: null,
commandLine: `spacetimedb-standalone.exe start --data-dir ${dataDir}`,
},
],
]);
}
test('api-server 可执行文件来自其它工作树时判定为不归属', () => {
const result = verifyAgcBackendOwnership({
...urls,
platform: 'win32',
expectedExePath,
expectedDataDir,
readPortOwners: () =>
ownerMap({
apiExe: resolve(
'.worktrees/other/server-rs/target/debug/api-server.exe',
),
}),
});
expect(result.ok).toBe(false);
expect(result.reason).toBe('api-server-owner-mismatch');
});
test('SpacetimeDB 使用其它 data dir 时判定为不归属', () => {
const result = verifyAgcBackendOwnership({
...urls,
platform: 'win32',
expectedExePath,
expectedDataDir,
readPortOwners: () =>
ownerMap({ dataDir: resolve('server-rs/.spacetimedb/local/data') }),
});
expect(result.ok).toBe(false);
expect(result.reason).toBe('spacetime-owner-mismatch');
});
test('可执行文件与 data dir 都匹配时允许复用', () => {
const result = verifyAgcBackendOwnership({
...urls,
platform: 'win32',
expectedExePath,
expectedDataDir,
readPortOwners: () => ownerMap(),
});
expect(result).toMatchObject({ ok: true, reason: 'owned' });
});
test('归属探测不可用时不阻断本地启动', () => {
const result = verifyAgcBackendOwnership({
...urls,
platform: 'win32',
expectedExePath,
expectedDataDir,
readPortOwners: () => null,
});
expect(result).toMatchObject({
ok: true,
reason: 'owner-probe-unavailable',
});
});
test('非 Windows 平台保持原有复用行为', () => {
const result = verifyAgcBackendOwnership({ ...urls, platform: 'linux' });
expect(result).toMatchObject({ ok: true, reason: 'platform-unsupported' });
});
test('可执行文件路径与 data dir 归属判定忽略大小写和 \\\\?\\ 前缀', () => {
expect(
isWorktreeApiServerOwner(
{
processId: 1,
executablePath: `\\\\?\\${expectedExePath.toUpperCase()}`,
},
{ expectedExePath },
),
).toBe(true);
expect(
isWorktreeSpacetimeOwner(
{
processId: 2,
name: 'spacetimedb-standalone.exe',
commandLine: `start --data-dir ${expectedDataDir.toUpperCase()}`,
},
{ expectedDataDir },
),
).toBe(true);
expect(
isWorktreeSpacetimeOwner(
{ processId: 3, name: 'node.exe', commandLine: expectedDataDir },
{ expectedDataDir },
),
).toBe(false);
});
test('端口监听进程探测解析 PowerShell 输出', () => {
const spawnImpl = vi.fn(() => ({
status: 0,
error: null,
stdout: JSON.stringify([
{
port: 8082,
processId: 4321,
name: 'api-server.exe',
executablePath: expectedExePath,
commandLine: null,
},
]),
}));
const owners = readWindowsPortOwnerIdentities([8082, 0], {
spawnImpl,
env: {},
});
expect(owners?.get(8082)).toMatchObject({ processId: 4321 });
expect(spawnImpl).toHaveBeenCalledWith(
'powershell.exe',
expect.any(Array),
expect.objectContaining({ env: { GENARRATIVE_QUERY_PORTS: '8082' } }),
);
});
test('探测失败时返回 null 以触发退化分支', () => {
expect(
readWindowsPortOwnerIdentities([8082], {
spawnImpl: () => ({ status: 1, error: null, stdout: '' }),
env: {},
}),
).toBeNull();
expect(readWindowsPortOwnerIdentities([], { env: {} })).toBeNull();
});
test('归属日志包含 pid 与进程标识', () => {
expect(
formatOwnerLabel({
processId: 4321,
executablePath: 'C:\\a\\api-server.exe',
}),
).toBe('pid=4321 C:\\a\\api-server.exe');
expect(formatOwnerLabel(null)).toBe('未知进程');
});
});
describe('AI 游戏创作启动子进程生命周期', () => {