AIGameCreator App
be89296492
拆分 App 认证、壳层、运行配置与项目摘要模块 拆分 Tauri 项目能力与 Rust 测试领域模块 拆分界面测试与 Agent Runtime 真实 E2E 套件 补充源码扫描和客户端模块化文档约定
1112 lines
36 KiB
JavaScript
1112 lines
36 KiB
JavaScript
import { assert, codedError, hashValue, sleep } from '../assertions/core.mjs';
|
|
import {
|
|
absolutePathVariants,
|
|
auditInputValue,
|
|
countBy,
|
|
countExactSecrets,
|
|
duplicateCount,
|
|
finalMessageId,
|
|
formalConfigPathVariants,
|
|
hasExactKeys,
|
|
isNonEmptyString,
|
|
receiptAuditIdentity,
|
|
sumObjectValues,
|
|
} from '../assertions/runtime.mjs';
|
|
import { fs, path, spawn } from '../dependencies.mjs';
|
|
import {
|
|
claimOwnedRunner,
|
|
ensureOwnedRunnerStableKillSupport,
|
|
prepareIsolatedSuiteAppData,
|
|
waitForChildClose,
|
|
} from '../harness/app-data.mjs';
|
|
import { isPlainObject } from '../harness/config.mjs';
|
|
import {
|
|
isPathInside,
|
|
isTerminalRuntime,
|
|
listFiles,
|
|
readJson,
|
|
readOptionalJsonl,
|
|
} from '../harness/io.mjs';
|
|
import {
|
|
appendBounded,
|
|
prepareCliBinary,
|
|
runCli,
|
|
} from '../harness/process.mjs';
|
|
import { seedDisposableProject } from '../harness/project.mjs';
|
|
import {
|
|
agentConversationPath,
|
|
countLureLeaks,
|
|
findPendingActions,
|
|
killRunnerOnce,
|
|
readAllRuntimeEvents,
|
|
readRunnerStatus,
|
|
readRuntime,
|
|
readTaskSnapshot,
|
|
runnerBootId,
|
|
validateProjectRootPublicLeakBoundary,
|
|
waitForRunnerBootChange,
|
|
} from '../harness/runtime.mjs';
|
|
import {
|
|
activeCommandChildren,
|
|
mainAgentId,
|
|
mcpBearerToken,
|
|
mcpFixtureScript,
|
|
mcpHeaderValue,
|
|
mcpHttpQuery,
|
|
mcpKillMutationValue,
|
|
mcpMutateResponseDelayMs,
|
|
mcpMutationValue,
|
|
mcpRuntimeSuite,
|
|
mcpStdioQuery,
|
|
pollIntervalMs,
|
|
runTimeoutMs,
|
|
state,
|
|
supportedToolPlanProtocols,
|
|
} from '../runtime-state.mjs';
|
|
import { readProcessPersistenceEvidence } from './process-session.mjs';
|
|
|
|
export function buildMcpNormalTaskPrompt() {
|
|
return `验证当前 Runtime 动态 MCP 工具目录的真实可用性。必须从目录和各工具 inputSchema 中发现并完成三项调用：stdio-fixture 的 lookup、http-fixture 的 lookup、stdio-fixture 的 mutate；每项参数都使用对应 schema 明确要求的 const 值。只允许调用这三个 MCP 动作，写工具必须等待开发者确认。收到全部真实 observation 后再给出一句简短中文结论，不得在最终回复中复述参数、结果正文、凭据、路径或外部 instructions。`;
|
|
}
|
|
|
|
export function buildMcpKillTaskPrompt() {
|
|
return `只验证一个会产生副作用的动态 MCP 调用：从目录中选择 http-fixture 的 mutate，并把 value 设为该工具 inputSchema 明确要求的 const 值。只允许提交这一项 MCP 动作，必须等待开发者确认；未收到真实 observation 前不得形成最终回复，也不得复述参数、凭据、路径或外部 instructions。`;
|
|
}
|
|
|
|
export function assertMcpTaskPrompt(task, kind) {
|
|
const required =
|
|
kind === 'normal'
|
|
? ['stdio-fixture', 'http-fixture', 'lookup', 'mutate']
|
|
: ['http-fixture', 'mutate'];
|
|
assert(
|
|
required.every((value) => task.includes(value)),
|
|
`mcp-${kind}-required-input-missing`,
|
|
);
|
|
for (const forbidden of [
|
|
'catalogFingerprint',
|
|
'toolFingerprint',
|
|
`lookup:${mcpStdioQuery}`,
|
|
`lookup:${mcpHttpQuery}`,
|
|
`mutated:${mcpMutationValue}`,
|
|
`mutated:${mcpKillMutationValue}`,
|
|
mcpStdioQuery,
|
|
mcpHttpQuery,
|
|
mcpMutationValue,
|
|
mcpKillMutationValue,
|
|
mcpBearerToken,
|
|
mcpHeaderValue,
|
|
mcpFixtureScript,
|
|
]) {
|
|
assert(!task.includes(forbidden), `mcp-${kind}-task-private-recipe-leak`);
|
|
}
|
|
}
|
|
|
|
export async function spawnMcpHttpFixture(appDataDir) {
|
|
assert(isMcpRuntimeSuite(), 'mcp-http-fixture-used-outside-suite');
|
|
state.mcp.normalMarkerPath = path.join(appDataDir, 'mcp-stdio-mutation.log');
|
|
state.mcp.killMarkerPath = path.join(appDataDir, 'mcp-http-mutation.log');
|
|
const child = spawn(
|
|
process.execPath,
|
|
[
|
|
mcpFixtureScript,
|
|
'http',
|
|
'0',
|
|
`--marker=${state.mcp.killMarkerPath}`,
|
|
`--mutate-response-delay-ms=${mcpMutateResponseDelayMs}`,
|
|
`--bearer-token=${mcpBearerToken}`,
|
|
`--fixture-header=${mcpHeaderValue}`,
|
|
`--lookup-value=${mcpHttpQuery}`,
|
|
`--mutate-value=${mcpKillMutationValue}`,
|
|
],
|
|
{
|
|
cwd: path.dirname(mcpFixtureScript),
|
|
env: { PATH: process.env.PATH ?? '' },
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
},
|
|
);
|
|
state.mcp.httpFixture = child;
|
|
activeCommandChildren.add(child);
|
|
child.once('close', () => activeCommandChildren.delete(child));
|
|
child.stderr.on('data', (chunk) => {
|
|
state.transcriptScanner?.scan('mcp-fixture-stderr', chunk);
|
|
state.formalConfigPathTranscriptScanner?.scan('mcp-fixture-stderr', chunk);
|
|
});
|
|
|
|
const port = await new Promise((resolve, reject) => {
|
|
let buffered = Buffer.alloc(0);
|
|
let settled = false;
|
|
const finish = (callback, value) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(timer);
|
|
child.off('error', onError);
|
|
child.off('close', onClose);
|
|
callback(value);
|
|
};
|
|
const onError = (error) =>
|
|
finish(reject, codedError('mcp-http-fixture-spawn-failed', error));
|
|
const onClose = () =>
|
|
finish(reject, codedError('mcp-http-fixture-closed-before-ready'));
|
|
const timer = setTimeout(
|
|
() => finish(reject, codedError('mcp-http-fixture-ready-timeout')),
|
|
10_000,
|
|
);
|
|
child.once('error', onError);
|
|
child.once('close', onClose);
|
|
child.stdout.on('data', (chunk) => {
|
|
state.transcriptScanner?.scan('mcp-fixture-stdout', chunk);
|
|
buffered = appendBounded(buffered, chunk, 8 * 1024);
|
|
const newline = buffered.indexOf(0x0a);
|
|
if (newline < 0) return;
|
|
let payload;
|
|
try {
|
|
payload = JSON.parse(buffered.subarray(0, newline).toString('utf8'));
|
|
} catch (error) {
|
|
finish(
|
|
reject,
|
|
codedError('mcp-http-fixture-ready-json-invalid', error),
|
|
);
|
|
return;
|
|
}
|
|
const candidate = Number(payload?.port);
|
|
if (!Number.isInteger(candidate) || candidate <= 0 || candidate > 65535) {
|
|
finish(reject, codedError('mcp-http-fixture-port-invalid'));
|
|
return;
|
|
}
|
|
finish(resolve, candidate);
|
|
});
|
|
});
|
|
state.mcp.httpPort = port;
|
|
return port;
|
|
}
|
|
|
|
export async function stopMcpHttpFixture() {
|
|
const child = state.mcp.httpFixture;
|
|
if (!child) return;
|
|
if (child.exitCode === null && child.signalCode === null) {
|
|
child.kill('SIGTERM');
|
|
try {
|
|
await waitForChildClose(child, 3_000);
|
|
} catch {
|
|
child.kill('SIGKILL');
|
|
await waitForChildClose(child, 3_000).catch(() => {});
|
|
}
|
|
}
|
|
activeCommandChildren.delete(child);
|
|
state.mcp.httpFixture = null;
|
|
}
|
|
|
|
export async function buildMcpConfigOverlay(appDataDir) {
|
|
const port = await spawnMcpHttpFixture(appDataDir);
|
|
return {
|
|
mcpServers: {
|
|
'stdio-fixture': {
|
|
required: true,
|
|
transport: 'stdio',
|
|
command: 'node',
|
|
args: [
|
|
mcpFixtureScript,
|
|
'stdio',
|
|
`--marker=${state.mcp.normalMarkerPath}`,
|
|
`--lookup-value=${mcpStdioQuery}`,
|
|
`--mutate-value=${mcpMutationValue}`,
|
|
],
|
|
startupTimeoutMs: 10_000,
|
|
toolTimeoutMs: 60_000,
|
|
enabledTools: ['lookup', 'mutate'],
|
|
defaultApprovalMode: 'writes',
|
|
},
|
|
'http-fixture': {
|
|
required: true,
|
|
transport: 'streamableHttp',
|
|
url: `http://127.0.0.1:${port}/mcp`,
|
|
bearerToken: mcpBearerToken,
|
|
httpHeaders: { 'X-MCP-Fixture': mcpHeaderValue },
|
|
allowInsecureLocalhost: true,
|
|
startupTimeoutMs: 10_000,
|
|
toolTimeoutMs: 60_000,
|
|
enabledTools: ['lookup', 'mutate'],
|
|
defaultApprovalMode: 'writes',
|
|
},
|
|
},
|
|
secrets: [mcpBearerToken, mcpHeaderValue],
|
|
};
|
|
}
|
|
|
|
export function mcpPendingCallInput(pending, codePrefix) {
|
|
const input = pending?.action?.input;
|
|
assert(
|
|
isPlainObject(input) &&
|
|
isPlainObject(input.arguments) &&
|
|
/^[0-9a-f]{64}$/u.test(input.catalogFingerprint ?? '') &&
|
|
/^[0-9a-f]{64}$/u.test(input.toolFingerprint ?? ''),
|
|
`${codePrefix}-pending-input-invalid`,
|
|
);
|
|
return input;
|
|
}
|
|
|
|
export function mcpResultSidecarPath(runId, actionId) {
|
|
return path.join(
|
|
state.projectRoot,
|
|
'.agent/runtime/mcp-results',
|
|
hashValue(mainAgentId),
|
|
hashValue(runId),
|
|
`${hashValue(actionId)}.json`,
|
|
);
|
|
}
|
|
|
|
export async function readMcpMarkerLines(markerPath) {
|
|
assert(
|
|
isNonEmptyString(markerPath) &&
|
|
isPathInside(state.isolatedRunner.appDataDir, markerPath),
|
|
'mcp-marker-path-invalid',
|
|
);
|
|
const metadata = await fs.lstat(markerPath).catch((error) => {
|
|
if (error?.code === 'ENOENT') return null;
|
|
throw error;
|
|
});
|
|
if (!metadata) return [];
|
|
assert(
|
|
metadata.isFile() && !metadata.isSymbolicLink(),
|
|
'mcp-marker-not-regular-file',
|
|
);
|
|
return (await fs.readFile(markerPath, 'utf8'))
|
|
.split('\n')
|
|
.filter((line) => line.length > 0);
|
|
}
|
|
|
|
export async function waitForMcpMarker(markerPath, expectedValue) {
|
|
const deadline = Date.now() + 60_000;
|
|
while (Date.now() < deadline) {
|
|
const lines = await readMcpMarkerLines(markerPath);
|
|
if (lines.length > 1) throw codedError('mcp-marker-replayed');
|
|
if (lines.length === 1) {
|
|
assert(lines[0] === expectedValue, 'mcp-marker-value-invalid');
|
|
return;
|
|
}
|
|
await sleep(25);
|
|
}
|
|
throw codedError('mcp-marker-timeout');
|
|
}
|
|
|
|
export async function waitForMcpRuntime(runId, { terminal = false } = {}) {
|
|
const deadline = Date.now() + 120_000;
|
|
while (Date.now() < deadline) {
|
|
const runtime = await readRuntime(mainAgentId).catch(() => null);
|
|
if (
|
|
runtime?.runId === runId &&
|
|
isNonEmptyString(runtime.sessionId) &&
|
|
(terminal || !isTerminalRuntime(runtime))
|
|
) {
|
|
return runtime;
|
|
}
|
|
await sleep(pollIntervalMs);
|
|
}
|
|
throw codedError('mcp-runtime-identity-timeout');
|
|
}
|
|
|
|
export async function driveMcpNormalRuntimeToCompletion() {
|
|
const deadline = Date.now() + runTimeoutMs;
|
|
while (Date.now() < deadline) {
|
|
const runtime = await readRuntime(mainAgentId).catch(() => null);
|
|
if (runtime?.runId !== state.mcp.normalRunId) {
|
|
await sleep(pollIntervalMs);
|
|
continue;
|
|
}
|
|
if (
|
|
runtime.phase === 'completed' &&
|
|
['completed', 'idle'].includes(runtime.status)
|
|
) {
|
|
return runtime;
|
|
}
|
|
if (
|
|
[
|
|
'failed',
|
|
'cancelled',
|
|
'budget-exhausted',
|
|
'needs-reconciliation',
|
|
].includes(runtime.phase)
|
|
) {
|
|
throw codedError('mcp-normal-runtime-failed');
|
|
}
|
|
const pending = (await findPendingActions()).filter(
|
|
(candidate) => candidate.runId === state.mcp.normalRunId,
|
|
);
|
|
assert(pending.length <= 1, 'mcp-normal-pending-count-invalid');
|
|
if (pending.length === 1) {
|
|
const target = pending[0];
|
|
assert(target.tool === 'mcp.call', 'mcp-normal-pending-tool-invalid');
|
|
const input = mcpPendingCallInput(target, 'mcp-normal');
|
|
assert(
|
|
input.server === 'stdio-fixture' &&
|
|
input.tool === 'mutate' &&
|
|
input.arguments.value === mcpMutationValue,
|
|
'mcp-normal-confirmation-target-invalid',
|
|
);
|
|
assert(
|
|
(await readMcpMarkerLines(state.mcp.normalMarkerPath)).length === 0,
|
|
'mcp-normal-marker-before-confirmation',
|
|
);
|
|
await runCli(
|
|
[
|
|
'--agent-confirm',
|
|
state.projectRoot,
|
|
target.agentId,
|
|
target.runId,
|
|
target.actionId,
|
|
],
|
|
{ timeoutMs: 120_000 },
|
|
);
|
|
state.confirmedActionIds.add(target.actionId);
|
|
state.mcp.normalActionIds.push(target.actionId);
|
|
}
|
|
await sleep(100);
|
|
}
|
|
throw codedError('mcp-normal-runtime-timeout');
|
|
}
|
|
|
|
export async function waitForMcpKillPendingAction() {
|
|
const deadline = Date.now() + runTimeoutMs;
|
|
while (Date.now() < deadline) {
|
|
const pending = (await findPendingActions()).filter(
|
|
(candidate) => candidate.runId === state.mcp.killRunId,
|
|
);
|
|
assert(pending.length <= 1, 'mcp-kill-pending-count-invalid');
|
|
if (pending.length === 1) {
|
|
const target = pending[0];
|
|
assert(target.tool === 'mcp.call', 'mcp-kill-pending-tool-invalid');
|
|
const input = mcpPendingCallInput(target, 'mcp-kill');
|
|
assert(
|
|
input.server === 'http-fixture' &&
|
|
input.tool === 'mutate' &&
|
|
input.arguments.value === mcpKillMutationValue,
|
|
'mcp-kill-confirmation-target-invalid',
|
|
);
|
|
return target;
|
|
}
|
|
const runtime = await readRuntime(mainAgentId).catch(() => null);
|
|
if (
|
|
runtime?.runId === state.mcp.killRunId &&
|
|
['failed', 'cancelled', 'budget-exhausted', 'completed'].includes(
|
|
runtime.phase,
|
|
)
|
|
) {
|
|
throw codedError('mcp-kill-runtime-ended-before-confirmation');
|
|
}
|
|
await sleep(pollIntervalMs);
|
|
}
|
|
throw codedError('mcp-kill-confirmation-timeout');
|
|
}
|
|
|
|
export async function waitForMcpKillReconciliation() {
|
|
const deadline = Date.now() + 120_000;
|
|
while (Date.now() < deadline) {
|
|
const [runtime, taskSnapshot] = await Promise.all([
|
|
readRuntime(mainAgentId).catch(() => null),
|
|
readTaskSnapshot(),
|
|
]);
|
|
const task = taskSnapshot.latest.find(
|
|
(candidate) =>
|
|
candidate.agentId === mainAgentId &&
|
|
candidate.runId === state.mcp.killRunId,
|
|
);
|
|
if (
|
|
runtime?.runId === state.mcp.killRunId &&
|
|
runtime.sessionId === state.mcp.sessionId &&
|
|
runtime.status === 'failed' &&
|
|
runtime.phase === 'needs-reconciliation' &&
|
|
task?.status === 'failed' &&
|
|
task.phase === 'needs-reconciliation'
|
|
) {
|
|
return runtime;
|
|
}
|
|
await sleep(pollIntervalMs);
|
|
}
|
|
throw codedError('mcp-kill-reconciliation-timeout');
|
|
}
|
|
|
|
export async function runMcpRuntimeE2e() {
|
|
await ensureOwnedRunnerStableKillSupport();
|
|
await seedDisposableProject();
|
|
state.cliBinary = await prepareCliBinary();
|
|
await prepareIsolatedSuiteAppData({
|
|
mcpConfigFactory: buildMcpConfigOverlay,
|
|
});
|
|
state.isolatedRunner.launchAttempted = true;
|
|
|
|
const normalTask = buildMcpNormalTaskPrompt();
|
|
assertMcpTaskPrompt(normalTask, 'normal');
|
|
state.initialTask = {
|
|
chars: [...normalTask].length,
|
|
sha256: hashValue(normalTask),
|
|
};
|
|
state.initialRunId = state.mcp.normalRunId;
|
|
await runCli(
|
|
[
|
|
'--agent-enqueue',
|
|
'--init',
|
|
state.projectRoot,
|
|
mainAgentId,
|
|
state.mcp.normalRunId,
|
|
normalTask,
|
|
],
|
|
{ timeoutMs: 120_000 },
|
|
);
|
|
await claimOwnedRunner();
|
|
const normalRuntime = await waitForMcpRuntime(state.mcp.normalRunId);
|
|
state.mcp.sessionId = normalRuntime.sessionId;
|
|
state.initialSessionId = normalRuntime.sessionId;
|
|
const completed = await driveMcpNormalRuntimeToCompletion();
|
|
assert(
|
|
completed.sessionId === state.mcp.sessionId &&
|
|
completed.runId === state.mcp.normalRunId,
|
|
'mcp-normal-runtime-identity-invalid',
|
|
);
|
|
await waitForMcpMarker(state.mcp.normalMarkerPath, mcpMutationValue);
|
|
|
|
const killTask = buildMcpKillTaskPrompt();
|
|
assertMcpTaskPrompt(killTask, 'kill');
|
|
await runCli(
|
|
[
|
|
'--agent-enqueue',
|
|
state.projectRoot,
|
|
mainAgentId,
|
|
state.mcp.killRunId,
|
|
killTask,
|
|
],
|
|
{ timeoutMs: 120_000 },
|
|
);
|
|
const killRuntime = await waitForMcpRuntime(state.mcp.killRunId);
|
|
assert(
|
|
killRuntime.sessionId === state.mcp.sessionId,
|
|
'mcp-kill-session-changed',
|
|
);
|
|
const pending = await waitForMcpKillPendingAction();
|
|
state.mcp.killActionId = pending.actionId;
|
|
const killSidecar = mcpResultSidecarPath(
|
|
state.mcp.killRunId,
|
|
pending.actionId,
|
|
);
|
|
assert(
|
|
(await readMcpMarkerLines(state.mcp.killMarkerPath)).length === 0 &&
|
|
!(await fs.lstat(killSidecar).catch(() => null)),
|
|
'mcp-kill-side-effect-before-confirmation',
|
|
);
|
|
const beforeKill = await readRunnerStatus();
|
|
state.mcp.oldRunnerBootId = runnerBootId(beforeKill);
|
|
assert(
|
|
isNonEmptyString(state.mcp.oldRunnerBootId),
|
|
'mcp-kill-runner-boot-missing',
|
|
);
|
|
await claimOwnedRunner(beforeKill);
|
|
await runCli(
|
|
[
|
|
'--agent-confirm',
|
|
state.projectRoot,
|
|
pending.agentId,
|
|
pending.runId,
|
|
pending.actionId,
|
|
],
|
|
{ timeoutMs: 120_000 },
|
|
);
|
|
state.confirmedActionIds.add(pending.actionId);
|
|
await waitForMcpMarker(state.mcp.killMarkerPath, mcpKillMutationValue);
|
|
assert(
|
|
!(await fs.lstat(killSidecar).catch(() => null)),
|
|
'mcp-kill-sidecar-landed-before-runner-kill',
|
|
);
|
|
|
|
await killRunnerOnce();
|
|
assert(
|
|
!(await fs.lstat(killSidecar).catch(() => null)),
|
|
'mcp-kill-sidecar-landed-after-runner-kill',
|
|
);
|
|
await runCli(['--agent-resume', state.projectRoot], { timeoutMs: 120_000 });
|
|
state.resumed = true;
|
|
const restarted = await waitForRunnerBootChange(state.mcp.oldRunnerBootId);
|
|
state.mcp.newRunnerBootId = runnerBootId(restarted);
|
|
await claimOwnedRunner(restarted);
|
|
await waitForMcpKillReconciliation();
|
|
await sleep(mcpMutateResponseDelayMs + 500);
|
|
await waitForMcpMarker(state.mcp.killMarkerPath, mcpKillMutationValue);
|
|
assert(
|
|
!(await fs.lstat(killSidecar).catch(() => null)),
|
|
'mcp-kill-sidecar-created-during-recovery',
|
|
);
|
|
state.identityStable = true;
|
|
state.evidence = await validateMcpRuntimeEvidence();
|
|
assert(state.evidence.secretLeakCount === 0, 'loaded-key-leak-detected');
|
|
}
|
|
|
|
export async function readMcpPersistenceEvidence() {
|
|
const persistence = await readProcessPersistenceEvidence();
|
|
const sidecarFiles = (
|
|
await listFiles(path.join(state.projectRoot, '.agent/runtime/mcp-results'))
|
|
).filter((file) => file.endsWith('.json'));
|
|
const sidecars = [];
|
|
for (const file of sidecarFiles) {
|
|
sidecars.push({ file, value: await readJson(file) });
|
|
}
|
|
return { ...persistence, sidecars };
|
|
}
|
|
|
|
export function validateMcpReceipt(record, expectedRunId) {
|
|
assert(
|
|
record.recordType === 'agent.runtime.action_receipt' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === expectedRunId &&
|
|
record.sessionId === state.mcp.sessionId &&
|
|
record.tool === 'mcp.call' &&
|
|
record.status === 'ok' &&
|
|
/^[0-9a-f]{64}$/u.test(record.actionFingerprint ?? '') &&
|
|
isNonEmptyString(record.inputSummary) &&
|
|
record.detailUnavailable === false &&
|
|
isNonEmptyString(record.safeDetail),
|
|
'mcp-normal-receipt-identity-invalid',
|
|
);
|
|
const server = auditInputValue(record.inputSummary, 'server');
|
|
const tool = auditInputValue(record.inputSummary, 'tool');
|
|
assert(
|
|
isNonEmptyString(server) &&
|
|
isNonEmptyString(tool) &&
|
|
isNonEmptyString(auditInputValue(record.inputSummary, 'argumentKeys')) &&
|
|
/^[0-9]+$/u.test(
|
|
auditInputValue(record.inputSummary, 'argumentsChars') ?? '',
|
|
) &&
|
|
/^[0-9a-f]{64}$/u.test(
|
|
auditInputValue(record.inputSummary, 'argumentsSha256') ?? '',
|
|
) &&
|
|
/^[0-9a-f]{12}$/u.test(
|
|
auditInputValue(record.inputSummary, 'catalog') ?? '',
|
|
) &&
|
|
/^[0-9a-f]{12}$/u.test(
|
|
auditInputValue(record.inputSummary, 'toolFingerprint') ?? '',
|
|
),
|
|
'mcp-normal-receipt-input-summary-invalid',
|
|
);
|
|
let detail;
|
|
try {
|
|
detail = JSON.parse(record.safeDetail);
|
|
} catch (error) {
|
|
throw codedError('mcp-normal-receipt-detail-invalid', error);
|
|
}
|
|
assert(
|
|
hasExactKeys(detail, [
|
|
'binaryBlockCount',
|
|
'contentBlockCount',
|
|
'isError',
|
|
'resultRef',
|
|
'resultSha256',
|
|
'server',
|
|
'structuredContentChars',
|
|
'textChars',
|
|
'tool',
|
|
]) &&
|
|
detail.server === server &&
|
|
detail.tool === tool &&
|
|
detail.isError === false &&
|
|
/^\.agent\/runtime\/mcp-results\/.+\.json$/u.test(
|
|
detail.resultRef ?? '',
|
|
) &&
|
|
/^[0-9a-f]{64}$/u.test(detail.resultSha256 ?? ''),
|
|
'mcp-normal-receipt-safe-detail-invalid',
|
|
);
|
|
return { server, tool, detail };
|
|
}
|
|
|
|
export function validateMcpPublicLeakBoundary(persistence) {
|
|
const surfaces = {
|
|
task: persistence.taskSnapshot.all,
|
|
event: persistence.events,
|
|
agentDb: persistence.agentDb,
|
|
conversation: persistence.conversations,
|
|
activity: persistence.activities,
|
|
output: persistence.outputs,
|
|
runtimeState: [persistence.runtimeState],
|
|
};
|
|
const groups = {
|
|
privateValue: mcpPrivateBodyValues(),
|
|
credential: [mcpBearerToken, mcpHeaderValue],
|
|
absolutePath: mcpPrivateAbsolutePathValues(),
|
|
};
|
|
const totals = {
|
|
privateValue: 0,
|
|
credential: 0,
|
|
absolutePath: 0,
|
|
};
|
|
for (const [surface, records] of Object.entries(surfaces)) {
|
|
const serialized = Buffer.from(
|
|
records.map((record) => JSON.stringify(record)).join('\n'),
|
|
);
|
|
for (const [group, values] of Object.entries(groups)) {
|
|
const count = countExactSecrets(serialized, values);
|
|
totals[group] += count;
|
|
assert(count === 0, `mcp-public-${surface}-${group}-leak`);
|
|
}
|
|
}
|
|
const projectPathCounts = validateProjectRootPublicLeakBoundary(
|
|
surfaces,
|
|
'mcp-public',
|
|
);
|
|
const formalConfigPathCounts = {};
|
|
for (const [surface, records] of Object.entries(surfaces)) {
|
|
const count = countExactSecrets(
|
|
Buffer.from(records.map((record) => JSON.stringify(record)).join('\n')),
|
|
formalConfigPathVariants(),
|
|
);
|
|
formalConfigPathCounts[surface] = count;
|
|
assert(count === 0, `mcp-public-${surface}-formal-config-path-leak`);
|
|
}
|
|
return {
|
|
privateValue: totals.privateValue,
|
|
credential: totals.credential,
|
|
absolutePath: totals.absolutePath,
|
|
projectPath: sumObjectValues(projectPathCounts),
|
|
projectPathSurfaceCount: Object.keys(projectPathCounts).length,
|
|
formalConfigPath: sumObjectValues(formalConfigPathCounts),
|
|
formalConfigPathSurfaceCount: Object.keys(formalConfigPathCounts).length,
|
|
};
|
|
}
|
|
|
|
export async function validateMcpRuntimeEvidence() {
|
|
const persistence = await readMcpPersistenceEvidence();
|
|
const normalTasks = persistence.taskSnapshot.all.filter(
|
|
(task) =>
|
|
task.agentId === mainAgentId && task.runId === state.mcp.normalRunId,
|
|
);
|
|
const killTasks = persistence.taskSnapshot.all.filter(
|
|
(task) =>
|
|
task.agentId === mainAgentId && task.runId === state.mcp.killRunId,
|
|
);
|
|
const normalCompleted = normalTasks.filter(
|
|
(task) => task.status === 'completed' && task.phase === 'completed',
|
|
);
|
|
const killReconciliation = killTasks.filter(
|
|
(task) => task.status === 'failed' && task.phase === 'needs-reconciliation',
|
|
);
|
|
assert(
|
|
normalCompleted.length === 1 && killReconciliation.length === 1,
|
|
'mcp-run-terminal-projection-count-invalid',
|
|
);
|
|
|
|
const normalReceipts = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.action_receipt' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.mcp.normalRunId &&
|
|
record.tool === 'mcp.call',
|
|
);
|
|
const killReceipts = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.action_receipt' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.mcp.killRunId &&
|
|
record.tool === 'mcp.call',
|
|
);
|
|
assert(
|
|
normalReceipts.length === 3 && killReceipts.length === 0,
|
|
'mcp-action-receipt-count-invalid',
|
|
);
|
|
const receiptDetails = normalReceipts.map((record) =>
|
|
validateMcpReceipt(record, state.mcp.normalRunId),
|
|
);
|
|
const receiptCombos = countBy(
|
|
receiptDetails.map(({ server, tool }) => `${server}/${tool}`),
|
|
);
|
|
assert(
|
|
receiptCombos.get('stdio-fixture/lookup') === 1 &&
|
|
receiptCombos.get('http-fixture/lookup') === 1 &&
|
|
receiptCombos.get('stdio-fixture/mutate') === 1 &&
|
|
receiptCombos.size === 3,
|
|
'mcp-normal-tool-coverage-invalid',
|
|
);
|
|
|
|
const normalSidecars = persistence.sidecars.filter(
|
|
({ value }) => value.runId === state.mcp.normalRunId,
|
|
);
|
|
const killSidecars = persistence.sidecars.filter(
|
|
({ value }) => value.runId === state.mcp.killRunId,
|
|
);
|
|
assert(
|
|
normalSidecars.length === 3 && killSidecars.length === 0,
|
|
'mcp-result-sidecar-count-invalid',
|
|
);
|
|
const sidecarCombos = countBy(
|
|
normalSidecars.map(({ value }) => `${value.server}/${value.tool}`),
|
|
);
|
|
assert(
|
|
sidecarCombos.get('stdio-fixture/lookup') === 1 &&
|
|
sidecarCombos.get('http-fixture/lookup') === 1 &&
|
|
sidecarCombos.get('stdio-fixture/mutate') === 1 &&
|
|
sidecarCombos.size === 3,
|
|
'mcp-sidecar-tool-coverage-invalid',
|
|
);
|
|
for (const { file, value } of normalSidecars) {
|
|
const matchingReceipt = normalReceipts.find(
|
|
(record) => record.actionId === value.actionId,
|
|
);
|
|
const serializedResult = JSON.stringify(value.result);
|
|
assert(
|
|
Boolean(matchingReceipt) &&
|
|
value.schemaVersion === 'game-creator-runtime-mcp-result.v1' &&
|
|
value.agentId === mainAgentId &&
|
|
value.sessionId === state.mcp.sessionId &&
|
|
value.runId === state.mcp.normalRunId &&
|
|
value.actionFingerprint === matchingReceipt.actionFingerprint &&
|
|
/^[0-9a-f]{64}$/u.test(value.argumentsSha256 ?? '') &&
|
|
/^[0-9a-f]{64}$/u.test(value.resultSha256 ?? '') &&
|
|
Number.isSafeInteger(value.argumentsChars) &&
|
|
value.argumentsChars > 0 &&
|
|
Number.isSafeInteger(value.resultBytes) &&
|
|
value.resultBytes > 0 &&
|
|
value.isError === false &&
|
|
path.resolve(file) ===
|
|
path.resolve(mcpResultSidecarPath(value.runId, value.actionId)) &&
|
|
(value.tool !== 'lookup' ||
|
|
(value.server === 'stdio-fixture'
|
|
? serializedResult.includes(`lookup:${mcpStdioQuery}`) &&
|
|
serializedResult.includes('"transport":"stdio"')
|
|
: serializedResult.includes(`lookup:${mcpHttpQuery}`) &&
|
|
serializedResult.includes('"transport":"http"'))) &&
|
|
(value.tool !== 'mutate' ||
|
|
serializedResult.includes(`mutated:${mcpMutationValue}`)),
|
|
'mcp-result-sidecar-content-invalid',
|
|
);
|
|
}
|
|
|
|
const normalApprovals = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.tool_confirmation.approved' &&
|
|
record.runId === state.mcp.normalRunId &&
|
|
record.tool === 'mcp.call',
|
|
);
|
|
const killApprovals = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.tool_confirmation.approved' &&
|
|
record.runId === state.mcp.killRunId &&
|
|
record.tool === 'mcp.call',
|
|
);
|
|
const killReconciliationAudits = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType ===
|
|
'agent.runtime.tool_confirmation.needs_reconciliation' &&
|
|
record.runId === state.mcp.killRunId &&
|
|
record.tool === 'mcp.call',
|
|
);
|
|
const killExecuting = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.tool_action.executing' &&
|
|
record.runId === state.mcp.killRunId &&
|
|
record.tool === 'mcp.call',
|
|
);
|
|
assert(
|
|
normalApprovals.length === 1 &&
|
|
killApprovals.length === 1 &&
|
|
killReconciliationAudits.length === 1 &&
|
|
killExecuting.length === 0 &&
|
|
normalApprovals[0].actionId === state.mcp.normalActionIds[0] &&
|
|
killApprovals[0].actionId === state.mcp.killActionId &&
|
|
killReconciliationAudits[0].actionId === state.mcp.killActionId &&
|
|
killReconciliationAudits[0].pendingStatus === 'executing',
|
|
'mcp-confirmation-and-reconciliation-identity-invalid',
|
|
);
|
|
|
|
const normalMessageId = finalMessageId(
|
|
mainAgentId,
|
|
state.mcp.sessionId,
|
|
state.mcp.normalRunId,
|
|
);
|
|
const killMessageId = finalMessageId(
|
|
mainAgentId,
|
|
state.mcp.sessionId,
|
|
state.mcp.killRunId,
|
|
);
|
|
const normalAssistants = persistence.conversations.filter(
|
|
(message) =>
|
|
message.role === 'assistant' && message.messageId === normalMessageId,
|
|
);
|
|
const killAssistants = persistence.conversations.filter(
|
|
(message) =>
|
|
message.role === 'assistant' && message.messageId === killMessageId,
|
|
);
|
|
const normalAssistantAudits = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'conversation.message' &&
|
|
record.role === 'assistant' &&
|
|
record.messageId === normalMessageId,
|
|
);
|
|
const killAssistantAudits = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'conversation.message' &&
|
|
record.role === 'assistant' &&
|
|
record.messageId === killMessageId,
|
|
);
|
|
assert(
|
|
normalAssistants.length === 1 &&
|
|
normalAssistantAudits.length === 1 &&
|
|
killAssistants.length === 0 &&
|
|
killAssistantAudits.length === 0,
|
|
'mcp-final-assistant-count-invalid',
|
|
);
|
|
|
|
const normalMarkerLines = await readMcpMarkerLines(
|
|
state.mcp.normalMarkerPath,
|
|
);
|
|
const killMarkerLines = await readMcpMarkerLines(state.mcp.killMarkerPath);
|
|
assert(
|
|
normalMarkerLines.length === 1 &&
|
|
normalMarkerLines[0] === mcpMutationValue &&
|
|
killMarkerLines.length === 1 &&
|
|
killMarkerLines[0] === mcpKillMutationValue,
|
|
'mcp-final-marker-count-invalid',
|
|
);
|
|
assert(
|
|
persistence.runtimeState.agentId === mainAgentId &&
|
|
persistence.runtimeState.runId === state.mcp.killRunId &&
|
|
persistence.runtimeState.sessionId === state.mcp.sessionId &&
|
|
persistence.runtimeState.status === 'failed' &&
|
|
persistence.runtimeState.phase === 'needs-reconciliation',
|
|
'mcp-final-runtime-state-invalid',
|
|
);
|
|
|
|
const normalProtocols = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.tool_plan.protocol' &&
|
|
record.runId === state.mcp.normalRunId &&
|
|
supportedToolPlanProtocols.has(record.protocol),
|
|
);
|
|
const killProtocols = persistence.agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.tool_plan.protocol' &&
|
|
record.runId === state.mcp.killRunId &&
|
|
supportedToolPlanProtocols.has(record.protocol),
|
|
);
|
|
assert(
|
|
normalProtocols.length > 0 && killProtocols.length > 0,
|
|
'mcp-provider-tool-plan-protocol-missing',
|
|
);
|
|
|
|
const duplicateActionCount = duplicateCount(
|
|
[
|
|
...normalReceipts.map((record) => record.actionId),
|
|
...killApprovals.map((record) => record.actionId),
|
|
].filter(Boolean),
|
|
);
|
|
const duplicateReceiptCount = duplicateCount(
|
|
normalReceipts.map(receiptAuditIdentity),
|
|
);
|
|
const duplicateMessageCount = duplicateCount(
|
|
persistence.conversations
|
|
.map((message) => message.messageId)
|
|
.filter(Boolean),
|
|
);
|
|
assert(
|
|
duplicateActionCount === 0 &&
|
|
duplicateReceiptCount === 0 &&
|
|
duplicateMessageCount === 0,
|
|
'mcp-duplicate-public-evidence-detected',
|
|
);
|
|
const publicLeaks = validateMcpPublicLeakBoundary(persistence);
|
|
state.lureLeakCount = await countLureLeaks();
|
|
assert(state.lureLeakCount === 0, 'sensitive-lure-leak-detected');
|
|
|
|
return {
|
|
scenario: 'mcp-transports-confirmation-and-runner-kill',
|
|
configuredServerCount: 2,
|
|
configuredToolCount: 4,
|
|
normalRunCompleted: true,
|
|
normalRunActionCount: normalReceipts.length,
|
|
normalRunReceiptCount: normalReceipts.length,
|
|
normalRunSidecarCount: normalSidecars.length,
|
|
normalRunAssistantCount: normalAssistants.length,
|
|
normalRunAssistantAuditCount: normalAssistantAudits.length,
|
|
normalRunConfirmationCount: normalApprovals.length,
|
|
stdioLookupCount: receiptCombos.get('stdio-fixture/lookup') ?? 0,
|
|
httpLookupCount: receiptCombos.get('http-fixture/lookup') ?? 0,
|
|
stdioMutationCount: receiptCombos.get('stdio-fixture/mutate') ?? 0,
|
|
normalMutationMarkerCount: normalMarkerLines.length,
|
|
killRunReconciliationCount: killReconciliationAudits.length,
|
|
killRunActionCount: killApprovals.length,
|
|
killRunReceiptCount: killReceipts.length,
|
|
killRunSidecarCount: killSidecars.length,
|
|
killRunAssistantCount: killAssistants.length,
|
|
killRunAssistantAuditCount: killAssistantAudits.length,
|
|
killRunConfirmationCount: killApprovals.length,
|
|
killMutationMarkerCount: killMarkerLines.length,
|
|
runnerBootChanged:
|
|
isNonEmptyString(state.mcp.oldRunnerBootId) &&
|
|
isNonEmptyString(state.mcp.newRunnerBootId) &&
|
|
state.mcp.oldRunnerBootId !== state.mcp.newRunnerBootId,
|
|
duplicateActionCount,
|
|
duplicateReceiptCount,
|
|
duplicateMessageCount,
|
|
publicPrivateValueLeakCount: publicLeaks.privateValue,
|
|
publicCredentialLeakCount: publicLeaks.credential,
|
|
publicAbsolutePathLeakCount: publicLeaks.absolutePath,
|
|
projectPathPublicLeakCount: publicLeaks.projectPath,
|
|
projectPathPublicSurfaceCount: publicLeaks.projectPathSurfaceCount,
|
|
formalConfigPathPublicLeakCount: publicLeaks.formalConfigPath,
|
|
formalConfigPathPublicSurfaceCount:
|
|
publicLeaks.formalConfigPathSurfaceCount,
|
|
taskCount: persistence.taskSnapshot.all.length,
|
|
eventCount: persistence.events.length,
|
|
agentDbRecordCount: persistence.agentDb.length,
|
|
conversationMessageCount: persistence.conversations.length,
|
|
actionReceiptCount: normalReceipts.length + killReceipts.length,
|
|
secretLeakCount: state.transcriptLeakCount + state.projectLeakCount,
|
|
lureLeakCount: state.lureLeakCount,
|
|
paths: [
|
|
'.agent/runtime/tasks',
|
|
'.agent/runtime/events',
|
|
'.agent/agent.db',
|
|
'.agent/runtime/mcp-results',
|
|
'.agent/conversations',
|
|
'.agent/activity.jsonl',
|
|
'.agent/output.jsonl',
|
|
`.agent/runtime/agents/${mainAgentId}.json`,
|
|
],
|
|
};
|
|
}
|
|
|
|
export function emptyMcpEvidence() {
|
|
return {
|
|
scenario: 'mcp-transports-confirmation-and-runner-kill',
|
|
isolatedAppDataUsed: false,
|
|
formalConfigCliCallCount: 0,
|
|
sourceRunnerEndpointUnchanged: false,
|
|
sourceConfigReplicaCount: 0,
|
|
sourceConfigReplicasVerified: false,
|
|
configuredServerCount: 2,
|
|
configuredToolCount: 4,
|
|
normalRunCompleted: false,
|
|
normalRunActionCount: 0,
|
|
normalRunReceiptCount: 0,
|
|
normalRunSidecarCount: 0,
|
|
normalRunAssistantCount: 0,
|
|
normalRunAssistantAuditCount: 0,
|
|
normalRunConfirmationCount: 0,
|
|
stdioLookupCount: 0,
|
|
httpLookupCount: 0,
|
|
stdioMutationCount: 0,
|
|
normalMutationMarkerCount: 0,
|
|
killRunReconciliationCount: 0,
|
|
killRunActionCount: 0,
|
|
killRunReceiptCount: 0,
|
|
killRunSidecarCount: 0,
|
|
killRunAssistantCount: 0,
|
|
killRunAssistantAuditCount: 0,
|
|
killRunConfirmationCount: 0,
|
|
killMutationMarkerCount: 0,
|
|
runnerBootChanged: false,
|
|
duplicateActionCount: 0,
|
|
duplicateReceiptCount: 0,
|
|
duplicateMessageCount: 0,
|
|
publicPrivateValueLeakCount: 0,
|
|
publicCredentialLeakCount: 0,
|
|
publicAbsolutePathLeakCount: 0,
|
|
projectPathPublicLeakCount: 0,
|
|
projectPathPublicSurfaceCount: 0,
|
|
formalConfigPathPublicLeakCount: 0,
|
|
formalConfigPathPublicSurfaceCount: 0,
|
|
taskCount: 0,
|
|
eventCount: 0,
|
|
agentDbRecordCount: 0,
|
|
conversationMessageCount: 0,
|
|
actionReceiptCount: 0,
|
|
mcpReportLeakCount: 0,
|
|
mcpRunnerKillMethod: null,
|
|
mcpRunnerPidfdClaimCount: 0,
|
|
mcpRunnerPidfdSignalCount: 0,
|
|
mcpRunnerStopped: false,
|
|
mcpAppDataCleanupPerformed: false,
|
|
httpFixtureStopped: false,
|
|
secretLeakCount: 0,
|
|
lureLeakCount: 0,
|
|
paths: [],
|
|
};
|
|
}
|
|
|
|
export async function collectPartialMcpEvidence() {
|
|
const [tasks, events, agentDb, conversations, normalLines, killLines] =
|
|
await Promise.all([
|
|
readTaskSnapshot().catch(() => ({ all: [], latest: [] })),
|
|
readAllRuntimeEvents().catch(() => []),
|
|
readOptionalJsonl(path.join(state.projectRoot, '.agent/agent.db')).catch(
|
|
() => [],
|
|
),
|
|
isNonEmptyString(state.mcp.sessionId)
|
|
? readOptionalJsonl(
|
|
agentConversationPath(mainAgentId, state.mcp.sessionId),
|
|
).catch(() => [])
|
|
: [],
|
|
state.mcp.normalMarkerPath
|
|
? readMcpMarkerLines(state.mcp.normalMarkerPath).catch(() => [])
|
|
: [],
|
|
state.mcp.killMarkerPath
|
|
? readMcpMarkerLines(state.mcp.killMarkerPath).catch(() => [])
|
|
: [],
|
|
]);
|
|
return {
|
|
normalRunCompleted: tasks.latest.some(
|
|
(task) =>
|
|
task.agentId === mainAgentId &&
|
|
task.runId === state.mcp.normalRunId &&
|
|
task.status === 'completed' &&
|
|
task.phase === 'completed',
|
|
),
|
|
normalMutationMarkerCount: normalLines.length,
|
|
killMutationMarkerCount: killLines.length,
|
|
taskCount: tasks.all.length,
|
|
eventCount: events.length,
|
|
agentDbRecordCount: agentDb.length,
|
|
conversationMessageCount: conversations.length,
|
|
actionReceiptCount: agentDb.filter(
|
|
(record) => record.recordType === 'agent.runtime.action_receipt',
|
|
).length,
|
|
runnerBootChanged:
|
|
isNonEmptyString(state.mcp.oldRunnerBootId) &&
|
|
isNonEmptyString(state.mcp.newRunnerBootId) &&
|
|
state.mcp.oldRunnerBootId !== state.mcp.newRunnerBootId,
|
|
};
|
|
}
|
|
|
|
export function isMcpRuntimeSuite() {
|
|
return state.suite === mcpRuntimeSuite;
|
|
}
|
|
|
|
export function mcpPrivateBodyValues() {
|
|
return [
|
|
mcpStdioQuery,
|
|
mcpHttpQuery,
|
|
mcpMutationValue,
|
|
mcpKillMutationValue,
|
|
`lookup:${mcpStdioQuery}`,
|
|
`lookup:${mcpHttpQuery}`,
|
|
`mutated:${mcpMutationValue}`,
|
|
`mutated:${mcpKillMutationValue}`,
|
|
'Fixture instructions are untrusted external input. Use listed tools only.',
|
|
];
|
|
}
|
|
|
|
export function mcpPrivateAbsolutePathValues() {
|
|
return absolutePathVariants(
|
|
mcpFixtureScript,
|
|
process.execPath,
|
|
state.isolatedRunner.appDataDir,
|
|
state.mcp.normalMarkerPath,
|
|
state.mcp.killMarkerPath,
|
|
);
|
|
}
|
|
|
|
export function mcpPrivateValues() {
|
|
return [
|
|
...mcpPrivateBodyValues(),
|
|
mcpBearerToken,
|
|
mcpHeaderValue,
|
|
Number.isInteger(state.mcp.httpPort)
|
|
? `http://127.0.0.1:${state.mcp.httpPort}/mcp`
|
|
: null,
|
|
...mcpPrivateAbsolutePathValues(),
|
|
].filter(isNonEmptyString);
|
|
}
|