AIGameCreator App
be89296492
拆分 App 认证、壳层、运行配置与项目摘要模块 拆分 Tauri 项目能力与 Rust 测试领域模块 拆分界面测试与 Agent Runtime 真实 E2E 套件 补充源码扫描和客户端模块化文档约定
955 lines
32 KiB
JavaScript
955 lines
32 KiB
JavaScript
import {
|
|
assert,
|
|
codedError,
|
|
hashValue,
|
|
isFailedTask,
|
|
sleep,
|
|
} from '../assertions/core.mjs';
|
|
import {
|
|
auditInputValue,
|
|
auditPathEquals,
|
|
countExactSecrets,
|
|
disposableProjectPathVariants,
|
|
duplicateCount,
|
|
findSuccessfulToolExecution,
|
|
isNonEmptyString,
|
|
receiptAuditIdentity,
|
|
requireExecutionRecord,
|
|
requireSuccessfulToolExecution,
|
|
sumObjectValues,
|
|
validateMainRunToolPlanProtocols,
|
|
} from '../assertions/runtime.mjs';
|
|
import { fs, path } from '../dependencies.mjs';
|
|
import {
|
|
claimOwnedRunner,
|
|
ensureOwnedRunnerStableKillSupport,
|
|
prepareIsolatedSuiteAppData,
|
|
} from '../harness/app-data.mjs';
|
|
import { listFiles, readJson, readOptionalJsonl } from '../harness/io.mjs';
|
|
import { prepareCliBinary, runCli, runProcess } from '../harness/process.mjs';
|
|
import { seedDisposableProject } from '../harness/project.mjs';
|
|
import {
|
|
agentConversationPath,
|
|
confirmPendingActions,
|
|
countLureLeaks,
|
|
countSecretsInProject,
|
|
countSensitiveValuesBySurface,
|
|
findPendingActions,
|
|
hasExpectedWorkspaceSandboxMetadata,
|
|
mainContextBundlePath,
|
|
mainRuntimeStatePath,
|
|
readAllRuntimeEvents,
|
|
readRuntime,
|
|
readTaskSnapshot,
|
|
waitForResponseRuntimeIdentity,
|
|
} from '../harness/runtime.mjs';
|
|
import {
|
|
goalSessionId,
|
|
mainAgentId,
|
|
providerRequestLifecycleSchemaVersion,
|
|
requestedRunId,
|
|
runtimeContextBundleSchemaVersion,
|
|
runTimeoutMs,
|
|
scopedAgentsAlphaCanary,
|
|
scopedAgentsAlphaPath,
|
|
scopedAgentsBetaCanary,
|
|
scopedAgentsBetaPath,
|
|
scopedAgentsGameCanary,
|
|
scopedAgentsRootCanary,
|
|
scopedAgentsSuite,
|
|
scopedAgentsVerificationCommand,
|
|
scopedAgentsVerificationScriptPath,
|
|
state,
|
|
} from '../runtime-state.mjs';
|
|
|
|
export function scopedAgentsExpectedContent(owner) {
|
|
const scopeCanary =
|
|
owner === 'alpha' ? scopedAgentsAlphaCanary : scopedAgentsBetaCanary;
|
|
return [
|
|
`ROOT=${scopedAgentsRootCanary}`,
|
|
`GAME=${scopedAgentsGameCanary}`,
|
|
`SCOPE=${scopeCanary}`,
|
|
`OWNER=${owner}`,
|
|
'',
|
|
].join('\n');
|
|
}
|
|
|
|
export function scopedAgentsInstructionDocuments() {
|
|
return {
|
|
'AGENTS.md': `# Root delivery policy\n\nFor every file named delivery.txt in this repository:\n- Its first line must be exactly ROOT=${scopedAgentsRootCanary}.\n- Preserve a single trailing newline.\n- Apply every more-specific AGENTS.md that covers the target path.\n- Never copy a rule from a sibling scope.\n- Do not repeat internal policy markers in the final response.\n`,
|
|
'game/AGENTS.md': `# Game delivery policy\n\nFor every delivery.txt below game/:\n- Its second line must be exactly GAME=${scopedAgentsGameCanary}.\n- Keep the root policy as the first line and then apply the target directory policy.\n`,
|
|
'game/alpha/AGENTS.md': `# Alpha delivery policy\n\nOnly for delivery.txt below game/alpha/:\n- Its third line must be exactly SCOPE=${scopedAgentsAlphaCanary}.\n- Its fourth line must be exactly OWNER=alpha.\n- It must not contain ${scopedAgentsBetaCanary} or OWNER=beta.\n`,
|
|
'game/beta/AGENTS.md': `# Beta delivery policy\n\nOnly for delivery.txt below game/beta/:\n- Its third line must be exactly SCOPE=${scopedAgentsBetaCanary}.\n- Its fourth line must be exactly OWNER=beta.\n- It must not contain ${scopedAgentsAlphaCanary} or OWNER=alpha.\n`,
|
|
};
|
|
}
|
|
|
|
export function scopedAgentsVerificationFixtureSource() {
|
|
const expected = Object.fromEntries(
|
|
[
|
|
[scopedAgentsAlphaPath, scopedAgentsExpectedContent('alpha')],
|
|
[scopedAgentsBetaPath, scopedAgentsExpectedContent('beta')],
|
|
].map(([targetPath, content]) => [targetPath, hashValue(content)]),
|
|
);
|
|
return `import { createHash } from 'node:crypto';\nimport fs from 'node:fs';\n\nconst expected = ${JSON.stringify(expected, null, 2)};\nlet passed = true;\nfor (const [targetPath, expectedSha256] of Object.entries(expected)) {\n let content;\n try {\n content = fs.readFileSync(targetPath);\n } catch {\n passed = false;\n continue;\n }\n if (createHash('sha256').update(content).digest('hex') !== expectedSha256) {\n passed = false;\n }\n}\nif (!passed) {\n console.error('scoped-agents=failed');\n process.exit(1);\n}\nconsole.log('scoped-agents=passed');\n`;
|
|
}
|
|
|
|
export async function seedScopedAgentsDisposableProject() {
|
|
await seedDisposableProject();
|
|
const documents = scopedAgentsInstructionDocuments();
|
|
const alphaExpected = scopedAgentsExpectedContent('alpha');
|
|
const betaExpected = scopedAgentsExpectedContent('beta');
|
|
state.scopedAgents.privateValues = [
|
|
scopedAgentsRootCanary,
|
|
scopedAgentsGameCanary,
|
|
scopedAgentsAlphaCanary,
|
|
scopedAgentsBetaCanary,
|
|
alphaExpected,
|
|
betaExpected,
|
|
...Object.values(documents),
|
|
];
|
|
const verificationSource = scopedAgentsVerificationFixtureSource();
|
|
assert(
|
|
countExactSecrets(
|
|
Buffer.from(verificationSource),
|
|
state.scopedAgents.privateValues,
|
|
) === 0,
|
|
'scoped-agents-verifier-reveals-instruction-content',
|
|
);
|
|
|
|
await Promise.all([
|
|
fs.mkdir(path.join(state.projectRoot, 'game/alpha'), { recursive: true }),
|
|
fs.mkdir(path.join(state.projectRoot, 'game/beta'), { recursive: true }),
|
|
]);
|
|
await Promise.all([
|
|
...Object.entries(documents).map(([relativePath, content]) =>
|
|
fs.writeFile(path.join(state.projectRoot, relativePath), content),
|
|
),
|
|
fs.writeFile(
|
|
path.join(state.projectRoot, scopedAgentsAlphaPath),
|
|
'seeded alpha delivery\n',
|
|
),
|
|
fs.writeFile(
|
|
path.join(state.projectRoot, scopedAgentsBetaPath),
|
|
'seeded beta delivery\n',
|
|
),
|
|
fs.writeFile(
|
|
path.join(state.projectRoot, scopedAgentsVerificationScriptPath),
|
|
verificationSource,
|
|
),
|
|
fs.writeFile(
|
|
path.join(state.projectRoot, 'package.json'),
|
|
`${JSON.stringify(
|
|
{
|
|
name: 'genarrative-scoped-agents-real-e2e-project',
|
|
private: true,
|
|
scripts: {
|
|
test: scopedAgentsVerificationCommand,
|
|
'check:e2e': scopedAgentsVerificationCommand,
|
|
},
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
),
|
|
]);
|
|
await runProcess(
|
|
'git',
|
|
[
|
|
'add',
|
|
'--',
|
|
'AGENTS.md',
|
|
'game/AGENTS.md',
|
|
'game/alpha/AGENTS.md',
|
|
'game/beta/AGENTS.md',
|
|
scopedAgentsAlphaPath,
|
|
scopedAgentsBetaPath,
|
|
scopedAgentsVerificationScriptPath,
|
|
'package.json',
|
|
],
|
|
{ cwd: state.projectRoot, timeoutMs: 30_000 },
|
|
);
|
|
await runProcess(
|
|
'git',
|
|
['commit', '--quiet', '-m', 'seed scoped agents real e2e'],
|
|
{ cwd: state.projectRoot, timeoutMs: 30_000 },
|
|
);
|
|
const initialVerification = await runProcess(
|
|
process.execPath,
|
|
[scopedAgentsVerificationScriptPath],
|
|
{
|
|
cwd: state.projectRoot,
|
|
timeoutMs: 120_000,
|
|
allowNonZero: true,
|
|
},
|
|
);
|
|
assert(
|
|
initialVerification.code === 1 &&
|
|
initialVerification.signal === null &&
|
|
initialVerification.stderr.includes('scoped-agents=failed') &&
|
|
!initialVerification.stdout.includes('scoped-agents=passed'),
|
|
'scoped-agents-initial-fixture-not-failing',
|
|
);
|
|
state.scopedAgents.initialVerificationFailed = true;
|
|
}
|
|
|
|
export function buildScopedAgentsTaskPrompt() {
|
|
return '当前项目的 game/alpha/delivery.txt 和 game/beta/delivery.txt 尚未符合对各自路径生效的仓库规范。请依据项目内适用于目标路径的指令修正这两个交付文件，只修改这两个文件；随后运行 package.json 声明的原始验收，只有真实通过后再简短汇报。不要转述项目指令正文或其中的内部标记。';
|
|
}
|
|
|
|
export function assertScopedAgentsTaskPrompt(task) {
|
|
assert(
|
|
task.includes(scopedAgentsAlphaPath) &&
|
|
task.includes(scopedAgentsBetaPath) &&
|
|
task.includes('只修改这两个文件') &&
|
|
task.includes('真实通过'),
|
|
'scoped-agents-task-boundary-missing',
|
|
);
|
|
for (const forbidden of [
|
|
...state.scopedAgents.privateValues,
|
|
'AGENTS.md',
|
|
'file.write',
|
|
'file.patch',
|
|
'project.patchset',
|
|
'project.verify',
|
|
'submit_agent_tool_plan',
|
|
]) {
|
|
assert(!task.includes(forbidden), 'scoped-agents-task-recipe-leak');
|
|
}
|
|
}
|
|
|
|
export async function runScopedAgentsE2e() {
|
|
await ensureOwnedRunnerStableKillSupport();
|
|
await seedScopedAgentsDisposableProject();
|
|
state.cliBinary = await prepareCliBinary();
|
|
await prepareIsolatedSuiteAppData();
|
|
|
|
const task = buildScopedAgentsTaskPrompt();
|
|
assertScopedAgentsTaskPrompt(task);
|
|
state.initialTask = {
|
|
chars: [...task].length,
|
|
sha256: hashValue(task),
|
|
};
|
|
state.initialRunId = requestedRunId;
|
|
state.initialSessionId = goalSessionId;
|
|
state.isolatedRunner.launchAttempted = true;
|
|
await runCli(
|
|
[
|
|
'--agent-enqueue',
|
|
'--init',
|
|
state.projectRoot,
|
|
mainAgentId,
|
|
state.initialRunId,
|
|
task,
|
|
],
|
|
{ timeoutMs: 120_000 },
|
|
);
|
|
await claimOwnedRunner();
|
|
const runtime = await waitForResponseRuntimeIdentity();
|
|
assert(
|
|
runtime.agentId === mainAgentId &&
|
|
runtime.runId === state.initialRunId &&
|
|
runtime.sessionId === state.initialSessionId,
|
|
'scoped-agents-runtime-identity-invalid',
|
|
);
|
|
state.identityStable = true;
|
|
|
|
await driveScopedAgentsRuntimeToCompletion();
|
|
state.evidence = await validateScopedAgentsEvidence();
|
|
assert(state.evidence.secretLeakCount === 0, 'loaded-key-leak-detected');
|
|
}
|
|
|
|
export function normalizeScopedAgentsTargetPath(value) {
|
|
assert(
|
|
isNonEmptyString(value) && !path.posix.isAbsolute(value),
|
|
'scoped-agents-pending-path-invalid',
|
|
);
|
|
const normalized = path.posix
|
|
.normalize(value.replaceAll('\\', '/'))
|
|
.replace(/^\.\//u, '');
|
|
assert(
|
|
[scopedAgentsAlphaPath, scopedAgentsBetaPath].includes(normalized),
|
|
'scoped-agents-pending-path-outside-deliveries',
|
|
);
|
|
return normalized;
|
|
}
|
|
|
|
export function validateScopedAgentsPendingAction(pending) {
|
|
assert(
|
|
pending.agentId === mainAgentId && pending.runId === state.initialRunId,
|
|
'scoped-agents-cross-run-pending-action',
|
|
);
|
|
const input = pending.action?.input ?? pending.record?.action?.input ?? {};
|
|
if (['file.write', 'file.patch'].includes(pending.tool)) {
|
|
normalizeScopedAgentsTargetPath(input.path);
|
|
return;
|
|
}
|
|
if (pending.tool === 'project.patchset') {
|
|
assert(
|
|
Array.isArray(input.changes) &&
|
|
input.changes.length > 0 &&
|
|
input.changes.length <= 2,
|
|
'scoped-agents-patchset-shape-invalid',
|
|
);
|
|
const targets = input.changes.map((change) => {
|
|
assert(
|
|
change?.operation === 'update',
|
|
'scoped-agents-patchset-operation-invalid',
|
|
);
|
|
return normalizeScopedAgentsTargetPath(change.path);
|
|
});
|
|
assert(
|
|
new Set(targets).size === targets.length,
|
|
'scoped-agents-patchset-duplicate-path',
|
|
);
|
|
return;
|
|
}
|
|
if (pending.tool === 'project.checkpoint') return;
|
|
if (pending.tool === 'project.verify') {
|
|
const script = input.script;
|
|
const expectedCommand = input.expectedCommand ?? input.expected_command;
|
|
assert(
|
|
['test', 'check:e2e'].includes(script) &&
|
|
expectedCommand === scopedAgentsVerificationCommand,
|
|
'scoped-agents-verification-request-invalid',
|
|
);
|
|
return;
|
|
}
|
|
throw codedError(`scoped-agents-pending-tool-not-allowed:${pending.tool}`);
|
|
}
|
|
|
|
export async function driveScopedAgentsRuntimeToCompletion() {
|
|
const deadline = Date.now() + runTimeoutMs;
|
|
const allowedTools = new Set([
|
|
'file.write',
|
|
'file.patch',
|
|
'project.patchset',
|
|
'project.checkpoint',
|
|
'project.verify',
|
|
]);
|
|
let quietPolls = 0;
|
|
while (Date.now() < deadline) {
|
|
const pending = await findPendingActions();
|
|
for (const action of pending) validateScopedAgentsPendingAction(action);
|
|
if (pending.length > 0) {
|
|
const before = state.confirmedActionIds.size;
|
|
await confirmPendingActions(
|
|
allowedTools,
|
|
(action) =>
|
|
action.agentId === mainAgentId && action.runId === state.initialRunId,
|
|
);
|
|
state.scopedAgents.confirmedActionCount +=
|
|
state.confirmedActionIds.size - before;
|
|
assert(
|
|
state.scopedAgents.confirmedActionCount <= 8,
|
|
'scoped-agents-confirmation-count-excessive',
|
|
);
|
|
await sleep(100);
|
|
continue;
|
|
}
|
|
|
|
const [runtime, tasks, conversations] = await Promise.all([
|
|
readRuntime(mainAgentId).catch(() => null),
|
|
readTaskSnapshot(),
|
|
readOptionalJsonl(
|
|
agentConversationPath(mainAgentId, state.initialSessionId),
|
|
),
|
|
]);
|
|
const task = tasks.latest.find(
|
|
(candidate) =>
|
|
candidate.agentId === mainAgentId &&
|
|
candidate.runId === state.initialRunId,
|
|
);
|
|
if (task && isFailedTask(task)) {
|
|
throw codedError('scoped-agents-runtime-failed');
|
|
}
|
|
if (runtime?.phase === 'needs-reconciliation') {
|
|
throw codedError('scoped-agents-runtime-needs-reconciliation');
|
|
}
|
|
const completed =
|
|
runtime?.runId === state.initialRunId &&
|
|
runtime?.sessionId === state.initialSessionId &&
|
|
runtime?.status === 'idle' &&
|
|
runtime?.phase === 'completed' &&
|
|
task?.status === 'completed' &&
|
|
task?.phase === 'completed' &&
|
|
conversations.filter((message) => message.role === 'assistant').length ===
|
|
1;
|
|
if (completed) {
|
|
quietPolls += 1;
|
|
if (quietPolls >= 3) return;
|
|
} else {
|
|
quietPolls = 0;
|
|
}
|
|
await sleep(250);
|
|
}
|
|
throw codedError('scoped-agents-runtime-timeout');
|
|
}
|
|
|
|
export async function readScopedAgentsPersistence() {
|
|
const [
|
|
taskSnapshot,
|
|
events,
|
|
agentDb,
|
|
conversations,
|
|
activity,
|
|
output,
|
|
runtimeState,
|
|
contextBundle,
|
|
] = await Promise.all([
|
|
readTaskSnapshot(),
|
|
readAllRuntimeEvents(),
|
|
readOptionalJsonl(path.join(state.projectRoot, '.agent/agent.db')),
|
|
readOptionalJsonl(
|
|
agentConversationPath(mainAgentId, state.initialSessionId),
|
|
),
|
|
readOptionalJsonl(path.join(state.projectRoot, '.agent/activity.jsonl')),
|
|
readOptionalJsonl(path.join(state.projectRoot, '.agent/output.jsonl')),
|
|
readJson(mainRuntimeStatePath()).catch(() => null),
|
|
readJson(mainContextBundlePath()).catch(() => null),
|
|
]);
|
|
return {
|
|
taskSnapshot,
|
|
events,
|
|
agentDb,
|
|
conversations,
|
|
activity,
|
|
output,
|
|
runtimeState,
|
|
contextBundle,
|
|
};
|
|
}
|
|
|
|
export function validateScopedAgentsProviderLifecycle(agentDb) {
|
|
const lifecycle = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.provider_request.lifecycle' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
const byRequest = new Map();
|
|
for (const record of lifecycle) {
|
|
assert(
|
|
record.auditSchemaVersion === providerRequestLifecycleSchemaVersion &&
|
|
isNonEmptyString(record.requestId) &&
|
|
isNonEmptyString(record.requestSlot) &&
|
|
['tool-plan', 'final-reply', 'context-compaction'].includes(
|
|
record.requestKind,
|
|
) &&
|
|
record.webSearchEnabled === false,
|
|
'scoped-agents-provider-lifecycle-record-invalid',
|
|
);
|
|
const records = byRequest.get(record.requestId) ?? [];
|
|
records.push(record);
|
|
byRequest.set(record.requestId, records);
|
|
}
|
|
for (const records of byRequest.values()) {
|
|
assert(
|
|
records.length === 2 &&
|
|
records[0].status === 'started' &&
|
|
records[1].status === 'completed' &&
|
|
records[0].requestKind === records[1].requestKind &&
|
|
records[0].requestSlot === records[1].requestSlot &&
|
|
records[0].sessionId === records[1].sessionId,
|
|
'scoped-agents-provider-lifecycle-sequence-invalid',
|
|
);
|
|
}
|
|
const started = lifecycle.filter((record) => record.status === 'started');
|
|
assert(
|
|
byRequest.size > 0 &&
|
|
started.length === byRequest.size &&
|
|
started.some((record) => record.requestKind === 'tool-plan'),
|
|
'scoped-agents-provider-request-count-invalid',
|
|
);
|
|
return {
|
|
requestIdentityCount: byRequest.size,
|
|
startedCount: started.length,
|
|
terminalCount: lifecycle.filter((record) => record.status === 'completed')
|
|
.length,
|
|
toolPlanCount: started.filter(
|
|
(record) => record.requestKind === 'tool-plan',
|
|
).length,
|
|
finalReplyCount: started.filter(
|
|
(record) => record.requestKind === 'final-reply',
|
|
).length,
|
|
};
|
|
}
|
|
|
|
export function scopedAgentsExecutionTouchesPath(execution, targetPath) {
|
|
return scopedAgentsExecutionPaths(execution).includes(targetPath);
|
|
}
|
|
|
|
export function scopedAgentsExecutionPaths(execution) {
|
|
if (['file.write', 'file.patch'].includes(execution.tool)) {
|
|
return [scopedAgentsAlphaPath, scopedAgentsBetaPath].filter((targetPath) =>
|
|
auditPathEquals(execution.inputSummary, targetPath),
|
|
);
|
|
}
|
|
if (execution.tool !== 'project.patchset') return [];
|
|
return auditInputValue(execution.inputSummary, 'paths')
|
|
.split(',')
|
|
.map((value) => value.trim())
|
|
.filter(Boolean)
|
|
.map((value) => value.slice(value.indexOf(':') + 1))
|
|
.map((value) =>
|
|
path.posix.normalize(value.replaceAll('\\', '/')).replace(/^\.\//u, ''),
|
|
);
|
|
}
|
|
|
|
export function findScopedAgentsMutationExecution(agentDb, targetPath) {
|
|
for (const tool of ['file.write', 'file.patch', 'project.patchset']) {
|
|
const execution = findSuccessfulToolExecution(
|
|
agentDb,
|
|
tool,
|
|
state.initialRunId,
|
|
(candidate) => scopedAgentsExecutionTouchesPath(candidate, targetPath),
|
|
);
|
|
if (execution) return execution;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function assertScopedAgentsMutationBoundaries(agentDb) {
|
|
const mutationTools = new Set([
|
|
'file.write',
|
|
'file.patch',
|
|
'file.delete',
|
|
'project.patchset',
|
|
'project.restore',
|
|
]);
|
|
const starts = agentDb.filter(
|
|
(record) =>
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId &&
|
|
mutationTools.has(record.tool) &&
|
|
[
|
|
'agent.runtime.tool_action.executing',
|
|
'agent.runtime.tool_confirmation.approved',
|
|
].includes(record.recordType),
|
|
);
|
|
assert(starts.length > 0, 'scoped-agents-mutation-action-missing');
|
|
for (const record of starts) {
|
|
assert(
|
|
['file.write', 'file.patch', 'project.patchset'].includes(record.tool),
|
|
'scoped-agents-unexpected-mutation-tool-executed',
|
|
);
|
|
const execution = { tool: record.tool, inputSummary: record.inputSummary };
|
|
const executionPaths = scopedAgentsExecutionPaths(execution);
|
|
assert(
|
|
executionPaths.length > 0 &&
|
|
executionPaths.every((targetPath) =>
|
|
[scopedAgentsAlphaPath, scopedAgentsBetaPath].includes(targetPath),
|
|
),
|
|
'scoped-agents-mutation-outside-deliveries',
|
|
);
|
|
}
|
|
return starts;
|
|
}
|
|
|
|
export async function validateScopedAgentsEvidence() {
|
|
const persistence = await readScopedAgentsPersistence();
|
|
const {
|
|
taskSnapshot,
|
|
events,
|
|
agentDb,
|
|
conversations,
|
|
activity,
|
|
output,
|
|
runtimeState,
|
|
contextBundle,
|
|
} = persistence;
|
|
const latest = taskSnapshot.latest.find(
|
|
(task) => task.agentId === mainAgentId && task.runId === state.initialRunId,
|
|
);
|
|
assert(
|
|
runtimeState?.agentId === mainAgentId &&
|
|
runtimeState.runId === state.initialRunId &&
|
|
runtimeState.sessionId === state.initialSessionId &&
|
|
runtimeState.status === 'idle' &&
|
|
runtimeState.phase === 'completed' &&
|
|
latest?.status === 'completed' &&
|
|
latest.phase === 'completed',
|
|
'scoped-agents-final-runtime-invalid',
|
|
);
|
|
const sourcePaths = contextBundle?.repositoryContextSourcePaths ?? [];
|
|
const requiredSourcePaths = [
|
|
'AGENTS.md',
|
|
'game/AGENTS.md',
|
|
'game/alpha/AGENTS.md',
|
|
'game/beta/AGENTS.md',
|
|
'package.json',
|
|
];
|
|
assert(
|
|
contextBundle?.schemaVersion === runtimeContextBundleSchemaVersion &&
|
|
contextBundle.agentId === mainAgentId &&
|
|
contextBundle.runId === state.initialRunId &&
|
|
/^[0-9a-f]{64}$/u.test(contextBundle.repositoryContextFingerprint) &&
|
|
requiredSourcePaths.every((sourcePath) =>
|
|
sourcePaths.includes(sourcePath),
|
|
),
|
|
'scoped-agents-context-bundle-invalid',
|
|
);
|
|
|
|
const [alphaContent, betaContent, changedFiles, hostVerification] =
|
|
await Promise.all([
|
|
fs.readFile(path.join(state.projectRoot, scopedAgentsAlphaPath), 'utf8'),
|
|
fs.readFile(path.join(state.projectRoot, scopedAgentsBetaPath), 'utf8'),
|
|
runProcess('git', ['diff', '--name-only', 'HEAD', '--'], {
|
|
cwd: state.projectRoot,
|
|
timeoutMs: 30_000,
|
|
}),
|
|
runProcess(process.execPath, [scopedAgentsVerificationScriptPath], {
|
|
cwd: state.projectRoot,
|
|
timeoutMs: 120_000,
|
|
}),
|
|
]);
|
|
const alphaExpected = scopedAgentsExpectedContent('alpha');
|
|
const betaExpected = scopedAgentsExpectedContent('beta');
|
|
assert(
|
|
alphaContent === alphaExpected && betaContent === betaExpected,
|
|
'scoped-agents-delivery-content-invalid',
|
|
);
|
|
assert(
|
|
!alphaContent.includes(scopedAgentsBetaCanary) &&
|
|
!alphaContent.includes('OWNER=beta') &&
|
|
!betaContent.includes(scopedAgentsAlphaCanary) &&
|
|
!betaContent.includes('OWNER=alpha'),
|
|
'scoped-agents-sibling-rule-crossed',
|
|
);
|
|
const changedPaths = changedFiles.stdout
|
|
.split(/\r?\n/u)
|
|
.map((value) => value.trim())
|
|
.filter(Boolean)
|
|
.sort();
|
|
assert(
|
|
JSON.stringify(changedPaths) ===
|
|
JSON.stringify([scopedAgentsAlphaPath, scopedAgentsBetaPath].sort()),
|
|
'scoped-agents-changed-path-set-invalid',
|
|
);
|
|
assert(
|
|
hostVerification.stdout.includes('scoped-agents=passed') &&
|
|
!hostVerification.stderr.includes('scoped-agents=failed'),
|
|
'scoped-agents-host-verification-failed',
|
|
);
|
|
|
|
const alphaExecution = findScopedAgentsMutationExecution(
|
|
agentDb,
|
|
scopedAgentsAlphaPath,
|
|
);
|
|
const betaExecution = findScopedAgentsMutationExecution(
|
|
agentDb,
|
|
scopedAgentsBetaPath,
|
|
);
|
|
assert(
|
|
alphaExecution && betaExecution,
|
|
'scoped-agents-target-mutation-evidence-missing',
|
|
);
|
|
const mutationStarts = assertScopedAgentsMutationBoundaries(agentDb);
|
|
const verificationExecution = requireSuccessfulToolExecution(
|
|
agentDb,
|
|
'project.verify',
|
|
state.initialRunId,
|
|
(execution) =>
|
|
['test', 'check:e2e'].includes(
|
|
auditInputValue(execution.inputSummary, 'script'),
|
|
) &&
|
|
auditInputValue(execution.inputSummary, 'expectedCommandSha256') ===
|
|
hashValue(scopedAgentsVerificationCommand) &&
|
|
auditInputValue(execution.inputSummary, 'timeoutSeconds') === '120',
|
|
'scoped-agents-project-verification-action-invalid',
|
|
);
|
|
const verificationAudit = requireExecutionRecord(
|
|
agentDb,
|
|
verificationExecution,
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.project.verify' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId &&
|
|
record.actionId === verificationExecution.actionId &&
|
|
['test', 'check:e2e'].includes(record.script) &&
|
|
record.expectedCommand === scopedAgentsVerificationCommand &&
|
|
record.status === 'completed' &&
|
|
record.exitCode === 0 &&
|
|
record.timedOut === false &&
|
|
hasExpectedWorkspaceSandboxMetadata(record),
|
|
'scoped-agents-project-verification-audit-invalid',
|
|
);
|
|
assert(
|
|
isNonEmptyString(verificationAudit.logPath),
|
|
'scoped-agents-verification-log-missing',
|
|
);
|
|
|
|
const userMessages = conversations.filter(
|
|
(message) => message.role === 'user',
|
|
);
|
|
const assistantMessages = conversations.filter(
|
|
(message) => message.role === 'assistant',
|
|
);
|
|
const completedAudits = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.completed' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
assert(
|
|
userMessages.length === 1 &&
|
|
assistantMessages.length === 1 &&
|
|
completedAudits.length === 1,
|
|
'scoped-agents-conversation-cardinality-invalid',
|
|
);
|
|
const assistantMarkerLeakCount = countExactSecrets(
|
|
Buffer.from(assistantMessages[0].content),
|
|
state.scopedAgents.privateValues,
|
|
);
|
|
assert(
|
|
assistantMarkerLeakCount === 0,
|
|
'scoped-agents-final-assistant-instruction-leak',
|
|
);
|
|
const duplicateMessageCount = duplicateCount(
|
|
conversations.map((message) => message.messageId).filter(Boolean),
|
|
);
|
|
const receipts = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.action_receipt' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
const duplicateReceiptCount = duplicateCount(
|
|
receipts.map(receiptAuditIdentity),
|
|
);
|
|
assert(
|
|
duplicateMessageCount === 0 && duplicateReceiptCount === 0,
|
|
'scoped-agents-duplicate-persistence-identity',
|
|
);
|
|
const finalizationFiles = (
|
|
await listFiles(
|
|
path.join(state.projectRoot, '.agent/runtime/finalizations'),
|
|
)
|
|
).filter((file) => file.endsWith('.json'));
|
|
assert(
|
|
finalizationFiles.length === 0,
|
|
'scoped-agents-finalization-journal-present',
|
|
);
|
|
|
|
const providerLifecycle = validateScopedAgentsProviderLifecycle(agentDb);
|
|
const toolPlanProtocolCount = validateMainRunToolPlanProtocols(agentDb);
|
|
const publicSurfaces = {
|
|
task: taskSnapshot.all,
|
|
event: events,
|
|
agentDb,
|
|
activity,
|
|
output,
|
|
runtimeState,
|
|
};
|
|
const instructionAuditSurfaces = {
|
|
task: taskSnapshot.all,
|
|
event: events,
|
|
agentDb,
|
|
activity,
|
|
output,
|
|
};
|
|
const instructionPublicCounts = countSensitiveValuesBySurface(
|
|
instructionAuditSurfaces,
|
|
state.scopedAgents.privateValues,
|
|
'scoped-agents-instruction-body-public',
|
|
);
|
|
const apiKeyPublicCounts = countSensitiveValuesBySurface(
|
|
publicSurfaces,
|
|
state.secrets,
|
|
'scoped-agents-api-key-public',
|
|
);
|
|
const projectPathPublicCounts = countSensitiveValuesBySurface(
|
|
publicSurfaces,
|
|
disposableProjectPathVariants(),
|
|
'scoped-agents-project-path-public',
|
|
);
|
|
state.lureLeakCount = await countLureLeaks();
|
|
assert(state.lureLeakCount === 0, 'sensitive-lure-leak-detected');
|
|
const projectSecretLeakCount = await countSecretsInProject(
|
|
state.projectRoot,
|
|
state.secrets,
|
|
);
|
|
const secretLeakCount =
|
|
(state.transcriptScanner?.count ?? 0) + projectSecretLeakCount;
|
|
assert(secretLeakCount === 0, 'loaded-key-leak-detected');
|
|
|
|
return {
|
|
scenario: 'root-parent-and-sibling-scoped-agents-behavior',
|
|
targetAgentId: mainAgentId,
|
|
providerModel: state.scopedAgents.effectiveModel,
|
|
providerApiKind: state.scopedAgents.effectiveApiKind,
|
|
isolatedAppDataUsed: true,
|
|
formalConfigCliCallCount: state.isolatedRunner.sourceConfigCliCallCount,
|
|
sourceRunnerEndpointUnchanged: false,
|
|
sourceConfigReplicaCount: state.isolatedRunner.configLinks.length,
|
|
sourceConfigReplicasVerified: false,
|
|
initialVerificationFailed:
|
|
state.scopedAgents.initialVerificationFailed === true,
|
|
taskCount: taskSnapshot.all.length,
|
|
eventCount: events.length,
|
|
agentDbRecordCount: agentDb.length,
|
|
conversationMessageCount: conversations.length,
|
|
targetRunCount: new Set(
|
|
taskSnapshot.all
|
|
.filter((task) => task.agentId === mainAgentId)
|
|
.map((task) => task.runId),
|
|
).size,
|
|
stableSessionCount: new Set(
|
|
taskSnapshot.all
|
|
.filter((task) => task.agentId === mainAgentId)
|
|
.map((task) => task.sessionId),
|
|
).size,
|
|
repositoryContextSourceCount: sourcePaths.length,
|
|
requiredScopedInstructionSourceCount: 4,
|
|
rootRuleApplied:
|
|
alphaContent.startsWith(`ROOT=${scopedAgentsRootCanary}`) &&
|
|
betaContent.startsWith(`ROOT=${scopedAgentsRootCanary}`),
|
|
parentRuleApplied:
|
|
alphaContent.includes(`GAME=${scopedAgentsGameCanary}`) &&
|
|
betaContent.includes(`GAME=${scopedAgentsGameCanary}`),
|
|
alphaRuleApplied: alphaContent === alphaExpected,
|
|
betaRuleApplied: betaContent === betaExpected,
|
|
siblingRuleCrossLeakCount: 0,
|
|
changedProjectFileCount: changedPaths.length,
|
|
targetMutationCoverageCount: 2,
|
|
mutationActionCount: new Set(
|
|
mutationStarts.map((record) => record.actionId),
|
|
).size,
|
|
confirmedActionCount: state.scopedAgents.confirmedActionCount,
|
|
verificationPassed: true,
|
|
hostVerificationPassed: true,
|
|
successfulToolExecutionCount: receipts.filter(
|
|
(record) => record.status === 'ok',
|
|
).length,
|
|
toolPlanProtocolCount,
|
|
providerRequestIdentityCount: providerLifecycle.requestIdentityCount,
|
|
providerLifecycleStartedCount: providerLifecycle.startedCount,
|
|
providerLifecycleTerminalCount: providerLifecycle.terminalCount,
|
|
toolPlanProviderRequestCount: providerLifecycle.toolPlanCount,
|
|
finalReplyProviderRequestCount: providerLifecycle.finalReplyCount,
|
|
providerFallbackReplayCount: 0,
|
|
finalAssistantCount: assistantMessages.length,
|
|
completedAuditCount: completedAudits.length,
|
|
duplicateMessageCount,
|
|
duplicateReceiptCount,
|
|
finalizationJournalCount: finalizationFiles.length,
|
|
assistantInstructionLeakCount: assistantMarkerLeakCount,
|
|
instructionPublicLeakCount: sumObjectValues(instructionPublicCounts),
|
|
apiKeyPublicLeakCount: sumObjectValues(apiKeyPublicCounts),
|
|
projectPathPublicLeakCount: sumObjectValues(projectPathPublicCounts),
|
|
projectPathPublicSurfaceCount: Object.keys(projectPathPublicCounts).length,
|
|
scopedAgentsReportLeakCount: state.scopedAgents.reportLeakCount,
|
|
scopedAgentsRunnerKillMethod: null,
|
|
scopedAgentsRunnerPidfdClaimCount: state.isolatedRunner.pidfdClaimCount,
|
|
scopedAgentsRunnerPidfdSignalCount: state.isolatedRunner.pidfdSignalCount,
|
|
scopedAgentsRunnerStopped: false,
|
|
scopedAgentsAppDataCleanupPerformed: false,
|
|
secretLeakCount,
|
|
lureLeakCount: state.lureLeakCount,
|
|
paths: [
|
|
'.agent/runtime/context-bundles',
|
|
'.agent/runtime/tasks',
|
|
'.agent/runtime/events',
|
|
'.agent/agent.db',
|
|
'.agent/conversations',
|
|
],
|
|
};
|
|
}
|
|
|
|
export async function collectPartialScopedAgentsEvidence() {
|
|
const persistence = await readScopedAgentsPersistence();
|
|
const [alphaContent, betaContent] = await Promise.all([
|
|
fs
|
|
.readFile(path.join(state.projectRoot, scopedAgentsAlphaPath), 'utf8')
|
|
.catch(() => ''),
|
|
fs
|
|
.readFile(path.join(state.projectRoot, scopedAgentsBetaPath), 'utf8')
|
|
.catch(() => ''),
|
|
]);
|
|
return {
|
|
providerModel: state.scopedAgents.effectiveModel,
|
|
providerApiKind: state.scopedAgents.effectiveApiKind,
|
|
initialVerificationFailed:
|
|
state.scopedAgents.initialVerificationFailed === true,
|
|
taskCount: persistence.taskSnapshot.all.length,
|
|
eventCount: persistence.events.length,
|
|
agentDbRecordCount: persistence.agentDb.length,
|
|
conversationMessageCount: persistence.conversations.length,
|
|
rootRuleApplied:
|
|
alphaContent.startsWith(`ROOT=${scopedAgentsRootCanary}`) &&
|
|
betaContent.startsWith(`ROOT=${scopedAgentsRootCanary}`),
|
|
parentRuleApplied:
|
|
alphaContent.includes(`GAME=${scopedAgentsGameCanary}`) &&
|
|
betaContent.includes(`GAME=${scopedAgentsGameCanary}`),
|
|
alphaRuleApplied: alphaContent === scopedAgentsExpectedContent('alpha'),
|
|
betaRuleApplied: betaContent === scopedAgentsExpectedContent('beta'),
|
|
finalAssistantCount: persistence.conversations.filter(
|
|
(message) => message.role === 'assistant',
|
|
).length,
|
|
};
|
|
}
|
|
|
|
export function emptyScopedAgentsEvidence() {
|
|
return {
|
|
scenario: 'root-parent-and-sibling-scoped-agents-behavior',
|
|
targetAgentId: mainAgentId,
|
|
providerModel: null,
|
|
providerApiKind: null,
|
|
isolatedAppDataUsed: false,
|
|
formalConfigCliCallCount: 0,
|
|
sourceRunnerEndpointUnchanged: false,
|
|
sourceConfigReplicaCount: 0,
|
|
sourceConfigReplicasVerified: false,
|
|
initialVerificationFailed: false,
|
|
taskCount: 0,
|
|
eventCount: 0,
|
|
agentDbRecordCount: 0,
|
|
conversationMessageCount: 0,
|
|
targetRunCount: 0,
|
|
stableSessionCount: 0,
|
|
repositoryContextSourceCount: 0,
|
|
requiredScopedInstructionSourceCount: 4,
|
|
rootRuleApplied: false,
|
|
parentRuleApplied: false,
|
|
alphaRuleApplied: false,
|
|
betaRuleApplied: false,
|
|
siblingRuleCrossLeakCount: 0,
|
|
changedProjectFileCount: 0,
|
|
targetMutationCoverageCount: 0,
|
|
mutationActionCount: 0,
|
|
confirmedActionCount: 0,
|
|
verificationPassed: false,
|
|
hostVerificationPassed: false,
|
|
successfulToolExecutionCount: 0,
|
|
toolPlanProtocolCount: 0,
|
|
providerRequestIdentityCount: 0,
|
|
providerLifecycleStartedCount: 0,
|
|
providerLifecycleTerminalCount: 0,
|
|
toolPlanProviderRequestCount: 0,
|
|
finalReplyProviderRequestCount: 0,
|
|
providerFallbackReplayCount: 0,
|
|
finalAssistantCount: 0,
|
|
completedAuditCount: 0,
|
|
duplicateMessageCount: 0,
|
|
duplicateReceiptCount: 0,
|
|
finalizationJournalCount: 0,
|
|
assistantInstructionLeakCount: 0,
|
|
instructionPublicLeakCount: 0,
|
|
apiKeyPublicLeakCount: 0,
|
|
projectPathPublicLeakCount: 0,
|
|
projectPathPublicSurfaceCount: 0,
|
|
scopedAgentsReportLeakCount: 0,
|
|
scopedAgentsRunnerKillMethod: null,
|
|
scopedAgentsRunnerPidfdClaimCount: 0,
|
|
scopedAgentsRunnerPidfdSignalCount: 0,
|
|
scopedAgentsRunnerStopped: false,
|
|
scopedAgentsAppDataCleanupPerformed: false,
|
|
secretLeakCount: 0,
|
|
lureLeakCount: 0,
|
|
paths: [],
|
|
};
|
|
}
|
|
|
|
export function isScopedAgentsSuite() {
|
|
return state.suite === scopedAgentsSuite;
|
|
}
|