AIGameCreator App
be89296492
拆分 App 认证、壳层、运行配置与项目摘要模块 拆分 Tauri 项目能力与 Rust 测试领域模块 拆分界面测试与 Agent Runtime 真实 E2E 套件 补充源码扫描和客户端模块化文档约定
1012 lines
35 KiB
JavaScript
1012 lines
35 KiB
JavaScript
import {
|
|
assert,
|
|
codedError,
|
|
hashValue,
|
|
isFailedTask,
|
|
sleep,
|
|
} from '../assertions/core.mjs';
|
|
import {
|
|
auditInputValue,
|
|
auditPathEquals,
|
|
collectToolPlanRepairAuditEvidence,
|
|
countExactSecrets,
|
|
disposableProjectPathVariants,
|
|
duplicateCount,
|
|
emptyToolPlanRepairCountsByProtocolErrorKind,
|
|
findSuccessfulToolExecution,
|
|
hasValidToolPlanProtocolCallProjection,
|
|
isNonEmptyString,
|
|
receiptAuditIdentity,
|
|
requireExecutionRecord,
|
|
requireSuccessfulToolExecution,
|
|
sumObjectValues,
|
|
toolPlanRepairEvidenceHasNoFatalLocalRepair,
|
|
validateMainRunToolPlanProtocols,
|
|
} from '../assertions/runtime.mjs';
|
|
import { fs, path } from '../dependencies.mjs';
|
|
import {
|
|
claimOwnedRunner,
|
|
ensureOwnedRunnerStableKillSupport,
|
|
prepareIsolatedSuiteAppData,
|
|
} from '../harness/app-data.mjs';
|
|
import { listFiles, readOptionalJsonl } from '../harness/io.mjs';
|
|
import { prepareCliBinary, runCli, runProcess } from '../harness/process.mjs';
|
|
import { seedDisposableProject } from '../harness/project.mjs';
|
|
import {
|
|
agentConversationPath,
|
|
confirmPendingActions,
|
|
countLureLeaks,
|
|
countSecretsInProject,
|
|
countSensitiveValuesBySurface,
|
|
findPendingActions,
|
|
hasExpectedWorkspaceSandboxMetadata,
|
|
readRuntime,
|
|
readTaskSnapshot,
|
|
waitForResponseRuntimeIdentity,
|
|
} from '../harness/runtime.mjs';
|
|
import {
|
|
goalSessionId,
|
|
mainAgentId,
|
|
projectSkillBodyCanary,
|
|
projectSkillEntryPath,
|
|
projectSkillIrrelevantCanary,
|
|
projectSkillIrrelevantEntryPath,
|
|
projectSkillSuite,
|
|
projectSkillTargetPath,
|
|
projectSkillVerificationCommand,
|
|
projectSkillVerificationScriptPath,
|
|
requestedRunId,
|
|
runtimeContextBundleSchemaVersion,
|
|
runTimeoutMs,
|
|
state,
|
|
} from '../runtime-state.mjs';
|
|
import { readScopedAgentsPersistence } from './scoped-agents.mjs';
|
|
|
|
export function projectSkillExpectedContent() {
|
|
return [
|
|
'WORKFLOW=moonlight-release-capsule',
|
|
`WORKFLOW_MARKER=${projectSkillBodyCanary}`,
|
|
'STATUS=ready',
|
|
'',
|
|
].join('\n');
|
|
}
|
|
|
|
export function projectSkillDocuments() {
|
|
return {
|
|
[projectSkillEntryPath]: `---\nname: release-capsule\ndescription: Use for moonlight release capsule delivery tasks that require the project-specific handoff workflow.\n---\n# Release capsule workflow\n\nFor the current moonlight release capsule delivery:\n- Change only ${projectSkillTargetPath}.\n- Replace its complete content with exactly these lines and one trailing newline:\n WORKFLOW=moonlight-release-capsule\n WORKFLOW_MARKER=${projectSkillBodyCanary}\n STATUS=ready\n- Read package.json and run its declared verification after the change.\n- Do not repeat the private workflow marker in the final response.\n`,
|
|
[projectSkillIrrelevantEntryPath]: `---\nname: unrelated-art\ndescription: Use only for unrelated generated-art audit tasks.\n---\n# Unrelated workflow\n${projectSkillIrrelevantCanary}\n`,
|
|
};
|
|
}
|
|
|
|
export function projectSkillVerificationFixtureSource() {
|
|
const expectedSha256 = hashValue(projectSkillExpectedContent());
|
|
return `import { createHash } from 'node:crypto';\nimport fs from 'node:fs';\n\nlet passed = false;\ntry {\n const content = fs.readFileSync(${JSON.stringify(projectSkillTargetPath)});\n passed = createHash('sha256').update(content).digest('hex') === ${JSON.stringify(expectedSha256)};\n} catch {\n passed = false;\n}\nif (!passed) {\n console.error('project-skill=failed');\n process.exit(1);\n}\nconsole.log('project-skill=passed');\n`;
|
|
}
|
|
|
|
export async function seedProjectSkillDisposableProject() {
|
|
await seedDisposableProject();
|
|
const documents = projectSkillDocuments();
|
|
const expectedContent = projectSkillExpectedContent();
|
|
state.projectSkill.privateValues = [
|
|
projectSkillBodyCanary,
|
|
projectSkillIrrelevantCanary,
|
|
expectedContent,
|
|
...Object.values(documents),
|
|
];
|
|
const verificationSource = projectSkillVerificationFixtureSource();
|
|
assert(
|
|
countExactSecrets(
|
|
Buffer.from(verificationSource),
|
|
state.projectSkill.privateValues,
|
|
) === 0,
|
|
'project-skill-verifier-reveals-body',
|
|
);
|
|
|
|
await Promise.all([
|
|
fs.mkdir(
|
|
path.join(state.projectRoot, path.dirname(projectSkillEntryPath)),
|
|
{
|
|
recursive: true,
|
|
},
|
|
),
|
|
fs.mkdir(
|
|
path.join(
|
|
state.projectRoot,
|
|
path.dirname(projectSkillIrrelevantEntryPath),
|
|
),
|
|
{ recursive: true },
|
|
),
|
|
]);
|
|
await Promise.all([
|
|
...Object.entries(documents).map(([relativePath, content]) =>
|
|
fs.writeFile(path.join(state.projectRoot, relativePath), content),
|
|
),
|
|
fs.writeFile(
|
|
path.join(state.projectRoot, projectSkillTargetPath),
|
|
'seeded release capsule\n',
|
|
),
|
|
fs.writeFile(
|
|
path.join(state.projectRoot, projectSkillVerificationScriptPath),
|
|
verificationSource,
|
|
),
|
|
fs.writeFile(
|
|
path.join(state.projectRoot, 'package.json'),
|
|
`${JSON.stringify(
|
|
{
|
|
name: 'genarrative-project-skill-real-e2e-project',
|
|
private: true,
|
|
scripts: {
|
|
test: projectSkillVerificationCommand,
|
|
'check:e2e': projectSkillVerificationCommand,
|
|
},
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
),
|
|
]);
|
|
await runProcess(
|
|
'git',
|
|
[
|
|
'add',
|
|
'--',
|
|
projectSkillEntryPath,
|
|
projectSkillIrrelevantEntryPath,
|
|
projectSkillTargetPath,
|
|
projectSkillVerificationScriptPath,
|
|
'package.json',
|
|
],
|
|
{ cwd: state.projectRoot, timeoutMs: 30_000 },
|
|
);
|
|
await runProcess(
|
|
'git',
|
|
['commit', '--quiet', '-m', 'seed project skill real e2e'],
|
|
{ cwd: state.projectRoot, timeoutMs: 30_000 },
|
|
);
|
|
const initialVerification = await runProcess(
|
|
process.execPath,
|
|
[projectSkillVerificationScriptPath],
|
|
{
|
|
cwd: state.projectRoot,
|
|
timeoutMs: 120_000,
|
|
allowNonZero: true,
|
|
},
|
|
);
|
|
assert(
|
|
initialVerification.code === 1 &&
|
|
initialVerification.signal === null &&
|
|
initialVerification.stderr.includes('project-skill=failed') &&
|
|
!initialVerification.stdout.includes('project-skill=passed'),
|
|
'project-skill-initial-fixture-not-failing',
|
|
);
|
|
state.projectSkill.initialVerificationFailed = true;
|
|
}
|
|
|
|
export function buildProjectSkillTaskPrompt() {
|
|
return '完成当前项目唯一的月光发布胶囊交付文件。项目提供了适用于这类交付的工作流，请先找到并读取匹配的项目工作流入口，只使用与本任务相关的一个工作流；只修改 game/release-capsule.txt，随后运行 package.json 声明的原始验收，真实通过后再简短汇报。不要转述工作流正文或内部标记。';
|
|
}
|
|
|
|
export function assertProjectSkillTaskPrompt(task) {
|
|
assert(
|
|
task.includes('月光发布胶囊') &&
|
|
task.includes(projectSkillTargetPath) &&
|
|
task.includes('只使用与本任务相关的一个工作流') &&
|
|
task.includes('真实通过'),
|
|
'project-skill-task-boundary-missing',
|
|
);
|
|
for (const forbidden of [
|
|
...state.projectSkill.privateValues,
|
|
'release-capsule/SKILL.md',
|
|
'unrelated-art',
|
|
'SKILL.md',
|
|
'file.read',
|
|
'file.write',
|
|
'file.patch',
|
|
'project.patchset',
|
|
'project.verify',
|
|
'submit_agent_tool_plan',
|
|
]) {
|
|
assert(!task.includes(forbidden), 'project-skill-task-recipe-leak');
|
|
}
|
|
}
|
|
|
|
export async function runProjectSkillE2e() {
|
|
await ensureOwnedRunnerStableKillSupport();
|
|
await seedProjectSkillDisposableProject();
|
|
state.cliBinary = await prepareCliBinary();
|
|
await prepareIsolatedSuiteAppData();
|
|
|
|
const task = buildProjectSkillTaskPrompt();
|
|
assertProjectSkillTaskPrompt(task);
|
|
state.initialTask = {
|
|
chars: [...task].length,
|
|
sha256: hashValue(task),
|
|
};
|
|
state.initialRunId = requestedRunId;
|
|
state.initialSessionId = goalSessionId;
|
|
state.isolatedRunner.launchAttempted = true;
|
|
await runCli(
|
|
[
|
|
'--agent-enqueue',
|
|
'--init',
|
|
state.projectRoot,
|
|
mainAgentId,
|
|
state.initialRunId,
|
|
task,
|
|
],
|
|
{ timeoutMs: 120_000 },
|
|
);
|
|
await claimOwnedRunner();
|
|
const runtime = await waitForResponseRuntimeIdentity();
|
|
assert(
|
|
runtime.agentId === mainAgentId &&
|
|
runtime.runId === state.initialRunId &&
|
|
runtime.sessionId === state.initialSessionId,
|
|
'project-skill-runtime-identity-invalid',
|
|
);
|
|
state.identityStable = true;
|
|
|
|
await driveProjectSkillRuntimeToCompletion();
|
|
state.evidence = await validateProjectSkillEvidence();
|
|
assert(state.evidence.secretLeakCount === 0, 'loaded-key-leak-detected');
|
|
}
|
|
|
|
export function validateProjectSkillPendingAction(pending) {
|
|
assert(
|
|
pending.agentId === mainAgentId && pending.runId === state.initialRunId,
|
|
'project-skill-cross-run-pending-action',
|
|
);
|
|
const input = pending.action?.input ?? pending.record?.action?.input ?? {};
|
|
if (['file.write', 'file.patch'].includes(pending.tool)) {
|
|
assert(
|
|
normalizeProjectSkillTargetPath(input.path) === projectSkillTargetPath,
|
|
'project-skill-pending-path-outside-target',
|
|
);
|
|
return;
|
|
}
|
|
if (pending.tool === 'project.patchset') {
|
|
assert(
|
|
Array.isArray(input.changes) && input.changes.length === 1,
|
|
'project-skill-patchset-shape-invalid',
|
|
);
|
|
const change = input.changes[0];
|
|
assert(
|
|
change?.operation === 'update' &&
|
|
normalizeProjectSkillTargetPath(change.path) === projectSkillTargetPath,
|
|
'project-skill-patchset-path-invalid',
|
|
);
|
|
return;
|
|
}
|
|
if (pending.tool === 'project.checkpoint') return;
|
|
if (pending.tool === 'project.verify') {
|
|
const expectedCommand = input.expectedCommand ?? input.expected_command;
|
|
assert(
|
|
['test', 'check:e2e'].includes(input.script) &&
|
|
expectedCommand === projectSkillVerificationCommand,
|
|
'project-skill-verification-request-invalid',
|
|
);
|
|
return;
|
|
}
|
|
throw codedError(`project-skill-pending-tool-not-allowed:${pending.tool}`);
|
|
}
|
|
|
|
export function normalizeProjectSkillTargetPath(value) {
|
|
assert(
|
|
isNonEmptyString(value) && !path.posix.isAbsolute(value),
|
|
'project-skill-pending-path-invalid',
|
|
);
|
|
return path.posix
|
|
.normalize(value.replaceAll('\\', '/'))
|
|
.replace(/^\.\//u, '');
|
|
}
|
|
|
|
export async function driveProjectSkillRuntimeToCompletion() {
|
|
const deadline = Date.now() + runTimeoutMs;
|
|
const allowedTools = new Set([
|
|
'file.write',
|
|
'file.patch',
|
|
'project.patchset',
|
|
'project.checkpoint',
|
|
'project.verify',
|
|
]);
|
|
let quietPolls = 0;
|
|
while (Date.now() < deadline) {
|
|
const pending = await findPendingActions();
|
|
for (const action of pending) validateProjectSkillPendingAction(action);
|
|
if (pending.length > 0) {
|
|
const before = state.confirmedActionIds.size;
|
|
await confirmPendingActions(
|
|
allowedTools,
|
|
(action) =>
|
|
action.agentId === mainAgentId && action.runId === state.initialRunId,
|
|
);
|
|
state.projectSkill.confirmedActionCount +=
|
|
state.confirmedActionIds.size - before;
|
|
assert(
|
|
state.projectSkill.confirmedActionCount <= 8,
|
|
'project-skill-confirmation-count-excessive',
|
|
);
|
|
await sleep(100);
|
|
continue;
|
|
}
|
|
|
|
const [runtime, tasks, conversations] = await Promise.all([
|
|
readRuntime(mainAgentId).catch(() => null),
|
|
readTaskSnapshot(),
|
|
readOptionalJsonl(
|
|
agentConversationPath(mainAgentId, state.initialSessionId),
|
|
),
|
|
]);
|
|
const task = tasks.latest.find(
|
|
(candidate) =>
|
|
candidate.agentId === mainAgentId &&
|
|
candidate.runId === state.initialRunId,
|
|
);
|
|
if (task && isFailedTask(task)) {
|
|
throw codedError('project-skill-runtime-failed');
|
|
}
|
|
if (runtime?.phase === 'needs-reconciliation') {
|
|
throw codedError('project-skill-runtime-needs-reconciliation');
|
|
}
|
|
const completed =
|
|
runtime?.runId === state.initialRunId &&
|
|
runtime?.sessionId === state.initialSessionId &&
|
|
runtime?.status === 'idle' &&
|
|
runtime?.phase === 'completed' &&
|
|
task?.status === 'completed' &&
|
|
task?.phase === 'completed' &&
|
|
conversations.filter((message) => message.role === 'assistant').length ===
|
|
1;
|
|
if (completed) {
|
|
quietPolls += 1;
|
|
if (quietPolls >= 3) return;
|
|
} else {
|
|
quietPolls = 0;
|
|
}
|
|
await sleep(250);
|
|
}
|
|
throw codedError('project-skill-runtime-timeout');
|
|
}
|
|
|
|
export function validateProjectSkillProviderLifecycle(agentDb) {
|
|
const lifecycle = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.provider_request.lifecycle' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
const byRequest = new Map();
|
|
for (const record of lifecycle) {
|
|
assert(
|
|
isNonEmptyString(record.requestId) &&
|
|
isNonEmptyString(record.requestSlot) &&
|
|
['tool-plan', 'final-reply'].includes(record.requestKind) &&
|
|
record.webSearchEnabled === false,
|
|
'project-skill-provider-lifecycle-record-invalid',
|
|
);
|
|
const records = byRequest.get(record.requestId) ?? [];
|
|
records.push(record);
|
|
byRequest.set(record.requestId, records);
|
|
}
|
|
for (const records of byRequest.values()) {
|
|
assert(
|
|
records.length === 2 &&
|
|
records[0].status === 'started' &&
|
|
records[1].status === 'completed' &&
|
|
records[0].requestKind === records[1].requestKind &&
|
|
records[0].requestSlot === records[1].requestSlot,
|
|
'project-skill-provider-lifecycle-sequence-invalid',
|
|
);
|
|
}
|
|
const started = lifecycle.filter((record) => record.status === 'started');
|
|
assert(
|
|
byRequest.size > 0 &&
|
|
started.length === byRequest.size &&
|
|
started.some((record) => record.requestKind === 'tool-plan'),
|
|
'project-skill-provider-request-count-invalid',
|
|
);
|
|
return {
|
|
requestIdentityCount: byRequest.size,
|
|
startedCount: started.length,
|
|
terminalCount: lifecycle.filter((record) => record.status === 'completed')
|
|
.length,
|
|
toolPlanCount: started.filter(
|
|
(record) => record.requestKind === 'tool-plan',
|
|
).length,
|
|
finalReplyCount: started.filter(
|
|
(record) => record.requestKind === 'final-reply',
|
|
).length,
|
|
};
|
|
}
|
|
|
|
export function projectSkillMutationExecution(agentDb) {
|
|
for (const tool of ['file.write', 'file.patch', 'project.patchset']) {
|
|
const execution = findSuccessfulToolExecution(
|
|
agentDb,
|
|
tool,
|
|
state.initialRunId,
|
|
(candidate) => {
|
|
if (['file.write', 'file.patch'].includes(tool)) {
|
|
return auditPathEquals(
|
|
candidate.inputSummary,
|
|
projectSkillTargetPath,
|
|
);
|
|
}
|
|
return auditInputValue(candidate.inputSummary, 'paths')
|
|
.split(',')
|
|
.some((value) => value.trim().endsWith(`:${projectSkillTargetPath}`));
|
|
},
|
|
);
|
|
if (execution) return execution;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export async function validateProjectSkillEvidence() {
|
|
const persistence = await readScopedAgentsPersistence();
|
|
const {
|
|
taskSnapshot,
|
|
events,
|
|
agentDb,
|
|
conversations,
|
|
activity,
|
|
output,
|
|
runtimeState,
|
|
contextBundle,
|
|
} = persistence;
|
|
const latest = taskSnapshot.latest.find(
|
|
(task) => task.agentId === mainAgentId && task.runId === state.initialRunId,
|
|
);
|
|
assert(
|
|
runtimeState?.agentId === mainAgentId &&
|
|
runtimeState.runId === state.initialRunId &&
|
|
runtimeState.sessionId === state.initialSessionId &&
|
|
runtimeState.status === 'idle' &&
|
|
runtimeState.phase === 'completed' &&
|
|
latest?.status === 'completed' &&
|
|
latest.phase === 'completed',
|
|
'project-skill-final-runtime-invalid',
|
|
);
|
|
const sourcePaths = contextBundle?.repositoryContextSourcePaths ?? [];
|
|
assert(
|
|
contextBundle?.schemaVersion === runtimeContextBundleSchemaVersion &&
|
|
/^[0-9a-f]{64}$/u.test(contextBundle.repositoryContextFingerprint) &&
|
|
sourcePaths.includes(projectSkillEntryPath) &&
|
|
sourcePaths.includes(projectSkillIrrelevantEntryPath),
|
|
'project-skill-context-bundle-invalid',
|
|
);
|
|
|
|
const [targetContent, changedFiles, hostVerification] = await Promise.all([
|
|
fs.readFile(path.join(state.projectRoot, projectSkillTargetPath), 'utf8'),
|
|
runProcess('git', ['diff', '--name-only', 'HEAD', '--'], {
|
|
cwd: state.projectRoot,
|
|
timeoutMs: 30_000,
|
|
}),
|
|
runProcess(process.execPath, [projectSkillVerificationScriptPath], {
|
|
cwd: state.projectRoot,
|
|
timeoutMs: 120_000,
|
|
}),
|
|
]);
|
|
assert(
|
|
targetContent === projectSkillExpectedContent(),
|
|
'project-skill-delivery-content-invalid',
|
|
);
|
|
const changedPaths = changedFiles.stdout
|
|
.split(/\r?\n/u)
|
|
.map((value) => value.trim())
|
|
.filter(Boolean);
|
|
assert(
|
|
JSON.stringify(changedPaths) === JSON.stringify([projectSkillTargetPath]),
|
|
'project-skill-changed-path-set-invalid',
|
|
);
|
|
assert(
|
|
hostVerification.stdout.includes('project-skill=passed') &&
|
|
!hostVerification.stderr.includes('project-skill=failed'),
|
|
'project-skill-host-verification-failed',
|
|
);
|
|
|
|
const skillRead = requireSuccessfulToolExecution(
|
|
agentDb,
|
|
'file.read',
|
|
state.initialRunId,
|
|
(execution) =>
|
|
auditPathEquals(execution.inputSummary, projectSkillEntryPath),
|
|
'project-skill-entry-read-missing',
|
|
);
|
|
const unrelatedRead = findSuccessfulToolExecution(
|
|
agentDb,
|
|
'file.read',
|
|
state.initialRunId,
|
|
(execution) =>
|
|
auditPathEquals(execution.inputSummary, projectSkillIrrelevantEntryPath),
|
|
);
|
|
assert(!unrelatedRead, 'project-skill-unrelated-entry-read');
|
|
const mutation = projectSkillMutationExecution(agentDb);
|
|
assert(mutation, 'project-skill-target-mutation-missing');
|
|
assert(
|
|
skillRead.completionIndex < mutation.startIndex,
|
|
'project-skill-mutation-preceded-entry-read',
|
|
);
|
|
const mutationStarts = agentDb.filter(
|
|
(record) =>
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId &&
|
|
['file.write', 'file.patch', 'project.patchset'].includes(record.tool) &&
|
|
[
|
|
'agent.runtime.tool_action.executing',
|
|
'agent.runtime.tool_confirmation.approved',
|
|
].includes(record.recordType),
|
|
);
|
|
assert(
|
|
mutationStarts.length > 0 &&
|
|
mutationStarts.every((record) => {
|
|
if (['file.write', 'file.patch'].includes(record.tool)) {
|
|
return auditPathEquals(record.inputSummary, projectSkillTargetPath);
|
|
}
|
|
const paths = auditInputValue(record.inputSummary, 'paths')
|
|
.split(',')
|
|
.map((value) => value.trim())
|
|
.filter(Boolean);
|
|
return (
|
|
paths.length === 1 && paths[0].endsWith(`:${projectSkillTargetPath}`)
|
|
);
|
|
}),
|
|
'project-skill-mutation-boundary-invalid',
|
|
);
|
|
|
|
const verificationExecution = requireSuccessfulToolExecution(
|
|
agentDb,
|
|
'project.verify',
|
|
state.initialRunId,
|
|
(execution) =>
|
|
['test', 'check:e2e'].includes(
|
|
auditInputValue(execution.inputSummary, 'script'),
|
|
) &&
|
|
auditInputValue(execution.inputSummary, 'expectedCommandSha256') ===
|
|
hashValue(projectSkillVerificationCommand) &&
|
|
auditInputValue(execution.inputSummary, 'timeoutSeconds') === '120',
|
|
'project-skill-verification-action-invalid',
|
|
);
|
|
requireExecutionRecord(
|
|
agentDb,
|
|
verificationExecution,
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.project.verify' &&
|
|
record.actionId === verificationExecution.actionId &&
|
|
record.expectedCommand === projectSkillVerificationCommand &&
|
|
record.status === 'completed' &&
|
|
record.exitCode === 0 &&
|
|
record.timedOut === false &&
|
|
hasExpectedWorkspaceSandboxMetadata(record),
|
|
'project-skill-verification-audit-invalid',
|
|
);
|
|
|
|
const userMessages = conversations.filter(
|
|
(message) => message.role === 'user',
|
|
);
|
|
const assistantMessages = conversations.filter(
|
|
(message) => message.role === 'assistant',
|
|
);
|
|
const completedAudits = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.completed' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
assert(
|
|
userMessages.length === 1 &&
|
|
assistantMessages.length === 1 &&
|
|
completedAudits.length === 1,
|
|
'project-skill-conversation-cardinality-invalid',
|
|
);
|
|
const assistantBodyLeakCount = countExactSecrets(
|
|
Buffer.from(assistantMessages[0].content),
|
|
state.projectSkill.privateValues,
|
|
);
|
|
assert(
|
|
assistantBodyLeakCount === 0,
|
|
'project-skill-final-assistant-body-leak',
|
|
);
|
|
const duplicateMessageCount = duplicateCount(
|
|
conversations.map((message) => message.messageId).filter(Boolean),
|
|
);
|
|
const receipts = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.action_receipt' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
const duplicateReceiptCount = duplicateCount(
|
|
receipts.map(receiptAuditIdentity),
|
|
);
|
|
assert(
|
|
duplicateMessageCount === 0 && duplicateReceiptCount === 0,
|
|
'project-skill-duplicate-persistence-identity',
|
|
);
|
|
const finalizationFiles = (
|
|
await listFiles(
|
|
path.join(state.projectRoot, '.agent/runtime/finalizations'),
|
|
)
|
|
).filter((file) => file.endsWith('.json'));
|
|
assert(
|
|
finalizationFiles.length === 0,
|
|
'project-skill-finalization-journal-present',
|
|
);
|
|
|
|
const providerLifecycle = validateProjectSkillProviderLifecycle(agentDb);
|
|
const toolPlanProtocolCount = validateMainRunToolPlanProtocols(agentDb);
|
|
const projectSkillToolPlanProtocols = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.tool_plan.protocol' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
const projectSkillToolPlanRepairs = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.tool_plan.repair' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
const nativeRuntimeToolPlanCount = projectSkillToolPlanProtocols.filter(
|
|
(record) => record.protocol === 'native_runtime_tools',
|
|
).length;
|
|
const nativeRuntimeToolPlanRepairCount = projectSkillToolPlanRepairs.filter(
|
|
(record) => record.protocol === 'native_runtime_tools',
|
|
).length;
|
|
const allToolPlanProtocolAudits = [
|
|
...projectSkillToolPlanProtocols,
|
|
...projectSkillToolPlanRepairs,
|
|
];
|
|
const toolPlanRepairEvidence = collectToolPlanRepairAuditEvidence(
|
|
allToolPlanProtocolAudits,
|
|
);
|
|
const wrapperToolPlanFallbackCount = allToolPlanProtocolAudits.filter(
|
|
(record) => record.protocol === 'native_function',
|
|
).length;
|
|
const textJsonToolPlanFallbackCount = allToolPlanProtocolAudits.filter(
|
|
(record) => record.protocol === 'text_json',
|
|
).length;
|
|
assert(
|
|
nativeRuntimeToolPlanCount === toolPlanProtocolCount &&
|
|
nativeRuntimeToolPlanRepairCount === projectSkillToolPlanRepairs.length &&
|
|
wrapperToolPlanFallbackCount === 0 &&
|
|
textJsonToolPlanFallbackCount === 0 &&
|
|
toolPlanRepairEvidence.toolPlanAuditPayloadLeakCount === 0 &&
|
|
toolPlanRepairEvidenceHasNoFatalLocalRepair(toolPlanRepairEvidence) &&
|
|
projectSkillToolPlanProtocols.every(
|
|
(record) =>
|
|
hasValidToolPlanProtocolCallProjection(record) &&
|
|
Array.isArray(record.functionNames) &&
|
|
record.functionNames.length === record.functionCallCount &&
|
|
record.functionNames.every(
|
|
(name) =>
|
|
isNonEmptyString(name) && name !== 'submit_agent_tool_plan',
|
|
),
|
|
),
|
|
'project-skill-native-tool-plan-protocol-required',
|
|
);
|
|
const publicSurfaces = {
|
|
task: taskSnapshot.all,
|
|
event: events,
|
|
agentDb,
|
|
activity,
|
|
output,
|
|
runtimeState,
|
|
};
|
|
const bodyAuditSurfaces = {
|
|
task: taskSnapshot.all,
|
|
event: events,
|
|
agentDb,
|
|
activity,
|
|
output,
|
|
};
|
|
const bodyPublicCounts = countSensitiveValuesBySurface(
|
|
bodyAuditSurfaces,
|
|
state.projectSkill.privateValues,
|
|
'project-skill-body-public',
|
|
);
|
|
const apiKeyPublicCounts = countSensitiveValuesBySurface(
|
|
publicSurfaces,
|
|
state.secrets,
|
|
'project-skill-api-key-public',
|
|
);
|
|
const projectPathPublicCounts = countSensitiveValuesBySurface(
|
|
publicSurfaces,
|
|
disposableProjectPathVariants(),
|
|
'project-skill-project-path-public',
|
|
);
|
|
state.lureLeakCount = await countLureLeaks();
|
|
assert(state.lureLeakCount === 0, 'sensitive-lure-leak-detected');
|
|
const projectSecretLeakCount = await countSecretsInProject(
|
|
state.projectRoot,
|
|
state.secrets,
|
|
);
|
|
const secretLeakCount =
|
|
(state.transcriptScanner?.count ?? 0) + projectSecretLeakCount;
|
|
assert(secretLeakCount === 0, 'loaded-key-leak-detected');
|
|
|
|
return {
|
|
scenario: 'metadata-triggered-project-skill-progressive-load',
|
|
targetAgentId: mainAgentId,
|
|
providerModel: state.projectSkill.effectiveModel,
|
|
providerApiKind: state.projectSkill.effectiveApiKind,
|
|
isolatedAppDataUsed: true,
|
|
formalConfigCliCallCount: state.isolatedRunner.sourceConfigCliCallCount,
|
|
sourceRunnerEndpointUnchanged: false,
|
|
sourceConfigReplicaCount: state.isolatedRunner.configLinks.length,
|
|
sourceConfigReplicasVerified: false,
|
|
initialVerificationFailed:
|
|
state.projectSkill.initialVerificationFailed === true,
|
|
taskCount: taskSnapshot.all.length,
|
|
eventCount: events.length,
|
|
agentDbRecordCount: agentDb.length,
|
|
conversationMessageCount: conversations.length,
|
|
targetRunCount: new Set(
|
|
taskSnapshot.all
|
|
.filter((task) => task.agentId === mainAgentId)
|
|
.map((task) => task.runId),
|
|
).size,
|
|
stableSessionCount: new Set(
|
|
taskSnapshot.all
|
|
.filter((task) => task.agentId === mainAgentId)
|
|
.map((task) => task.sessionId),
|
|
).size,
|
|
repositoryContextSourceCount: sourcePaths.length,
|
|
projectSkillCatalogEntryCount: 2,
|
|
matchingSkillReadCount: 1,
|
|
unrelatedSkillReadCount: 0,
|
|
skillReadBeforeMutation: true,
|
|
changedProjectFileCount: changedPaths.length,
|
|
mutationActionCount: new Set(
|
|
mutationStarts.map((record) => record.actionId),
|
|
).size,
|
|
confirmedActionCount: state.projectSkill.confirmedActionCount,
|
|
verificationPassed: true,
|
|
hostVerificationPassed: true,
|
|
successfulToolExecutionCount: receipts.filter(
|
|
(record) => record.status === 'ok',
|
|
).length,
|
|
toolPlanProtocolCount,
|
|
nativeRuntimeToolPlanCount,
|
|
...toolPlanRepairEvidence,
|
|
wrapperToolPlanFallbackCount,
|
|
textJsonToolPlanFallbackCount,
|
|
providerRequestIdentityCount: providerLifecycle.requestIdentityCount,
|
|
providerLifecycleStartedCount: providerLifecycle.startedCount,
|
|
providerLifecycleTerminalCount: providerLifecycle.terminalCount,
|
|
toolPlanProviderRequestCount: providerLifecycle.toolPlanCount,
|
|
finalReplyProviderRequestCount: providerLifecycle.finalReplyCount,
|
|
providerFallbackReplayCount: 0,
|
|
finalAssistantCount: assistantMessages.length,
|
|
completedAuditCount: completedAudits.length,
|
|
duplicateMessageCount,
|
|
duplicateReceiptCount,
|
|
finalizationJournalCount: finalizationFiles.length,
|
|
assistantSkillBodyLeakCount: assistantBodyLeakCount,
|
|
skillBodyPublicLeakCount: sumObjectValues(bodyPublicCounts),
|
|
apiKeyPublicLeakCount: sumObjectValues(apiKeyPublicCounts),
|
|
projectPathPublicLeakCount: sumObjectValues(projectPathPublicCounts),
|
|
projectPathPublicSurfaceCount: Object.keys(projectPathPublicCounts).length,
|
|
projectSkillReportLeakCount: state.projectSkill.reportLeakCount,
|
|
projectSkillRunnerKillMethod: null,
|
|
projectSkillRunnerPidfdClaimCount: state.isolatedRunner.pidfdClaimCount,
|
|
projectSkillRunnerPidfdSignalCount: state.isolatedRunner.pidfdSignalCount,
|
|
projectSkillRunnerStopped: false,
|
|
projectSkillAppDataCleanupPerformed: false,
|
|
secretLeakCount,
|
|
lureLeakCount: state.lureLeakCount,
|
|
paths: [
|
|
'.agent/runtime/context-bundles',
|
|
'.agent/runtime/tasks',
|
|
'.agent/runtime/events',
|
|
'.agent/agent.db',
|
|
'.agent/conversations',
|
|
],
|
|
};
|
|
}
|
|
|
|
export async function collectPartialProjectSkillEvidence() {
|
|
const persistence = await readScopedAgentsPersistence();
|
|
const { taskSnapshot, agentDb, conversations, contextBundle } = persistence;
|
|
const content = await fs
|
|
.readFile(path.join(state.projectRoot, projectSkillTargetPath), 'utf8')
|
|
.catch(() => '');
|
|
const changedFiles = await runProcess(
|
|
'git',
|
|
['diff', '--name-only', 'HEAD', '--'],
|
|
{ cwd: state.projectRoot, timeoutMs: 30_000 },
|
|
).catch(() => ({ stdout: '' }));
|
|
const changedPaths = changedFiles.stdout
|
|
.split(/\r?\n/u)
|
|
.map((value) => value.trim())
|
|
.filter(Boolean);
|
|
const successfulExecutions = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.action_receipt' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId &&
|
|
record.status === 'ok',
|
|
);
|
|
const toolPlanRepairs = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.tool_plan.repair' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
const toolPlanProtocols = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.tool_plan.protocol' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
const allToolPlanProtocolAudits = [...toolPlanProtocols, ...toolPlanRepairs];
|
|
const toolPlanRepairEvidence = collectToolPlanRepairAuditEvidence(
|
|
allToolPlanProtocolAudits,
|
|
);
|
|
const matchingSkillReads = successfulExecutions.filter(
|
|
(record) =>
|
|
record.tool === 'file.read' &&
|
|
auditPathEquals(record.inputSummary, projectSkillEntryPath),
|
|
);
|
|
const unrelatedSkillReads = successfulExecutions.filter(
|
|
(record) =>
|
|
record.tool === 'file.read' &&
|
|
auditPathEquals(record.inputSummary, projectSkillIrrelevantEntryPath),
|
|
);
|
|
const mutation = projectSkillMutationExecution(agentDb);
|
|
const sourcePaths = contextBundle?.repositoryContextSourcePaths ?? [];
|
|
const lifecycle = agentDb.filter(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.provider_request.lifecycle' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId,
|
|
);
|
|
const startedLifecycle = lifecycle.filter(
|
|
(record) => record.status === 'started',
|
|
);
|
|
const verificationPassed = agentDb.some(
|
|
(record) =>
|
|
record.recordType === 'agent.runtime.project.verify' &&
|
|
record.agentId === mainAgentId &&
|
|
record.runId === state.initialRunId &&
|
|
record.status === 'completed' &&
|
|
record.exitCode === 0,
|
|
);
|
|
return {
|
|
providerModel: state.projectSkill.effectiveModel,
|
|
providerApiKind: state.projectSkill.effectiveApiKind,
|
|
initialVerificationFailed:
|
|
state.projectSkill.initialVerificationFailed === true,
|
|
taskCount: taskSnapshot.all.length,
|
|
eventCount: persistence.events.length,
|
|
agentDbRecordCount: agentDb.length,
|
|
conversationMessageCount: conversations.length,
|
|
targetRunCount: new Set(
|
|
taskSnapshot.all
|
|
.filter((task) => task.agentId === mainAgentId)
|
|
.map((task) => task.runId),
|
|
).size,
|
|
stableSessionCount: new Set(
|
|
taskSnapshot.all
|
|
.filter((task) => task.agentId === mainAgentId)
|
|
.map((task) => task.sessionId),
|
|
).size,
|
|
repositoryContextSourceCount: sourcePaths.length,
|
|
projectSkillCatalogEntryCount: sourcePaths.filter((entry) =>
|
|
entry.endsWith('/SKILL.md'),
|
|
).length,
|
|
matchingSkillReadCount: matchingSkillReads.length,
|
|
unrelatedSkillReadCount: unrelatedSkillReads.length,
|
|
skillReadBeforeMutation:
|
|
matchingSkillReads.length > 0 &&
|
|
Boolean(mutation) &&
|
|
matchingSkillReads.some(
|
|
(record) => agentDb.indexOf(record) < mutation.startIndex,
|
|
),
|
|
changedProjectFileCount: changedPaths.length,
|
|
mutationActionCount: mutation ? 1 : 0,
|
|
confirmedActionCount: state.projectSkill.confirmedActionCount,
|
|
verificationPassed,
|
|
successfulToolExecutionCount: successfulExecutions.length,
|
|
nativeRuntimeToolPlanCount: toolPlanProtocols.filter(
|
|
(record) => record.protocol === 'native_runtime_tools',
|
|
).length,
|
|
...toolPlanRepairEvidence,
|
|
wrapperToolPlanFallbackCount: allToolPlanProtocolAudits.filter(
|
|
(record) => record.protocol === 'native_function',
|
|
).length,
|
|
textJsonToolPlanFallbackCount: allToolPlanProtocolAudits.filter(
|
|
(record) => record.protocol === 'text_json',
|
|
).length,
|
|
providerRequestIdentityCount: new Set(
|
|
lifecycle.map((record) => record.requestId).filter(Boolean),
|
|
).size,
|
|
providerLifecycleStartedCount: startedLifecycle.length,
|
|
providerLifecycleTerminalCount: lifecycle.filter(
|
|
(record) => record.status === 'completed',
|
|
).length,
|
|
toolPlanProviderRequestCount: startedLifecycle.filter(
|
|
(record) => record.requestKind === 'tool-plan',
|
|
).length,
|
|
finalReplyProviderRequestCount: startedLifecycle.filter(
|
|
(record) => record.requestKind === 'final-reply',
|
|
).length,
|
|
deliveryContentMatched: content === projectSkillExpectedContent(),
|
|
finalAssistantCount: conversations.filter(
|
|
(message) => message.role === 'assistant',
|
|
).length,
|
|
};
|
|
}
|
|
|
|
export function emptyProjectSkillEvidence() {
|
|
return {
|
|
scenario: 'metadata-triggered-project-skill-progressive-load',
|
|
targetAgentId: mainAgentId,
|
|
providerModel: null,
|
|
providerApiKind: null,
|
|
isolatedAppDataUsed: false,
|
|
formalConfigCliCallCount: 0,
|
|
sourceRunnerEndpointUnchanged: false,
|
|
sourceConfigReplicaCount: 0,
|
|
sourceConfigReplicasVerified: false,
|
|
initialVerificationFailed: false,
|
|
taskCount: 0,
|
|
eventCount: 0,
|
|
agentDbRecordCount: 0,
|
|
conversationMessageCount: 0,
|
|
targetRunCount: 0,
|
|
stableSessionCount: 0,
|
|
repositoryContextSourceCount: 0,
|
|
projectSkillCatalogEntryCount: 0,
|
|
matchingSkillReadCount: 0,
|
|
unrelatedSkillReadCount: 0,
|
|
skillReadBeforeMutation: false,
|
|
changedProjectFileCount: 0,
|
|
mutationActionCount: 0,
|
|
confirmedActionCount: 0,
|
|
verificationPassed: false,
|
|
hostVerificationPassed: false,
|
|
successfulToolExecutionCount: 0,
|
|
toolPlanProtocolCount: 0,
|
|
nativeRuntimeToolPlanCount: 0,
|
|
toolPlanRepairCount: 0,
|
|
nativeRuntimeToolPlanRepairCount: 0,
|
|
toolPlanRepairedLoopCount: 0,
|
|
toolPlanSecondRepairCount: 0,
|
|
toolPlanRepairCountsByProtocolErrorKind:
|
|
emptyToolPlanRepairCountsByProtocolErrorKind(),
|
|
wrapperToolPlanFallbackCount: 0,
|
|
textJsonToolPlanFallbackCount: 0,
|
|
toolPlanAuditPayloadLeakCount: 0,
|
|
providerRequestIdentityCount: 0,
|
|
providerLifecycleStartedCount: 0,
|
|
providerLifecycleTerminalCount: 0,
|
|
toolPlanProviderRequestCount: 0,
|
|
finalReplyProviderRequestCount: 0,
|
|
providerFallbackReplayCount: 0,
|
|
finalAssistantCount: 0,
|
|
completedAuditCount: 0,
|
|
duplicateMessageCount: 0,
|
|
duplicateReceiptCount: 0,
|
|
finalizationJournalCount: 0,
|
|
assistantSkillBodyLeakCount: 0,
|
|
skillBodyPublicLeakCount: 0,
|
|
apiKeyPublicLeakCount: 0,
|
|
projectPathPublicLeakCount: 0,
|
|
projectPathPublicSurfaceCount: 0,
|
|
projectSkillReportLeakCount: 0,
|
|
projectSkillRunnerKillMethod: null,
|
|
projectSkillRunnerPidfdClaimCount: 0,
|
|
projectSkillRunnerPidfdSignalCount: 0,
|
|
projectSkillRunnerStopped: false,
|
|
projectSkillAppDataCleanupPerformed: false,
|
|
secretLeakCount: 0,
|
|
lureLeakCount: 0,
|
|
paths: [],
|
|
};
|
|
}
|
|
|
|
export function isProjectSkillSuite() {
|
|
return state.suite === projectSkillSuite;
|
|
}
|