补齐自主可玩项目确定性门禁
- 新增确定性 OpenAI Chat Provider 与安全 E2E wrapper - 修复成功试玩后历史失败 observation 仍阻断收束的问题 - 覆盖塔防专业返工、双视口和 37 条试玩断言并同步文档
This commit is contained in:
AIGameCreator App
@@ -19,6 +19,8 @@
|
||||
"agent-runtime:mixed-swarm-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-static-isolated-autonomous-chat",
|
||||
"agent-runtime:supervisor-swarm-autonomous-chat-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-autonomous-chat",
|
||||
"agent-runtime:supervisor-autonomous-playable-lane-defense-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-autonomous-playable-lane-defense",
|
||||
"agent-runtime:supervisor-autonomous-playable-lane-defense-deterministic-e2e": "node scripts/agent-runtime-deterministic-playable-e2e.mjs",
|
||||
"agent-runtime:supervisor-autonomous-playable-lane-defense-deterministic-self-test": "node scripts/agent-runtime-deterministic-playable-e2e.mjs --self-test",
|
||||
"agent-runtime:supervisor-swarm-transient-retry-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-transient-retry",
|
||||
"agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-final-reply-transient-retry",
|
||||
"agent-runtime:supervisor-swarm-tool-plan-handoff-runner-kill-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-tool-plan-handoff-runner-kill",
|
||||
|
||||
@@ -0,0 +1,395 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import {
|
||||
createDeterministicLaneDefenseRouter,
|
||||
deterministicLaneDefenseInitialHtml,
|
||||
deterministicLaneDefenseModel,
|
||||
hiddenCanvasCss,
|
||||
startDeterministicLaneDefenseProvider,
|
||||
visibleCanvasCss,
|
||||
} from './deterministic-lane-defense-provider.mjs';
|
||||
import { withLoopbackNoProxy } from './llm-transient-fault-proxy.mjs';
|
||||
|
||||
const appRoot = path.resolve(fileURLToPath(new URL('..', import.meta.url)));
|
||||
const repoRoot = path.resolve(appRoot, '../..');
|
||||
const realE2eScript = path.join(appRoot, 'scripts/agent-runtime-real-e2e.mjs');
|
||||
const suite = 'supervisor-autonomous-playable-lane-defense';
|
||||
const wrapperSuite =
|
||||
'supervisor-autonomous-playable-lane-defense-deterministic';
|
||||
const configFileName = 'game-creator.config.json';
|
||||
const configSentinelName = '.deterministic-provider-e2e.json';
|
||||
const configSentinelSchema = 'genarrative-deterministic-provider-e2e-config.v1';
|
||||
const outputLimit = 32 * 1024 * 1024;
|
||||
|
||||
function hashValue(value) {
|
||||
return createHash('sha256').update(value).digest('hex');
|
||||
}
|
||||
|
||||
function assert(condition, code) {
|
||||
if (condition) return;
|
||||
const error = new Error(code);
|
||||
error.code = code;
|
||||
throw error;
|
||||
}
|
||||
|
||||
function parseArguments(args) {
|
||||
let keepProject = false;
|
||||
let selfTest = false;
|
||||
for (const arg of args) {
|
||||
if (arg === '--keep-project') keepProject = true;
|
||||
else if (arg === '--self-test') selfTest = true;
|
||||
else throw new Error('unknown-argument');
|
||||
}
|
||||
assert(!(keepProject && selfTest), 'self-test-keep-project-conflict');
|
||||
return { keepProject, selfTest };
|
||||
}
|
||||
|
||||
function appendBounded(current, chunk) {
|
||||
const combined = Buffer.concat([current, chunk]);
|
||||
if (combined.length > outputLimit) throw new Error('child-output-too-large');
|
||||
return combined;
|
||||
}
|
||||
|
||||
function runChild(args, environment) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(process.execPath, args, {
|
||||
cwd: appRoot,
|
||||
env: environment,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let stdout = Buffer.alloc(0);
|
||||
let stderr = Buffer.alloc(0);
|
||||
let outputError = null;
|
||||
child.stdout.on('data', (chunk) => {
|
||||
try {
|
||||
stdout = appendBounded(stdout, chunk);
|
||||
} catch (error) {
|
||||
outputError = error;
|
||||
child.kill('SIGTERM');
|
||||
}
|
||||
});
|
||||
child.stderr.on('data', (chunk) => {
|
||||
try {
|
||||
stderr = appendBounded(stderr, chunk);
|
||||
} catch (error) {
|
||||
outputError = error;
|
||||
child.kill('SIGTERM');
|
||||
}
|
||||
});
|
||||
child.once('error', (error) =>
|
||||
resolve({ code: null, signal: null, error, stdout, stderr }),
|
||||
);
|
||||
child.once('close', (code, signal) =>
|
||||
resolve({ code, signal, error: outputError, stdout, stderr }),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function safeChildDiagnostic(result) {
|
||||
return {
|
||||
exitCode: Number.isInteger(result.code) ? result.code : null,
|
||||
signal: typeof result.signal === 'string' ? result.signal : null,
|
||||
errorCode:
|
||||
typeof result.error?.code === 'string' ? result.error.code : null,
|
||||
stdoutBytes: result.stdout.length,
|
||||
stdoutSha256: hashValue(result.stdout),
|
||||
stderrBytes: result.stderr.length,
|
||||
stderrSha256: hashValue(result.stderr),
|
||||
};
|
||||
}
|
||||
|
||||
function parseChildReport(result) {
|
||||
try {
|
||||
const report = JSON.parse(result.stdout.toString('utf8'));
|
||||
assert(report && typeof report === 'object', 'child-report-root-invalid');
|
||||
return report;
|
||||
} catch (error) {
|
||||
if (error?.code) throw error;
|
||||
const wrapped = new Error('child-report-json-invalid');
|
||||
wrapped.code = 'child-report-json-invalid';
|
||||
throw wrapped;
|
||||
}
|
||||
}
|
||||
|
||||
function expectedProviderStats(stats) {
|
||||
return (
|
||||
stats.requestCount === 17 &&
|
||||
stats.planningRequestCount === 17 &&
|
||||
stats.finalReplyRequestCount === 0 &&
|
||||
stats.initialDelegationCount === 2 &&
|
||||
stats.followupDelegationCount === 1 &&
|
||||
stats.runStatusCount === 2 &&
|
||||
stats.sourceWriteCount === 1 &&
|
||||
stats.sourcePatchCount === 1 &&
|
||||
stats.staticSmokeCount === 4 &&
|
||||
stats.previewValidationCount === 2 &&
|
||||
stats.supervisorDirectMutationAttemptCount === 1 &&
|
||||
stats.unexpectedRequestCount === 0 &&
|
||||
Object.keys(stats.rejectionCodes ?? {}).length === 0 &&
|
||||
stats.byAgent?.['project-supervisor']?.planning === 9 &&
|
||||
stats.byAgent?.['project-supervisor']?.finalReply === 0 &&
|
||||
stats.byAgent?.['code-prototype']?.planning === 6 &&
|
||||
stats.byAgent?.['code-prototype']?.finalReply === 0 &&
|
||||
stats.byAgent?.['quality-review']?.planning === 2 &&
|
||||
stats.byAgent?.['quality-review']?.finalReply === 0
|
||||
);
|
||||
}
|
||||
|
||||
function syntheticPayload(agentId, runId, tools, extraContext = '') {
|
||||
return {
|
||||
model: deterministicLaneDefenseModel,
|
||||
stream: false,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: `- templateAgentId: ${agentId}\n- runId: ${runId}\n${extraContext}`,
|
||||
},
|
||||
],
|
||||
tools: tools.map((name) => ({ type: 'function', function: { name } })),
|
||||
};
|
||||
}
|
||||
|
||||
function responseFunctionNames(response) {
|
||||
return response.choices[0].message.tool_calls.map(
|
||||
(call) => call.function.name,
|
||||
);
|
||||
}
|
||||
|
||||
async function runSelfTest() {
|
||||
const html = deterministicLaneDefenseInitialHtml();
|
||||
assert([...html].length <= 8_000, 'self-test-html-source-budget-invalid');
|
||||
assert(
|
||||
html.includes(hiddenCanvasCss) &&
|
||||
!html.includes(visibleCanvasCss) &&
|
||||
html.includes('playable-web-game-state.v1') &&
|
||||
html.includes('data-playtest-id="next-level"') &&
|
||||
html.includes('Goal: defend the garden and win every wave.') &&
|
||||
html.includes('requestAnimationFrame'),
|
||||
'self-test-html-contract-invalid',
|
||||
);
|
||||
const apiKey = `deterministic-self-test-${randomUUID()}`;
|
||||
const router = createDeterministicLaneDefenseRouter({ apiKey });
|
||||
const allTools = [
|
||||
'update_agent_plan',
|
||||
'respond_to_user',
|
||||
'runtime_tool_agent_delegate',
|
||||
'runtime_tool_agent_run_status',
|
||||
'runtime_tool_command_run_limited',
|
||||
'runtime_tool_file_patch',
|
||||
'runtime_tool_file_write',
|
||||
'runtime_tool_preview_validate',
|
||||
];
|
||||
const route = (agentId, runId, tools = allTools, extraContext = '') =>
|
||||
router.route({
|
||||
authorization: `Bearer ${apiKey}`,
|
||||
payload: syntheticPayload(agentId, runId, tools, extraContext),
|
||||
});
|
||||
assert(
|
||||
responseFunctionNames(route('project-supervisor', 'parent-run')).join(
|
||||
',',
|
||||
) === 'runtime_tool_agent_delegate,runtime_tool_agent_delegate',
|
||||
'self-test-initial-delegation-invalid',
|
||||
);
|
||||
const qualityPlan =
|
||||
'计划进度：\n- #1 [in_progress] 核对完整玩法\n- #2 [pending] 回传结论\n工具策略：auto=无';
|
||||
const builderPlan =
|
||||
'计划进度：\n- #1 [completed] 生成入口\n- #2 [completed] 静态验证\n- #3 [in_progress] 回传结论\n工具策略：auto=无';
|
||||
route('quality-review', 'quality-run', allTools, qualityPlan);
|
||||
route('code-prototype', 'initial-code-run');
|
||||
route('code-prototype', 'initial-code-run');
|
||||
assert(
|
||||
responseFunctionNames(
|
||||
route('code-prototype', 'initial-code-run', allTools, builderPlan),
|
||||
).join(',') === 'update_agent_plan,respond_to_user',
|
||||
'self-test-builder-plan-completion-invalid',
|
||||
);
|
||||
assert(
|
||||
responseFunctionNames(
|
||||
route('quality-review', 'quality-run', allTools, qualityPlan),
|
||||
).join(',') === 'update_agent_plan,respond_to_user',
|
||||
'self-test-quality-stale-replan-invalid',
|
||||
);
|
||||
route('project-supervisor', 'parent-run');
|
||||
route('project-supervisor', 'parent-run');
|
||||
route('project-supervisor', 'parent-run');
|
||||
assert(
|
||||
responseFunctionNames(route('project-supervisor', 'parent-run'))[0] ===
|
||||
'runtime_tool_file_patch',
|
||||
'self-test-forbidden-parent-mutation-invalid',
|
||||
);
|
||||
assert(
|
||||
responseFunctionNames(
|
||||
route(
|
||||
'project-supervisor',
|
||||
'parent-run',
|
||||
['runtime_tool_agent_delegate'],
|
||||
'当前父 run 已进入只编排模式，本次修复的原生工具目录只保留 agent.delegate',
|
||||
),
|
||||
)[0] === 'runtime_tool_agent_delegate',
|
||||
'self-test-followup-delegation-invalid',
|
||||
);
|
||||
route('code-prototype', 'repair-code-run');
|
||||
route('code-prototype', 'repair-code-run');
|
||||
route('code-prototype', 'repair-code-run', allTools, builderPlan);
|
||||
assert(
|
||||
responseFunctionNames(route('project-supervisor', 'parent-run')).join(
|
||||
',',
|
||||
) === 'runtime_tool_command_run_limited,runtime_tool_preview_validate',
|
||||
'self-test-repair-verification-batch-invalid',
|
||||
);
|
||||
route('project-supervisor', 'parent-run');
|
||||
route('project-supervisor', 'parent-run');
|
||||
const stats = router.getStats();
|
||||
assert(expectedProviderStats(stats), 'self-test-provider-stats-invalid');
|
||||
|
||||
const rootPackage = JSON.parse(
|
||||
await fs.readFile(path.join(repoRoot, 'package.json'), 'utf8'),
|
||||
);
|
||||
const shellPackage = JSON.parse(
|
||||
await fs.readFile(path.join(appRoot, 'package.json'), 'utf8'),
|
||||
);
|
||||
assert(
|
||||
shellPackage.scripts?.[
|
||||
'agent-runtime:supervisor-autonomous-playable-lane-defense-deterministic-e2e'
|
||||
] === 'node scripts/agent-runtime-deterministic-playable-e2e.mjs' &&
|
||||
rootPackage.scripts?.[
|
||||
'ai-game-creator-shell:agent-runtime:supervisor-autonomous-playable-lane-defense-deterministic-e2e'
|
||||
] ===
|
||||
'npm --prefix apps/ai-game-creator-shell run agent-runtime:supervisor-autonomous-playable-lane-defense-deterministic-e2e --',
|
||||
'self-test-package-command-invalid',
|
||||
);
|
||||
return {
|
||||
status: 'PASS',
|
||||
suite: `${wrapperSuite}-self-test`,
|
||||
providerUsed: false,
|
||||
htmlChars: [...html].length,
|
||||
providerStats: stats,
|
||||
packageCommandsRegistered: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function runE2e(options) {
|
||||
const token = randomUUID();
|
||||
const apiKey = `deterministic-runtime-${randomUUID()}`;
|
||||
const configDir = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'genarrative-deterministic-provider-config-'),
|
||||
);
|
||||
let provider = null;
|
||||
let childResult = null;
|
||||
let childReport = null;
|
||||
let configRemoved = false;
|
||||
let providerStats = null;
|
||||
let failureCode = null;
|
||||
try {
|
||||
if (process.platform !== 'win32') await fs.chmod(configDir, 0o700);
|
||||
await fs.writeFile(
|
||||
path.join(configDir, configSentinelName),
|
||||
`${JSON.stringify({ schemaVersion: configSentinelSchema, token })}\n`,
|
||||
{ flag: 'wx', mode: 0o600 },
|
||||
);
|
||||
provider = await startDeterministicLaneDefenseProvider({ apiKey });
|
||||
const config = {
|
||||
llm: {
|
||||
apiKey,
|
||||
baseUrl: provider.baseUrl,
|
||||
model: deterministicLaneDefenseModel,
|
||||
apiKind: 'openai_chat',
|
||||
reasoningEffort: 'low',
|
||||
stream: false,
|
||||
requestTimeoutMs: 30_000,
|
||||
maxRetries: 0,
|
||||
retryBackoffMs: 100,
|
||||
},
|
||||
};
|
||||
await fs.writeFile(
|
||||
path.join(configDir, configFileName),
|
||||
`${JSON.stringify(config)}\n`,
|
||||
{ flag: 'wx', mode: 0o600 },
|
||||
);
|
||||
const childArgs = [
|
||||
realE2eScript,
|
||||
'--suite',
|
||||
suite,
|
||||
'--config-dir',
|
||||
configDir,
|
||||
];
|
||||
if (options.keepProject) childArgs.push('--keep-project');
|
||||
childResult = await runChild(
|
||||
childArgs,
|
||||
withLoopbackNoProxy({ ...process.env, NO_COLOR: '1' }),
|
||||
);
|
||||
childReport = parseChildReport(childResult);
|
||||
} catch (error) {
|
||||
failureCode = error?.code ?? 'deterministic-e2e-unexpected-error';
|
||||
} finally {
|
||||
if (provider) {
|
||||
try {
|
||||
await provider.stop();
|
||||
providerStats = provider.getStats();
|
||||
} catch {
|
||||
failureCode ??= 'deterministic-provider-stop-failed';
|
||||
}
|
||||
}
|
||||
try {
|
||||
const sentinel = JSON.parse(
|
||||
await fs.readFile(path.join(configDir, configSentinelName), 'utf8'),
|
||||
);
|
||||
assert(
|
||||
sentinel.schemaVersion === configSentinelSchema &&
|
||||
sentinel.token === token,
|
||||
'deterministic-config-sentinel-invalid',
|
||||
);
|
||||
await fs.rm(configDir, { recursive: true, force: false });
|
||||
configRemoved = true;
|
||||
} catch (error) {
|
||||
failureCode ??= error?.code ?? 'deterministic-config-cleanup-failed';
|
||||
}
|
||||
}
|
||||
|
||||
const childPassed =
|
||||
childResult?.code === 0 &&
|
||||
childResult?.signal === null &&
|
||||
!childResult?.error &&
|
||||
childReport?.status === 'PASS' &&
|
||||
childReport?.suite === suite &&
|
||||
childReport?.cleanup?.performed === !options.keepProject &&
|
||||
childReport?.cleanup?.kept === options.keepProject;
|
||||
const providerPassed =
|
||||
providerStats?.stopped === true && expectedProviderStats(providerStats);
|
||||
const status =
|
||||
!failureCode && childPassed && providerPassed && configRemoved
|
||||
? 'PASS'
|
||||
: 'FAIL';
|
||||
if (status !== 'PASS' && !failureCode) {
|
||||
failureCode = !childPassed
|
||||
? 'deterministic-child-e2e-failed'
|
||||
: !providerPassed
|
||||
? 'deterministic-provider-contract-failed'
|
||||
: 'deterministic-config-not-cleaned';
|
||||
}
|
||||
return {
|
||||
status,
|
||||
suite: wrapperSuite,
|
||||
providerMode: 'deterministic-loopback-openai-chat',
|
||||
delegatedSuite: suite,
|
||||
child: childReport,
|
||||
provider: providerStats,
|
||||
cleanup: {
|
||||
providerStopped: providerStats?.stopped === true,
|
||||
configRemoved,
|
||||
projectKept: options.keepProject,
|
||||
},
|
||||
childDiagnostic: childResult ? safeChildDiagnostic(childResult) : null,
|
||||
failureCode,
|
||||
};
|
||||
}
|
||||
|
||||
const options = parseArguments(process.argv.slice(2));
|
||||
const report = options.selfTest ? await runSelfTest() : await runE2e(options);
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
process.exitCode = report.status === 'PASS' ? 0 : 1;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17397,6 +17397,74 @@ mod autonomous_completion_contract_tests {
|
||||
.iter()
|
||||
.any(|message| message.role == "assistant" && message.content == response));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn autonomous_playtest_liveness_only_enforces_the_latest_preview_result() {
|
||||
let verification_gate = AgentRuntimeVerificationGate {
|
||||
schema_version: AGENT_RUNTIME_VERIFICATION_GATE_SCHEMA_VERSION.to_string(),
|
||||
project_id: "autonomous-playtest-liveness-project".to_string(),
|
||||
agent_id: GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID.to_string(),
|
||||
run_id: "autonomous-playtest-liveness-run".to_string(),
|
||||
requires_verification: false,
|
||||
mutation_revision: None,
|
||||
verified_revision: Some(2),
|
||||
last_mutation_tool: None,
|
||||
last_verification_tool: Some("game.static_smoke".to_string()),
|
||||
last_verification_status: Some(AGENT_RUNTIME_VERIFICATION_STATUS_PASSED.to_string()),
|
||||
failed_playtest_revision: None,
|
||||
updated_at: 0,
|
||||
};
|
||||
let failed_preview = AgentRuntimeToolObservation {
|
||||
tool: "preview.validate".to_string(),
|
||||
status: "failed".to_string(),
|
||||
summary: "浏览器验证未通过，请根据诊断修复后重试".to_string(),
|
||||
detail: None,
|
||||
};
|
||||
let passed_preview = AgentRuntimeToolObservation {
|
||||
tool: "preview.validate".to_string(),
|
||||
status: "ok".to_string(),
|
||||
summary: "浏览器验证已通过，已生成桌面与移动证据".to_string(),
|
||||
detail: None,
|
||||
};
|
||||
let non_progress = || AgentRuntimeToolObservation {
|
||||
tool: "memory.read".to_string(),
|
||||
status: "ok".to_string(),
|
||||
summary: "已读取记忆".to_string(),
|
||||
detail: None,
|
||||
};
|
||||
let response_plan = AgentRuntimeToolPlan {
|
||||
response: "项目已完成真实试玩验证。".to_string(),
|
||||
..AgentRuntimeToolPlan::default()
|
||||
};
|
||||
|
||||
let mut superseded_failure = vec![failed_preview.clone(), passed_preview.clone()];
|
||||
superseded_failure.extend((0..4).map(|_| non_progress()));
|
||||
assert!(validate_agent_runtime_autonomous_plan_liveness(
|
||||
GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID,
|
||||
7,
|
||||
2,
|
||||
&verification_gate,
|
||||
&superseded_failure,
|
||||
&response_plan,
|
||||
true,
|
||||
)
|
||||
.is_ok());
|
||||
|
||||
let mut latest_failure = vec![passed_preview, failed_preview];
|
||||
latest_failure.extend((0..4).map(|_| non_progress()));
|
||||
let error = validate_agent_runtime_autonomous_plan_liveness(
|
||||
GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID,
|
||||
7,
|
||||
2,
|
||||
&verification_gate,
|
||||
&latest_failure,
|
||||
&response_plan,
|
||||
true,
|
||||
)
|
||||
.expect_err("latest failed preview must still require delegated repair");
|
||||
assert!(error
|
||||
.starts_with(AGENT_RUNTIME_AUTONOMOUS_DELEGATED_PLAYTEST_REPAIR_LIVENESS_ERROR_PREFIX));
|
||||
}
|
||||
}
|
||||
|
||||
fn game_creator_agent_runtime_verification_gate_relative_path(
|
||||
@@ -28438,13 +28506,19 @@ fn validate_agent_runtime_autonomous_plan_liveness(
|
||||
));
|
||||
}
|
||||
}
|
||||
let Some(failed_playtest_index) = observations.iter().rposition(|observation| {
|
||||
observation.tool == "preview.validate"
|
||||
&& observation.status == "failed"
|
||||
&& observation.summary == "浏览器验证未通过，请根据诊断修复后重试"
|
||||
}) else {
|
||||
let Some(latest_playtest_index) = observations
|
||||
.iter()
|
||||
.rposition(|observation| observation.tool == "preview.validate")
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let latest_playtest = &observations[latest_playtest_index];
|
||||
if latest_playtest.status != "failed"
|
||||
|| latest_playtest.summary != "浏览器验证未通过，请根据诊断修复后重试"
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let failed_playtest_index = latest_playtest_index;
|
||||
let observations_after_failure = &observations[failed_playtest_index + 1..];
|
||||
if observations_after_failure
|
||||
.iter()
|
||||
|
||||
Reference in New Issue
Block a user