清理提示词脆弱测试并修复分片失败日志

删除绑定提示词措辞的断言和测试专用的重复 Direct 回合编排
通过生产函数验证消息转发、文件登记和版本幂等行为
保留参数修复和上下文压缩测试并更新提示词来源校验
保留分片失败详情并区分选中用例数与失败数,补充脚本测试
同步测试边界和定向验证文档
This commit is contained in:
2026-09-20 17:25:57 +08:00
parent 4358a3f7e0
commit d69668e078
9 changed files with 169 additions and 264 deletions
@@ -312,34 +312,15 @@ function runShard(executable, shardIndex, shardCount, shardTestNames) {
},
);
const failureLines = [];
let inFailureList = false;
// Rust 的首个 failures: 后有空行,不能按空行结束采集,否则会丢掉 panic 详情。
// 只保存有界尾部;成功时不输出,失败时优先输出完整失败段。
let stdoutTail = '';
let stderr = '';
const consumeLine = (rawLine) => {
const line = rawLine.replace(/\r$/, '');
if (line.includes('failures:')) {
inFailureList = true;
return;
}
if (inFailureList) {
if (line.trim().length === 0) {
inFailureList = false;
return;
}
failureLines.push(line.trim());
}
};
child.stdout.setEncoding('utf8');
child.stderr.setEncoding('utf8');
let stdoutBuffer = '';
child.stdout.on('data', (chunk) => {
stdoutBuffer += chunk;
const lines = stdoutBuffer.split('\n');
stdoutBuffer = lines.pop() ?? '';
for (const line of lines) {
consumeLine(line);
}
stdoutTail = (stdoutTail + chunk).slice(-64_000);
});
child.stderr.on('data', (chunk) => {
stderr += chunk;
@@ -356,12 +337,17 @@ function runShard(executable, shardIndex, shardCount, shardTestNames) {
});
});
child.on('close', (code) => {
const output = stdoutTail.replace(/\r\n/g, '\n');
const failureStart = output.indexOf('failures:\n');
resolve({
label,
ok: code === 0,
durationMs: Date.now() - startedAt,
testCount: shardTestNames.length,
failures: failureLines,
failures: output
.slice(failureStart < 0 ? 0 : failureStart)
.trim()
.split('\n'),
stderr,
});
});
@@ -438,7 +424,7 @@ async function main() {
}
failed = true;
console.error(
`[rust-shards] ${result.label} FAILED: ${result.testCount} test(s) in ${formatDuration(result.durationMs)}`,
`[rust-shards] ${result.label} FAILED (selected ${result.testCount} test(s)) in ${formatDuration(result.durationMs)}`,
);
for (const failure of result.failures) {
console.error(`[rust-shards] ${failure}`);
@@ -0,0 +1,70 @@
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { fileURLToPath } from 'node:url';
const runner = fileURLToPath(
new URL('./run-rust-shell-test-shards.mjs', import.meta.url),
);
function runFixture(t, source) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agc-shard-output-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
fs.mkdirSync(path.join(root, 'src'));
fs.writeFileSync(
path.join(root, 'Cargo.toml'),
'[package]\nname = "shard-output-fixture"\nversion = "0.1.0"\nedition = "2021"\n',
);
fs.writeFileSync(path.join(root, 'src/lib.rs'), source);
const result = spawnSync(
process.execPath,
[
runner,
`--manifest=${path.join(root, 'Cargo.toml')}`,
'--target-kind=lib',
'--no-locked',
'--shards=1',
`--shard-tmp-root=${path.join(root, 'tmp')}`,
],
{
encoding: 'utf8',
timeout: 60_000,
windowsHide: true,
env: { ...process.env, CARGO_TARGET_DIR: path.join(root, 'target') },
},
);
assert.ifError(result.error);
return { status: result.status, output: result.stdout + result.stderr };
}
test('failed shard retains panic details and separates selected count from failures', (t) => {
const result = runFixture(
t,
`
#[test]
fn passing_case() {}
#[test]
fn failing_case() {
assert_eq!(1, 2, "shard panic evidence");
}
`,
);
assert.equal(result.status, 1, result.output);
assert.match(result.output, /FAILED \(selected 2 test\(s\)\)/);
assert.match(result.output, /failing_case/);
assert.match(result.output, /panicked at src[\\/]lib\.rs:/);
assert.match(result.output, /shard panic evidence/);
assert.match(result.output, /left: 1/);
assert.match(result.output, /right: 2/);
assert.match(result.output, /1 passed; 1 failed/);
});
test('successful shard keeps its compact summary', (t) => {
const result = runFixture(t, '#[test]\nfn passing_case() {}\n');
assert.equal(result.status, 0, result.output);
assert.match(result.output, /shard 1\/1 ok: 1 test\(s\)/);
assert.doesNotMatch(result.output, /test passing_case \.\.\. ok/);
});
@@ -5225,10 +5225,17 @@ mod tests {
#[test]
fn direct_codex_turn_keeps_system_prompt_out_of_user_input() {
let request = LlmRunRequest::single_turn("AGC 系统规则", "制作一个可玩的游戏");
assert_eq!(direct_codex_base_instructions(&request), "AGC 系统规则");
assert_eq!(direct_codex_user_prompt(&request), "制作一个可玩的游戏");
assert!(!direct_codex_user_prompt(&request).contains("AGC 系统规则"));
let system =
crate::agent::direct_runtime::build_direct_codex_system_prompt_with_creation_type(
Path::new("."),
Some("art"),
)
.expect("creation context");
for prompt in ["你好", "今天多少号?", "画一个橙色陶罐角色"] {
let request = LlmRunRequest::single_turn(system.clone(), prompt);
assert_eq!(direct_codex_base_instructions(&request), system);
assert_eq!(direct_codex_user_prompt(&request), prompt);
}
}
#[tokio::test]
@@ -5283,67 +5283,6 @@ mod direct_turn_stream_writer_tests {
}
}
/// Default product path: one user message becomes one turn on the same
/// project-bound Codex app-server thread. The client does not classify the
/// intent or perform hidden art, preview, repair, or another LLM workflow. If
/// Codex actually changes the canonical game files, the client performs only
/// deterministic resource/version projection so its own workspace reflects
/// the files now on disk.
async fn run_direct_game_creator_turn_with<F, Fut>(
root: &Path,
prompt: &str,
run_turn: F,
) -> Result<String, DirectCodexTurnFailure>
where
F: FnOnce(String, String) -> Fut,
Fut: Future<Output = Result<String, String>>,
{
run_direct_game_creator_turn_with_creation_type(root, prompt, None, run_turn).await
}
async fn run_direct_game_creator_turn_with_creation_type<F, Fut>(
root: &Path,
prompt: &str,
creation_type: Option<&str>,
run_turn: F,
) -> Result<String, DirectCodexTurnFailure>
where
F: FnOnce(String, String) -> Fut,
Fut: Future<Output = Result<String, String>>,
{
emit_direct_game_creator_progress(root, "codex.turn", "陶泥儿正在处理这条消息");
let previous_output_fingerprint = direct_codex_output_fingerprint(root);
let base_system_prompt =
build_direct_codex_system_prompt_with_creation_type(root, creation_type).map_err(
|error| DirectCodexTurnFailure::new(DirectCodexFailureStage::CodeGeneration, error),
)?;
let engine_contract = direct_engine_three_dimensional_contract(root, prompt);
let system_prompt = match engine_contract.as_deref() {
Some(contract) => format!("{contract}\n{base_system_prompt}")
.chars()
.take(MAX_DIRECT_SYSTEM_PROMPT_CHARS)
.collect(),
None => base_system_prompt,
};
let reply = run_turn(system_prompt, prompt.to_string())
.await
.map_err(|error| {
DirectCodexTurnFailure::new(DirectCodexFailureStage::CodeGeneration, error)
})?;
if direct_codex_output_fingerprint(root) != previous_output_fingerprint {
emit_direct_game_creator_progress(
root,
"project.sync",
"检测到游戏文件更新,正在同步客户端资源",
);
sync_direct_codex_project_file_projection_at(root, Some(&previous_output_fingerprint))
.map_err(|error| {
DirectCodexTurnFailure::new(DirectCodexFailureStage::VersionRegistration, error)
})?;
}
Ok(reply)
}
#[allow(dead_code)]
async fn run_direct_game_creator_turn_with_private_editor_credentials(
root: &Path,
@@ -6442,159 +6381,27 @@ mod tests {
.is_err());
}
#[tokio::test]
async fn direct_creation_type_keeps_the_original_user_prompt_unchanged() {
let root = tempfile::tempdir().expect("temp dir");
init_local_game_project_at(root.path(), "direct-art-context", "素材上下文测试")
.expect("init project");
let reply = run_direct_game_creator_turn_with_creation_type(
root.path(),
"画一个橙色陶罐角色",
Some("art"),
|system, prompt| async move {
assert!(system.contains("art / 做素材"));
assert_eq!(prompt, "画一个橙色陶罐角色");
assert!(!prompt.contains("初始意图"));
Ok("已理解素材需求。".to_string())
},
)
.await
.expect("direct creation type turn");
assert_eq!(reply, "已理解素材需求。");
}
#[tokio::test]
async fn default_direct_turn_forwards_a_greeting_without_client_generation_workflow() {
let root = tempfile::tempdir().expect("temp dir");
init_local_game_project_at(root.path(), "direct-chat-only", "纯对话测试")
.expect("init project");
std::fs::write(
root.path().join("game/index.html"),
"<!doctype html><title>before</title>",
)
.expect("index");
std::fs::write(root.path().join("game/style.css"), "body { color: black; }")
.expect("style");
std::fs::write(root.path().join("game/game.js"), "console.info('before');")
.expect("script");
let manifest_before =
std::fs::read(root.path().join(".agent/manifest.json")).expect("manifest before");
let index_before =
std::fs::read(root.path().join("game/index.html")).expect("index before");
let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let captured = std::sync::Arc::clone(&seen);
let reply =
run_direct_game_creator_turn_with(root.path(), "你好", move |system, prompt| {
let captured = std::sync::Arc::clone(&captured);
async move {
captured
.lock()
.expect("capture direct handoff")
.push((system, prompt));
Ok("你好!我是陶泥儿。".to_string())
}
})
.await
.expect("greeting reply");
assert_eq!(reply, "你好!我是陶泥儿。");
let calls = seen.lock().expect("read direct handoff");
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].1, "你好");
assert!(calls[0].0.contains("普通对话"));
assert!(calls[0].0.contains("不触碰工作区"));
assert_eq!(
std::fs::read(root.path().join("game/index.html")).expect("index after"),
index_before
);
assert_eq!(
std::fs::read(root.path().join(".agent/manifest.json")).expect("manifest after"),
manifest_before
);
assert!(
!root
.path()
.join(".agent/runtime/direct-codex-browser-validation")
.exists(),
"ordinary chat must not start client browser validation"
);
}
#[tokio::test]
async fn default_direct_turn_forwards_a_date_question_without_client_art_or_version_side_effects(
) {
let root = tempfile::tempdir().expect("temp dir");
init_local_game_project_at(root.path(), "direct-date-only", "日期对话测试")
.expect("init project");
let manifest_before =
std::fs::read(root.path().join(".agent/manifest.json")).expect("manifest before");
let reply = run_direct_game_creator_turn_with(
root.path(),
"今天多少号?",
|system, prompt| async move {
assert!(system.contains("普通对话"));
assert!(system.contains("不触碰工作区"));
assert_eq!(prompt, "今天多少号?");
Ok("今天是测试日期。".to_string())
},
)
.await
.expect("date reply");
assert_eq!(reply, "今天是测试日期。");
assert_eq!(
std::fs::read(root.path().join(".agent/manifest.json")).expect("manifest after"),
manifest_before
);
assert!(
!root.path().join("assets/art-spec.png").exists()
&& !root
.path()
.join("assets/direct-game-background.png")
.exists()
&& !root.path().join("assets/art-spritesheet.png").exists(),
"ordinary chat must not prepare a TaoNier art package"
);
}
#[tokio::test]
async fn default_direct_turn_projects_changed_game_files_without_supervisor_or_art() {
#[test]
fn direct_file_projection_registers_changes_without_art_and_preserves_unchanged_versions() {
let root = tempfile::tempdir().expect("temp dir");
init_local_game_project_at(root.path(), "direct-file-projection", "文件投影测试")
.expect("init project");
let write_root = root.path().to_path_buf();
let before = direct_codex_output_fingerprint(root.path());
let files = [
("game/index.html", "<!doctype html><canvas></canvas>"),
("game/style.css", "canvas { background: red; }"),
("game/game.js", "requestAnimationFrame(() => {});"),
];
for (path, content) in files {
std::fs::write(root.path().join(path), content).expect("game file");
}
let reply = run_direct_game_creator_turn_with(
root.path(),
"创建一个纯色方块游戏",
move |system, prompt| {
let write_root = write_root.clone();
async move {
assert!(system.contains("唯一执行主体"));
assert_eq!(prompt, "创建一个纯色方块游戏");
std::fs::write(
write_root.join("game/index.html"),
"<!doctype html><link rel=\"stylesheet\" href=\"style.css\"><canvas></canvas><script src=\"game.js\"></script>",
)
.expect("index");
std::fs::write(write_root.join("game/style.css"), "canvas { background: red; }")
.expect("style");
std::fs::write(write_root.join("game/game.js"), "requestAnimationFrame(() => {});")
.expect("script");
Ok("已写入三个游戏文件。".to_string())
}
},
)
.await
.expect("direct file projection");
sync_direct_codex_project_file_projection_at(root.path(), Some(&before))
.expect("project changed files");
assert_eq!(reply, "已写入三个游戏文件。");
let manifest =
read_manifest(&root.path().join(".agent/manifest.json")).expect("projected manifest");
for (local_path, kind, media_type) in direct_codex_game_outputs(&root.path()) {
for (local_path, kind, media_type) in direct_codex_game_outputs(root.path()) {
assert!(manifest.assets.iter().any(|asset| {
asset.local_path == local_path
&& asset.kind == kind
@@ -6611,6 +6418,28 @@ mod tests {
.map(|task| task.status.clone()),
Some(GameCreationAppTaskStatus::Completed)
);
let revision = read_game_creator_agent_runtime_project_revision(root.path())
.expect("project revision");
let unchanged = direct_codex_output_fingerprint(root.path());
sync_direct_codex_project_file_projection_at(root.path(), Some(&unchanged))
.expect("project unchanged files");
let after =
read_manifest(&root.path().join(".agent/manifest.json")).expect("unchanged manifest");
assert_eq!(after.assets, manifest.assets);
assert_eq!(after.versions, manifest.versions);
assert_eq!(
read_game_creator_agent_runtime_project_revision(root.path())
.expect("unchanged revision")
.revision,
revision.revision
);
for (path, content) in files {
assert_eq!(
std::fs::read_to_string(root.path().join(path))
.expect("game file after projection"),
content
);
}
assert!(!root.path().join("assets/art-spec.png").exists());
assert!(!root.path().join(".agent/runtime/runtimes").exists());
}
@@ -1338,16 +1338,25 @@ mod supervisor_collaboration_repair_tests {
#[test]
fn collaboration_repair_instruction_composes_the_generated_fragment() {
let protocol_error = "missing collaboration";
let fragment = required_runtime_prompt_section(
RUNTIME_PROMPT_PROVIDER_AUTONOMOUS_INITIAL_COLLABORATION_REPAIR_SECTION,
)
.trim();
let instruction = provider_collaboration_repair_instruction(
"missing collaboration",
protocol_error,
RUNTIME_PROMPT_PROVIDER_AUTONOMOUS_INITIAL_COLLABORATION_REPAIR_SECTION,
);
assert!(instruction.starts_with(
"上一条输出不符合工具计划协议:missing collaboration\n本片段适用于 GUI / CLI"
));
assert!(instruction.contains("本次修复原生工具目录"));
assert_eq!(instruction.matches("一次性建立完整首批合同").count(), 1);
assert_eq!(
instruction,
format!(
prompt_text!("recovery.collaboration_protocol"),
fragment,
protocol_error = protocol_error
)
);
assert_eq!(instruction.matches(fragment).count(), 1);
assert!(instruction.contains("design-director"));
assert!(instruction.contains("art-director"));
assert!(instruction.contains("code-director"));
@@ -798,25 +798,6 @@ fn provider_transient_retry_uses_configured_max_retries_for_every_run_profile()
fs::remove_dir_all(root).ok();
}
#[test]
fn autonomous_game_build_tool_plan_guidance_bounds_each_source_payload() {
let guidance = AGENT_RUNTIME_AUTONOMOUS_TOOL_PLAN_PAYLOAD_GUIDANCE;
for expected in [
"最多提交一个",
"不得超过 8000 字符",
"合计不得超过 10000 字符",
"可运行且保留扩展点的紧凑 scaffold",
"后续 planning 轮次",
"闭合的 <script> 和 </html>",
"不得在一个 function arguments 中输出完整大型游戏",
] {
assert!(
guidance.contains(expected),
"autonomous payload guidance missing {expected}"
);
}
}
#[test]
fn autonomous_game_build_tool_plan_payload_is_enforced_before_actions() {
let plan_with = |actions| AgentRuntimeToolPlan {
@@ -5403,7 +5384,6 @@ async fn background_agent_runtime_repairs_malformed_native_function_arguments()
assert!(initial_request.contains("必须直接调用当前请求提供的原生函数"));
assert!(!initial_request.contains("Legacy text JSON schema"));
assert!(initial_request.contains("arguments.input"));
assert!(initial_request.contains("禁止把 input 扁平到 arguments 顶层"));
assert!(initial_request.contains("必须调用 respond_to_user"));
let repair_request = receiver
.recv_timeout(Duration::from_secs(2))
@@ -210,9 +210,23 @@ async fn background_agent_runtime_auto_compacts_before_over_budget_planning() {
.recv_timeout(Duration::from_secs(4))
.expect("post-compaction planning Provider request");
assert!(receiver.recv_timeout(Duration::from_millis(200)).is_err());
assert!(compaction_request.contains("你负责压缩 Agent 的旧历史"));
assert!(compaction_request.contains("AUTO_CONTEXT_MARKER_0"));
let compaction_json = mock_http_request_json(&compaction_request);
let compaction_system = compaction_json["input"]
.as_array()
.expect("compaction responses input")
.iter()
.find(|message| message["role"] == "system")
.and_then(|message| message["content"].as_array())
.and_then(|content| content.iter().find(|part| part["type"] == "input_text"))
.and_then(|part| part["text"].as_str())
.expect("compaction system input text");
assert_eq!(
compaction_system,
prompt_text!("interaction.compaction_system")
);
assert!(compaction_json["input"]
.to_string()
.contains("AUTO_CONTEXT_MARKER_0"));
assert!(compaction_json
.get("tools")
.and_then(Value::as_array)
@@ -55,6 +55,10 @@
提示词外置变更运行 `runtime_prompt_bundle_build``prompt_source_boundaries` 两个 Rust 集成测试,验证编译期文本、目录登记和源码边界;现有 `agc-rust-shard-1` 本地/CI 入口先执行这组检查,再运行分片单测。
提示词测试验证实际请求中的片段来源、动态参数和工具结构;措辞不作为逐字契约。已有行为测试覆盖的限制不再另设整段文案检查。Direct 回合测试复用生产的消息转换和文件投影函数,不维护仅供测试调用的回合编排副本。
Rust 分片失败日志保留有界的失败详情,包括 panic 位置、断言和最终通过/失败数量;分片选中数量标为 selected,避免误读为失败数量。修改分片日志时运行 `node --test apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.test.mjs`,用最小 Rust fixture 验证失败详情和成功摘要。
AGC 运行时配置默认值调整时,同步核对 Rust 默认值、分发配置模板、设置弹窗默认草稿和 `runtime-settings.suite.ts` 的恢复默认断言;显式传入旧值的配置读取用例仍验证原值保留,不批量替换测试数据。
AGC 测试构造单 HTML 项目时,必须在初始化之前写入 HTML,避免自动建立 npm 工程;npm 预览和导出测试应提供 dist 产物。已有图片生成 pending/operation 属于持久化恢复合同,修改工具默认参数后仍须验证旧动作恢复不重复提交、不因默认值变化被误判为新意图。
@@ -73,6 +73,12 @@ Rust 侧在 `server-rs/crates/shared-contracts` 维护唯一权威 `GameCreation
客户端平台服务固定为 dev,登录页不提供服务器选择。凭据按 origin 隔离迁移;官网下载入口汇总最新渠道清单,自动展示已有首装包的 Windows/Mac 平台及架构。完整合同见 [AGC 客户端更新检查与下载](./【技术方案】AGC客户端更新检查与下载-2026-08-31.md) 的“官网下载与客户端服务地址”。
## Agent 提示词与回合测试边界
提示词拼装测试核对动态错误、外置片段和实际 Provider 请求结构,避免绑定可自由润色的中文原句。源码载荷限制由实际动作校验与 Provider 修复测试覆盖;工具参数自动修复和超预算上下文压缩继续验证恢复结果、状态及私有信息隔离。Direct 消息原样转发复用生产请求转换测试,文件变化与版本幂等直接测试生产文件投影函数,删除测试专用的重复回合编排。
Rust 分片日志在失败时输出有界 stdout 尾部中的失败段,保留 panic 位置、断言详情和 Rust 汇总;selected 只表示该分片选中的用例数量。成功分片保留简短摘要。定向验证使用相关 Rust 用例与 `node --test apps/ai-game-creator-shell/scripts/run-rust-shell-test-shards.test.mjs`
## 策划 Agent 批量局部修改
`patch_file` 的所有 edits 均匹配同一份原文件,参数顺序不影响结果。完成唯一匹配与不重叠校验后,按原文起点升序拼接未修改片段与替换文本,最后一次性写入;任一校验失败时不写文件。回归用例覆盖乱序 edits、中文内容与替换长度增减,并核对完整落盘内容。此行为仅属于策划 Agent 文件工具。